diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 2aa47b1ae932..845f49da5fd4 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1210,6 +1210,35 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_bitwise_identities_tests EXCLUDE_FROM_ALL tests/BitwiseIdentities.cpp) +target_link_libraries(agc_bitwise_identities_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_bitwise_identities_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_bitwise_identities_tests) +if(BUILD_TESTING) + set_target_properties(agc_bitwise_identities_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_bitwise_identities COMMAND agc_bitwise_identities_tests) + if(WIN32) + set_tests_properties(agc_bitwise_identities PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + +add_test_executable(agc_single_lane_tests EXCLUDE_FROM_ALL tests/SingleLane.cpp) +target_link_libraries(agc_single_lane_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_single_lane_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_single_lane_tests) +if(BUILD_TESTING) + set_target_properties(agc_single_lane_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_single_lane COMMAND agc_single_lane_tests) + add_test(NAME agc_single_lane_off COMMAND agc_single_lane_tests) + set_tests_properties(agc_single_lane PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1") + set_tests_properties(agc_single_lane_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_SINGLE_LANE_RULE=1") + if(WIN32) + set_tests_properties(agc_single_lane agc_single_lane_off PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp b/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp new file mode 100644 index 000000000000..cd5b75c5f383 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp @@ -0,0 +1,210 @@ +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "Optimization/ConstantFolder.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +constexpr IrOpcode And = IrOpcode::BitwiseAnd32; +constexpr IrOpcode Or = IrOpcode::BitwiseOr32; +constexpr IrOpcode Xor = IrOpcode::BitwiseXor32; +constexpr IrOpcode Shift = IrOpcode::ShiftRightLogical32; + +struct Builder { + IrProgram program; + IrBuilder ir; + IrBlock* block = nullptr; + std::vector kept; + + Builder() : ir(program) { + block = &program.CreateBlock(); + program.SetEntryBlock(*block); + program.BlockOrder().push_back(block); + ir.SetInsertionPoint(*block); + } + + IrValue& Emit(IrOpcode opcode, std::initializer_list arguments) { + return ir.Emit(opcode, IrType::U32, arguments); + } + + IrValue& Value() { + return Emit(IrOpcode::LaneId, {}); + } + + IrValue& Other(IrValue& value) { + return Emit(IrOpcode::IAdd32, {&value, &ir.Constant(1u)}); + } + + IrValue& Not(IrValue& value) { + return Emit(IrOpcode::BitwiseNot32, {&value}); + } + + IrValue& Same(IrValue& value) { + auto& identity = program.CreateValue(IrOpcode::Identity, IrType::U32); + identity.AddArgument(&value); + block->AppendInstruction(&identity); + return identity; + } + + IrValue& Binary(IrOpcode opcode, IrValue& lhs, IrValue& rhs) { + return Emit(opcode, {&lhs, &rhs}); + } + + IrValue& Binary(IrOpcode opcode, IrValue& lhs, std::uint32_t rhs) { + return Binary(opcode, lhs, ir.Constant(rhs)); + } + + IrValue& Binary(IrOpcode opcode, std::uint32_t lhs, IrValue& rhs) { + return Binary(opcode, ir.Constant(lhs), rhs); + } + + std::size_t Keep(IrValue& value) { + kept.push_back(&ir.Emit(IrOpcode::ReferenceU32, IrType::Void, {&value})); + return kept.size() - 1u; + } + + void Fold() { + static_cast(ir.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder().Fold(program); + } + + const IrValue& Result(std::size_t index) const { + return *kept.at(index)->Argument(0)->Resolve(); + } + + bool Folds(std::size_t index, std::uint32_t expected) const { + const IrValue& value = Result(index); + return value.HasImmediate() && value.Type() == IrType::U32 && value.ImmediateU32() == expected; + } + + bool Becomes(std::size_t index, const IrValue& expected) const { + return &Result(index) == &expected; + } + + bool Keeps(std::size_t index, IrOpcode opcode) const { + return Result(index).Opcode() == opcode; + } +}; + +bool expect(const char* name, bool condition) { + if (!condition) std::fprintf(stderr, "%s\n", name); + return condition; +} + +bool run(const char* name, const std::function& test) { + try { + return expect(name, test()); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("and and or of a value with itself give the value and xor gives zero", [] { + Builder b; + auto& value = b.Value(); + const auto both = b.Keep(b.Binary(And, value, value)); + const auto either = b.Keep(b.Binary(Or, value, value)); + const auto differ = b.Keep(b.Binary(Xor, value, value)); + b.Fold(); + return b.Becomes(both, value) && b.Becomes(either, value) && b.Folds(differ, 0u); + }); + + passed &= run("a value and its complement give zero for and and all ones for or and xor in either order", [] { + Builder b; + auto& value = b.Value(); + auto& inverse = b.Not(value); + const auto andLeft = b.Keep(b.Binary(And, value, inverse)); + const auto andRight = b.Keep(b.Binary(And, inverse, value)); + const auto orLeft = b.Keep(b.Binary(Or, value, inverse)); + const auto orRight = b.Keep(b.Binary(Or, inverse, value)); + const auto xorLeft = b.Keep(b.Binary(Xor, value, inverse)); + const auto xorRight = b.Keep(b.Binary(Xor, inverse, value)); + b.Fold(); + return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu) && b.Folds(xorLeft, 0xffffffffu) && b.Folds(xorRight, 0xffffffffu); + }); + + passed &= run("and with zero and or with all ones absorb the other operand in either order", [] { + Builder b; + auto& value = b.Value(); + const auto andLeft = b.Keep(b.Binary(And, value, 0u)); + const auto andRight = b.Keep(b.Binary(And, 0u, value)); + const auto orLeft = b.Keep(b.Binary(Or, value, 0xffffffffu)); + const auto orRight = b.Keep(b.Binary(Or, 0xffffffffu, value)); + b.Fold(); + return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu); + }); + + passed &= run("operands match through identities", [] { + Builder b; + auto& value = b.Value(); + auto& inverse = b.Same(b.Not(b.Same(value))); + const auto both = b.Keep(b.Binary(And, value, b.Same(value))); + const auto differ = b.Keep(b.Binary(Xor, b.Same(value), value)); + const auto complement = b.Keep(b.Binary(Or, inverse, value)); + b.Fold(); + return b.Becomes(both, value) && b.Folds(differ, 0u) && b.Folds(complement, 0xffffffffu); + }); + + passed &= run("different values, the complement of another value and xor with all ones are kept", [] { + Builder b; + auto& value = b.Value(); + auto& other = b.Other(value); + auto& inverse = b.Not(other); + const auto both = b.Keep(b.Binary(And, value, other)); + const auto either = b.Keep(b.Binary(Or, value, other)); + const auto differ = b.Keep(b.Binary(Xor, value, other)); + const auto andInverse = b.Keep(b.Binary(And, value, inverse)); + const auto orInverse = b.Keep(b.Binary(Or, inverse, value)); + const auto xorInverse = b.Keep(b.Binary(Xor, value, inverse)); + const auto flipped = b.Keep(b.Binary(Xor, value, 0xffffffffu)); + b.Fold(); + return b.Keeps(both, And) && b.Keeps(either, Or) && b.Keeps(differ, Xor) && b.Keeps(andInverse, And) && b.Keeps(orInverse, Or) && b.Keeps(xorInverse, Xor) && b.Keeps(flipped, Xor); + }); + + passed &= run("bit 0 of all ones shifted right by an amount masked to 31 is one in either order", [] { + Builder b; + auto& lane = b.Value(); + auto& maskRight = b.Binary(And, lane, 31u); + auto& maskLeft = b.Binary(And, 31u, lane); + const auto bitRight = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, maskRight), 1u)); + const auto bitLeft = b.Keep(b.Binary(And, 1u, b.Binary(Shift, 0xffffffffu, maskLeft))); + b.Fold(); + return b.Folds(bitRight, 1u) && b.Folds(bitLeft, 1u); + }); + + passed &= run("bit 0 of an unmasked or wider shift, of another word, and bit 1 are kept", [] { + Builder b; + auto& lane = b.Value(); + auto& masked = b.Binary(And, lane, 31u); + const auto unmasked = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, lane), 1u)); + const auto wider = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, b.Binary(And, lane, 63u)), 1u)); + const auto word = b.Keep(b.Binary(And, b.Binary(Shift, 0x7fffffffu, masked), 1u)); + const auto second = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, masked), 2u)); + b.Fold(); + return b.Keeps(unmasked, And) && b.Keeps(wider, And) && b.Keeps(word, And) && b.Keeps(second, And); + }); + + passed &= run("zero shifted right by any amount is zero and another word is kept", [] { + Builder b; + auto& lane = b.Value(); + const auto zero = b.Keep(b.Binary(Shift, 0u, lane)); + const auto word = b.Keep(b.Binary(Shift, b.Other(lane), lane)); + b.Fold(); + return b.Folds(zero, 0u) && b.Keeps(word, Shift); + }); + + return passed ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 3efcc4e97acf..9cc080349dc8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -5428,6 +5428,18 @@ void verifyHalfWaveReduction() { expectFailure([&] { static_cast(pixel(upperHelperLoop, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: the low-word loop proof was reused for helper-inclusive upper lanes"); require(reductions(pixel(loopScan, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: a loop-carried saved EXEC mask exhausted the entry-lane proof on 32 lanes"); require(halves(pixel(loopScan, 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: a loop-carried saved EXEC mask did not preserve the two half-wave UMin reductions"); + auto computeScan = scan(0xffffffffu, 0x26000000u, 0x83800000u); + const auto exported = std::find(computeScan.begin(), computeScan.end(), 0xf800180fu); + exported[0] = 0xe0700000u; + exported[1] = 0x80000100u; + static constexpr std::array computeBuffer{0x10000000u, 0u, 0x40u, 0x00027facu}; + static constexpr std::array computeCapabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic}; + auto computeRequest = FrontendRequest(computeScan, computeBuffer); + computeRequest.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + computeRequest.target.supportedCapabilities = computeCapabilities; + const auto twoLane = Recompile(computeRequest); + require(twoLane.hostSubgroupSize == 32u && reductions(twoLane.spirv, spv::OpGroupNonUniformUMin) == 0u, "half-wave reduction: a two-lane wave64 compute UMin scan under s_or_saveexec_b64 -1 reached the half-wave scan matcher"); + require(halves(pixel(scan(0xffffffffu, 0x26000000u, 0x83800000u), 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: the wave64 UMin scan under s_or_saveexec_b64 -1 did not read lanes 31 and 63 as two half-wave UMin reductions on 64 lanes"); auto comparedScan = scan(0u, 0x38000000u, 0x88000000u); *std::find(comparedScan.begin(), comparedScan.end(), 0xbe98047eu) = 0x8a986a7eu; comparedScan.insert(comparedScan.begin(), 0x7c080080u); @@ -9315,8 +9327,10 @@ int main(int argc, char** argv) { try { #ifdef _WIN32 _putenv_s("ANYPS5_NO_SHADER_CACHE", "1"); + _putenv_s("APS5_NO_SINGLE_LANE_RULE", "1"); #else setenv("ANYPS5_NO_SHADER_CACHE", "1", 1); + setenv("APS5_NO_SINGLE_LANE_RULE", "1", 1); #endif using namespace ShaderRecompiler; if (argc == 2 && std::string_view(argv[1]) == "--specialized-bda-abi") { diff --git a/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp b/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp new file mode 100644 index 000000000000..96e4cb400925 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp @@ -0,0 +1,643 @@ +#include "IntermediateRepresentation/IrProgram.hpp" +#include "IntermediateRepresentation/IrMetadata/StageIO.hpp" +#include "Optimization/ConstantFolder.hpp" +#include "Optimization/SingleLaneAnalysis.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +struct Builder { + IrProgram program; + IrBlock* block = nullptr; + ShaderWorkgroupInputInfo workgroup; + + explicit Builder(std::uint32_t width = 16u, std::uint32_t height = 16u) { + block = &program.CreateBlock(); + program.BlockOrder().push_back(block); + program.SetEntryBlock(*block); + BlockInfo info; + info.id = 0; + program.Metadata().blockInfo.push_back(info); + program.Resources().stage = IrShaderStage::Compute; + program.SetWaveSize(64u); + workgroup.threadsNum[0] = width; + workgroup.threadsNum[1] = height; + workgroup.threadsNum[2] = 1u; + } + + IrValue& constant(std::uint32_t value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::U32); + created.SetImmediateU32(value); + return created; + } + + IrValue& boolean(bool value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::Bool); + created.SetImmediateBool(value); + return created; + } + + IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list arguments) { + auto& created = program.CreateValue(opcode, type); + for (auto* argument : arguments) created.AddArgument(argument); + block->AppendInstruction(&created); + return created; + } + + IrValue& binary(IrOpcode opcode, IrValue& left, IrValue& right) { + return emit(opcode, IrType::U32, {&left, &right}); + } + + IrValue& lane() { + return emit(IrOpcode::LaneId, IrType::U32, {}); + } + + IrValue& localId(std::uint32_t axis) { + return emit(IrOpcode::GetBuiltin, IrType::U32, {&constant(static_cast(StageInputKind::LocalInvocationId)), &constant(axis)}); + } + + IrValue& userData() { + return emit(IrOpcode::GetUserData, IrType::U32, {}); + } + + IrValue& laneDword() { + auto& row = binary(IrOpcode::IMul32, constant(16u), emit(IrOpcode::BitFieldUExtract, IrType::U32, {&localId(1), &constant(0u), &constant(24u)})); + return binary(IrOpcode::ShiftLeftLogical32, binary(IrOpcode::IAdd32, row, localId(0)), constant(2u)); + } + + IrValue& ballot(IrValue& predicate, std::uint32_t half) { + auto& mask = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + return emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&mask, &constant(half)}); + } + + IrValue& threadBit(IrValue& low, IrValue& high) { + auto& laneId = lane(); + auto& word = emit(IrOpcode::SelectU32, IrType::U32, {&emit(IrOpcode::ULessThan32, IrType::Bool, {&laneId, &constant(32u)}), &low, &high}); + auto& shift = binary(IrOpcode::BitwiseAnd32, laneId, constant(31u)); + auto& bit = binary(IrOpcode::BitwiseAnd32, binary(IrOpcode::ShiftRightLogical32, word, shift), constant(1u)); + return emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &constant(0u)}); + } + + IrValue& lds(IrOpcode opcode, IrType type, std::initializer_list arguments, std::uint32_t offset) { + MemoryInfo memory; + memory.kind = ResourceKind::Lds; + memory.offset = offset; + const auto index = static_cast(program.Resources().memoryInfo.size()); + program.Resources().memoryInfo.push_back(memory); + auto& created = emit(opcode, type, arguments); + created.SetFlags(MemoryFlags{index, 0u}); + return created; + } + + void keep(IrValue& value) { + (void)emit(value.Type() == IrType::Bool ? IrOpcode::Reference : IrOpcode::ReferenceU32, IrType::Void, {&value}); + } + + void branch(BranchCondition condition, IrValue& predicate, IrValue* exec) { + auto& info = program.Metadata().blockInfo.front(); + info.terminator.kind = TerminatorKind::ConditionalBranch; + info.terminator.condition = condition; + info.condition = &predicate; + info.exec = exec; + } + + IrBlock& addBlock() { + auto& created = program.CreateBlock(); + program.BlockOrder().push_back(&created); + BlockInfo info; + info.id = static_cast(program.Metadata().blockInfo.size()); + program.Metadata().blockInfo.push_back(info); + return created; + } + + void at(IrBlock& target) { + block = ⌖ + } + + BlockInfo& infoOf(const IrBlock& target) { + const auto& order = program.BlockOrder(); + return program.Metadata().blockInfo.at(static_cast(std::find(order.begin(), order.end(), &target) - order.begin())); + } + + void jump(IrBlock& from, IrBlock& to) { + auto& info = infoOf(from); + info.terminator.kind = TerminatorKind::Branch; + info.terminator.trueBlock = infoOf(to).id; + from.AddSuccessor(&to); + to.AddPredecessor(&from); + } + + void split(IrBlock& from, BranchCondition condition, IrValue& predicate, IrValue* exec, IrBlock& taken, IrBlock& otherwise) { + auto& info = infoOf(from); + info.terminator.kind = TerminatorKind::ConditionalBranch; + info.terminator.condition = condition; + info.terminator.trueBlock = infoOf(taken).id; + info.terminator.falseBlock = infoOf(otherwise).id; + info.condition = &predicate; + info.exec = exec; + for (IrBlock* target : {&taken, &otherwise}) { + from.AddSuccessor(target); + target->AddPredecessor(&from); + } + } + + IrValue& phi(IrType type, std::initializer_list> incoming) { + auto& created = program.CreateValue(IrOpcode::Phi, type); + for (const auto& [from, value] : incoming) created.AddPhiOperand(from, value); + block->AppendInstruction(&created); + return created; + } + + void fold() { + (void)emit(IrOpcode::Return, IrType::Void, {}); + ConstantFolder{}.Fold(program); + } + + SingleLaneVerdict analyze() const { + return AnalyzeSingleLane(program, workgroup); + } +}; + +bool eligible(const SingleLaneVerdict& verdict) { + if (!verdict.eligible) std::fprintf(stderr, " rejected: %s\n", verdict.reason); + return verdict.eligible; +} + +bool rejected(const SingleLaneVerdict& verdict, const char* reason) { + if (verdict.eligible) { + std::fprintf(stderr, " accepted, expected \"%s\"\n", reason); + return false; + } + if (std::strstr(verdict.reason, reason) != nullptr) return true; + std::fprintf(stderr, " rejected with \"%s\", expected \"%s\"\n", verdict.reason, reason); + return false; +} + +bool run(const char* name, const std::function& test) { + try { + if (test()) return true; + std::fprintf(stderr, "%s\n", name); + return false; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("a select on a lane's bit of ~exec | exec folds to its true arm", [] { + Builder b; + auto& low = b.ballot(b.boolean(true), 0u); + auto& high = b.ballot(b.boolean(true), 1u); + auto& allLow = b.binary(IrOpcode::BitwiseOr32, b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&low}), low); + auto& allHigh = b.binary(IrOpcode::BitwiseOr32, b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&high}), high); + auto& active = b.localId(0); + auto& selected = b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(allLow, allHigh), &active, &b.constant(0u)}); + b.keep(selected); + b.fold(); + return selected.Resolve() == &active; + }); + + passed &= run("a lane-private LDS array (tid * 4 + k * 1024) runs at one lane per invocation", [] { + Builder b; + auto& address = b.laneDword(); + auto& value = b.localId(0); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &value, &b.boolean(true)}, 0u); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &value, &b.boolean(true)}, 1024u); + b.keep(b.lds(IrOpcode::LoadSharedU32, IrType::U32, {&address, &b.boolean(true)}, 1024u)); + return eligible(b.analyze()); + }); + + passed &= run("a two-dimensional private LDS layout (4x + 1024y) runs at one lane", [] { + Builder b; + auto& address = b.binary(IrOpcode::IAdd32, b.binary(IrOpcode::ShiftLeftLogical32, b.localId(0), b.constant(2u)), b.binary(IrOpcode::ShiftLeftLogical32, b.localId(1), b.constant(10u))); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return eligible(b.analyze()); + }); + + passed &= run("lanes of one column sharing an LDS word keep two lanes", [] { + Builder b; + auto& address = b.binary(IrOpcode::ShiftLeftLogical32, b.localId(0), b.constant(2u)); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(1), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "share LDS"); + }); + + passed &= run("an LDS read reaching the next lane's dword keeps two lanes", [] { + Builder b; + auto& address = b.laneDword(); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + b.keep(b.lds(IrOpcode::LoadSharedU32x2, IrType::U32x2, {&address, &b.boolean(true)}, 0u)); + return rejected(b.analyze(), "share LDS"); + }); + + passed &= run("LDS addressed by a loaded value keeps two lanes", [] { + Builder b; + auto& address = b.userData(); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "not affine"); + }); + + passed &= run("LDS writes in a one-wave workgroup keep two lanes", [] { + Builder b(8u, 8u); + auto& address = b.binary(IrOpcode::ShiftLeftLogical32, b.binary(IrOpcode::IAdd32, b.binary(IrOpcode::ShiftLeftLogical32, b.localId(1), b.constant(3u)), b.localId(0)), b.constant(2u)); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "one-wave"); + }); + + passed &= run("a lane's own ballot bit runs at one lane", [] { + Builder b; + auto& predicate = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(5u)}); + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + auto& low = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}); + auto& high = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}); + auto& mask = b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&low}); + auto& maskHigh = b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&high}); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(mask, maskHigh), &b.constant(1u), &b.constant(0u)})); + return eligible(b.analyze()); + }); + + passed &= run("a counted ballot keeps two lanes", [] { + Builder b; + auto& low = b.ballot(b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(5u)}), 0u); + b.keep(b.emit(IrOpcode::BitCount32, IrType::U32, {&low})); + return rejected(b.analyze(), "lane mask read as a number"); + }); + + passed &= run("a lane mask whose halves are different expressions keeps two lanes", [] { + Builder b; + auto& low = b.ballot(b.boolean(true), 0u); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(low, b.constant(0u)), &b.constant(1u), &b.constant(0u)})); + return rejected(b.analyze(), "halves differ"); + }); + + passed &= run("the lane id read as data keeps two lanes", [] { + Builder b; + b.keep(b.binary(IrOpcode::IAdd32, b.lane(), b.constant(1u))); + return rejected(b.analyze(), "lane id read as a number"); + }); + + passed &= run("a workgroup of at most 32 lanes runs the same checks as a larger one", [] { + Builder b(16u, 2u); + b.keep(b.binary(IrOpcode::IAdd32, b.lane(), b.constant(1u))); + return rejected(b.analyze(), "lane id read as a number"); + }); + + passed &= run("a 32-thread group reading lane 63 in whole-wave mode keeps two lanes", [] { + Builder b(32u, 1u); + b.keep(b.emit(IrOpcode::ReadLane, IrType::U32, {&b.localId(0), &b.constant(63u)})); + return rejected(b.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a barrier beside s_cbranch_execz in a 32-thread group keeps two lanes", [] { + Builder b(32u, 1u); + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.keep(skip); + b.branch(BranchCondition::ExecZero, skip, nullptr); + return rejected(b.analyze(), "barrier"); + }); + + passed &= run("readfirstlane of user data runs at one lane, of a local id keeps two", [] { + Builder uniform; + uniform.keep(uniform.emit(IrOpcode::ReadFirstLane, IrType::U32, {&uniform.userData()})); + Builder varying; + varying.keep(varying.emit(IrOpcode::ReadFirstLane, IrType::U32, {&varying.localId(0)})); + return eligible(uniform.analyze()) && rejected(varying.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a lane's vector window indexed by user data runs at one lane", [] { + Builder b; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&b.userData(), &b.localId(0)}); + b.keep(b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&b.userData()})); + return eligible(b.analyze()); + }); + + passed &= run("readfirstlane of a vector window load keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&b.constant(0u), &b.localId(0)}); + b.keep(b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&b.constant(0u)})})); + return rejected(b.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a VCC branch on a uniform compare under the branch's exec runs at one lane", [] { + Builder b; + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&vcc}); + b.keep(condition); + b.keep(exec); + b.branch(BranchCondition::VccZero, condition, &exec); + return eligible(b.analyze()); + }); + + passed &= run("a VCC branch on a lane compare keeps two lanes", [] { + Builder b; + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + b.keep(vcc); + b.keep(exec); + b.branch(BranchCondition::VccNonZero, vcc, &exec); + return rejected(b.analyze(), "VCC branch on lane-varying bits"); + }); + + passed &= run("a VCC branch whose bits came from a narrower exec keeps two lanes", [] { + Builder b; + auto& wide = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& narrow = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&wide, &b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)})}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&narrow, &compare}); + b.keep(vcc); + b.keep(wide); + b.branch(BranchCondition::VccNonZero, vcc, &wide); + return rejected(b.analyze(), "VCC branch on lane-varying bits"); + }); + + passed &= run("a VCC branch without its exec mask keeps two lanes", [] { + Builder b; + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + b.keep(compare); + b.branch(BranchCondition::VccNonZero, compare, nullptr); + return rejected(b.analyze(), "without its exec mask"); + }); + + passed &= run("a barrier beside a VCC branch on exec-implied bits keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + b.keep(vcc); + b.keep(exec); + b.branch(BranchCondition::VccNonZero, vcc, &exec); + return rejected(b.analyze(), "barrier"); + }); + + const auto skippable = [](Builder& b, IrBlock*& region, IrBlock*& merge) -> IrValue& { + IrBlock& entry = *b.block; + region = &b.addBlock(); + merge = &b.addBlock(); + auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(32u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.split(entry, BranchCondition::ExecZero, skip, nullptr, *merge, *region); + b.at(*region); + return exec; + }; + + passed &= run("a scalar written under s_cbranch_execz and read per lane after the merge keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.userData(); + (void)skippable(b, region, merge); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &b.constant(1u)}})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("an exec-masked VGPR written under s_cbranch_execz runs at one lane", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + auto& exec = skippable(b, region, merge); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(1u)), &old}); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &written}})); + return eligible(b.analyze()); + }); + + passed &= run("a whole-wave VGPR write under s_cbranch_execz keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + (void)skippable(b, region, merge); + auto& written = b.binary(IrOpcode::IAdd32, old, b.constant(1u)); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &written}})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("a VCC written under s_cbranch_execz and read after the merge keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& before = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(3u)})}); + auto& beforeLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&before, &b.constant(0u)}); + auto& beforeHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&before, &b.constant(1u)}); + auto& exec = skippable(b, region, merge); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)}); + auto& after = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&compare, &exec})}); + auto& afterLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&after, &b.constant(0u)}); + auto& afterHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&after, &b.constant(1u)}); + b.jump(*region, *merge); + b.at(*merge); + auto& low = b.phi(IrType::U32, {{entry, &beforeLow}, {region, &afterLow}}); + auto& high = b.phi(IrType::U32, {{entry, &beforeHigh}, {region, &afterHigh}}); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(low, high), &b.constant(1u), &b.constant(0u)})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("a vector window store under s_cbranch_execz must keep the skipped lanes' value", [&] { + const auto build = [&](bool masked) { + Builder b(64u, 1u); + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& exec = skippable(b, region, merge); + auto& index = b.constant(0u); + auto& value = b.binary(IrOpcode::IAdd32, b.localId(0), b.constant(1u)); + auto& stored = masked ? b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &value, &b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&index})}) : value; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&index, &stored}); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&index})); + return b.analyze(); + }; + return eligible(build(true)) && rejected(build(false), "vector window store one half can skip"); + }); + + const auto loop = [](bool counterUsedAfter) { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& body = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& entryExec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(40u)}); + auto& start = b.localId(1); + b.jump(entry, header); + b.at(header); + auto& exec = b.phi(IrType::Bool, {{&entry, &entryExec}}); + auto& counter = b.phi(IrType::U32, {{&entry, &b.constant(0u)}}); + auto& carried = b.phi(IrType::U32, {{&entry, &start}}); + auto& done = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.split(header, BranchCondition::ExecZero, done, nullptr, exit, body); + b.at(body); + auto& next = b.binary(IrOpcode::IAdd32, counter, b.constant(1u)); + auto& nextExec = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &next})}); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&nextExec, &b.binary(IrOpcode::IAdd32, carried, b.constant(2u)), &carried}); + exec.AddPhiOperand(&body, &nextExec); + counter.AddPhiOperand(&body, &next); + carried.AddPhiOperand(&body, &written); + b.jump(body, header); + b.at(exit); + b.keep(counterUsedAfter ? counter : carried); + return b.analyze(); + }; + + passed &= run("an s_cbranch_execz loop with a scalar counter used after it keeps two lanes", [&] { + return rejected(loop(true), "leaves a loop one half can leave early"); + }); + + passed &= run("an s_cbranch_execz loop carrying an exec-masked VGPR out runs at one lane", [&] { + return eligible(loop(false)); + }); + + passed &= run("a scalar written under s_cbranch_execz keeps two lanes only when an instruction reads it", [&] { + const auto build = [&](bool read) { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& region = b.addBlock(); + IrBlock& merge = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& old = b.userData(); + auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(32u)}); + b.jump(entry, header); + b.at(header); + auto& carried = b.phi(IrType::U32, {{&entry, &old}}); + b.split(header, BranchCondition::ExecZero, b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}), nullptr, merge, region); + b.jump(region, merge); + b.at(merge); + auto& merged = b.phi(IrType::U32, {{&header, &carried}, {®ion, &b.constant(1u)}}); + carried.AddPhiOperand(&merge, &merged); + b.split(merge, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}), nullptr, header, exit); + b.at(exit); + if (read) b.keep(merged); + return b.analyze(); + }; + return eligible(build(false)) && rejected(build(true), "leaves a region one half can skip"); + }); + + passed &= run("an exec-masked VGPR carried through a loop inside an s_cbranch_execz region runs at one lane", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + auto& exec = skippable(b, region, merge); + b.at(*entry); + auto& kept = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(7u)), &old}); + IrBlock& header = b.addBlock(); + IrBlock& choose = b.addBlock(); + IrBlock& write = b.addBlock(); + IrBlock& join = b.addBlock(); + IrBlock& leave = b.addBlock(); + b.at(*region); + auto& first = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(3u)), &old}); + b.jump(*region, header); + b.at(header); + auto& carried = b.phi(IrType::U32, {{region, &first}}); + b.split(header, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(2u)}), nullptr, leave, choose); + b.at(choose); + b.split(choose, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(3u)}), nullptr, write, join); + b.at(write); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, carried, b.constant(1u)), &carried}); + b.jump(write, join); + b.at(join); + auto& joined = b.phi(IrType::U32, {{&choose, &carried}, {&write, &written}}); + carried.AddPhiOperand(&join, &joined); + b.split(join, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(4u)}), nullptr, header, leave); + b.at(leave); + auto& left = b.phi(IrType::U32, {{&header, &carried}, {&join, &joined}}); + b.jump(leave, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &kept}, {&leave, &left}})); + return eligible(b.analyze()); + }); + + passed &= run("a loop whose exit test reads fewer lanes than its body runs keeps two lanes", [] { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& body = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& entryAlive = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(40u)}); + auto& entryState = b.binary(IrOpcode::BitwiseAnd32, b.localId(0), b.constant(1u)); + b.jump(entry, header); + b.at(header); + auto& alive = b.phi(IrType::Bool, {{&entry, &entryAlive}}); + auto& state = b.phi(IrType::U32, {{&entry, &entryState}}); + auto& exec = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&alive, &b.emit(IrOpcode::IEqual32, IrType::Bool, {&state, &b.constant(0u)})}); + b.split(header, BranchCondition::ExecZero, b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}), nullptr, exit, body); + b.at(body); + auto& waiting = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&alive, &b.emit(IrOpcode::IEqual32, IrType::Bool, {&state, &b.constant(1u)})}); + auto& next = b.emit(IrOpcode::SelectU32, IrType::U32, {&waiting, &b.constant(0u), &b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.constant(2u), &state})}); + alive.AddPhiOperand(&body, &alive); + state.AddPhiOperand(&body, &next); + b.jump(body, header); + b.at(exit); + b.keep(state); + return rejected(b.analyze(), "exec can widen"); + }); + + passed &= run("the recompiler lays a qualifying wave64 program out at one lane unless APS5_NO_SINGLE_LANE_RULE is set", [] { + const std::vector code{0xbf810000u}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x100000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.compute = ShaderComputeStageInfo{{64u, 2u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32u; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + const bool off = std::getenv("APS5_NO_SINGLE_LANE_RULE") != nullptr; + return Recompile(request).hostSubgroupSize == (off ? 32u : 64u); + }); + + passed &= run("a DPP move keeps two lanes", [] { + Builder b; + b.keep(b.emit(IrOpcode::DppMoveU32, IrType::U32, {&b.localId(0)})); + return rejected(b.analyze(), "operation across lanes"); + }); + + passed &= run("a barrier in a program with exec branches keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.keep(skip); + b.branch(BranchCondition::ExecZero, skip, nullptr); + return rejected(b.analyze(), "barrier"); + }); + + if (!passed) return 1; + std::printf("single lane tests passed\n"); + return 0; +} diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 031d897b2b5b..6c3bd66e1535 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -106,6 +106,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/HostInterpolationChecker.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ReadLaneEliminator.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/MaskedSelectEliminator.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SingleLaneAnalysis.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/TessellationLowering.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceTracker.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceMaterializer.cpp diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 32f615dfcbf1..333602663982 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -17,6 +17,7 @@ struct BlockInfo { Terminator terminator; IrValue* condition = nullptr; IrValue* indirectTarget = nullptr; + IrValue* exec = nullptr; }; struct KeyDomain { diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp index eee7578d208c..b527f36d626f 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp @@ -32,6 +32,8 @@ struct IrProgramMetadata { std::uint32_t vectorWindowBase = 0; std::uint32_t vectorWindowSize = 0; bool vectorWindowIndexedWrites = false; + bool skippedRegionsChecked = false; + std::string skippedRegionRejection; std::uint32_t tessellationControlStride = 0; std::uint32_t tessellationPatchStride = 0; std::uint32_t tessellationSharedStride = 0; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp index f724becd4b7a..89d7d99efd28 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp @@ -49,6 +49,7 @@ class IrProgram { }; std::string ProgramToString(const IrProgram& program); +std::string DescribeValue(const IrValue* value, std::uint32_t depth); void ValidateProgram(const IrProgram& program, bool requireSsa); void ResolveControlFlowIdentities(IrProgram& program); bool EquivalentValue(const IrResourcePlan& program, const IrValue* left, const IrValue* right); diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp index 4c5e1fc57c98..987dd9678c8e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp @@ -255,7 +255,7 @@ std::size_t IrProgram::ReclaimUnusedValues() { for (const auto& read : resourcePlan.srtReads) keep(read.value); for (const auto& block : resourcePlan.controlFlow) keep(block.condition); for (IrValue* value : resourcePlan.uniformFill.values) keep(value); - for (const auto& block : metadata.blockInfo) { keep(block.condition); keep(block.indirectTarget); } + for (const auto& block : metadata.blockInfo) { keep(block.condition); keep(block.indirectTarget); keep(block.exec); } for (IrValue* value : metadata.dynamicReads) keep(value); while (!pending.empty()) { IrValue* value = pending.back(); @@ -392,6 +392,20 @@ const std::vector& IrProgram::BlockOrder() const { return blockOrder; } +std::string DescribeValue(const IrValue* value, std::uint32_t depth) { + if (value == nullptr) return "null"; + value = value->Resolve(); + std::string text(IrOpcodeName(value->Opcode())); + if (value->HasImmediate() && value->Type() == IrType::U32) return text + "(" + std::to_string(value->ImmediateU32()) + ")"; + if (depth == 0 || value->ArgumentCount() == 0) return text; + text += "("; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { + if (index != 0) text += ", "; + text += DescribeValue(value->Argument(index), depth - 1); + } + return text + ")"; +} + std::string ProgramToString(const IrProgram& program) { std::map ids; std::size_t nextId = 1; @@ -811,6 +825,9 @@ void ResolveControlFlowIdentities(IrProgram& program) { if (info.indirectTarget != nullptr) { info.indirectTarget = info.indirectTarget->Resolve(); } + if (info.exec != nullptr) { + info.exec = info.exec->Resolve(); + } } } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp b/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp new file mode 100644 index 000000000000..5dab4df31d8a --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp @@ -0,0 +1,20 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_SINGLELANEANALYSIS_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_SINGLELANEANALYSIS_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/ShaderStageInputInfo.hpp" + +namespace ShaderRecompiler { + +struct SingleLaneVerdict { + bool eligible = false; + const char* reason = nullptr; + const IrValue* at = nullptr; +}; + +[[nodiscard]] SingleLaneVerdict AnalyzeSingleLane(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup); +void RecordSkippedRegions(IrProgram& program); + +} + +#endif diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index 43db702ad8bd..a15841e025d9 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -664,6 +664,40 @@ bool replaceBinaryIdentity(IrValue& inst, IrType type, std::uint64_t identity) { return false; } +bool foldBitwiseSelf(IrBuilder& builder, IrValue& inst) { + auto& lhs = resolveArg(inst, 0); + auto& rhs = resolveArg(inst, 1); + const auto opcode = inst.Opcode(); + const std::uint32_t absorbing = opcode == IrOpcode::BitwiseOr32 ? 0xffffffffu : 0u; + if (opcode != IrOpcode::BitwiseXor32 && (isImmediateU32(lhs, absorbing) || isImmediateU32(rhs, absorbing))) { + replaceWith(inst, builder.Constant(absorbing)); + return true; + } + if (&lhs == &rhs) { + if (opcode == IrOpcode::BitwiseXor32) replaceWith(inst, builder.Constant(0u)); + else replaceWith(inst, lhs); + return true; + } + const bool complement = (lhs.Opcode() == IrOpcode::BitwiseNot32 && &resolveArg(lhs, 0) == &rhs) || (rhs.Opcode() == IrOpcode::BitwiseNot32 && &resolveArg(rhs, 0) == &lhs); + if (!complement) return false; + replaceWith(inst, builder.Constant(opcode == IrOpcode::BitwiseAnd32 ? 0u : 0xffffffffu)); + return true; +} + +bool foldLowBitOfOnes(IrBuilder& builder, IrValue& inst) { + auto& lhs = resolveArg(inst, 0); + auto& rhs = resolveArg(inst, 1); + const bool lowBit = isImmediateU32(rhs, 1u); + auto& shifted = lowBit ? lhs : rhs; + if (!lowBit && !isImmediateU32(lhs, 1u)) return false; + if (shifted.Opcode() != IrOpcode::ShiftRightLogical32 || !isImmediateU32(resolveArg(shifted, 0), 0xffffffffu)) return false; + auto& amount = resolveArg(shifted, 1); + if (amount.Opcode() != IrOpcode::BitwiseAnd32) return false; + if (!isImmediateU32(resolveArg(amount, 0), 31u) && !isImmediateU32(resolveArg(amount, 1), 31u)) return false; + replaceWith(inst, builder.Constant(1u)); + return true; +} + bool foldSelect(IrValue& inst) { auto& condition = resolveArg(inst, 0); if (isImmediate(condition, IrType::U1)) { @@ -1118,6 +1152,10 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { replaceWith(value, resolveArg(value, 0)); return true; } + if (isImmediateU32(resolveArg(value, 0), 0u)) { + replaceWith(value, resolveArg(value, 0)); + return true; + } if (!isImmediate(shift, IrType::U32)) return false; const auto amount = shift.ImmediateU32() & 31u; readThroughUnusedBits(value, 0, UINT32_MAX << amount); @@ -1170,6 +1208,7 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { return true; } if (replaceBinaryIdentity(value, IrType::U32, 0xffffffffu)) return true; + if (foldBitwiseSelf(builder, value) || foldLowBitOfOnes(builder, value)) return true; if (foldRedundantMask(value)) return true; bool changed = false; for (std::size_t side = 0; side < 2u; ++side) { @@ -1191,6 +1230,7 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { return true; } if (replaceBinaryIdentity(value, IrType::U32, 0u)) return true; + if (foldBitwiseSelf(builder, value)) return true; return opcode == IrOpcode::BitwiseOr32 && mergeMaskedParts(program, value); } case IrOpcode::BitwiseNot32: { diff --git a/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp b/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp index df61fa400d80..751261c3e547 100644 --- a/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp @@ -326,6 +326,7 @@ RedundantValueEliminationStats RedundantValueEliminator::Eliminate(IrProgram& pr for (auto& info : program.Metadata().blockInfo) { replaced(info.condition); replaced(info.indirectTarget); + replaced(info.exec); } for (auto*& read : program.Metadata().dynamicReads) replaced(read); for (auto& descriptor : program.Resources().descriptorSources) { diff --git a/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp b/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp new file mode 100644 index 000000000000..4e1dbed4c4c3 --- /dev/null +++ b/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp @@ -0,0 +1,1542 @@ +#include "Optimization/SingleLaneAnalysis.hpp" +#include "IntermediateRepresentation/IrBlock.hpp" +#include "IntermediateRepresentation/IrMetadata/StageIO.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +constexpr std::size_t PairLimit = std::size_t{1} << 18u; +constexpr std::uint32_t DepthLimit = 64; +constexpr std::uint64_t LaneOffsetLimit = std::uint64_t{1} << 16u; +constexpr std::size_t FootprintLimit = 64; + +struct Rejection { + const char* reason; + const IrValue* at; +}; + +bool isImmediateU32(const IrValue* value, std::uint32_t immediate) { + return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == immediate; +} + +bool isLane(const IrValue* value) { + return value->Resolve()->Opcode() == IrOpcode::LaneId; +} + +bool isHalfSelector(const IrValue* value) { + value = value->Resolve(); + return value->Opcode() == IrOpcode::ULessThan32 && isLane(value->Argument(0)) && isImmediateU32(value->Argument(1)->Resolve(), 32u); +} + +bool isBitSelector(const IrValue* value) { + value = value->Resolve(); + if (value->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + return (isLane(left) && isImmediateU32(right, 31u)) || (isLane(right) && isImmediateU32(left, 31u)); +} + +bool isHalfWord(const IrValue* value) { + value = value->Resolve(); + return value->Opcode() == IrOpcode::SelectU32 && isHalfSelector(value->Argument(0)); +} + +const IrValue* otherThan(const IrValue* value, std::uint32_t immediate) { + if (value->ArgumentCount() != 2u) return nullptr; + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + if (isImmediateU32(right, immediate)) return left; + if (isImmediateU32(left, immediate)) return right; + return nullptr; +} + +const IrValue* ownBitWord(const IrValue* value) { + value = value->Resolve(); + if (value->Opcode() != IrOpcode::INotEqual32) return nullptr; + const IrValue* bit = otherThan(value, 0u); + if (bit == nullptr || bit->Opcode() != IrOpcode::BitwiseAnd32) return nullptr; + const IrValue* shifted = otherThan(bit, 1u); + if (shifted == nullptr || shifted->Opcode() != IrOpcode::ShiftRightLogical32 || !isBitSelector(shifted->Argument(1))) return nullptr; + return shifted->Argument(0)->Resolve(); +} + +bool isBallotWord(const IrValue* value, std::uint32_t half) { + return value->Opcode() == IrOpcode::CompositeExtractU32x4 && value->Argument(0)->Resolve()->Opcode() == IrOpcode::Ballot && isImmediateU32(value->Argument(1)->Resolve(), half); +} + +const IrValue* ballotOfHalves(const IrValue* low, const IrValue* high) { + if (!isBallotWord(low, 0u) || !isBallotWord(high, 1u)) return nullptr; + const IrValue* lowBallot = low->Argument(0)->Resolve(); + const IrValue* highBallot = high->Argument(0)->Resolve(); + if (lowBallot == highBallot) return lowBallot; + if (lowBallot->Parent() != highBallot->Parent()) return nullptr; + const IrValue* lowPredicate = lowBallot->Argument(0)->Resolve(); + const IrValue* highPredicate = highBallot->Argument(0)->Resolve(); + if (lowPredicate == highPredicate) return lowBallot; + const bool sameImmediate = lowPredicate->HasImmediate() && highPredicate->HasImmediate() && lowPredicate->Type() == IrType::Bool && highPredicate->Type() == IrType::Bool && lowPredicate->ImmediateBool() == highPredicate->ImmediateBool(); + return sameImmediate ? lowBallot : nullptr; +} + +bool isLaneMaskOp(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + case IrOpcode::BitwiseNot32: + case IrOpcode::SelectU32: + case IrOpcode::Phi: + return true; + default: + return false; + } +} + +bool isLaneLocal(IrOpcode opcode) { + if (BufferAccessOf(opcode) != BufferAccess::None || AddressOpcodeInfoOf(opcode).access != AddressAccess::None || ImageOpcodeInfoOf(opcode).access != ImageAccess::None) return true; + switch (opcode) { + case IrOpcode::Identity: + case IrOpcode::Phi: + case IrOpcode::Void: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + case IrOpcode::IMul32: + case IrOpcode::IDiv32: + case IrOpcode::IMod32: + case IrOpcode::IAnd32: + case IrOpcode::IOr32: + case IrOpcode::IXor32: + case IrOpcode::IShiftLeft32: + case IrOpcode::IShiftRightLogical32: + case IrOpcode::IShiftRightArithmetic32: + case IrOpcode::INegate32: + case IrOpcode::INot32: + case IrOpcode::FAdd32: + case IrOpcode::FSub32: + case IrOpcode::FMul32: + case IrOpcode::FDiv32: + case IrOpcode::FNegate32: + case IrOpcode::FAbs32: + case IrOpcode::FFma32: + case IrOpcode::FSqrt32: + case IrOpcode::FRcp32: + case IrOpcode::FRsq32: + case IrOpcode::FMin32: + case IrOpcode::FMax32: + case IrOpcode::FFloor32: + case IrOpcode::FCeil32: + case IrOpcode::FFract32: + case IrOpcode::ConvertFToI32: + case IrOpcode::ConvertIToF32: + case IrOpcode::ConvertFToU32: + case IrOpcode::ConvertUToF32: + case IrOpcode::CompareEqual: + case IrOpcode::CompareNotEqual: + case IrOpcode::CompareLessThan: + case IrOpcode::CompareGreaterThan: + case IrOpcode::Select: + case IrOpcode::Barrier: + case IrOpcode::Reference: + case IrOpcode::ReferenceU32: + case IrOpcode::GetUserData: + case IrOpcode::GetShaderBase: + case IrOpcode::ShaderClock: + case IrOpcode::RealtimeClock: + case IrOpcode::UndefU1: + case IrOpcode::UndefU8: + case IrOpcode::UndefU16: + case IrOpcode::UndefU32: + case IrOpcode::UndefU64: + case IrOpcode::BitCastU16F16: + case IrOpcode::BitCastF16U16: + case IrOpcode::BitCastU32F32: + case IrOpcode::BitCastF32U32: + case IrOpcode::ConvertU16U32: + case IrOpcode::ConvertU32U16: + case IrOpcode::ConvertU8U32: + case IrOpcode::ConvertU32U8: + case IrOpcode::ConvertF32F16: + case IrOpcode::ConvertF16F32: + case IrOpcode::ConvertS32F32: + case IrOpcode::ConvertU32F32: + case IrOpcode::ConvertF32S32: + case IrOpcode::ConvertF32U32: + case IrOpcode::CompositeConstructU64: + case IrOpcode::CompositeConstructU32x2: + case IrOpcode::CompositeConstructU32x3: + case IrOpcode::CompositeConstructF32x2: + case IrOpcode::CompositeConstructU32x4: + case IrOpcode::CompositeExtractU64: + case IrOpcode::CompositeExtractU32x2: + case IrOpcode::CompositeExtractU32x3: + case IrOpcode::CompositeExtractU32x4: + case IrOpcode::PackHalf2x16: + case IrOpcode::PackFloat2x16Rtz: + case IrOpcode::FPAbs32: + case IrOpcode::FPNeg32: + case IrOpcode::FPSaturate32: + case IrOpcode::BitFieldInsert: + case IrOpcode::BitFieldUExtract: + case IrOpcode::BitFieldSExtract: + case IrOpcode::SelectU1: + case IrOpcode::SelectF32: + case IrOpcode::SelectU32: + case IrOpcode::IAdd64: + case IrOpcode::IAddCarry32: + case IrOpcode::ISub64: + case IrOpcode::IMul64: + case IrOpcode::UDiv32: + case IrOpcode::SMulHi: + case IrOpcode::UMulHi: + case IrOpcode::IAbs32: + case IrOpcode::ShiftLeftLogical32: + case IrOpcode::ShiftLeftLogical64: + case IrOpcode::ShiftRightLogical32: + case IrOpcode::ShiftRightLogical64: + case IrOpcode::ShiftRightArithmetic32: + case IrOpcode::ShiftRightArithmetic64: + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseAnd64: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + case IrOpcode::BitwiseNot32: + case IrOpcode::BitReverse32: + case IrOpcode::BitCount32: + case IrOpcode::BitCount64: + case IrOpcode::FindUMsb32: + case IrOpcode::FindUMsb64: + case IrOpcode::FindILsb32: + case IrOpcode::SMin32: + case IrOpcode::UMin32: + case IrOpcode::SMax32: + case IrOpcode::UMax32: + case IrOpcode::SMinTri32: + case IrOpcode::UMinTri32: + case IrOpcode::SMaxTri32: + case IrOpcode::UMaxTri32: + case IrOpcode::SMedTri32: + case IrOpcode::UMedTri32: + case IrOpcode::SLessThan32: + case IrOpcode::SLessThan64: + case IrOpcode::ULessThan32: + case IrOpcode::ULessThan64: + case IrOpcode::IEqual32: + case IrOpcode::IEqual64: + case IrOpcode::SLessThanEqual32: + case IrOpcode::ULessThanEqual32: + case IrOpcode::SGreaterThan32: + case IrOpcode::UGreaterThan32: + case IrOpcode::UGreaterThan64: + case IrOpcode::INotEqual32: + case IrOpcode::INotEqual64: + case IrOpcode::SGreaterThanEqual32: + case IrOpcode::UGreaterThanEqual32: + case IrOpcode::LogicalOr: + case IrOpcode::LogicalAnd: + case IrOpcode::LogicalXor: + case IrOpcode::LogicalNot: + case IrOpcode::FPOrdEqual32: + case IrOpcode::FPUnordEqual32: + case IrOpcode::FPOrdNotEqual32: + case IrOpcode::FPUnordNotEqual32: + case IrOpcode::FPOrdLessThan32: + case IrOpcode::FPUnordLessThan32: + case IrOpcode::FPOrdGreaterThan32: + case IrOpcode::FPUnordGreaterThan32: + case IrOpcode::FPOrdLessThanEqual32: + case IrOpcode::FPUnordLessThanEqual32: + case IrOpcode::FPOrdGreaterThanEqual32: + case IrOpcode::FPUnordGreaterThanEqual32: + case IrOpcode::FPIsNan32: + case IrOpcode::FPCmpClass32: + case IrOpcode::FPAdd32: + case IrOpcode::FPSub32: + case IrOpcode::FPFma32: + case IrOpcode::FPMad32: + case IrOpcode::FPMul32: + case IrOpcode::FPMin32: + case IrOpcode::FPMax32: + case IrOpcode::FPMinTri32: + case IrOpcode::FPMaxTri32: + case IrOpcode::FPMedTri32: + case IrOpcode::FPRecip32: + case IrOpcode::FPRecipIFlag32: + case IrOpcode::FPRecipSqrt32: + case IrOpcode::FPSqrt: + case IrOpcode::FPSin: + case IrOpcode::FPCos: + case IrOpcode::FPExp2: + case IrOpcode::FPLog2: + case IrOpcode::FPRoundEven32: + case IrOpcode::FPFloor32: + case IrOpcode::FPCeil32: + case IrOpcode::FPTrunc32: + case IrOpcode::FPFract32: + case IrOpcode::FPAdd64: + case IrOpcode::FPMul64: + case IrOpcode::FPFma64: + case IrOpcode::FPFmaScale64: + case IrOpcode::FPMin64: + case IrOpcode::FPMax64: + case IrOpcode::FPSaturate64: + case IrOpcode::FPLdexp64: + case IrOpcode::FPRoundEven64: + case IrOpcode::FPFloor64: + case IrOpcode::FPCeil64: + case IrOpcode::FPTrunc64: + case IrOpcode::FPFract64: + case IrOpcode::FPFrexpMant64: + case IrOpcode::FPFrexpExp64: + case IrOpcode::FPRcp64: + case IrOpcode::FPRsq64: + case IrOpcode::FPSqrt64: + case IrOpcode::FPTrigPreop64: + case IrOpcode::FPDot2F32F16: + case IrOpcode::ConvertF32F64: + case IrOpcode::ConvertF64F32: + case IrOpcode::ConvertF64S32: + case IrOpcode::ConvertF64U32: + case IrOpcode::ConvertS32F64: + case IrOpcode::ConvertU32F64: + case IrOpcode::GetSrtResource: + case IrOpcode::GetBufferResource: + case IrOpcode::GetAddressResource: + case IrOpcode::GetScratchResource: + case IrOpcode::GetImageResource: + case IrOpcode::GetSamplerResource: + case IrOpcode::MakeImageAddress: + case IrOpcode::ReadConst: + case IrOpcode::ReadConstBuffer: + case IrOpcode::ImageQueryDimensions: + case IrOpcode::ImageQueryLod: + case IrOpcode::ImageBvhIntersectRay: + case IrOpcode::ControlNop: + case IrOpcode::Waitcnt: + case IrOpcode::InstPrefetch: + case IrOpcode::LoadVectorWindow: + case IrOpcode::StoreVectorWindow: + return true; + default: + return false; + } +} + +std::uint32_t sharedBytes(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::LoadSharedU8: + case IrOpcode::WriteSharedU8: + return 1u; + case IrOpcode::LoadSharedU16: + case IrOpcode::WriteSharedU16: + return 2u; + case IrOpcode::LoadSharedU32: + case IrOpcode::WriteSharedU32: + case IrOpcode::SharedAtomicFMin32: + case IrOpcode::SharedAtomicFMax32: + case IrOpcode::SharedAtomicSwap32: + case IrOpcode::SharedAtomicIAdd32: + case IrOpcode::SharedAtomicISub32: + case IrOpcode::SharedAtomicInc32: + case IrOpcode::SharedAtomicDec32: + case IrOpcode::SharedAtomicSMin32: + case IrOpcode::SharedAtomicUMin32: + case IrOpcode::SharedAtomicSMax32: + case IrOpcode::SharedAtomicUMax32: + case IrOpcode::SharedAtomicAnd32: + case IrOpcode::SharedAtomicOr32: + case IrOpcode::SharedAtomicXor32: + case IrOpcode::SharedAtomicRsub32: + case IrOpcode::SharedAtomicFAdd32: + case IrOpcode::SharedAtomicCmpst32: + case IrOpcode::SharedAtomicCmpstF32: + case IrOpcode::SharedAtomicMskor32: + case IrOpcode::SharedAtomicWrap32: + return 4u; + case IrOpcode::LoadSharedU32x2: + case IrOpcode::WriteSharedU32x2: + return 8u; + case IrOpcode::LoadSharedU32x3: + case IrOpcode::WriteSharedU32x3: + return 12u; + case IrOpcode::LoadSharedU32x4: + case IrOpcode::WriteSharedU32x4: + return 16u; + default: + return 0u; + } +} + +class Uniformity { +public: + explicit Uniformity(const IrProgram& program) { + bool changed = true; + while (changed) { + changed = false; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if (varying.contains(inst) || !varies(program, *inst)) continue; + varying.insert(inst); + changed = true; + } + } + } + } + + [[nodiscard]] bool Uniform(const IrValue* value) const { + value = value->Resolve(); + return value->HasImmediate() || !varying.contains(value); + } + +private: + bool varies(const IrProgram& program, const IrValue& inst) const { + switch (inst.Opcode()) { + case IrOpcode::GetUserData: + case IrOpcode::GetShaderBase: + case IrOpcode::UndefU1: + case IrOpcode::UndefU8: + case IrOpcode::UndefU16: + case IrOpcode::UndefU32: + case IrOpcode::UndefU64: + return false; + case IrOpcode::LaneId: + case IrOpcode::Ballot: + case IrOpcode::ShaderClock: + case IrOpcode::RealtimeClock: + case IrOpcode::LoadVectorWindow: + return true; + case IrOpcode::GetBuiltin: { + const IrValue* kind = inst.Argument(0)->Resolve(); + if (!kind->HasImmediate()) return true; + const auto input = static_cast(kind->ImmediateU32()); + return input != StageInputKind::WorkgroupId && input != StageInputKind::DispatchThreadLimit; + } + case IrOpcode::ReadConst: + case IrOpcode::ReadConstBuffer: + return arguments(inst); + default: + break; + } + if (SharedAccessOf(inst.Opcode()) != SharedAccess::None || BufferAccessOf(inst.Opcode()) != BufferAccess::None || ImageOpcodeInfoOf(inst.Opcode()).access != ImageAccess::None) return true; + if (AddressOpcodeInfoOf(inst.Opcode()).access != AddressAccess::None) { + const auto index = inst.Flags().index; + const auto& memory = program.Resources().memoryInfo; + const bool scalarRead = AddressOpcodeInfoOf(inst.Opcode()).access == AddressAccess::Read && index < memory.size() && memory[index].kind == ResourceKind::ScalarAddress; + if (!scalarRead) return true; + } + return arguments(inst); + } + + bool arguments(const IrValue& inst) const { + for (std::size_t index = 0; index < inst.ArgumentCount(); ++index) { + const IrValue* argument = inst.Argument(index); + if (argument != nullptr && !Uniform(argument)) return true; + } + return false; + } + + std::unordered_set varying; +}; + +class PairSymmetry { +public: + [[nodiscard]] bool Symmetric(const IrValue* low, const IrValue* high) { + const auto root = intern(low->Resolve(), high->Resolve()); + std::vector> children; + while (!unexpanded.empty() && !overflow) { + const auto current = unexpanded.back(); + unexpanded.pop_back(); + children.clear(); + if (!expand(pairs[current].low, pairs[current].high, children)) { + pairs[current].broken = true; + pending.push_back(current); + continue; + } + for (const auto& [childLow, childHigh] : children) { + const auto child = intern(childLow, childHigh); + pairs[child].parents.push_back(current); + if (pairs[child].broken) pending.push_back(child); + } + } + if (overflow) return false; + settle(); + return !pairs[root].broken; + } + +private: + struct Pair { + const IrValue* low = nullptr; + const IrValue* high = nullptr; + std::vector parents; + bool broken = false; + }; + + std::size_t intern(const IrValue* low, const IrValue* high) { + const auto [found, inserted] = index.try_emplace({low, high}, pairs.size()); + if (inserted) { + pairs.push_back({low, high, {}, false}); + unexpanded.push_back(found->second); + overflow |= pairs.size() > PairLimit; + } + return found->second; + } + + static bool expand(const IrValue* low, const IrValue* high, std::vector>& children) { + if (low == high) return true; + if (low->HasImmediate() || high->HasImmediate()) return low->HasImmediate() && high->HasImmediate() && low->Type() == high->Type() && low->Type() == IrType::U32 && low->ImmediateU32() == high->ImmediateU32(); + if (isBallotWord(low, 0u) && isBallotWord(high, 1u)) return ballotOfHalves(low, high) != nullptr; + if (low->Opcode() != high->Opcode()) return false; + switch (low->Opcode()) { + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + children.emplace_back(low->Argument(0)->Resolve(), high->Argument(0)->Resolve()); + children.emplace_back(low->Argument(1)->Resolve(), high->Argument(1)->Resolve()); + return true; + case IrOpcode::BitwiseNot32: + children.emplace_back(low->Argument(0)->Resolve(), high->Argument(0)->Resolve()); + return true; + case IrOpcode::SelectU32: + if (low->Argument(0)->Resolve() != high->Argument(0)->Resolve()) return false; + children.emplace_back(low->Argument(1)->Resolve(), high->Argument(1)->Resolve()); + children.emplace_back(low->Argument(2)->Resolve(), high->Argument(2)->Resolve()); + return true; + case IrOpcode::Phi: { + if (low->Parent() != high->Parent() || low->PhiBlockCount() != high->PhiBlockCount()) return false; + for (std::size_t incoming = 0; incoming < low->PhiBlockCount(); ++incoming) { + const IrValue* match = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == low->PhiBlock(incoming)) match = high->Argument(other)->Resolve(); + } + if (match == nullptr) return false; + children.emplace_back(low->Argument(incoming)->Resolve(), match); + } + return true; + } + default: + return false; + } + } + + void settle() { + while (!pending.empty()) { + const auto broken = pending.back(); + pending.pop_back(); + for (const auto parent : pairs[broken].parents) { + if (pairs[parent].broken) continue; + pairs[parent].broken = true; + pending.push_back(parent); + } + } + } + + std::map, std::size_t> index; + std::vector pairs; + std::vector unexpanded; + std::vector pending; + bool overflow = false; +}; + +struct Term { + const IrValue* value = nullptr; + const IrValue* high = nullptr; + + bool operator==(const Term& other) const = default; +}; + +void collectTerms(const IrValue* predicate, std::vector& terms, bool& never, std::uint32_t depth = 0); + +void collectPairTerms(const IrValue* low, const IrValue* high, std::vector& terms, bool& never, std::uint32_t depth) { + if (low->HasImmediate() && high->HasImmediate() && low->Type() == IrType::U32 && high->Type() == IrType::U32 && low->ImmediateU32() == high->ImmediateU32()) { + if (low->ImmediateU32() == 0u) never = true; + else if (low->ImmediateU32() != 0xffffffffu) terms.push_back({low, high}); + return; + } + if (const IrValue* ballot = depth < DepthLimit ? ballotOfHalves(low, high) : nullptr; ballot != nullptr) { + collectTerms(ballot->Argument(0), terms, never, depth + 1u); + return; + } + if (depth < DepthLimit && low->Opcode() == IrOpcode::BitwiseAnd32 && high->Opcode() == IrOpcode::BitwiseAnd32) { + collectPairTerms(low->Argument(0)->Resolve(), high->Argument(0)->Resolve(), terms, never, depth + 1u); + collectPairTerms(low->Argument(1)->Resolve(), high->Argument(1)->Resolve(), terms, never, depth + 1u); + return; + } + terms.push_back({low, high}); +} + +void collectTerms(const IrValue* predicate, std::vector& terms, bool& never, std::uint32_t depth) { + predicate = predicate->Resolve(); + if (predicate->HasImmediate() && predicate->Type() == IrType::Bool) { + if (!predicate->ImmediateBool()) never = true; + return; + } + if (depth < DepthLimit && predicate->Opcode() == IrOpcode::LogicalAnd) { + collectTerms(predicate->Argument(0), terms, never, depth + 1u); + collectTerms(predicate->Argument(1), terms, never, depth + 1u); + return; + } + if (const IrValue* word = ownBitWord(predicate); word != nullptr && depth < DepthLimit) { + if (isHalfWord(word)) { + collectPairTerms(word->Argument(1)->Resolve(), word->Argument(2)->Resolve(), terms, never, depth + 1u); + return; + } + if (word->HasImmediate()) { + collectPairTerms(word, word, terms, never, depth + 1u); + return; + } + } + terms.push_back({predicate, nullptr}); +} + +class Implication { +public: + [[nodiscard]] bool Implies(const IrValue* premise, const Term& conclusion) { + steps = 0; + return bit(premise, conclusion, 0); + } + + [[nodiscard]] bool HalvesImply(const IrValue* low, const IrValue* high, const Term& conclusion) { + steps = 0; + return halves(low->Resolve(), high->Resolve(), conclusion, 0); + } + +private: + static constexpr std::size_t StepLimit = std::size_t{1} << 16u; + + bool bit(const IrValue* premise, const Term& conclusion, std::uint32_t depth) { + premise = premise->Resolve(); + if (++steps > StepLimit || depth > DepthLimit) return false; + if (conclusion.high == nullptr && premise == conclusion.value) return true; + if (premise->HasImmediate()) return premise->Type() == IrType::Bool && !premise->ImmediateBool(); + if (premise->Opcode() == IrOpcode::LogicalAnd) return bit(premise->Argument(0), conclusion, depth + 1u) || bit(premise->Argument(1), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::LogicalOr) return bit(premise->Argument(0), conclusion, depth + 1u) && bit(premise->Argument(1), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::SelectU1) return bit(premise->Argument(1), conclusion, depth + 1u) && bit(premise->Argument(2), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::Phi) { + if (!openBits.insert(premise).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < premise->PhiBlockCount(); ++incoming) all = bit(premise->Argument(incoming), conclusion, depth + 1u); + openBits.erase(premise); + return all; + } + const IrValue* word = ownBitWord(premise); + if (word == nullptr) return false; + if (isHalfWord(word)) return halves(word->Argument(1)->Resolve(), word->Argument(2)->Resolve(), conclusion, depth + 1u); + return isImmediateU32(word, 0u); + } + + bool halves(const IrValue* low, const IrValue* high, const Term& conclusion, std::uint32_t depth) { + if (++steps > StepLimit || depth > DepthLimit) return false; + if (low == conclusion.value && high == conclusion.high) return true; + if (low->HasImmediate() || high->HasImmediate()) return isImmediateU32(low, 0u) && isImmediateU32(high, 0u); + if (const IrValue* ballot = ballotOfHalves(low, high)) return bit(ballot->Argument(0), conclusion, depth + 1u); + if (low->Opcode() != high->Opcode()) return false; + const auto argument = [](const IrValue* value, std::size_t index) { return value->Argument(index)->Resolve(); }; + switch (low->Opcode()) { + case IrOpcode::BitwiseAnd32: + return halves(argument(low, 0), argument(high, 0), conclusion, depth + 1u) || halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u); + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + return halves(argument(low, 0), argument(high, 0), conclusion, depth + 1u) && halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u); + case IrOpcode::SelectU32: + return argument(low, 0) == argument(high, 0) && halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u) && halves(argument(low, 2), argument(high, 2), conclusion, depth + 1u); + case IrOpcode::Phi: { + if (low->Parent() != high->Parent() || low->PhiBlockCount() != high->PhiBlockCount()) return false; + const std::pair key{low, high}; + if (open.contains(key)) return true; + open.insert(key); + bool all = true; + for (std::size_t incoming = 0; all && incoming < low->PhiBlockCount(); ++incoming) { + const IrValue* match = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == low->PhiBlock(incoming)) match = argument(high, other); + } + all = match != nullptr && halves(argument(low, incoming), match, conclusion, depth + 1u); + } + open.erase(key); + return all; + } + default: + return false; + } + } + + std::set> open; + std::set openBits; + std::size_t steps = 0; +}; + +std::optional checkVccBranch(const BlockInfo& info, const Uniformity& uniformity) { + if (info.condition == nullptr) return std::nullopt; + const IrValue* condition = info.condition->Resolve(); + if (info.exec == nullptr) return Rejection{"a VCC branch without its exec mask", condition}; + const IrValue* exec = info.exec->Resolve(); + if (exec->Opcode() == IrOpcode::Unreachable) return Rejection{"a VCC branch whose exec mask was removed", condition}; + const IrValue* predicate = condition; + if (info.terminator.condition == BranchCondition::VccZero) { + if (condition->Opcode() != IrOpcode::LogicalNot) return uniformity.Uniform(condition) ? std::nullopt : std::optional(Rejection{"a VCC branch on lane-varying bits", condition}); + predicate = condition->Argument(0)->Resolve(); + } + std::vector terms; + bool never = false; + collectTerms(predicate, terms, never); + if (never) return std::nullopt; + Implication implication; + for (const auto& term : terms) { + if (term.high == nullptr && uniformity.Uniform(term.value)) continue; + if (implication.Implies(exec, term)) continue; + return Rejection{"a VCC branch on lane-varying bits", term.value}; + } + return std::nullopt; +} + +std::optional checkLaneMasks(const IrProgram& program) { + std::vector masks; + std::unordered_set isMask; + std::vector halfWords; + std::unordered_set isHalf; + std::vector lanes; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::LaneId) lanes.push_back(inst); + if (inst->Opcode() != IrOpcode::Ballot) continue; + for (const IrUse& use : inst->OperandUses()) { + if (!isBallotWord(use.user, 0u) && !isBallotWord(use.user, 1u)) return Rejection{"a ballot read other than as a lane mask", use.user}; + if (isMask.insert(use.user).second) masks.push_back(use.user); + } + } + } + for (std::size_t next = 0; next < masks.size(); ++next) { + for (const IrUse& use : masks[next]->OperandUses()) { + const IrValue* user = use.user; + if (isHalfWord(user) && use.operand != 0u) { + if (isHalf.insert(user).second) halfWords.push_back(user); + continue; + } + const bool combines = user->Opcode() == IrOpcode::Identity || (isLaneMaskOp(user->Opcode()) && (user->Opcode() != IrOpcode::SelectU32 || use.operand != 0u)); + if (!combines) return Rejection{"a lane mask read as a number", user}; + if (isMask.insert(user).second) masks.push_back(user); + } + } + for (const IrValue* lane : lanes) { + for (const IrUse& laneUse : lane->OperandUses()) { + const IrValue* user = laneUse.user; + if (isHalfSelector(user)) { + for (const IrUse& use : user->OperandUses()) { + if (!isHalfWord(use.user) || use.operand != 0u) return Rejection{"the lane id read as a number", use.user}; + if (isHalf.insert(use.user).second) halfWords.push_back(use.user); + } + continue; + } + if (!isBitSelector(user)) return Rejection{"the lane id read as a number", user}; + for (const IrUse& shift : user->OperandUses()) { + const IrValue* shifted = shift.user; + if (shifted->Opcode() != IrOpcode::ShiftRightLogical32 || shift.operand != 1u) return Rejection{"the lane id read as a number", shifted}; + const IrValue* word = shifted->Argument(0)->Resolve(); + if (!isHalfWord(word) && !word->HasImmediate()) return Rejection{"a lane's bit read from one half of a lane mask", shifted}; + for (const IrUse& bit : shifted->OperandUses()) { + if (bit.user->Opcode() != IrOpcode::BitwiseAnd32 || otherThan(bit.user, 1u) != shifted) return Rejection{"a lane mask shifted by the lane id read as a number", bit.user}; + } + } + } + } + PairSymmetry symmetry; + for (const IrValue* word : halfWords) { + for (const IrUse& use : word->OperandUses()) { + if (use.user->Opcode() != IrOpcode::ShiftRightLogical32 || use.operand != 0u || !isBitSelector(use.user->Argument(1))) return Rejection{"half of a lane mask read as a number", use.user}; + } + if (!symmetry.Symmetric(word->Argument(1), word->Argument(2))) return Rejection{"a lane mask whose halves differ", word}; + } + return std::nullopt; +} + +struct Affine { + std::array coefficients{}; + std::uint64_t constant = 0; +}; + +class AffineEvaluator { +public: + explicit AffineEvaluator(const std::array& sizes) : sizes(sizes) {} + + [[nodiscard]] std::uint64_t Max(const Affine& affine) const { + std::uint64_t result = affine.constant; + for (std::size_t axis = 0; axis < 3u; ++axis) result += affine.coefficients[axis] * (sizes[axis] - 1u); + return result; + } + + [[nodiscard]] std::optional Of(const IrValue* value, std::uint32_t depth = 0) const { + value = value->Resolve(); + if (value->HasImmediate()) return value->Type() == IrType::U32 ? std::optional(Affine{{}, value->ImmediateU32()}) : std::nullopt; + if (depth > DepthLimit) return std::nullopt; + const auto argument = [&](std::size_t index) { return Of(value->Argument(index), depth + 1u); }; + const auto immediate = [&](std::size_t index) -> std::optional { + const IrValue* operand = value->Argument(index)->Resolve(); + return operand->HasImmediate() && operand->Type() == IrType::U32 ? std::optional(operand->ImmediateU32()) : std::nullopt; + }; + std::optional result; + switch (value->Opcode()) { + case IrOpcode::GetBuiltin: { + const auto kind = immediate(0); + if (!kind) return std::nullopt; + if (static_cast(*kind) == StageInputKind::LocalInvocationIndex) return Affine{{1u, sizes[0], std::uint64_t{sizes[0]} * sizes[1]}, 0u}; + const auto axis = immediate(1); + if (static_cast(*kind) != StageInputKind::LocalInvocationId || !axis || *axis >= 3u) return std::nullopt; + Affine id; + id.coefficients[*axis] = 1u; + return id; + } + case IrOpcode::IAdd32: { + const auto a = argument(0), b = argument(1); + if (!a || !b) return std::nullopt; + result = sum(*a, *b); + break; + } + case IrOpcode::IMul32: { + auto a = argument(0), b = argument(1); + if (!a || !b) return std::nullopt; + if (!isConstant(*a)) std::swap(a, b); + if (!isConstant(*a)) return std::nullopt; + result = scaled(*b, a->constant); + break; + } + case IrOpcode::ShiftLeftLogical32: { + const auto a = argument(0); + const auto shift = immediate(1); + if (!a || !shift || *shift >= 32u) return std::nullopt; + result = scaled(*a, std::uint64_t{1} << *shift); + break; + } + case IrOpcode::BitFieldUExtract: { + const auto a = argument(0); + const auto offset = immediate(1), count = immediate(2); + if (!a || offset != 0u || !count || *count > 32u || (*count < 32u && Max(*a) >> *count != 0u)) return std::nullopt; + result = a; + break; + } + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: { + auto mask = immediate(1); + auto a = argument(0); + if (!mask) { + mask = immediate(0); + a = argument(1); + } + if (!a || !mask) return std::nullopt; + if (value->Opcode() == IrOpcode::BitwiseAnd32) { + if ((*mask & (*mask + 1u)) != 0u || Max(*a) > *mask) return std::nullopt; + result = a; + } else { + std::uint32_t zeros = 32u; + if (a->constant != 0u) zeros = std::min(zeros, static_cast(std::countr_zero(a->constant))); + for (const auto coefficient : a->coefficients) { + if (coefficient != 0u) zeros = std::min(zeros, static_cast(std::countr_zero(coefficient))); + } + if (zeros < 32u && (*mask >> zeros) != 0u) return std::nullopt; + result = sum(*a, Affine{{}, *mask}); + } + break; + } + default: + return std::nullopt; + } + if (!result || Max(*result) > 0xffffffffull) return std::nullopt; + return result; + } + +private: + static bool isConstant(const Affine& affine) { + return affine.coefficients == std::array{}; + } + + static Affine sum(const Affine& a, const Affine& b) { + Affine result; + for (std::size_t axis = 0; axis < 3u; ++axis) result.coefficients[axis] = a.coefficients[axis] + b.coefficients[axis]; + result.constant = a.constant + b.constant; + return result; + } + + std::optional scaled(const Affine& affine, std::uint64_t factor) const { + if (factor > 0xffffffffull) return std::nullopt; + Affine result; + for (std::size_t axis = 0; axis < 3u; ++axis) { + if (affine.coefficients[axis] > 0xffffffffull) return std::nullopt; + result.coefficients[axis] = affine.coefficients[axis] * factor; + } + if (affine.constant > 0xffffffffull) return std::nullopt; + result.constant = affine.constant * factor; + return result; + } + + std::array sizes; +}; + +std::optional checkLds(const IrProgram& program, const std::array& sizes, bool& writes) { + const AffineEvaluator evaluator(sizes); + std::optional> stride; + std::vector> footprint; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + const auto access = SharedAccessOf(inst->Opcode()); + if (access == SharedAccess::None) continue; + const auto index = inst->Flags().index; + if (index >= program.Resources().memoryInfo.size()) return Rejection{"an LDS access without memory metadata", inst}; + const auto& memory = program.Resources().memoryInfo[index]; + if (memory.kind != ResourceKind::Lds) return Rejection{"a GDS access", inst}; + const auto bytes = sharedBytes(inst->Opcode()); + if (bytes == 0u || inst->ArgumentCount() == 0u) return Rejection{"an LDS access that counts or locks across the wave", inst}; + writes |= access != SharedAccess::Read; + auto affine = evaluator.Of(inst->Argument(0)); + if (!affine) return Rejection{"an LDS address that is not affine in the local invocation id", inst}; + for (std::size_t axis = 0; axis < 3u; ++axis) { + if (sizes[axis] == 1u) affine->coefficients[axis] = 0u; + } + if (stride && *stride != affine->coefficients) return Rejection{"LDS accesses at different strides", inst}; + stride = affine->coefficients; + const auto begin = affine->constant + memory.offset; + if (evaluator.Max(*affine) + memory.offset + bytes > 0x100000000ull) return Rejection{"an LDS address that wraps", inst}; + footprint.emplace_back(begin, begin + bytes); + } + } + if (!stride) return std::nullopt; + std::sort(footprint.begin(), footprint.end()); + std::vector> merged; + for (const auto& range : footprint) { + if (!merged.empty() && range.first <= merged.back().second) merged.back().second = std::max(merged.back().second, range.second); + else merged.push_back(range); + } + if (merged.size() > FootprintLimit) return Rejection{"an LDS footprint of too many ranges", nullptr}; + std::uint64_t offsets = 1; + for (const auto size : sizes) offsets *= 2u * size - 1u; + if (offsets > LaneOffsetLimit) return Rejection{"a workgroup too large to check LDS privacy", nullptr}; + std::array delta{}; + for (std::int64_t x = 1 - static_cast(sizes[0]); x < static_cast(sizes[0]); ++x) { + for (std::int64_t y = 1 - static_cast(sizes[1]); y < static_cast(sizes[1]); ++y) { + for (std::int64_t z = 1 - static_cast(sizes[2]); z < static_cast(sizes[2]); ++z) { + if (x == 0 && y == 0 && z == 0) continue; + delta = {x, y, z}; + std::int64_t shift = 0; + for (std::size_t axis = 0; axis < 3u; ++axis) shift += delta[axis] * static_cast((*stride)[axis]); + for (const auto& moved : merged) { + for (const auto& fixed : merged) { + const auto begin = static_cast(moved.first) + shift; + const auto end = static_cast(moved.second) + shift; + if (begin < static_cast(fixed.second) && static_cast(fixed.first) < end) return Rejection{"lanes that share LDS bytes", nullptr}; + } + } + } + } + } + return std::nullopt; +} + +bool sameWord(const IrValue* left, const IrValue* right) { + left = left->Resolve(); + right = right->Resolve(); + if (left == right) return true; + return left->HasImmediate() && right->HasImmediate() && left->Type() == IrType::U32 && right->Type() == IrType::U32 && left->ImmediateU32() == right->ImmediateU32(); +} + +bool isSelect(IrOpcode opcode) { + return opcode == IrOpcode::Select || opcode == IrOpcode::SelectU1 || opcode == IrOpcode::SelectF32 || opcode == IrOpcode::SelectU32; +} + +bool writesMemory(IrOpcode opcode) { + const auto buffer = BufferAccessOf(opcode); + const auto address = AddressOpcodeInfoOf(opcode).access; + const auto image = ImageOpcodeInfoOf(opcode).access; + const auto shared = SharedAccessOf(opcode); + return buffer == BufferAccess::Write || buffer == BufferAccess::Atomic || address == AddressAccess::Write || address == AddressAccess::Atomic || image == ImageAccess::Write || image == ImageAccess::Atomic || (shared != SharedAccess::None && shared != SharedAccess::Read); +} + +const IrValue* lanePredicate(const IrValue& inst) { + for (std::size_t index = inst.ArgumentCount(); index > 0u; --index) { + const IrValue* argument = inst.Argument(index - 1u); + if (argument != nullptr && argument->Resolve()->Type() == IrType::Bool) return argument->Resolve(); + } + return nullptr; +} + +const IrValue* branchExec(const BlockInfo& info) { + if (info.exec != nullptr) return info.exec->Resolve(); + const IrValue* condition = info.condition->Resolve(); + if (info.terminator.condition == BranchCondition::ExecNonZero) return condition; + if (info.terminator.condition == BranchCondition::ExecZero && condition->Opcode() == IrOpcode::LogicalNot) return condition->Argument(0)->Resolve(); + return nullptr; +} + +bool pureLaneOp(IrOpcode opcode) { + if (opcode == IrOpcode::ReadConst || opcode == IrOpcode::ReadConstBuffer) return true; + if (!isLaneLocal(opcode) || opcode == IrOpcode::Phi || opcode == IrOpcode::ShaderClock || opcode == IrOpcode::RealtimeClock || opcode == IrOpcode::LoadVectorWindow || opcode == IrOpcode::StoreVectorWindow) return false; + return BufferAccessOf(opcode) == BufferAccess::None && AddressOpcodeInfoOf(opcode).access == AddressAccess::None && ImageOpcodeInfoOf(opcode).access == ImageAccess::None && SharedAccessOf(opcode) == SharedAccess::None; +} + +bool halfDecided(const BlockInfo& info, const Uniformity& uniformity) { + if (info.terminator.kind != TerminatorKind::ConditionalBranch || info.condition == nullptr) return false; + switch (info.terminator.condition) { + case BranchCondition::ExecZero: + case BranchCondition::ExecNonZero: + return true; + case BranchCondition::VccZero: + case BranchCondition::VccNonZero: { + const IrValue* predicate = info.condition->Resolve(); + if (info.terminator.condition == BranchCondition::VccZero && predicate->Opcode() == IrOpcode::LogicalNot) predicate = predicate->Argument(0)->Resolve(); + std::vector terms; + bool never = false; + collectTerms(predicate, terms, never); + return std::ranges::any_of(terms, [&](const Term& term) { return term.high != nullptr || !uniformity.Uniform(term.value); }); + } + default: + return false; + } +} + +class SkippedRegions { +public: + SkippedRegions(const IrProgram& program, const Uniformity& uniformity) : program(program), uniformity(uniformity), order(program.BlockOrder()) { + const auto& infos = program.Metadata().blockInfo; + for (std::size_t index = 0; index < order.size(); ++index) { + indices.emplace(order[index], index); + if (index < infos.size()) ids.emplace(infos[index].id, index); + } + postDominate(); + observe(); + } + + [[nodiscard]] std::optional Check() { + const auto& infos = program.Metadata().blockInfo; + for (std::size_t branch = 0; branch < order.size() && branch < infos.size(); ++branch) { + if (!halfDecided(infos[branch], uniformity)) continue; + if (auto rejected = checkBranch(branch, infos[branch])) return rejected; + } + return std::nullopt; + } + +private: + using Bits = std::vector; + + static bool test(const Bits& bits, std::size_t index) { + return ((bits[index / 64u] >> (index % 64u)) & 1u) != 0u; + } + + static void set(Bits& bits, std::size_t index) { + bits[index / 64u] |= std::uint64_t{1} << (index % 64u); + } + + std::size_t blockIndex(const IrBlock* block) const { + const auto found = indices.find(block); + return found == indices.end() ? order.size() : found->second; + } + + std::size_t blockOf(std::uint32_t id) const { + const auto found = ids.find(id); + return found == ids.end() ? order.size() : found->second; + } + + bool inRegion(const IrBlock* block) const { + const auto index = blockIndex(block); + return index < order.size() && region[index] != 0; + } + + bool definedInRegion(const IrValue* value) const { + return !value->HasImmediate() && inRegion(value->Parent()); + } + + void postDominate() { + const std::size_t exit = order.size(); + const std::size_t words = (exit + 64u) / 64u; + const Bits all(words, ~std::uint64_t{0}); + dominators.assign(exit + 1u, all); + dominators[exit].assign(words, 0u); + set(dominators[exit], exit); + bool changed = true; + while (changed) { + changed = false; + for (std::size_t index = exit; index-- > 0u;) { + Bits next = order[index]->Successors().empty() ? dominators[exit] : all; + for (const IrBlock* successor : order[index]->Successors()) { + const auto& after = dominators[blockIndex(successor)]; + for (std::size_t word = 0; word < words; ++word) next[word] &= after[word]; + } + set(next, index); + if (next == dominators[index]) continue; + dominators[index] = std::move(next); + changed = true; + } + } + reachesEnd.assign(exit, 0); + std::vector pending; + for (std::size_t index = 0; index < exit; ++index) { + if (!order[index]->Successors().empty()) continue; + reachesEnd[index] = 1; + pending.push_back(index); + } + while (!pending.empty()) { + const auto current = pending.back(); + pending.pop_back(); + for (const IrBlock* predecessor : order[current]->Predecessors()) { + const auto index = blockIndex(predecessor); + if (index >= exit || reachesEnd[index]) continue; + reachesEnd[index] = 1; + pending.push_back(index); + } + } + } + + void observe() { + std::vector pending; + const auto mark = [&](const IrValue* value) { + if (value == nullptr) return; + value = value->Resolve(); + if (value->HasImmediate() || !observed.insert(value).second) return; + if (value->Opcode() == IrOpcode::Phi) pending.push_back(value); + }; + for (const IrBlock* block : order) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::Phi) continue; + for (std::size_t index = 0; index < inst->ArgumentCount(); ++index) mark(inst->Argument(index)); + } + } + while (!pending.empty()) { + const IrValue* phi = pending.back(); + pending.pop_back(); + for (std::size_t incoming = 0; incoming < phi->PhiBlockCount(); ++incoming) mark(phi->Argument(incoming)); + } + } + + std::size_t immediatePostDominator(std::size_t block) const { + std::size_t best = order.size(); + std::size_t bestCount = 0; + for (std::size_t candidate = 0; candidate < order.size(); ++candidate) { + if (candidate == block || !test(dominators[block], candidate)) continue; + std::size_t count = 0; + for (const auto word : dominators[candidate]) count += static_cast(std::popcount(word)); + if (count <= bestCount) continue; + best = candidate; + bestCount = count; + } + return best; + } + + std::vector reach(std::size_t from, std::size_t stop) const { + std::vector seen(order.size(), 0); + if (from >= order.size() || from == stop) return seen; + std::vector pending{from}; + seen[from] = 1; + while (!pending.empty()) { + const auto current = pending.back(); + pending.pop_back(); + for (const IrBlock* successor : order[current]->Successors()) { + const auto next = blockIndex(successor); + if (next >= order.size() || next == stop || seen[next]) continue; + seen[next] = 1; + pending.push_back(next); + } + } + return seen; + } + + std::optional checkBranch(std::size_t branch, const BlockInfo& info) { + const IrValue* condition = info.condition->Resolve(); + exec = branchExec(info); + if (exec == nullptr) return Rejection{"a branch one half can decide alone without its exec mask", condition}; + terms.clear(); + bool never = false; + collectTerms(exec, terms, never); + if (never || terms.empty()) return std::nullopt; + if (!reachesEnd[branch]) return Rejection{"a branch one half can decide alone that never reaches the end", condition}; + const bool skipOnFalse = info.terminator.condition == BranchCondition::ExecNonZero || info.terminator.condition == BranchCondition::VccNonZero; + const auto skip = blockOf(skipOnFalse ? info.terminator.falseBlock : info.terminator.trueBlock); + const auto wave = blockOf(skipOnFalse ? info.terminator.trueBlock : info.terminator.falseBlock); + const auto merge = immediatePostDominator(branch); + branchIndex = branch; + skipIndex = skip; + mergeIndex = merge; + region = reach(wave, merge); + skippedRegion = reach(skip, merge); + for (std::size_t index = 0; index < region.size(); ++index) region[index] = region[index] != 0 && skippedRegion[index] == 0 ? 1 : 0; + cyclic = region[branch] != 0; + stability.clear(); + skippedValues.clear(); + if (cyclic && !monotone()) return Rejection{"a loop left through a branch one half can decide alone whose exec can widen", condition}; + for (std::size_t index = 0; index < order.size(); ++index) { + if (region[index] != 0) { + for (const IrValue* inst : order[index]->Instructions()) { + if (auto rejected = checkValue(*inst)) return rejected; + } + continue; + } + if (std::ranges::none_of(order[index]->Predecessors(), [&](const IrBlock* predecessor) { return inRegion(predecessor); })) continue; + for (const IrValue* inst : order[index]->Instructions()) { + if (inst->Opcode() != IrOpcode::Phi || !observed.contains(inst)) continue; + if (auto rejected = checkExit(*inst)) return rejected; + } + } + return std::nullopt; + } + + std::optional checkValue(const IrValue& inst) { + const auto opcode = inst.Opcode(); + if (opcode == IrOpcode::StoreVectorWindow && !windowKept(inst.Argument(1), inst.Argument(0)->Resolve(), 0)) return Rejection{"a vector window store one half can skip", &inst}; + if (writesMemory(opcode)) { + const IrValue* predicate = lanePredicate(inst); + if (predicate == nullptr || !impliesExec(predicate)) return Rejection{"a memory write one half can skip outside the branch's exec", &inst}; + } + for (const IrUse& use : inst.OperandUses()) { + const IrValue* user = use.user; + if (user->Opcode() == IrOpcode::Phi || inRegion(user->Parent())) continue; + if (cyclic && stable(&inst, 0)) continue; + return Rejection{cyclic ? "a value that leaves a loop one half can leave early" : "a value that leaves a region one half can skip", &inst}; + } + return std::nullopt; + } + + std::optional checkExit(const IrValue& phi) { + if (!cyclic) return merges(phi) ? std::nullopt : std::optional(Rejection{"a value that leaves a region one half can skip", &phi}); + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (inRegion(phi.PhiBlock(incoming)) && !stable(phi.Argument(incoming), 0)) return Rejection{"a value that leaves a loop one half can leave early", &phi}; + } + return std::nullopt; + } + + bool impliesExec(const IrValue* predicate) { + Implication implication; + return std::ranges::all_of(terms, [&](const Term& term) { return implication.Implies(predicate, term); }); + } + + bool halvesImplyExec(const IrValue* low, const IrValue* high) { + Implication implication; + return std::ranges::all_of(terms, [&](const Term& term) { return implication.HalvesImply(low, high, term); }); + } + + bool monotone() { + if (!definedInRegion(exec)) return true; + for (const Term& term : terms) { + const IrValue* low = term.value->Resolve(); + const IrValue* high = term.high == nullptr ? nullptr : term.high->Resolve(); + if (low->Opcode() != IrOpcode::Phi || !definedInRegion(low)) continue; + if (high != nullptr && (high->Opcode() != IrOpcode::Phi || high->Parent() != low->Parent())) continue; + bool entered = false; + bool narrows = true; + for (std::size_t incoming = 0; narrows && incoming < low->PhiBlockCount(); ++incoming) { + const IrBlock* from = low->PhiBlock(incoming); + if (!inRegion(from)) { + entered = true; + continue; + } + if (high == nullptr) { + narrows = impliesExec(low->Argument(incoming)); + continue; + } + const IrValue* matching = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == from) matching = high->Argument(other); + } + narrows = matching != nullptr && halvesImplyExec(low->Argument(incoming), matching); + } + if (entered && narrows) return true; + } + return false; + } + + bool skipSide(const IrBlock* from) const { + const auto index = blockIndex(from); + return index < order.size() && (skippedRegion[index] != 0 || (index == branchIndex && skipIndex == mergeIndex)); + } + + bool merges(const IrValue& phi) { + bool skippedPath = false; + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (!skipSide(phi.PhiBlock(incoming))) continue; + skippedPath = true; + const IrValue* kept = phi.Argument(incoming)->Resolve(); + for (std::size_t other = 0; other < phi.PhiBlockCount(); ++other) { + if (!inRegion(phi.PhiBlock(other))) continue; + if (!sameWord(skippedValue(phi.Argument(other)), skippedValue(kept)) && !chainTo(phi.Argument(other), kept, 0)) return false; + } + } + return skippedPath; + } + + const IrValue* settled(const IrValue* value) const { + for (std::uint32_t step = 0; step <= DepthLimit; ++step) { + const auto found = skippedValues.find(value); + if (found == skippedValues.end() || found->second == value) return value; + value = found->second; + } + return value; + } + + const IrValue* skippedValue(const IrValue* value) { + value = value->Resolve(); + if (const auto found = skippedValues.find(value); found != skippedValues.end()) return settled(found->second); + const std::size_t depth = passing.size(); + if (const auto open = passing.find(value); open != passing.end()) { + lowestOpen = std::min(lowestOpen, open->second); + return value; + } + passing.emplace(value, depth); + const std::size_t outer = std::exchange(lowestOpen, NoOpenCycle); + const IrValue* result = value; + if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u) { + const IrValue* kept = skippedValue(value->Argument(2)); + if (impliesExec(value->Argument(0)) || sameWord(skippedValue(value->Argument(1)), kept)) result = kept; + } else if (value->Opcode() == IrOpcode::Phi && definedInRegion(value)) { + const IrValue* base = nullptr; + bool agree = true; + for (std::size_t incoming = 0; agree && incoming < value->PhiBlockCount(); ++incoming) { + const IrValue* candidate = skippedValue(value->Argument(incoming)); + if (candidate == value) continue; + if (base == nullptr) base = candidate; + else agree = sameWord(base, candidate); + } + if (agree && base != nullptr) result = base; + } + passing.erase(value); + const bool pending = lowestOpen < depth; + if (!pending || result != value) skippedValues.emplace(value, result); + lowestOpen = std::min(outer, pending ? lowestOpen : NoOpenCycle); + return settled(result); + } + + bool chainTo(const IrValue* value, const IrValue* kept, std::uint32_t depth) { + value = value->Resolve(); + if (value == kept) return true; + if (depth > DepthLimit) return false; + if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u && chainTo(value->Argument(2), kept, depth + 1u) && (impliesExec(value->Argument(0)) || chainTo(value->Argument(1), kept, depth + 1u))) return true; + if (value->Opcode() == IrOpcode::Phi && definedInRegion(value)) { + if (!visiting.insert(value).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < value->PhiBlockCount(); ++incoming) all = chainTo(value->Argument(incoming), kept, depth + 1u); + visiting.erase(value); + if (all) return true; + } + if (value->Type() == IrType::Bool && kept->Type() == IrType::Bool) return impliesExec(value) && impliesExec(kept); + if (value->Type() != IrType::U32 || kept->Type() != IrType::U32) return false; + if (value->ArgumentCount() == 2u) { + const IrValue* left = value->Argument(0); + const IrValue* right = value->Argument(1); + if (value->Opcode() == IrOpcode::BitwiseAnd32 && ((chainTo(left, kept, depth + 1u) && fullWord(right, 0)) || (chainTo(right, kept, depth + 1u) && fullWord(left, 0)))) return true; + if ((value->Opcode() == IrOpcode::BitwiseOr32 || value->Opcode() == IrOpcode::BitwiseXor32) && ((chainTo(left, kept, depth + 1u) && boundedWord(right, 0)) || (chainTo(right, kept, depth + 1u) && boundedWord(left, 0)))) return true; + } + return boundedWord(value, 0) && boundedWord(kept, 0); + } + + bool fullWord(const IrValue* word, std::uint32_t depth) { + word = word->Resolve(); + if (word->HasImmediate()) return word->Type() == IrType::U32 && word->ImmediateU32() == 0xffffffffu; + if (depth > DepthLimit) return false; + switch (word->Opcode()) { + case IrOpcode::BitwiseNot32: + return boundedWord(word->Argument(0), depth + 1u); + case IrOpcode::BitwiseOr32: + return fullWord(word->Argument(0), depth + 1u) || fullWord(word->Argument(1), depth + 1u); + case IrOpcode::BitwiseAnd32: + return fullWord(word->Argument(0), depth + 1u) && fullWord(word->Argument(1), depth + 1u); + default: + return false; + } + } + + bool windowKept(const IrValue* value, const IrValue* index, std::uint32_t depth) { + value = value->Resolve(); + if (value->Opcode() == IrOpcode::LoadVectorWindow) return sameWord(value->Argument(0)->Resolve(), index); + if (depth > DepthLimit || !isSelect(value->Opcode()) || value->ArgumentCount() != 3u) return false; + if (!windowKept(value->Argument(2), index, depth + 1u)) return false; + return impliesExec(value->Argument(0)) || windowKept(value->Argument(1), index, depth + 1u); + } + + bool boundedWord(const IrValue* word, std::uint32_t depth) { + word = word->Resolve(); + if (word->HasImmediate()) return word->Type() == IrType::U32 && word->ImmediateU32() == 0u; + if (depth > DepthLimit) return false; + if (terms.size() == 1u && terms.front().high != nullptr && (terms.front().value->Resolve() == word || terms.front().high->Resolve() == word)) return true; + if (isBallotWord(word, 0u) || isBallotWord(word, 1u)) return impliesExec(word->Argument(0)->Resolve()->Argument(0)); + const auto argument = [&](std::size_t index) { return word->Argument(index); }; + switch (word->Opcode()) { + case IrOpcode::BitwiseAnd32: + return boundedWord(argument(0), depth + 1u) || boundedWord(argument(1), depth + 1u); + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + return boundedWord(argument(0), depth + 1u) && boundedWord(argument(1), depth + 1u); + case IrOpcode::SelectU32: + return boundedWord(argument(1), depth + 1u) && boundedWord(argument(2), depth + 1u); + case IrOpcode::Phi: { + if (!visiting.insert(word).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < word->PhiBlockCount(); ++incoming) all = boundedWord(argument(incoming), depth + 1u); + visiting.erase(word); + return all; + } + default: + return false; + } + } + + bool stable(const IrValue* value, std::uint32_t depth) { + value = value->Resolve(); + if (!definedInRegion(value)) return true; + if (depth > DepthLimit) return false; + if (const auto found = stability.find(value); found != stability.end()) return found->second; + stability.emplace(value, true); + const auto arguments = [&] { + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { + if (!stable(value->Argument(index), depth + 1u)) return false; + } + return true; + }; + bool result = false; + if (value->Opcode() == IrOpcode::Phi) { + result = stablePhi(*value, depth); + } else if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u) { + result = (impliesExec(value->Argument(0)) && stable(value->Argument(2), depth + 1u)) || arguments(); + } else if (value->Opcode() == IrOpcode::LoadVectorWindow) { + result = stable(value->Argument(0), depth + 1u); + } else if (value->Type() == IrType::Bool && impliesExec(value)) { + result = true; + } else if (pureLaneOp(value->Opcode())) { + result = arguments(); + } else { + result = boundedWord(value, 0); + } + stability[value] = result; + return result; + } + + bool stablePhi(const IrValue& phi, std::uint32_t depth) { + const bool entered = std::ranges::any_of(phi.Parent()->Predecessors(), [&](const IrBlock* predecessor) { return !inRegion(predecessor); }); + if (entered) { + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (inRegion(phi.PhiBlock(incoming)) && !chainTo(phi.Argument(incoming), &phi, 0)) return false; + } + return true; + } + for (std::size_t base = 0; base < phi.PhiBlockCount(); ++base) { + const IrValue* kept = phi.Argument(base)->Resolve(); + bool all = true; + for (std::size_t incoming = 0; all && incoming < phi.PhiBlockCount(); ++incoming) all = incoming == base || chainTo(phi.Argument(incoming), kept, 0); + if (all && stable(kept, depth + 1u)) return true; + } + return false; + } + + const IrProgram& program; + const Uniformity& uniformity; + const std::vector& order; + std::unordered_map indices; + std::unordered_map ids; + std::vector dominators; + std::vector reachesEnd; + std::vector region; + std::vector skippedRegion; + std::size_t branchIndex = 0; + std::size_t skipIndex = 0; + std::size_t mergeIndex = 0; + bool cyclic = false; + const IrValue* exec = nullptr; + std::vector terms; + std::unordered_set visiting; + std::unordered_map stability; + static constexpr std::size_t NoOpenCycle = ~std::size_t{0}; + + std::unordered_map skippedValues; + std::unordered_map passing; + std::size_t lowestOpen = NoOpenCycle; + std::unordered_set observed; +}; + +std::optional analyze(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup) { + if (program.Resources().stage != IrShaderStage::Compute || program.WaveSize() != 64u) return Rejection{"not a wave64 compute program", nullptr}; + const std::array sizes{std::max(workgroup.threadsNum[0], 1u), std::max(workgroup.threadsNum[1], 1u), std::max(workgroup.threadsNum[2], 1u)}; + bool reads = false; + bool barrier = false; + const auto& memory = program.Resources().memoryInfo; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + const auto opcode = inst->Opcode(); + reads |= opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::ReadLane; + barrier |= opcode == IrOpcode::Barrier; + if (opcode == IrOpcode::GetBuiltin) { + const IrValue* kind = inst->Argument(0)->Resolve(); + const auto input = kind->HasImmediate() ? static_cast(kind->ImmediateU32()) : StageInputKind::Parameter; + if (input != StageInputKind::WorkgroupId && input != StageInputKind::LocalInvocationId && input != StageInputKind::LocalInvocationIndex && input != StageInputKind::GlobalInvocationId && input != StageInputKind::DispatchThreadLimit) return Rejection{"a builtin outside compute dispatch ids", inst}; + continue; + } + const bool wave = opcode == IrOpcode::LaneId || opcode == IrOpcode::Ballot || opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::ReadLane; + if (!wave && !isLaneLocal(opcode) && SharedAccessOf(opcode) == SharedAccess::None) return Rejection{"an operation across lanes", inst}; + if (opcode == IrOpcode::ReadConstBuffer || (BufferAccessOf(opcode) == BufferAccess::None && AddressOpcodeInfoOf(opcode).access == AddressAccess::None && ImageOpcodeInfoOf(opcode).access == ImageAccess::None)) continue; + const auto index = inst->Flags().index; + if (index >= memory.size()) continue; + if (memory[index].gpuDescriptor) return Rejection{"a GPU-selected buffer descriptor", inst}; + } + } + if (auto rejected = checkLaneMasks(program)) return rejected; + bool writes = false; + if (auto rejected = checkLds(program, sizes, writes)) return rejected; + if (writes && sizes[0] * sizes[1] * sizes[2] <= program.WaveSize()) return Rejection{"LDS writes in a one-wave workgroup", nullptr}; + std::optional uniformity; + const auto uniform = [&]() -> const Uniformity& { + if (!uniformity) uniformity.emplace(program); + return *uniformity; + }; + if (reads) { + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if ((inst->Opcode() == IrOpcode::ReadFirstLane || inst->Opcode() == IrOpcode::ReadLane) && !uniform().Uniform(inst->Argument(0))) return Rejection{"a lane read of a lane-varying value", inst}; + } + } + } + bool execBranches = false; + for (const auto& info : program.Metadata().blockInfo) { + if (info.indirectTarget != nullptr && !uniform().Uniform(info.indirectTarget)) return Rejection{"an indirect branch on a lane-varying value", info.indirectTarget}; + if (info.terminator.kind != TerminatorKind::ConditionalBranch || info.condition == nullptr) continue; + switch (info.terminator.condition) { + case BranchCondition::Always: + break; + case BranchCondition::ExecZero: + case BranchCondition::ExecNonZero: + execBranches = true; + break; + case BranchCondition::VccZero: + case BranchCondition::VccNonZero: + if (auto rejected = checkVccBranch(info, uniform())) return rejected; + execBranches |= halfDecided(info, uniform()); + break; + default: + if (!uniform().Uniform(info.condition)) return Rejection{"a scalar branch on a lane-varying value", info.condition->Resolve()}; + break; + } + } + if (barrier && execBranches) return Rejection{"a barrier in a program with exec branches", nullptr}; + const auto& metadata = program.Metadata(); + if (metadata.skippedRegionsChecked) return metadata.skippedRegionRejection.empty() ? std::nullopt : std::optional(Rejection{metadata.skippedRegionRejection.c_str(), nullptr}); + if (execBranches) return SkippedRegions(program, uniform()).Check(); + return std::nullopt; +} + +} + +SingleLaneVerdict AnalyzeSingleLane(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup) { + const auto rejected = analyze(program, workgroup); + if (!rejected) return {true, nullptr, nullptr}; + return {false, rejected->reason, rejected->at}; +} + +void RecordSkippedRegions(IrProgram& program) { + auto& metadata = program.Metadata(); + if (metadata.skippedRegionsChecked || program.Resources().stage != IrShaderStage::Compute || program.WaveSize() != 64u) return; + const Uniformity uniformity(program); + const auto rejected = SkippedRegions(program, uniformity).Check(); + metadata.skippedRegionsChecked = true; + if (rejected) metadata.skippedRegionRejection = std::string(rejected->reason) + (rejected->at == nullptr ? std::string() : " at " + DescribeValue(rejected->at, 3)); +} + +} diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 74739dfae658..32700dc3f814 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -21,20 +21,6 @@ std::string& failureReason() { return HostThreadLocal(); } -std::string DescribeValue(const IrValue* value, std::uint32_t depth) { - if (value == nullptr) return "null"; - value = value->Resolve(); - std::string text(IrOpcodeName(value->Opcode())); - if (value->HasImmediate() && value->Type() == IrType::U32) return text + "(" + std::to_string(value->ImmediateU32()) + ")"; - if (depth == 0 || value->ArgumentCount() == 0) return text; - text += "("; - for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { - if (index != 0) text += ", "; - text += DescribeValue(value->Argument(index), depth - 1); - } - return text + ")"; -} - bool Fail(std::string reason) { failureReason() = std::move(reason); return false; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 437d284e5e5e..b38bbff8c453 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -42,6 +42,7 @@ #include "Optimization/include/Optimization/ResourceTracker.hpp" #include "Optimization/include/Optimization/VectorWindowStoreEliminator.hpp" #include "Optimization/include/Optimization/ShaderInfoCollector.hpp" +#include "Optimization/include/Optimization/SingleLaneAnalysis.hpp" #include "Optimization/include/Optimization/SrtWalker.hpp" #include "Optimization/include/Optimization/SsaBuilder.hpp" #include "Optimization/include/Optimization/TessellationLowering.hpp" @@ -108,6 +109,15 @@ std::uint32_t HostSubgroupSize(const RecompileRequest& request) { return request.target.subgroupSize; } +bool SingleLaneCandidate(const RecompileRequest& request) { + static const bool enabled = std::getenv("APS5_NO_SINGLE_LANE_RULE") == nullptr; + return enabled && request.shader.stage == ShaderStage::Compute && request.context.waveSize == 64u && HostSubgroupSize(request) == 32u; +} + +std::uint32_t CacheLayout(const RecompileRequest& request) { + return HostSubgroupSize(request) | (SingleLaneCandidate(request) ? 0x100u : 0u); +} + bool CapturesGeometryInput(const RecompileRequest& request) { return request.shader.stage == ShaderStage::TessellationEvaluation && request.graphics && request.graphics->mesh; } @@ -308,6 +318,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat deadCodeEliminator.Eliminate(program); } + if (SingleLaneCandidate(request)) RecordSkippedRegions(program); if (vectorWindowStoreEliminator.Eliminate(program).removedStores != 0u) { deadCodeEliminator.Eliminate(program); } @@ -597,11 +608,9 @@ void prepareInterface(const RecompileRequest& request, IrProgram& program) { resourceMaterializer.ApplyStaticInterface(program, nativeSampleOffsets); constexpr ConstantFolder constantFolder; - if (request.shader.stage == ShaderStage::Vertex && constantFolder.FoldVertexLaneBounds(program, HostSubgroupSize(request), request.target.spirvVersion)) { - constantFolder.Fold(program); - ResolveControlFlowIdentities(program); - } + if (request.shader.stage == ShaderStage::Vertex && constantFolder.FoldVertexLaneBounds(program, HostSubgroupSize(request), request.target.spirvVersion)) constantFolder.Fold(program); + ResolveControlFlowIdentities(program); deadCodeEliminator.RemoveIdentities(program); constexpr RedundantValueEliminator redundantValueEliminator; static_cast(redundantValueEliminator.Eliminate(program)); @@ -619,6 +628,25 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) const bool stencilReplay = request.shader.stage == ShaderStage::Fragment && std::ranges::any_of(program.Info().outputs, [](const StageOutput& output) { return output.kind == StageOutputKind::StencilRef; }) && std::ranges::find(request.target.supportedCapabilities, static_cast(spv::CapabilityStencilExportEXT)) == request.target.supportedCapabilities.end(); auto bindings = bindingAllocator.Allocate(program, request.layout, registration, stencilReplay); + auto emitInfo = inputInfo; + ShaderComputeInputInfo singleLane; + auto hostSubgroupSize = HostSubgroupSize(request); + if (SingleLaneCandidate(request) && inputInfo.compute != nullptr) { + const auto verdict = AnalyzeSingleLane(program, *inputInfo.compute); + static const bool trace = std::getenv("APS5_TRACE_SINGLE_LANE") != nullptr; + if (trace) { + const auto& threads = inputInfo.compute->threadsNum; + const std::string at = verdict.at == nullptr ? std::string() : " at " + DescribeValue(verdict.at, 3); + std::fprintf(stderr, "[single-lane] program 0x%llx (%ux%ux%u threads): %s%s%s\n", static_cast(request.shader.codeAddress), threads[0], threads[1], threads[2], verdict.eligible ? "one lane per invocation" : "two lanes per invocation: ", verdict.eligible ? "" : verdict.reason, at.c_str()); + } + if (verdict.eligible) { + singleLane = *inputInfo.compute; + singleLane.hostSubgroupSize = 64u; + emitInfo.compute = &singleLane; + hostSubgroupSize = 64u; + } + } + SpirvTargetOptions targetOptions {}; targetOptions.vulkanVersion = request.target.vulkanVersion; targetOptions.spirvVersion = request.target.spirvVersion; @@ -683,18 +711,18 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.variantId = nextVariantId(); SpirvEmitInfo emitted; try { - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } catch (const SpannedWaveUnsupported& error) { std::fprintf(stderr, "[spanned-wave] 0x%llx falls back to two lanes per invocation: %s\n", static_cast(request.shader.codeAddress), error.what()); targetOptions.spannedWaves = false; - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } if (registration && result.spirv.size() > request.target.registrationWordLimit) { registration = false; program.Metadata().bindingLayoutComplete = false; bindings = bindingAllocator.Allocate(program, request.layout, false, stencilReplay); targetOptions.registrationModules = false; - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } result.registrationModule = registration; if (request.shader.stage == ShaderStage::Mesh && request.graphics) result.meshBarycentrics = request.graphics->meshBarycentrics; @@ -713,7 +741,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.shaderDataDwords = bindings.layout.ShaderDataDwords(); result.imageMetadataDword = bindings.layout.ImageMetadataDword(); result.runtimeImageCount = bindings.layout.runtimeImageCount; - result.hostSubgroupSize = HostSubgroupSize(request); + result.hostSubgroupSize = hostSubgroupSize; result.fullSubgroups = emitted.fullSubgroups; result.functionLdsBuffer = emitted.functionLdsBuffer; result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; @@ -1381,7 +1409,7 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, std::vector diskKey; std::shared_ptr variant; if (disk) { - ShaderDiskCache::BuildKey(request, HostSubgroupSize(request), diskKey); + ShaderDiskCache::BuildKey(request, CacheLayout(request), diskKey); CompiledVariant loaded; if (ShaderDiskCache::Load(diskKey, loaded)) { loaded.layout = request.layout; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 98a5c4d1bcc9..56067b0e3be4 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -311,6 +311,7 @@ bool AllOnesWord(const IrValue* word) { bool AllLanesBit(const IrValue* bit) { bit = Resolved(bit); + if (bit != nullptr && bit->HasImmediate()) return bit->Type() == IrType::Bool && bit->ImmediateBool(); if (Is(bit, IrOpcode::LogicalOr) && EitherOrder(bit, [](const IrValue* inverted, const IrValue* other) { return Is(inverted, IrOpcode::LogicalNot) && Resolved(inverted->Argument(0)) == other; })) return true; const auto words = LaneBitWords(bit); return words && AllOnesWord((*words)[0]) && AllOnesWord((*words)[1]); @@ -460,20 +461,14 @@ class EntryLaneWalk { std::uint32_t budget = 2048u; }; -bool EveryLaneBit(const IrValue* bit, bool constantTrue) { - const auto* resolved = Resolved(bit); - if (constantTrue && resolved != nullptr && resolved->HasImmediate()) return resolved->Type() == IrType::Bool && resolved->ImmediateBool(); - return AllLanesBit(resolved); -} - bool WritesEveryLane(const IrValue& update) { const auto flags = update.Flags(); - return flags.boundControl && flags.rowMask == 0xfu && flags.bankMask == 0xfu && EveryLaneBit(update.Argument(2), true); + return flags.boundControl && flags.rowMask == 0xfu && flags.bankMask == 0xfu && AllLanesBit(update.Argument(2)); } bool KeepsNoOldValue(const IrValue& permlane) { const auto flags = permlane.Flags(); - return flags.fetchInactive || flags.boundControl || EveryLaneBit(permlane.Argument(3), true); + return flags.fetchInactive || flags.boundControl || AllLanesBit(permlane.Argument(3)); } bool UpperLaneTest(const IrValue* bit) { @@ -592,40 +587,40 @@ std::optional> ReductionOperands(const IrValue* co const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control, bool narrow) { const bool zeroFill = narrow && reduction.identity == 0u && !reduction.floating; - if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control, zeroFill) || !EveryLaneBit(step->Argument(2), narrow)) return nullptr; + if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control, zeroFill) || !AllLanesBit(step->Argument(2))) return nullptr; const auto* previous = zeroFill && WritesEveryLane(*step) ? nullptr : Resolved(step->Argument(1)); const IrValue* input = nullptr; const auto operands = ReductionOperands(Resolved(step->Argument(0)), reduction); const bool matched = operands && EitherOrder(*operands, [&](const IrValue* moved, const IrValue* own) { input = own; - return (previous == nullptr || own == previous) && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control, zeroFill) && Resolved(moved->Argument(0)) == own && EveryLaneBit(moved->Argument(1), narrow); + return (previous == nullptr || own == previous) && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control, zeroFill) && Resolved(moved->Argument(0)) == own && AllLanesBit(moved->Argument(1)); }); return matched ? input : nullptr; } -const IrValue* Unmasked(const IrValue* value, bool narrow) { - while (Is(value, IrOpcode::SelectU32) && EveryLaneBit(value->Argument(0), narrow)) value = Resolved(value->Argument(1)); +const IrValue* Unmasked(const IrValue* value) { + while (Is(value, IrOpcode::SelectU32) && AllLanesBit(value->Argument(0))) value = Resolved(value->Argument(1)); return value; } -bool CrossesRowsOf(const IrValue* value, const IrValue* scan, bool narrow) { - const auto* permlane = Unmasked(value, narrow); +bool CrossesRowsOf(const IrValue* value, const IrValue* scan) { + const auto* permlane = Unmasked(value); if (!Is(permlane, IrOpcode::Permlane16U32)) return false; const auto flags = permlane->Flags(); - return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && EveryLaneBit(permlane->Argument(3), narrow); + return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && AllLanesBit(permlane->Argument(3)); } std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStage stage, bool narrow) { - const auto* combined = Unmasked(Resolved(value), narrow); + const auto* combined = Unmasked(Resolved(value)); std::optional> operands; const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return (operands = ReductionOperands(combined, candidate)).has_value(); }); const IrValue* scan = nullptr; - if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own, narrow); })) return std::nullopt; + if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; for (const auto control : {0x118u, 0x114u, 0x112u, 0x111u}) { scan = RowScanStepInput(scan, *reduction, control, narrow); if (scan == nullptr) return std::nullopt; } - const auto* source = Unmasked(scan, narrow); + const auto* source = Unmasked(scan); const bool entryKeys = Is(source, IrOpcode::SelectU32) && IsU32(source->Argument(2), reduction->identity) && EntryLaneWalk{stage}.ZeroBit(source->Argument(0)); const bool upperKeys = narrow && reduction->identity == 0u && !reduction->floating && UpperLaneWalk{stage}.ZeroValue(source); if (!entryKeys && !upperKeys) return std::nullopt; @@ -682,7 +677,7 @@ bool ActiveMaskedBit(const IrValue* bit) { } std::optional MatchRowScan(const IrValue* value, IrShaderStage stage, bool narrow) { - const auto* top = Unmasked(Resolved(value), narrow); + const auto* top = Unmasked(Resolved(value)); for (const auto& reduction : WaveReductions) { if (reduction.floating) continue; const IrValue* scan = top; @@ -691,7 +686,7 @@ std::optional MatchRowScan(const IrValue* value, IrShaderStage sta if (scan == nullptr) break; } if (scan == nullptr) continue; - const auto* source = Unmasked(scan, narrow); + const auto* source = Unmasked(scan); if (Is(source, IrOpcode::SelectU32) && IsU32(source->Argument(2), reduction.identity) && (ActiveMaskedBit(source->Argument(0)) || EntryLaneWalk{stage}.ZeroBit(source->Argument(0)))) return HalfWaveScan{&reduction, source}; } return std::nullopt; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp index 5a87cd02e408..71a400eaef2d 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp @@ -214,6 +214,11 @@ void TranslationContext::AddBranchCondition(const BasicBlock& source, BlockInfo& } info.condition = &condition.Value(); (void)ir.Emit(IrOpcode::Reference, IrType::Void, {info.condition}); + const auto kind = source.terminator.condition; + if (kind == BranchCondition::VccZero || kind == BranchCondition::VccNonZero || kind == BranchCondition::ExecZero || kind == BranchCondition::ExecNonZero) { + info.exec = &ir.GetExec(); + (void)ir.Emit(IrOpcode::Reference, IrType::Void, {info.exec}); + } } } diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 05678c738d62..6fd5fd3841c4 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -48,6 +48,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PERF_DOM_INTERSECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | computes dominator intersections into a fresh vector instead of intersecting in place | | `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them | | `APS5_NO_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes a fresh result for every recompile instead of reusing the result memo | +| `APS5_NO_SINGLE_LANE_RULE` | presence | `core/shader/recompiler/Recompiler.cpp` | | keeps every wave64 compute program on a 32-lane host at two guest lanes per invocation, even where the single-lane analysis proves one lane gives the same results | | `APS5_NO_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | runs the runtime-source walk of a plan in the IR evaluator instead of the compiled tape | | `APS5_NO_TABLE_RESOLUTION_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | stops a result memo miss from reusing an unchanged image table's resolution, so the table is resolved again | | `APS5_NO_TABLE_SNAPSHOT_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | turns off the image table snapshot reuse, so every capture reads and compares each table column | @@ -62,6 +63,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_TRACE_BDA` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | prints the first few accesses that make a program take the address-based path | | `APS5_TRACE_BUFFER_WRITTEN` | presence | `core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp` | | prints how many guest buffer elements a program may store to and how many it only loads | | `APS5_TRACE_IMAGE_TABLE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | traces image-table resolution | +| `APS5_TRACE_SINGLE_LANE` | presence | `core/shader/recompiler/Recompiler.cpp` | | prints each wave64 compute program's single-lane verdict and, for a program kept at two lanes, the reason and the value it was found at | | `APS5_VERIFY_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | also materializes every patched result memo hit, prints the first twenty whose patched result differs and counts them in the result memo report of `APS5_PROFILE_DRAW` | | `APS5_VERIFY_SRT_COMPACT` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp` | | walks every SRT descriptor a second time through the IR evaluator and throws when the two walks differ | | `APS5_VERIFY_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | walks a plan's runtime sources a second time through the evaluator, adopts its answers where the two differ and prints the first twenty differences |