From 433fe37a09853d1a9ffb9902e00b7d35e0afa51b Mon Sep 17 00:00:00 2001 From: Mehran Date: Wed, 28 Oct 2026 10:07:08 +0300 Subject: [PATCH 1/2] perf(shader): fold bitwise self and complement identities Fold And32, Or32 and Xor32 of a value with itself, with its complement and with an absorbing constant, bit 0 of all ones shifted right by an amount masked to 0-31, and 0 shifted right by any amount. The lane masks the translator builds reduce this way: s_orn2_saveexec_b64 of exec gives exec | ~exec, all ones, so its lane bits and the exec selects they guard fold away. The half-wave scan matcher (24894376ab) took a constant-true lane bit only on hosts narrower than the wave and otherwise required the exec | ~exec structure, which no longer survives the fold. A constant-true bit now counts as every lane on every host, the same proof as x | ~x, so EveryLaneBit and its narrow-only flag go. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 13 ++ .../tests/BitwiseIdentities.cpp | 210 ++++++++++++++++++ .../Optimization/src/ConstantFolder.cpp | 40 ++++ .../src/SpirvAlu/SpirvAluEmitterMath.cpp | 35 ++- 4 files changed, 278 insertions(+), 20 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 2aa47b1ae932..666dfb379f14 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1210,6 +1210,19 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_bitwise_identities_tests EXCLUDE_FROM_ALL tests/BitwiseIdentities.cpp) +target_link_libraries(agc_bitwise_identities_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_bitwise_identities_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_bitwise_identities_tests) +if(BUILD_TESTING) + set_target_properties(agc_bitwise_identities_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_bitwise_identities COMMAND agc_bitwise_identities_tests) + if(WIN32) + set_tests_properties(agc_bitwise_identities PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp b/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp new file mode 100644 index 000000000000..cd5b75c5f383 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp @@ -0,0 +1,210 @@ +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "Optimization/ConstantFolder.hpp" +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +constexpr IrOpcode And = IrOpcode::BitwiseAnd32; +constexpr IrOpcode Or = IrOpcode::BitwiseOr32; +constexpr IrOpcode Xor = IrOpcode::BitwiseXor32; +constexpr IrOpcode Shift = IrOpcode::ShiftRightLogical32; + +struct Builder { + IrProgram program; + IrBuilder ir; + IrBlock* block = nullptr; + std::vector kept; + + Builder() : ir(program) { + block = &program.CreateBlock(); + program.SetEntryBlock(*block); + program.BlockOrder().push_back(block); + ir.SetInsertionPoint(*block); + } + + IrValue& Emit(IrOpcode opcode, std::initializer_list arguments) { + return ir.Emit(opcode, IrType::U32, arguments); + } + + IrValue& Value() { + return Emit(IrOpcode::LaneId, {}); + } + + IrValue& Other(IrValue& value) { + return Emit(IrOpcode::IAdd32, {&value, &ir.Constant(1u)}); + } + + IrValue& Not(IrValue& value) { + return Emit(IrOpcode::BitwiseNot32, {&value}); + } + + IrValue& Same(IrValue& value) { + auto& identity = program.CreateValue(IrOpcode::Identity, IrType::U32); + identity.AddArgument(&value); + block->AppendInstruction(&identity); + return identity; + } + + IrValue& Binary(IrOpcode opcode, IrValue& lhs, IrValue& rhs) { + return Emit(opcode, {&lhs, &rhs}); + } + + IrValue& Binary(IrOpcode opcode, IrValue& lhs, std::uint32_t rhs) { + return Binary(opcode, lhs, ir.Constant(rhs)); + } + + IrValue& Binary(IrOpcode opcode, std::uint32_t lhs, IrValue& rhs) { + return Binary(opcode, ir.Constant(lhs), rhs); + } + + std::size_t Keep(IrValue& value) { + kept.push_back(&ir.Emit(IrOpcode::ReferenceU32, IrType::Void, {&value})); + return kept.size() - 1u; + } + + void Fold() { + static_cast(ir.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder().Fold(program); + } + + const IrValue& Result(std::size_t index) const { + return *kept.at(index)->Argument(0)->Resolve(); + } + + bool Folds(std::size_t index, std::uint32_t expected) const { + const IrValue& value = Result(index); + return value.HasImmediate() && value.Type() == IrType::U32 && value.ImmediateU32() == expected; + } + + bool Becomes(std::size_t index, const IrValue& expected) const { + return &Result(index) == &expected; + } + + bool Keeps(std::size_t index, IrOpcode opcode) const { + return Result(index).Opcode() == opcode; + } +}; + +bool expect(const char* name, bool condition) { + if (!condition) std::fprintf(stderr, "%s\n", name); + return condition; +} + +bool run(const char* name, const std::function& test) { + try { + return expect(name, test()); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("and and or of a value with itself give the value and xor gives zero", [] { + Builder b; + auto& value = b.Value(); + const auto both = b.Keep(b.Binary(And, value, value)); + const auto either = b.Keep(b.Binary(Or, value, value)); + const auto differ = b.Keep(b.Binary(Xor, value, value)); + b.Fold(); + return b.Becomes(both, value) && b.Becomes(either, value) && b.Folds(differ, 0u); + }); + + passed &= run("a value and its complement give zero for and and all ones for or and xor in either order", [] { + Builder b; + auto& value = b.Value(); + auto& inverse = b.Not(value); + const auto andLeft = b.Keep(b.Binary(And, value, inverse)); + const auto andRight = b.Keep(b.Binary(And, inverse, value)); + const auto orLeft = b.Keep(b.Binary(Or, value, inverse)); + const auto orRight = b.Keep(b.Binary(Or, inverse, value)); + const auto xorLeft = b.Keep(b.Binary(Xor, value, inverse)); + const auto xorRight = b.Keep(b.Binary(Xor, inverse, value)); + b.Fold(); + return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu) && b.Folds(xorLeft, 0xffffffffu) && b.Folds(xorRight, 0xffffffffu); + }); + + passed &= run("and with zero and or with all ones absorb the other operand in either order", [] { + Builder b; + auto& value = b.Value(); + const auto andLeft = b.Keep(b.Binary(And, value, 0u)); + const auto andRight = b.Keep(b.Binary(And, 0u, value)); + const auto orLeft = b.Keep(b.Binary(Or, value, 0xffffffffu)); + const auto orRight = b.Keep(b.Binary(Or, 0xffffffffu, value)); + b.Fold(); + return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu); + }); + + passed &= run("operands match through identities", [] { + Builder b; + auto& value = b.Value(); + auto& inverse = b.Same(b.Not(b.Same(value))); + const auto both = b.Keep(b.Binary(And, value, b.Same(value))); + const auto differ = b.Keep(b.Binary(Xor, b.Same(value), value)); + const auto complement = b.Keep(b.Binary(Or, inverse, value)); + b.Fold(); + return b.Becomes(both, value) && b.Folds(differ, 0u) && b.Folds(complement, 0xffffffffu); + }); + + passed &= run("different values, the complement of another value and xor with all ones are kept", [] { + Builder b; + auto& value = b.Value(); + auto& other = b.Other(value); + auto& inverse = b.Not(other); + const auto both = b.Keep(b.Binary(And, value, other)); + const auto either = b.Keep(b.Binary(Or, value, other)); + const auto differ = b.Keep(b.Binary(Xor, value, other)); + const auto andInverse = b.Keep(b.Binary(And, value, inverse)); + const auto orInverse = b.Keep(b.Binary(Or, inverse, value)); + const auto xorInverse = b.Keep(b.Binary(Xor, value, inverse)); + const auto flipped = b.Keep(b.Binary(Xor, value, 0xffffffffu)); + b.Fold(); + return b.Keeps(both, And) && b.Keeps(either, Or) && b.Keeps(differ, Xor) && b.Keeps(andInverse, And) && b.Keeps(orInverse, Or) && b.Keeps(xorInverse, Xor) && b.Keeps(flipped, Xor); + }); + + passed &= run("bit 0 of all ones shifted right by an amount masked to 31 is one in either order", [] { + Builder b; + auto& lane = b.Value(); + auto& maskRight = b.Binary(And, lane, 31u); + auto& maskLeft = b.Binary(And, 31u, lane); + const auto bitRight = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, maskRight), 1u)); + const auto bitLeft = b.Keep(b.Binary(And, 1u, b.Binary(Shift, 0xffffffffu, maskLeft))); + b.Fold(); + return b.Folds(bitRight, 1u) && b.Folds(bitLeft, 1u); + }); + + passed &= run("bit 0 of an unmasked or wider shift, of another word, and bit 1 are kept", [] { + Builder b; + auto& lane = b.Value(); + auto& masked = b.Binary(And, lane, 31u); + const auto unmasked = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, lane), 1u)); + const auto wider = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, b.Binary(And, lane, 63u)), 1u)); + const auto word = b.Keep(b.Binary(And, b.Binary(Shift, 0x7fffffffu, masked), 1u)); + const auto second = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, masked), 2u)); + b.Fold(); + return b.Keeps(unmasked, And) && b.Keeps(wider, And) && b.Keeps(word, And) && b.Keeps(second, And); + }); + + passed &= run("zero shifted right by any amount is zero and another word is kept", [] { + Builder b; + auto& lane = b.Value(); + const auto zero = b.Keep(b.Binary(Shift, 0u, lane)); + const auto word = b.Keep(b.Binary(Shift, b.Other(lane), lane)); + b.Fold(); + return b.Folds(zero, 0u) && b.Keeps(word, Shift); + }); + + return passed ? 0 : 1; +} diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index 43db702ad8bd..a15841e025d9 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -664,6 +664,40 @@ bool replaceBinaryIdentity(IrValue& inst, IrType type, std::uint64_t identity) { return false; } +bool foldBitwiseSelf(IrBuilder& builder, IrValue& inst) { + auto& lhs = resolveArg(inst, 0); + auto& rhs = resolveArg(inst, 1); + const auto opcode = inst.Opcode(); + const std::uint32_t absorbing = opcode == IrOpcode::BitwiseOr32 ? 0xffffffffu : 0u; + if (opcode != IrOpcode::BitwiseXor32 && (isImmediateU32(lhs, absorbing) || isImmediateU32(rhs, absorbing))) { + replaceWith(inst, builder.Constant(absorbing)); + return true; + } + if (&lhs == &rhs) { + if (opcode == IrOpcode::BitwiseXor32) replaceWith(inst, builder.Constant(0u)); + else replaceWith(inst, lhs); + return true; + } + const bool complement = (lhs.Opcode() == IrOpcode::BitwiseNot32 && &resolveArg(lhs, 0) == &rhs) || (rhs.Opcode() == IrOpcode::BitwiseNot32 && &resolveArg(rhs, 0) == &lhs); + if (!complement) return false; + replaceWith(inst, builder.Constant(opcode == IrOpcode::BitwiseAnd32 ? 0u : 0xffffffffu)); + return true; +} + +bool foldLowBitOfOnes(IrBuilder& builder, IrValue& inst) { + auto& lhs = resolveArg(inst, 0); + auto& rhs = resolveArg(inst, 1); + const bool lowBit = isImmediateU32(rhs, 1u); + auto& shifted = lowBit ? lhs : rhs; + if (!lowBit && !isImmediateU32(lhs, 1u)) return false; + if (shifted.Opcode() != IrOpcode::ShiftRightLogical32 || !isImmediateU32(resolveArg(shifted, 0), 0xffffffffu)) return false; + auto& amount = resolveArg(shifted, 1); + if (amount.Opcode() != IrOpcode::BitwiseAnd32) return false; + if (!isImmediateU32(resolveArg(amount, 0), 31u) && !isImmediateU32(resolveArg(amount, 1), 31u)) return false; + replaceWith(inst, builder.Constant(1u)); + return true; +} + bool foldSelect(IrValue& inst) { auto& condition = resolveArg(inst, 0); if (isImmediate(condition, IrType::U1)) { @@ -1118,6 +1152,10 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { replaceWith(value, resolveArg(value, 0)); return true; } + if (isImmediateU32(resolveArg(value, 0), 0u)) { + replaceWith(value, resolveArg(value, 0)); + return true; + } if (!isImmediate(shift, IrType::U32)) return false; const auto amount = shift.ImmediateU32() & 31u; readThroughUnusedBits(value, 0, UINT32_MAX << amount); @@ -1170,6 +1208,7 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { return true; } if (replaceBinaryIdentity(value, IrType::U32, 0xffffffffu)) return true; + if (foldBitwiseSelf(builder, value) || foldLowBitOfOnes(builder, value)) return true; if (foldRedundantMask(value)) return true; bool changed = false; for (std::size_t side = 0; side < 2u; ++side) { @@ -1191,6 +1230,7 @@ bool tryFoldValue(IrProgram& program, IrValue& value, OperandFacts& facts) { return true; } if (replaceBinaryIdentity(value, IrType::U32, 0u)) return true; + if (foldBitwiseSelf(builder, value)) return true; return opcode == IrOpcode::BitwiseOr32 && mergeMaskedParts(program, value); } case IrOpcode::BitwiseNot32: { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 98a5c4d1bcc9..56067b0e3be4 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -311,6 +311,7 @@ bool AllOnesWord(const IrValue* word) { bool AllLanesBit(const IrValue* bit) { bit = Resolved(bit); + if (bit != nullptr && bit->HasImmediate()) return bit->Type() == IrType::Bool && bit->ImmediateBool(); if (Is(bit, IrOpcode::LogicalOr) && EitherOrder(bit, [](const IrValue* inverted, const IrValue* other) { return Is(inverted, IrOpcode::LogicalNot) && Resolved(inverted->Argument(0)) == other; })) return true; const auto words = LaneBitWords(bit); return words && AllOnesWord((*words)[0]) && AllOnesWord((*words)[1]); @@ -460,20 +461,14 @@ class EntryLaneWalk { std::uint32_t budget = 2048u; }; -bool EveryLaneBit(const IrValue* bit, bool constantTrue) { - const auto* resolved = Resolved(bit); - if (constantTrue && resolved != nullptr && resolved->HasImmediate()) return resolved->Type() == IrType::Bool && resolved->ImmediateBool(); - return AllLanesBit(resolved); -} - bool WritesEveryLane(const IrValue& update) { const auto flags = update.Flags(); - return flags.boundControl && flags.rowMask == 0xfu && flags.bankMask == 0xfu && EveryLaneBit(update.Argument(2), true); + return flags.boundControl && flags.rowMask == 0xfu && flags.bankMask == 0xfu && AllLanesBit(update.Argument(2)); } bool KeepsNoOldValue(const IrValue& permlane) { const auto flags = permlane.Flags(); - return flags.fetchInactive || flags.boundControl || EveryLaneBit(permlane.Argument(3), true); + return flags.fetchInactive || flags.boundControl || AllLanesBit(permlane.Argument(3)); } bool UpperLaneTest(const IrValue* bit) { @@ -592,40 +587,40 @@ std::optional> ReductionOperands(const IrValue* co const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control, bool narrow) { const bool zeroFill = narrow && reduction.identity == 0u && !reduction.floating; - if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control, zeroFill) || !EveryLaneBit(step->Argument(2), narrow)) return nullptr; + if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control, zeroFill) || !AllLanesBit(step->Argument(2))) return nullptr; const auto* previous = zeroFill && WritesEveryLane(*step) ? nullptr : Resolved(step->Argument(1)); const IrValue* input = nullptr; const auto operands = ReductionOperands(Resolved(step->Argument(0)), reduction); const bool matched = operands && EitherOrder(*operands, [&](const IrValue* moved, const IrValue* own) { input = own; - return (previous == nullptr || own == previous) && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control, zeroFill) && Resolved(moved->Argument(0)) == own && EveryLaneBit(moved->Argument(1), narrow); + return (previous == nullptr || own == previous) && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control, zeroFill) && Resolved(moved->Argument(0)) == own && AllLanesBit(moved->Argument(1)); }); return matched ? input : nullptr; } -const IrValue* Unmasked(const IrValue* value, bool narrow) { - while (Is(value, IrOpcode::SelectU32) && EveryLaneBit(value->Argument(0), narrow)) value = Resolved(value->Argument(1)); +const IrValue* Unmasked(const IrValue* value) { + while (Is(value, IrOpcode::SelectU32) && AllLanesBit(value->Argument(0))) value = Resolved(value->Argument(1)); return value; } -bool CrossesRowsOf(const IrValue* value, const IrValue* scan, bool narrow) { - const auto* permlane = Unmasked(value, narrow); +bool CrossesRowsOf(const IrValue* value, const IrValue* scan) { + const auto* permlane = Unmasked(value); if (!Is(permlane, IrOpcode::Permlane16U32)) return false; const auto flags = permlane->Flags(); - return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && EveryLaneBit(permlane->Argument(3), narrow); + return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && AllLanesBit(permlane->Argument(3)); } std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStage stage, bool narrow) { - const auto* combined = Unmasked(Resolved(value), narrow); + const auto* combined = Unmasked(Resolved(value)); std::optional> operands; const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return (operands = ReductionOperands(combined, candidate)).has_value(); }); const IrValue* scan = nullptr; - if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own, narrow); })) return std::nullopt; + if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; for (const auto control : {0x118u, 0x114u, 0x112u, 0x111u}) { scan = RowScanStepInput(scan, *reduction, control, narrow); if (scan == nullptr) return std::nullopt; } - const auto* source = Unmasked(scan, narrow); + const auto* source = Unmasked(scan); const bool entryKeys = Is(source, IrOpcode::SelectU32) && IsU32(source->Argument(2), reduction->identity) && EntryLaneWalk{stage}.ZeroBit(source->Argument(0)); const bool upperKeys = narrow && reduction->identity == 0u && !reduction->floating && UpperLaneWalk{stage}.ZeroValue(source); if (!entryKeys && !upperKeys) return std::nullopt; @@ -682,7 +677,7 @@ bool ActiveMaskedBit(const IrValue* bit) { } std::optional MatchRowScan(const IrValue* value, IrShaderStage stage, bool narrow) { - const auto* top = Unmasked(Resolved(value), narrow); + const auto* top = Unmasked(Resolved(value)); for (const auto& reduction : WaveReductions) { if (reduction.floating) continue; const IrValue* scan = top; @@ -691,7 +686,7 @@ std::optional MatchRowScan(const IrValue* value, IrShaderStage sta if (scan == nullptr) break; } if (scan == nullptr) continue; - const auto* source = Unmasked(scan, narrow); + const auto* source = Unmasked(scan); if (Is(source, IrOpcode::SelectU32) && IsU32(source->Argument(2), reduction.identity) && (ActiveMaskedBit(source->Argument(0)) || EntryLaneWalk{stage}.ZeroBit(source->Argument(0)))) return HalfWaveScan{&reduction, source}; } return std::nullopt; From 7a8da113db96ef882b362d52af10e10ee08a9773 Mon Sep 17 00:00:00 2001 From: Mehran Date: Wed, 28 Oct 2026 10:52:26 +0300 Subject: [PATCH 2/2] perf(shader): run wave64 compute at one lane where no lane can tell On a 32-lane host a wave64 compute program runs two guest lanes per invocation, so each invocation does every ALU operation, load and sample twice. SingleLaneAnalysis proves on the IR that the program gives every lane the same results at one guest lane per invocation, where each host subgroup holds half a wave, and the emitter then lays it out that way. A program qualifies when: - no lane reads another lane's value by index: readlane and readfirstlane read only uniform values, and there are no permutes, DPP, swizzles or GPU-selected buffer descriptors; - lane masks are combined bit by bit and read only as each lane's own bit, with the same expression over both 32-bit halves; - LDS is lane-private: every access is one affine function of the local invocation id and no two lanes' footprints share a byte, and a one-wave workgroup does not write it; - each term of a VCC branch's predicate is uniform or implied by the exec mask at the branch; - nothing leaves a region that one half of the wave can skip unless it is the same whether that half ran the region or not; - no barrier shares the program with a branch one half can decide alone. A half decides an exec branch, and a VCC branch on lane-varying terms, by itself. For each such branch the skipped region is what the taken side reaches before the immediate post-dominator and the other side does not. A value leaving it qualifies when it is a VGPR written under the branch's exec, a lane mask word that keeps its bits outside that exec, or a vector window store that keeps the old value outside it; scalar, VCC and goto flag values written in the region reject. A value no instruction reads, such as a phi cycle that only feeds itself, leaves nothing to compare. A memory write in the region needs a lane predicate inside the branch's exec. A loop left through such a branch must carry values that stay the same per lane and an exec that only narrows. Workgroups of at most 32 lanes get the same checks, since whole-wave mode still decides branches per half. The region check runs in the first simplify pass, before the vector window store and masked select eliminators drop exec selects whose inactive lanes nothing reads, and the program metadata keeps its verdict for the layout. Relatively addressed VGPRs live in a function-scope window per lane (LoadVectorWindow, StoreVectorWindow), so window accesses are lane-local; a window load counts as lane-varying, so readfirstlane of one keeps two lanes. BlockInfo::exec records the exec mask at each VCC and exec branch; the IR reclaim, identity resolution and redundant value elimination keep it current. A proven program needs no spanned layout, so APS5_SPANNED_WAVE applies only to programs the analysis rejects. The disk cache key marks the requests the rule lays out, so a run with APS5_NO_SINGLE_LANE_RULE=1 does not load their results. APS5_TRACE_SINGLE_LANE=1 prints each program's verdict and the reason a program keeps two lanes. --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 16 + .../libSceAgcDriver/tests/ShaderMemory.cpp | 14 + .../prx/libSceAgcDriver/tests/SingleLane.cpp | 643 +++++++ core/shader/recompiler/CMakeLists.txt | 1 + .../IrMetadata/ControlFlowInfo.hpp | 1 + .../IrMetadata/ProgramMetadata.hpp | 2 + .../IntermediateRepresentation/IrProgram.hpp | 1 + .../src/IrProgram.cpp | 19 +- .../Optimization/SingleLaneAnalysis.hpp | 20 + .../src/RedundantValueEliminator.cpp | 1 + .../Optimization/src/SingleLaneAnalysis.cpp | 1542 +++++++++++++++++ .../src/SrtWalker/SrtDescriptorEvaluation.cpp | 14 - core/shader/recompiler/Recompiler.cpp | 46 +- .../src/TranslationContext/ExecMask.cpp | 5 + docs/dev/ENVIRONMENT.md | 2 + 15 files changed, 2303 insertions(+), 24 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp create mode 100644 core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp create mode 100644 core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 666dfb379f14..845f49da5fd4 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1223,6 +1223,22 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_single_lane_tests EXCLUDE_FROM_ALL tests/SingleLane.cpp) +target_link_libraries(agc_single_lane_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_single_lane_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_single_lane_tests) +if(BUILD_TESTING) + set_target_properties(agc_single_lane_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_single_lane COMMAND agc_single_lane_tests) + add_test(NAME agc_single_lane_off COMMAND agc_single_lane_tests) + set_tests_properties(agc_single_lane PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1") + set_tests_properties(agc_single_lane_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_SINGLE_LANE_RULE=1") + if(WIN32) + set_tests_properties(agc_single_lane agc_single_lane_off PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 3efcc4e97acf..9cc080349dc8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -5428,6 +5428,18 @@ void verifyHalfWaveReduction() { expectFailure([&] { static_cast(pixel(upperHelperLoop, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: the low-word loop proof was reused for helper-inclusive upper lanes"); require(reductions(pixel(loopScan, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: a loop-carried saved EXEC mask exhausted the entry-lane proof on 32 lanes"); require(halves(pixel(loopScan, 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: a loop-carried saved EXEC mask did not preserve the two half-wave UMin reductions"); + auto computeScan = scan(0xffffffffu, 0x26000000u, 0x83800000u); + const auto exported = std::find(computeScan.begin(), computeScan.end(), 0xf800180fu); + exported[0] = 0xe0700000u; + exported[1] = 0x80000100u; + static constexpr std::array computeBuffer{0x10000000u, 0u, 0x40u, 0x00027facu}; + static constexpr std::array computeCapabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic}; + auto computeRequest = FrontendRequest(computeScan, computeBuffer); + computeRequest.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + computeRequest.target.supportedCapabilities = computeCapabilities; + const auto twoLane = Recompile(computeRequest); + require(twoLane.hostSubgroupSize == 32u && reductions(twoLane.spirv, spv::OpGroupNonUniformUMin) == 0u, "half-wave reduction: a two-lane wave64 compute UMin scan under s_or_saveexec_b64 -1 reached the half-wave scan matcher"); + require(halves(pixel(scan(0xffffffffu, 0x26000000u, 0x83800000u), 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: the wave64 UMin scan under s_or_saveexec_b64 -1 did not read lanes 31 and 63 as two half-wave UMin reductions on 64 lanes"); auto comparedScan = scan(0u, 0x38000000u, 0x88000000u); *std::find(comparedScan.begin(), comparedScan.end(), 0xbe98047eu) = 0x8a986a7eu; comparedScan.insert(comparedScan.begin(), 0x7c080080u); @@ -9315,8 +9327,10 @@ int main(int argc, char** argv) { try { #ifdef _WIN32 _putenv_s("ANYPS5_NO_SHADER_CACHE", "1"); + _putenv_s("APS5_NO_SINGLE_LANE_RULE", "1"); #else setenv("ANYPS5_NO_SHADER_CACHE", "1", 1); + setenv("APS5_NO_SINGLE_LANE_RULE", "1", 1); #endif using namespace ShaderRecompiler; if (argc == 2 && std::string_view(argv[1]) == "--specialized-bda-abi") { diff --git a/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp b/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp new file mode 100644 index 000000000000..96e4cb400925 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SingleLane.cpp @@ -0,0 +1,643 @@ +#include "IntermediateRepresentation/IrProgram.hpp" +#include "IntermediateRepresentation/IrMetadata/StageIO.hpp" +#include "Optimization/ConstantFolder.hpp" +#include "Optimization/SingleLaneAnalysis.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +struct Builder { + IrProgram program; + IrBlock* block = nullptr; + ShaderWorkgroupInputInfo workgroup; + + explicit Builder(std::uint32_t width = 16u, std::uint32_t height = 16u) { + block = &program.CreateBlock(); + program.BlockOrder().push_back(block); + program.SetEntryBlock(*block); + BlockInfo info; + info.id = 0; + program.Metadata().blockInfo.push_back(info); + program.Resources().stage = IrShaderStage::Compute; + program.SetWaveSize(64u); + workgroup.threadsNum[0] = width; + workgroup.threadsNum[1] = height; + workgroup.threadsNum[2] = 1u; + } + + IrValue& constant(std::uint32_t value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::U32); + created.SetImmediateU32(value); + return created; + } + + IrValue& boolean(bool value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::Bool); + created.SetImmediateBool(value); + return created; + } + + IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list arguments) { + auto& created = program.CreateValue(opcode, type); + for (auto* argument : arguments) created.AddArgument(argument); + block->AppendInstruction(&created); + return created; + } + + IrValue& binary(IrOpcode opcode, IrValue& left, IrValue& right) { + return emit(opcode, IrType::U32, {&left, &right}); + } + + IrValue& lane() { + return emit(IrOpcode::LaneId, IrType::U32, {}); + } + + IrValue& localId(std::uint32_t axis) { + return emit(IrOpcode::GetBuiltin, IrType::U32, {&constant(static_cast(StageInputKind::LocalInvocationId)), &constant(axis)}); + } + + IrValue& userData() { + return emit(IrOpcode::GetUserData, IrType::U32, {}); + } + + IrValue& laneDword() { + auto& row = binary(IrOpcode::IMul32, constant(16u), emit(IrOpcode::BitFieldUExtract, IrType::U32, {&localId(1), &constant(0u), &constant(24u)})); + return binary(IrOpcode::ShiftLeftLogical32, binary(IrOpcode::IAdd32, row, localId(0)), constant(2u)); + } + + IrValue& ballot(IrValue& predicate, std::uint32_t half) { + auto& mask = emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + return emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&mask, &constant(half)}); + } + + IrValue& threadBit(IrValue& low, IrValue& high) { + auto& laneId = lane(); + auto& word = emit(IrOpcode::SelectU32, IrType::U32, {&emit(IrOpcode::ULessThan32, IrType::Bool, {&laneId, &constant(32u)}), &low, &high}); + auto& shift = binary(IrOpcode::BitwiseAnd32, laneId, constant(31u)); + auto& bit = binary(IrOpcode::BitwiseAnd32, binary(IrOpcode::ShiftRightLogical32, word, shift), constant(1u)); + return emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &constant(0u)}); + } + + IrValue& lds(IrOpcode opcode, IrType type, std::initializer_list arguments, std::uint32_t offset) { + MemoryInfo memory; + memory.kind = ResourceKind::Lds; + memory.offset = offset; + const auto index = static_cast(program.Resources().memoryInfo.size()); + program.Resources().memoryInfo.push_back(memory); + auto& created = emit(opcode, type, arguments); + created.SetFlags(MemoryFlags{index, 0u}); + return created; + } + + void keep(IrValue& value) { + (void)emit(value.Type() == IrType::Bool ? IrOpcode::Reference : IrOpcode::ReferenceU32, IrType::Void, {&value}); + } + + void branch(BranchCondition condition, IrValue& predicate, IrValue* exec) { + auto& info = program.Metadata().blockInfo.front(); + info.terminator.kind = TerminatorKind::ConditionalBranch; + info.terminator.condition = condition; + info.condition = &predicate; + info.exec = exec; + } + + IrBlock& addBlock() { + auto& created = program.CreateBlock(); + program.BlockOrder().push_back(&created); + BlockInfo info; + info.id = static_cast(program.Metadata().blockInfo.size()); + program.Metadata().blockInfo.push_back(info); + return created; + } + + void at(IrBlock& target) { + block = ⌖ + } + + BlockInfo& infoOf(const IrBlock& target) { + const auto& order = program.BlockOrder(); + return program.Metadata().blockInfo.at(static_cast(std::find(order.begin(), order.end(), &target) - order.begin())); + } + + void jump(IrBlock& from, IrBlock& to) { + auto& info = infoOf(from); + info.terminator.kind = TerminatorKind::Branch; + info.terminator.trueBlock = infoOf(to).id; + from.AddSuccessor(&to); + to.AddPredecessor(&from); + } + + void split(IrBlock& from, BranchCondition condition, IrValue& predicate, IrValue* exec, IrBlock& taken, IrBlock& otherwise) { + auto& info = infoOf(from); + info.terminator.kind = TerminatorKind::ConditionalBranch; + info.terminator.condition = condition; + info.terminator.trueBlock = infoOf(taken).id; + info.terminator.falseBlock = infoOf(otherwise).id; + info.condition = &predicate; + info.exec = exec; + for (IrBlock* target : {&taken, &otherwise}) { + from.AddSuccessor(target); + target->AddPredecessor(&from); + } + } + + IrValue& phi(IrType type, std::initializer_list> incoming) { + auto& created = program.CreateValue(IrOpcode::Phi, type); + for (const auto& [from, value] : incoming) created.AddPhiOperand(from, value); + block->AppendInstruction(&created); + return created; + } + + void fold() { + (void)emit(IrOpcode::Return, IrType::Void, {}); + ConstantFolder{}.Fold(program); + } + + SingleLaneVerdict analyze() const { + return AnalyzeSingleLane(program, workgroup); + } +}; + +bool eligible(const SingleLaneVerdict& verdict) { + if (!verdict.eligible) std::fprintf(stderr, " rejected: %s\n", verdict.reason); + return verdict.eligible; +} + +bool rejected(const SingleLaneVerdict& verdict, const char* reason) { + if (verdict.eligible) { + std::fprintf(stderr, " accepted, expected \"%s\"\n", reason); + return false; + } + if (std::strstr(verdict.reason, reason) != nullptr) return true; + std::fprintf(stderr, " rejected with \"%s\", expected \"%s\"\n", verdict.reason, reason); + return false; +} + +bool run(const char* name, const std::function& test) { + try { + if (test()) return true; + std::fprintf(stderr, "%s\n", name); + return false; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("a select on a lane's bit of ~exec | exec folds to its true arm", [] { + Builder b; + auto& low = b.ballot(b.boolean(true), 0u); + auto& high = b.ballot(b.boolean(true), 1u); + auto& allLow = b.binary(IrOpcode::BitwiseOr32, b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&low}), low); + auto& allHigh = b.binary(IrOpcode::BitwiseOr32, b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&high}), high); + auto& active = b.localId(0); + auto& selected = b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(allLow, allHigh), &active, &b.constant(0u)}); + b.keep(selected); + b.fold(); + return selected.Resolve() == &active; + }); + + passed &= run("a lane-private LDS array (tid * 4 + k * 1024) runs at one lane per invocation", [] { + Builder b; + auto& address = b.laneDword(); + auto& value = b.localId(0); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &value, &b.boolean(true)}, 0u); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &value, &b.boolean(true)}, 1024u); + b.keep(b.lds(IrOpcode::LoadSharedU32, IrType::U32, {&address, &b.boolean(true)}, 1024u)); + return eligible(b.analyze()); + }); + + passed &= run("a two-dimensional private LDS layout (4x + 1024y) runs at one lane", [] { + Builder b; + auto& address = b.binary(IrOpcode::IAdd32, b.binary(IrOpcode::ShiftLeftLogical32, b.localId(0), b.constant(2u)), b.binary(IrOpcode::ShiftLeftLogical32, b.localId(1), b.constant(10u))); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return eligible(b.analyze()); + }); + + passed &= run("lanes of one column sharing an LDS word keep two lanes", [] { + Builder b; + auto& address = b.binary(IrOpcode::ShiftLeftLogical32, b.localId(0), b.constant(2u)); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(1), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "share LDS"); + }); + + passed &= run("an LDS read reaching the next lane's dword keeps two lanes", [] { + Builder b; + auto& address = b.laneDword(); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + b.keep(b.lds(IrOpcode::LoadSharedU32x2, IrType::U32x2, {&address, &b.boolean(true)}, 0u)); + return rejected(b.analyze(), "share LDS"); + }); + + passed &= run("LDS addressed by a loaded value keeps two lanes", [] { + Builder b; + auto& address = b.userData(); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "not affine"); + }); + + passed &= run("LDS writes in a one-wave workgroup keep two lanes", [] { + Builder b(8u, 8u); + auto& address = b.binary(IrOpcode::ShiftLeftLogical32, b.binary(IrOpcode::IAdd32, b.binary(IrOpcode::ShiftLeftLogical32, b.localId(1), b.constant(3u)), b.localId(0)), b.constant(2u)); + (void)b.lds(IrOpcode::WriteSharedU32, IrType::Void, {&address, &b.localId(0), &b.boolean(true)}, 0u); + return rejected(b.analyze(), "one-wave"); + }); + + passed &= run("a lane's own ballot bit runs at one lane", [] { + Builder b; + auto& predicate = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(5u)}); + auto& ballot = b.emit(IrOpcode::Ballot, IrType::U32x4, {&predicate}); + auto& low = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(0u)}); + auto& high = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&ballot, &b.constant(1u)}); + auto& mask = b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&low}); + auto& maskHigh = b.emit(IrOpcode::BitwiseNot32, IrType::U32, {&high}); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(mask, maskHigh), &b.constant(1u), &b.constant(0u)})); + return eligible(b.analyze()); + }); + + passed &= run("a counted ballot keeps two lanes", [] { + Builder b; + auto& low = b.ballot(b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(5u)}), 0u); + b.keep(b.emit(IrOpcode::BitCount32, IrType::U32, {&low})); + return rejected(b.analyze(), "lane mask read as a number"); + }); + + passed &= run("a lane mask whose halves are different expressions keeps two lanes", [] { + Builder b; + auto& low = b.ballot(b.boolean(true), 0u); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(low, b.constant(0u)), &b.constant(1u), &b.constant(0u)})); + return rejected(b.analyze(), "halves differ"); + }); + + passed &= run("the lane id read as data keeps two lanes", [] { + Builder b; + b.keep(b.binary(IrOpcode::IAdd32, b.lane(), b.constant(1u))); + return rejected(b.analyze(), "lane id read as a number"); + }); + + passed &= run("a workgroup of at most 32 lanes runs the same checks as a larger one", [] { + Builder b(16u, 2u); + b.keep(b.binary(IrOpcode::IAdd32, b.lane(), b.constant(1u))); + return rejected(b.analyze(), "lane id read as a number"); + }); + + passed &= run("a 32-thread group reading lane 63 in whole-wave mode keeps two lanes", [] { + Builder b(32u, 1u); + b.keep(b.emit(IrOpcode::ReadLane, IrType::U32, {&b.localId(0), &b.constant(63u)})); + return rejected(b.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a barrier beside s_cbranch_execz in a 32-thread group keeps two lanes", [] { + Builder b(32u, 1u); + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.keep(skip); + b.branch(BranchCondition::ExecZero, skip, nullptr); + return rejected(b.analyze(), "barrier"); + }); + + passed &= run("readfirstlane of user data runs at one lane, of a local id keeps two", [] { + Builder uniform; + uniform.keep(uniform.emit(IrOpcode::ReadFirstLane, IrType::U32, {&uniform.userData()})); + Builder varying; + varying.keep(varying.emit(IrOpcode::ReadFirstLane, IrType::U32, {&varying.localId(0)})); + return eligible(uniform.analyze()) && rejected(varying.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a lane's vector window indexed by user data runs at one lane", [] { + Builder b; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&b.userData(), &b.localId(0)}); + b.keep(b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&b.userData()})); + return eligible(b.analyze()); + }); + + passed &= run("readfirstlane of a vector window load keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&b.constant(0u), &b.localId(0)}); + b.keep(b.emit(IrOpcode::ReadFirstLane, IrType::U32, {&b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&b.constant(0u)})})); + return rejected(b.analyze(), "lane read of a lane-varying value"); + }); + + passed &= run("a VCC branch on a uniform compare under the branch's exec runs at one lane", [] { + Builder b; + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + auto& condition = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&vcc}); + b.keep(condition); + b.keep(exec); + b.branch(BranchCondition::VccZero, condition, &exec); + return eligible(b.analyze()); + }); + + passed &= run("a VCC branch on a lane compare keeps two lanes", [] { + Builder b; + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + b.keep(vcc); + b.keep(exec); + b.branch(BranchCondition::VccNonZero, vcc, &exec); + return rejected(b.analyze(), "VCC branch on lane-varying bits"); + }); + + passed &= run("a VCC branch whose bits came from a narrower exec keeps two lanes", [] { + Builder b; + auto& wide = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& narrow = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&wide, &b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)})}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&narrow, &compare}); + b.keep(vcc); + b.keep(wide); + b.branch(BranchCondition::VccNonZero, vcc, &wide); + return rejected(b.analyze(), "VCC branch on lane-varying bits"); + }); + + passed &= run("a VCC branch without its exec mask keeps two lanes", [] { + Builder b; + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + b.keep(compare); + b.branch(BranchCondition::VccNonZero, compare, nullptr); + return rejected(b.analyze(), "without its exec mask"); + }); + + passed &= run("a barrier beside a VCC branch on exec-implied bits keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}); + auto& vcc = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &compare}); + b.keep(vcc); + b.keep(exec); + b.branch(BranchCondition::VccNonZero, vcc, &exec); + return rejected(b.analyze(), "barrier"); + }); + + const auto skippable = [](Builder& b, IrBlock*& region, IrBlock*& merge) -> IrValue& { + IrBlock& entry = *b.block; + region = &b.addBlock(); + merge = &b.addBlock(); + auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(32u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.split(entry, BranchCondition::ExecZero, skip, nullptr, *merge, *region); + b.at(*region); + return exec; + }; + + passed &= run("a scalar written under s_cbranch_execz and read per lane after the merge keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.userData(); + (void)skippable(b, region, merge); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &b.constant(1u)}})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("an exec-masked VGPR written under s_cbranch_execz runs at one lane", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + auto& exec = skippable(b, region, merge); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(1u)), &old}); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &written}})); + return eligible(b.analyze()); + }); + + passed &= run("a whole-wave VGPR write under s_cbranch_execz keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + (void)skippable(b, region, merge); + auto& written = b.binary(IrOpcode::IAdd32, old, b.constant(1u)); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &old}, {region, &written}})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("a VCC written under s_cbranch_execz and read after the merge keeps two lanes", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& before = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(3u)})}); + auto& beforeLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&before, &b.constant(0u)}); + auto& beforeHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&before, &b.constant(1u)}); + auto& exec = skippable(b, region, merge); + auto& compare = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(1), &b.constant(5u)}); + auto& after = b.emit(IrOpcode::Ballot, IrType::U32x4, {&b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&compare, &exec})}); + auto& afterLow = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&after, &b.constant(0u)}); + auto& afterHigh = b.emit(IrOpcode::CompositeExtractU32x4, IrType::U32, {&after, &b.constant(1u)}); + b.jump(*region, *merge); + b.at(*merge); + auto& low = b.phi(IrType::U32, {{entry, &beforeLow}, {region, &afterLow}}); + auto& high = b.phi(IrType::U32, {{entry, &beforeHigh}, {region, &afterHigh}}); + b.keep(b.emit(IrOpcode::SelectU32, IrType::U32, {&b.threadBit(low, high), &b.constant(1u), &b.constant(0u)})); + return rejected(b.analyze(), "leaves a region one half can skip"); + }); + + passed &= run("a vector window store under s_cbranch_execz must keep the skipped lanes' value", [&] { + const auto build = [&](bool masked) { + Builder b(64u, 1u); + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& exec = skippable(b, region, merge); + auto& index = b.constant(0u); + auto& value = b.binary(IrOpcode::IAdd32, b.localId(0), b.constant(1u)); + auto& stored = masked ? b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &value, &b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&index})}) : value; + (void)b.emit(IrOpcode::StoreVectorWindow, IrType::Void, {&index, &stored}); + b.jump(*region, *merge); + b.at(*merge); + b.keep(b.emit(IrOpcode::LoadVectorWindow, IrType::U32, {&index})); + return b.analyze(); + }; + return eligible(build(true)) && rejected(build(false), "vector window store one half can skip"); + }); + + const auto loop = [](bool counterUsedAfter) { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& body = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& entryExec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(40u)}); + auto& start = b.localId(1); + b.jump(entry, header); + b.at(header); + auto& exec = b.phi(IrType::Bool, {{&entry, &entryExec}}); + auto& counter = b.phi(IrType::U32, {{&entry, &b.constant(0u)}}); + auto& carried = b.phi(IrType::U32, {{&entry, &start}}); + auto& done = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.split(header, BranchCondition::ExecZero, done, nullptr, exit, body); + b.at(body); + auto& next = b.binary(IrOpcode::IAdd32, counter, b.constant(1u)); + auto& nextExec = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&exec, &b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &next})}); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&nextExec, &b.binary(IrOpcode::IAdd32, carried, b.constant(2u)), &carried}); + exec.AddPhiOperand(&body, &nextExec); + counter.AddPhiOperand(&body, &next); + carried.AddPhiOperand(&body, &written); + b.jump(body, header); + b.at(exit); + b.keep(counterUsedAfter ? counter : carried); + return b.analyze(); + }; + + passed &= run("an s_cbranch_execz loop with a scalar counter used after it keeps two lanes", [&] { + return rejected(loop(true), "leaves a loop one half can leave early"); + }); + + passed &= run("an s_cbranch_execz loop carrying an exec-masked VGPR out runs at one lane", [&] { + return eligible(loop(false)); + }); + + passed &= run("a scalar written under s_cbranch_execz keeps two lanes only when an instruction reads it", [&] { + const auto build = [&](bool read) { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& region = b.addBlock(); + IrBlock& merge = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& old = b.userData(); + auto& exec = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(32u)}); + b.jump(entry, header); + b.at(header); + auto& carried = b.phi(IrType::U32, {{&entry, &old}}); + b.split(header, BranchCondition::ExecZero, b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}), nullptr, merge, region); + b.jump(region, merge); + b.at(merge); + auto& merged = b.phi(IrType::U32, {{&header, &carried}, {®ion, &b.constant(1u)}}); + carried.AddPhiOperand(&merge, &merged); + b.split(merge, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(5u)}), nullptr, header, exit); + b.at(exit); + if (read) b.keep(merged); + return b.analyze(); + }; + return eligible(build(false)) && rejected(build(true), "leaves a region one half can skip"); + }); + + passed &= run("an exec-masked VGPR carried through a loop inside an s_cbranch_execz region runs at one lane", [&] { + Builder b(64u, 1u); + IrBlock* entry = b.block; + IrBlock* region = nullptr; + IrBlock* merge = nullptr; + auto& old = b.localId(0); + auto& exec = skippable(b, region, merge); + b.at(*entry); + auto& kept = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(7u)), &old}); + IrBlock& header = b.addBlock(); + IrBlock& choose = b.addBlock(); + IrBlock& write = b.addBlock(); + IrBlock& join = b.addBlock(); + IrBlock& leave = b.addBlock(); + b.at(*region); + auto& first = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, old, b.constant(3u)), &old}); + b.jump(*region, header); + b.at(header); + auto& carried = b.phi(IrType::U32, {{region, &first}}); + b.split(header, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(2u)}), nullptr, leave, choose); + b.at(choose); + b.split(choose, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(3u)}), nullptr, write, join); + b.at(write); + auto& written = b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.binary(IrOpcode::IAdd32, carried, b.constant(1u)), &carried}); + b.jump(write, join); + b.at(join); + auto& joined = b.phi(IrType::U32, {{&choose, &carried}, {&write, &written}}); + carried.AddPhiOperand(&join, &joined); + b.split(join, BranchCondition::SccNonZero, b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.userData(), &b.constant(4u)}), nullptr, header, leave); + b.at(leave); + auto& left = b.phi(IrType::U32, {{&header, &carried}, {&join, &joined}}); + b.jump(leave, *merge); + b.at(*merge); + b.keep(b.phi(IrType::U32, {{entry, &kept}, {&leave, &left}})); + return eligible(b.analyze()); + }); + + passed &= run("a loop whose exit test reads fewer lanes than its body runs keeps two lanes", [] { + Builder b(64u, 1u); + IrBlock& entry = *b.block; + IrBlock& header = b.addBlock(); + IrBlock& body = b.addBlock(); + IrBlock& exit = b.addBlock(); + auto& entryAlive = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.localId(0), &b.constant(40u)}); + auto& entryState = b.binary(IrOpcode::BitwiseAnd32, b.localId(0), b.constant(1u)); + b.jump(entry, header); + b.at(header); + auto& alive = b.phi(IrType::Bool, {{&entry, &entryAlive}}); + auto& state = b.phi(IrType::U32, {{&entry, &entryState}}); + auto& exec = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&alive, &b.emit(IrOpcode::IEqual32, IrType::Bool, {&state, &b.constant(0u)})}); + b.split(header, BranchCondition::ExecZero, b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}), nullptr, exit, body); + b.at(body); + auto& waiting = b.emit(IrOpcode::LogicalAnd, IrType::Bool, {&alive, &b.emit(IrOpcode::IEqual32, IrType::Bool, {&state, &b.constant(1u)})}); + auto& next = b.emit(IrOpcode::SelectU32, IrType::U32, {&waiting, &b.constant(0u), &b.emit(IrOpcode::SelectU32, IrType::U32, {&exec, &b.constant(2u), &state})}); + alive.AddPhiOperand(&body, &alive); + state.AddPhiOperand(&body, &next); + b.jump(body, header); + b.at(exit); + b.keep(state); + return rejected(b.analyze(), "exec can widen"); + }); + + passed &= run("the recompiler lays a qualifying wave64 program out at one lane unless APS5_NO_SINGLE_LANE_RULE is set", [] { + const std::vector code{0xbf810000u}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x100000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.compute = ShaderComputeStageInfo{{64u, 2u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32u; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + const bool off = std::getenv("APS5_NO_SINGLE_LANE_RULE") != nullptr; + return Recompile(request).hostSubgroupSize == (off ? 32u : 64u); + }); + + passed &= run("a DPP move keeps two lanes", [] { + Builder b; + b.keep(b.emit(IrOpcode::DppMoveU32, IrType::U32, {&b.localId(0)})); + return rejected(b.analyze(), "operation across lanes"); + }); + + passed &= run("a barrier in a program with exec branches keeps two lanes", [] { + Builder b; + (void)b.emit(IrOpcode::Barrier, IrType::Void, {}); + auto& exec = b.emit(IrOpcode::INotEqual32, IrType::Bool, {&b.localId(0), &b.constant(3u)}); + auto& skip = b.emit(IrOpcode::LogicalNot, IrType::Bool, {&exec}); + b.keep(skip); + b.branch(BranchCondition::ExecZero, skip, nullptr); + return rejected(b.analyze(), "barrier"); + }); + + if (!passed) return 1; + std::printf("single lane tests passed\n"); + return 0; +} diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index 031d897b2b5b..6c3bd66e1535 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -106,6 +106,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/HostInterpolationChecker.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ReadLaneEliminator.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/MaskedSelectEliminator.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/SingleLaneAnalysis.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/TessellationLowering.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceTracker.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Optimization/src/ResourceMaterializer.cpp diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 32f615dfcbf1..333602663982 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -17,6 +17,7 @@ struct BlockInfo { Terminator terminator; IrValue* condition = nullptr; IrValue* indirectTarget = nullptr; + IrValue* exec = nullptr; }; struct KeyDomain { diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp index eee7578d208c..b527f36d626f 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp @@ -32,6 +32,8 @@ struct IrProgramMetadata { std::uint32_t vectorWindowBase = 0; std::uint32_t vectorWindowSize = 0; bool vectorWindowIndexedWrites = false; + bool skippedRegionsChecked = false; + std::string skippedRegionRejection; std::uint32_t tessellationControlStride = 0; std::uint32_t tessellationPatchStride = 0; std::uint32_t tessellationSharedStride = 0; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp index f724becd4b7a..89d7d99efd28 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp @@ -49,6 +49,7 @@ class IrProgram { }; std::string ProgramToString(const IrProgram& program); +std::string DescribeValue(const IrValue* value, std::uint32_t depth); void ValidateProgram(const IrProgram& program, bool requireSsa); void ResolveControlFlowIdentities(IrProgram& program); bool EquivalentValue(const IrResourcePlan& program, const IrValue* left, const IrValue* right); diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp index 4c5e1fc57c98..987dd9678c8e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp @@ -255,7 +255,7 @@ std::size_t IrProgram::ReclaimUnusedValues() { for (const auto& read : resourcePlan.srtReads) keep(read.value); for (const auto& block : resourcePlan.controlFlow) keep(block.condition); for (IrValue* value : resourcePlan.uniformFill.values) keep(value); - for (const auto& block : metadata.blockInfo) { keep(block.condition); keep(block.indirectTarget); } + for (const auto& block : metadata.blockInfo) { keep(block.condition); keep(block.indirectTarget); keep(block.exec); } for (IrValue* value : metadata.dynamicReads) keep(value); while (!pending.empty()) { IrValue* value = pending.back(); @@ -392,6 +392,20 @@ const std::vector& IrProgram::BlockOrder() const { return blockOrder; } +std::string DescribeValue(const IrValue* value, std::uint32_t depth) { + if (value == nullptr) return "null"; + value = value->Resolve(); + std::string text(IrOpcodeName(value->Opcode())); + if (value->HasImmediate() && value->Type() == IrType::U32) return text + "(" + std::to_string(value->ImmediateU32()) + ")"; + if (depth == 0 || value->ArgumentCount() == 0) return text; + text += "("; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { + if (index != 0) text += ", "; + text += DescribeValue(value->Argument(index), depth - 1); + } + return text + ")"; +} + std::string ProgramToString(const IrProgram& program) { std::map ids; std::size_t nextId = 1; @@ -811,6 +825,9 @@ void ResolveControlFlowIdentities(IrProgram& program) { if (info.indirectTarget != nullptr) { info.indirectTarget = info.indirectTarget->Resolve(); } + if (info.exec != nullptr) { + info.exec = info.exec->Resolve(); + } } } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp b/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp new file mode 100644 index 000000000000..5dab4df31d8a --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/SingleLaneAnalysis.hpp @@ -0,0 +1,20 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_SINGLELANEANALYSIS_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_SINGLELANEANALYSIS_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/ShaderStageInputInfo.hpp" + +namespace ShaderRecompiler { + +struct SingleLaneVerdict { + bool eligible = false; + const char* reason = nullptr; + const IrValue* at = nullptr; +}; + +[[nodiscard]] SingleLaneVerdict AnalyzeSingleLane(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup); +void RecordSkippedRegions(IrProgram& program); + +} + +#endif diff --git a/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp b/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp index df61fa400d80..751261c3e547 100644 --- a/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp +++ b/core/shader/recompiler/Optimization/src/RedundantValueEliminator.cpp @@ -326,6 +326,7 @@ RedundantValueEliminationStats RedundantValueEliminator::Eliminate(IrProgram& pr for (auto& info : program.Metadata().blockInfo) { replaced(info.condition); replaced(info.indirectTarget); + replaced(info.exec); } for (auto*& read : program.Metadata().dynamicReads) replaced(read); for (auto& descriptor : program.Resources().descriptorSources) { diff --git a/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp b/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp new file mode 100644 index 000000000000..4e1dbed4c4c3 --- /dev/null +++ b/core/shader/recompiler/Optimization/src/SingleLaneAnalysis.cpp @@ -0,0 +1,1542 @@ +#include "Optimization/SingleLaneAnalysis.hpp" +#include "IntermediateRepresentation/IrBlock.hpp" +#include "IntermediateRepresentation/IrMetadata/StageIO.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +constexpr std::size_t PairLimit = std::size_t{1} << 18u; +constexpr std::uint32_t DepthLimit = 64; +constexpr std::uint64_t LaneOffsetLimit = std::uint64_t{1} << 16u; +constexpr std::size_t FootprintLimit = 64; + +struct Rejection { + const char* reason; + const IrValue* at; +}; + +bool isImmediateU32(const IrValue* value, std::uint32_t immediate) { + return value->HasImmediate() && value->Type() == IrType::U32 && value->ImmediateU32() == immediate; +} + +bool isLane(const IrValue* value) { + return value->Resolve()->Opcode() == IrOpcode::LaneId; +} + +bool isHalfSelector(const IrValue* value) { + value = value->Resolve(); + return value->Opcode() == IrOpcode::ULessThan32 && isLane(value->Argument(0)) && isImmediateU32(value->Argument(1)->Resolve(), 32u); +} + +bool isBitSelector(const IrValue* value) { + value = value->Resolve(); + if (value->Opcode() != IrOpcode::BitwiseAnd32) return false; + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + return (isLane(left) && isImmediateU32(right, 31u)) || (isLane(right) && isImmediateU32(left, 31u)); +} + +bool isHalfWord(const IrValue* value) { + value = value->Resolve(); + return value->Opcode() == IrOpcode::SelectU32 && isHalfSelector(value->Argument(0)); +} + +const IrValue* otherThan(const IrValue* value, std::uint32_t immediate) { + if (value->ArgumentCount() != 2u) return nullptr; + const IrValue* left = value->Argument(0)->Resolve(); + const IrValue* right = value->Argument(1)->Resolve(); + if (isImmediateU32(right, immediate)) return left; + if (isImmediateU32(left, immediate)) return right; + return nullptr; +} + +const IrValue* ownBitWord(const IrValue* value) { + value = value->Resolve(); + if (value->Opcode() != IrOpcode::INotEqual32) return nullptr; + const IrValue* bit = otherThan(value, 0u); + if (bit == nullptr || bit->Opcode() != IrOpcode::BitwiseAnd32) return nullptr; + const IrValue* shifted = otherThan(bit, 1u); + if (shifted == nullptr || shifted->Opcode() != IrOpcode::ShiftRightLogical32 || !isBitSelector(shifted->Argument(1))) return nullptr; + return shifted->Argument(0)->Resolve(); +} + +bool isBallotWord(const IrValue* value, std::uint32_t half) { + return value->Opcode() == IrOpcode::CompositeExtractU32x4 && value->Argument(0)->Resolve()->Opcode() == IrOpcode::Ballot && isImmediateU32(value->Argument(1)->Resolve(), half); +} + +const IrValue* ballotOfHalves(const IrValue* low, const IrValue* high) { + if (!isBallotWord(low, 0u) || !isBallotWord(high, 1u)) return nullptr; + const IrValue* lowBallot = low->Argument(0)->Resolve(); + const IrValue* highBallot = high->Argument(0)->Resolve(); + if (lowBallot == highBallot) return lowBallot; + if (lowBallot->Parent() != highBallot->Parent()) return nullptr; + const IrValue* lowPredicate = lowBallot->Argument(0)->Resolve(); + const IrValue* highPredicate = highBallot->Argument(0)->Resolve(); + if (lowPredicate == highPredicate) return lowBallot; + const bool sameImmediate = lowPredicate->HasImmediate() && highPredicate->HasImmediate() && lowPredicate->Type() == IrType::Bool && highPredicate->Type() == IrType::Bool && lowPredicate->ImmediateBool() == highPredicate->ImmediateBool(); + return sameImmediate ? lowBallot : nullptr; +} + +bool isLaneMaskOp(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + case IrOpcode::BitwiseNot32: + case IrOpcode::SelectU32: + case IrOpcode::Phi: + return true; + default: + return false; + } +} + +bool isLaneLocal(IrOpcode opcode) { + if (BufferAccessOf(opcode) != BufferAccess::None || AddressOpcodeInfoOf(opcode).access != AddressAccess::None || ImageOpcodeInfoOf(opcode).access != ImageAccess::None) return true; + switch (opcode) { + case IrOpcode::Identity: + case IrOpcode::Phi: + case IrOpcode::Void: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + case IrOpcode::IMul32: + case IrOpcode::IDiv32: + case IrOpcode::IMod32: + case IrOpcode::IAnd32: + case IrOpcode::IOr32: + case IrOpcode::IXor32: + case IrOpcode::IShiftLeft32: + case IrOpcode::IShiftRightLogical32: + case IrOpcode::IShiftRightArithmetic32: + case IrOpcode::INegate32: + case IrOpcode::INot32: + case IrOpcode::FAdd32: + case IrOpcode::FSub32: + case IrOpcode::FMul32: + case IrOpcode::FDiv32: + case IrOpcode::FNegate32: + case IrOpcode::FAbs32: + case IrOpcode::FFma32: + case IrOpcode::FSqrt32: + case IrOpcode::FRcp32: + case IrOpcode::FRsq32: + case IrOpcode::FMin32: + case IrOpcode::FMax32: + case IrOpcode::FFloor32: + case IrOpcode::FCeil32: + case IrOpcode::FFract32: + case IrOpcode::ConvertFToI32: + case IrOpcode::ConvertIToF32: + case IrOpcode::ConvertFToU32: + case IrOpcode::ConvertUToF32: + case IrOpcode::CompareEqual: + case IrOpcode::CompareNotEqual: + case IrOpcode::CompareLessThan: + case IrOpcode::CompareGreaterThan: + case IrOpcode::Select: + case IrOpcode::Barrier: + case IrOpcode::Reference: + case IrOpcode::ReferenceU32: + case IrOpcode::GetUserData: + case IrOpcode::GetShaderBase: + case IrOpcode::ShaderClock: + case IrOpcode::RealtimeClock: + case IrOpcode::UndefU1: + case IrOpcode::UndefU8: + case IrOpcode::UndefU16: + case IrOpcode::UndefU32: + case IrOpcode::UndefU64: + case IrOpcode::BitCastU16F16: + case IrOpcode::BitCastF16U16: + case IrOpcode::BitCastU32F32: + case IrOpcode::BitCastF32U32: + case IrOpcode::ConvertU16U32: + case IrOpcode::ConvertU32U16: + case IrOpcode::ConvertU8U32: + case IrOpcode::ConvertU32U8: + case IrOpcode::ConvertF32F16: + case IrOpcode::ConvertF16F32: + case IrOpcode::ConvertS32F32: + case IrOpcode::ConvertU32F32: + case IrOpcode::ConvertF32S32: + case IrOpcode::ConvertF32U32: + case IrOpcode::CompositeConstructU64: + case IrOpcode::CompositeConstructU32x2: + case IrOpcode::CompositeConstructU32x3: + case IrOpcode::CompositeConstructF32x2: + case IrOpcode::CompositeConstructU32x4: + case IrOpcode::CompositeExtractU64: + case IrOpcode::CompositeExtractU32x2: + case IrOpcode::CompositeExtractU32x3: + case IrOpcode::CompositeExtractU32x4: + case IrOpcode::PackHalf2x16: + case IrOpcode::PackFloat2x16Rtz: + case IrOpcode::FPAbs32: + case IrOpcode::FPNeg32: + case IrOpcode::FPSaturate32: + case IrOpcode::BitFieldInsert: + case IrOpcode::BitFieldUExtract: + case IrOpcode::BitFieldSExtract: + case IrOpcode::SelectU1: + case IrOpcode::SelectF32: + case IrOpcode::SelectU32: + case IrOpcode::IAdd64: + case IrOpcode::IAddCarry32: + case IrOpcode::ISub64: + case IrOpcode::IMul64: + case IrOpcode::UDiv32: + case IrOpcode::SMulHi: + case IrOpcode::UMulHi: + case IrOpcode::IAbs32: + case IrOpcode::ShiftLeftLogical32: + case IrOpcode::ShiftLeftLogical64: + case IrOpcode::ShiftRightLogical32: + case IrOpcode::ShiftRightLogical64: + case IrOpcode::ShiftRightArithmetic32: + case IrOpcode::ShiftRightArithmetic64: + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseAnd64: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + case IrOpcode::BitwiseNot32: + case IrOpcode::BitReverse32: + case IrOpcode::BitCount32: + case IrOpcode::BitCount64: + case IrOpcode::FindUMsb32: + case IrOpcode::FindUMsb64: + case IrOpcode::FindILsb32: + case IrOpcode::SMin32: + case IrOpcode::UMin32: + case IrOpcode::SMax32: + case IrOpcode::UMax32: + case IrOpcode::SMinTri32: + case IrOpcode::UMinTri32: + case IrOpcode::SMaxTri32: + case IrOpcode::UMaxTri32: + case IrOpcode::SMedTri32: + case IrOpcode::UMedTri32: + case IrOpcode::SLessThan32: + case IrOpcode::SLessThan64: + case IrOpcode::ULessThan32: + case IrOpcode::ULessThan64: + case IrOpcode::IEqual32: + case IrOpcode::IEqual64: + case IrOpcode::SLessThanEqual32: + case IrOpcode::ULessThanEqual32: + case IrOpcode::SGreaterThan32: + case IrOpcode::UGreaterThan32: + case IrOpcode::UGreaterThan64: + case IrOpcode::INotEqual32: + case IrOpcode::INotEqual64: + case IrOpcode::SGreaterThanEqual32: + case IrOpcode::UGreaterThanEqual32: + case IrOpcode::LogicalOr: + case IrOpcode::LogicalAnd: + case IrOpcode::LogicalXor: + case IrOpcode::LogicalNot: + case IrOpcode::FPOrdEqual32: + case IrOpcode::FPUnordEqual32: + case IrOpcode::FPOrdNotEqual32: + case IrOpcode::FPUnordNotEqual32: + case IrOpcode::FPOrdLessThan32: + case IrOpcode::FPUnordLessThan32: + case IrOpcode::FPOrdGreaterThan32: + case IrOpcode::FPUnordGreaterThan32: + case IrOpcode::FPOrdLessThanEqual32: + case IrOpcode::FPUnordLessThanEqual32: + case IrOpcode::FPOrdGreaterThanEqual32: + case IrOpcode::FPUnordGreaterThanEqual32: + case IrOpcode::FPIsNan32: + case IrOpcode::FPCmpClass32: + case IrOpcode::FPAdd32: + case IrOpcode::FPSub32: + case IrOpcode::FPFma32: + case IrOpcode::FPMad32: + case IrOpcode::FPMul32: + case IrOpcode::FPMin32: + case IrOpcode::FPMax32: + case IrOpcode::FPMinTri32: + case IrOpcode::FPMaxTri32: + case IrOpcode::FPMedTri32: + case IrOpcode::FPRecip32: + case IrOpcode::FPRecipIFlag32: + case IrOpcode::FPRecipSqrt32: + case IrOpcode::FPSqrt: + case IrOpcode::FPSin: + case IrOpcode::FPCos: + case IrOpcode::FPExp2: + case IrOpcode::FPLog2: + case IrOpcode::FPRoundEven32: + case IrOpcode::FPFloor32: + case IrOpcode::FPCeil32: + case IrOpcode::FPTrunc32: + case IrOpcode::FPFract32: + case IrOpcode::FPAdd64: + case IrOpcode::FPMul64: + case IrOpcode::FPFma64: + case IrOpcode::FPFmaScale64: + case IrOpcode::FPMin64: + case IrOpcode::FPMax64: + case IrOpcode::FPSaturate64: + case IrOpcode::FPLdexp64: + case IrOpcode::FPRoundEven64: + case IrOpcode::FPFloor64: + case IrOpcode::FPCeil64: + case IrOpcode::FPTrunc64: + case IrOpcode::FPFract64: + case IrOpcode::FPFrexpMant64: + case IrOpcode::FPFrexpExp64: + case IrOpcode::FPRcp64: + case IrOpcode::FPRsq64: + case IrOpcode::FPSqrt64: + case IrOpcode::FPTrigPreop64: + case IrOpcode::FPDot2F32F16: + case IrOpcode::ConvertF32F64: + case IrOpcode::ConvertF64F32: + case IrOpcode::ConvertF64S32: + case IrOpcode::ConvertF64U32: + case IrOpcode::ConvertS32F64: + case IrOpcode::ConvertU32F64: + case IrOpcode::GetSrtResource: + case IrOpcode::GetBufferResource: + case IrOpcode::GetAddressResource: + case IrOpcode::GetScratchResource: + case IrOpcode::GetImageResource: + case IrOpcode::GetSamplerResource: + case IrOpcode::MakeImageAddress: + case IrOpcode::ReadConst: + case IrOpcode::ReadConstBuffer: + case IrOpcode::ImageQueryDimensions: + case IrOpcode::ImageQueryLod: + case IrOpcode::ImageBvhIntersectRay: + case IrOpcode::ControlNop: + case IrOpcode::Waitcnt: + case IrOpcode::InstPrefetch: + case IrOpcode::LoadVectorWindow: + case IrOpcode::StoreVectorWindow: + return true; + default: + return false; + } +} + +std::uint32_t sharedBytes(IrOpcode opcode) { + switch (opcode) { + case IrOpcode::LoadSharedU8: + case IrOpcode::WriteSharedU8: + return 1u; + case IrOpcode::LoadSharedU16: + case IrOpcode::WriteSharedU16: + return 2u; + case IrOpcode::LoadSharedU32: + case IrOpcode::WriteSharedU32: + case IrOpcode::SharedAtomicFMin32: + case IrOpcode::SharedAtomicFMax32: + case IrOpcode::SharedAtomicSwap32: + case IrOpcode::SharedAtomicIAdd32: + case IrOpcode::SharedAtomicISub32: + case IrOpcode::SharedAtomicInc32: + case IrOpcode::SharedAtomicDec32: + case IrOpcode::SharedAtomicSMin32: + case IrOpcode::SharedAtomicUMin32: + case IrOpcode::SharedAtomicSMax32: + case IrOpcode::SharedAtomicUMax32: + case IrOpcode::SharedAtomicAnd32: + case IrOpcode::SharedAtomicOr32: + case IrOpcode::SharedAtomicXor32: + case IrOpcode::SharedAtomicRsub32: + case IrOpcode::SharedAtomicFAdd32: + case IrOpcode::SharedAtomicCmpst32: + case IrOpcode::SharedAtomicCmpstF32: + case IrOpcode::SharedAtomicMskor32: + case IrOpcode::SharedAtomicWrap32: + return 4u; + case IrOpcode::LoadSharedU32x2: + case IrOpcode::WriteSharedU32x2: + return 8u; + case IrOpcode::LoadSharedU32x3: + case IrOpcode::WriteSharedU32x3: + return 12u; + case IrOpcode::LoadSharedU32x4: + case IrOpcode::WriteSharedU32x4: + return 16u; + default: + return 0u; + } +} + +class Uniformity { +public: + explicit Uniformity(const IrProgram& program) { + bool changed = true; + while (changed) { + changed = false; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if (varying.contains(inst) || !varies(program, *inst)) continue; + varying.insert(inst); + changed = true; + } + } + } + } + + [[nodiscard]] bool Uniform(const IrValue* value) const { + value = value->Resolve(); + return value->HasImmediate() || !varying.contains(value); + } + +private: + bool varies(const IrProgram& program, const IrValue& inst) const { + switch (inst.Opcode()) { + case IrOpcode::GetUserData: + case IrOpcode::GetShaderBase: + case IrOpcode::UndefU1: + case IrOpcode::UndefU8: + case IrOpcode::UndefU16: + case IrOpcode::UndefU32: + case IrOpcode::UndefU64: + return false; + case IrOpcode::LaneId: + case IrOpcode::Ballot: + case IrOpcode::ShaderClock: + case IrOpcode::RealtimeClock: + case IrOpcode::LoadVectorWindow: + return true; + case IrOpcode::GetBuiltin: { + const IrValue* kind = inst.Argument(0)->Resolve(); + if (!kind->HasImmediate()) return true; + const auto input = static_cast(kind->ImmediateU32()); + return input != StageInputKind::WorkgroupId && input != StageInputKind::DispatchThreadLimit; + } + case IrOpcode::ReadConst: + case IrOpcode::ReadConstBuffer: + return arguments(inst); + default: + break; + } + if (SharedAccessOf(inst.Opcode()) != SharedAccess::None || BufferAccessOf(inst.Opcode()) != BufferAccess::None || ImageOpcodeInfoOf(inst.Opcode()).access != ImageAccess::None) return true; + if (AddressOpcodeInfoOf(inst.Opcode()).access != AddressAccess::None) { + const auto index = inst.Flags().index; + const auto& memory = program.Resources().memoryInfo; + const bool scalarRead = AddressOpcodeInfoOf(inst.Opcode()).access == AddressAccess::Read && index < memory.size() && memory[index].kind == ResourceKind::ScalarAddress; + if (!scalarRead) return true; + } + return arguments(inst); + } + + bool arguments(const IrValue& inst) const { + for (std::size_t index = 0; index < inst.ArgumentCount(); ++index) { + const IrValue* argument = inst.Argument(index); + if (argument != nullptr && !Uniform(argument)) return true; + } + return false; + } + + std::unordered_set varying; +}; + +class PairSymmetry { +public: + [[nodiscard]] bool Symmetric(const IrValue* low, const IrValue* high) { + const auto root = intern(low->Resolve(), high->Resolve()); + std::vector> children; + while (!unexpanded.empty() && !overflow) { + const auto current = unexpanded.back(); + unexpanded.pop_back(); + children.clear(); + if (!expand(pairs[current].low, pairs[current].high, children)) { + pairs[current].broken = true; + pending.push_back(current); + continue; + } + for (const auto& [childLow, childHigh] : children) { + const auto child = intern(childLow, childHigh); + pairs[child].parents.push_back(current); + if (pairs[child].broken) pending.push_back(child); + } + } + if (overflow) return false; + settle(); + return !pairs[root].broken; + } + +private: + struct Pair { + const IrValue* low = nullptr; + const IrValue* high = nullptr; + std::vector parents; + bool broken = false; + }; + + std::size_t intern(const IrValue* low, const IrValue* high) { + const auto [found, inserted] = index.try_emplace({low, high}, pairs.size()); + if (inserted) { + pairs.push_back({low, high, {}, false}); + unexpanded.push_back(found->second); + overflow |= pairs.size() > PairLimit; + } + return found->second; + } + + static bool expand(const IrValue* low, const IrValue* high, std::vector>& children) { + if (low == high) return true; + if (low->HasImmediate() || high->HasImmediate()) return low->HasImmediate() && high->HasImmediate() && low->Type() == high->Type() && low->Type() == IrType::U32 && low->ImmediateU32() == high->ImmediateU32(); + if (isBallotWord(low, 0u) && isBallotWord(high, 1u)) return ballotOfHalves(low, high) != nullptr; + if (low->Opcode() != high->Opcode()) return false; + switch (low->Opcode()) { + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + children.emplace_back(low->Argument(0)->Resolve(), high->Argument(0)->Resolve()); + children.emplace_back(low->Argument(1)->Resolve(), high->Argument(1)->Resolve()); + return true; + case IrOpcode::BitwiseNot32: + children.emplace_back(low->Argument(0)->Resolve(), high->Argument(0)->Resolve()); + return true; + case IrOpcode::SelectU32: + if (low->Argument(0)->Resolve() != high->Argument(0)->Resolve()) return false; + children.emplace_back(low->Argument(1)->Resolve(), high->Argument(1)->Resolve()); + children.emplace_back(low->Argument(2)->Resolve(), high->Argument(2)->Resolve()); + return true; + case IrOpcode::Phi: { + if (low->Parent() != high->Parent() || low->PhiBlockCount() != high->PhiBlockCount()) return false; + for (std::size_t incoming = 0; incoming < low->PhiBlockCount(); ++incoming) { + const IrValue* match = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == low->PhiBlock(incoming)) match = high->Argument(other)->Resolve(); + } + if (match == nullptr) return false; + children.emplace_back(low->Argument(incoming)->Resolve(), match); + } + return true; + } + default: + return false; + } + } + + void settle() { + while (!pending.empty()) { + const auto broken = pending.back(); + pending.pop_back(); + for (const auto parent : pairs[broken].parents) { + if (pairs[parent].broken) continue; + pairs[parent].broken = true; + pending.push_back(parent); + } + } + } + + std::map, std::size_t> index; + std::vector pairs; + std::vector unexpanded; + std::vector pending; + bool overflow = false; +}; + +struct Term { + const IrValue* value = nullptr; + const IrValue* high = nullptr; + + bool operator==(const Term& other) const = default; +}; + +void collectTerms(const IrValue* predicate, std::vector& terms, bool& never, std::uint32_t depth = 0); + +void collectPairTerms(const IrValue* low, const IrValue* high, std::vector& terms, bool& never, std::uint32_t depth) { + if (low->HasImmediate() && high->HasImmediate() && low->Type() == IrType::U32 && high->Type() == IrType::U32 && low->ImmediateU32() == high->ImmediateU32()) { + if (low->ImmediateU32() == 0u) never = true; + else if (low->ImmediateU32() != 0xffffffffu) terms.push_back({low, high}); + return; + } + if (const IrValue* ballot = depth < DepthLimit ? ballotOfHalves(low, high) : nullptr; ballot != nullptr) { + collectTerms(ballot->Argument(0), terms, never, depth + 1u); + return; + } + if (depth < DepthLimit && low->Opcode() == IrOpcode::BitwiseAnd32 && high->Opcode() == IrOpcode::BitwiseAnd32) { + collectPairTerms(low->Argument(0)->Resolve(), high->Argument(0)->Resolve(), terms, never, depth + 1u); + collectPairTerms(low->Argument(1)->Resolve(), high->Argument(1)->Resolve(), terms, never, depth + 1u); + return; + } + terms.push_back({low, high}); +} + +void collectTerms(const IrValue* predicate, std::vector& terms, bool& never, std::uint32_t depth) { + predicate = predicate->Resolve(); + if (predicate->HasImmediate() && predicate->Type() == IrType::Bool) { + if (!predicate->ImmediateBool()) never = true; + return; + } + if (depth < DepthLimit && predicate->Opcode() == IrOpcode::LogicalAnd) { + collectTerms(predicate->Argument(0), terms, never, depth + 1u); + collectTerms(predicate->Argument(1), terms, never, depth + 1u); + return; + } + if (const IrValue* word = ownBitWord(predicate); word != nullptr && depth < DepthLimit) { + if (isHalfWord(word)) { + collectPairTerms(word->Argument(1)->Resolve(), word->Argument(2)->Resolve(), terms, never, depth + 1u); + return; + } + if (word->HasImmediate()) { + collectPairTerms(word, word, terms, never, depth + 1u); + return; + } + } + terms.push_back({predicate, nullptr}); +} + +class Implication { +public: + [[nodiscard]] bool Implies(const IrValue* premise, const Term& conclusion) { + steps = 0; + return bit(premise, conclusion, 0); + } + + [[nodiscard]] bool HalvesImply(const IrValue* low, const IrValue* high, const Term& conclusion) { + steps = 0; + return halves(low->Resolve(), high->Resolve(), conclusion, 0); + } + +private: + static constexpr std::size_t StepLimit = std::size_t{1} << 16u; + + bool bit(const IrValue* premise, const Term& conclusion, std::uint32_t depth) { + premise = premise->Resolve(); + if (++steps > StepLimit || depth > DepthLimit) return false; + if (conclusion.high == nullptr && premise == conclusion.value) return true; + if (premise->HasImmediate()) return premise->Type() == IrType::Bool && !premise->ImmediateBool(); + if (premise->Opcode() == IrOpcode::LogicalAnd) return bit(premise->Argument(0), conclusion, depth + 1u) || bit(premise->Argument(1), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::LogicalOr) return bit(premise->Argument(0), conclusion, depth + 1u) && bit(premise->Argument(1), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::SelectU1) return bit(premise->Argument(1), conclusion, depth + 1u) && bit(premise->Argument(2), conclusion, depth + 1u); + if (premise->Opcode() == IrOpcode::Phi) { + if (!openBits.insert(premise).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < premise->PhiBlockCount(); ++incoming) all = bit(premise->Argument(incoming), conclusion, depth + 1u); + openBits.erase(premise); + return all; + } + const IrValue* word = ownBitWord(premise); + if (word == nullptr) return false; + if (isHalfWord(word)) return halves(word->Argument(1)->Resolve(), word->Argument(2)->Resolve(), conclusion, depth + 1u); + return isImmediateU32(word, 0u); + } + + bool halves(const IrValue* low, const IrValue* high, const Term& conclusion, std::uint32_t depth) { + if (++steps > StepLimit || depth > DepthLimit) return false; + if (low == conclusion.value && high == conclusion.high) return true; + if (low->HasImmediate() || high->HasImmediate()) return isImmediateU32(low, 0u) && isImmediateU32(high, 0u); + if (const IrValue* ballot = ballotOfHalves(low, high)) return bit(ballot->Argument(0), conclusion, depth + 1u); + if (low->Opcode() != high->Opcode()) return false; + const auto argument = [](const IrValue* value, std::size_t index) { return value->Argument(index)->Resolve(); }; + switch (low->Opcode()) { + case IrOpcode::BitwiseAnd32: + return halves(argument(low, 0), argument(high, 0), conclusion, depth + 1u) || halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u); + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + return halves(argument(low, 0), argument(high, 0), conclusion, depth + 1u) && halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u); + case IrOpcode::SelectU32: + return argument(low, 0) == argument(high, 0) && halves(argument(low, 1), argument(high, 1), conclusion, depth + 1u) && halves(argument(low, 2), argument(high, 2), conclusion, depth + 1u); + case IrOpcode::Phi: { + if (low->Parent() != high->Parent() || low->PhiBlockCount() != high->PhiBlockCount()) return false; + const std::pair key{low, high}; + if (open.contains(key)) return true; + open.insert(key); + bool all = true; + for (std::size_t incoming = 0; all && incoming < low->PhiBlockCount(); ++incoming) { + const IrValue* match = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == low->PhiBlock(incoming)) match = argument(high, other); + } + all = match != nullptr && halves(argument(low, incoming), match, conclusion, depth + 1u); + } + open.erase(key); + return all; + } + default: + return false; + } + } + + std::set> open; + std::set openBits; + std::size_t steps = 0; +}; + +std::optional checkVccBranch(const BlockInfo& info, const Uniformity& uniformity) { + if (info.condition == nullptr) return std::nullopt; + const IrValue* condition = info.condition->Resolve(); + if (info.exec == nullptr) return Rejection{"a VCC branch without its exec mask", condition}; + const IrValue* exec = info.exec->Resolve(); + if (exec->Opcode() == IrOpcode::Unreachable) return Rejection{"a VCC branch whose exec mask was removed", condition}; + const IrValue* predicate = condition; + if (info.terminator.condition == BranchCondition::VccZero) { + if (condition->Opcode() != IrOpcode::LogicalNot) return uniformity.Uniform(condition) ? std::nullopt : std::optional(Rejection{"a VCC branch on lane-varying bits", condition}); + predicate = condition->Argument(0)->Resolve(); + } + std::vector terms; + bool never = false; + collectTerms(predicate, terms, never); + if (never) return std::nullopt; + Implication implication; + for (const auto& term : terms) { + if (term.high == nullptr && uniformity.Uniform(term.value)) continue; + if (implication.Implies(exec, term)) continue; + return Rejection{"a VCC branch on lane-varying bits", term.value}; + } + return std::nullopt; +} + +std::optional checkLaneMasks(const IrProgram& program) { + std::vector masks; + std::unordered_set isMask; + std::vector halfWords; + std::unordered_set isHalf; + std::vector lanes; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::LaneId) lanes.push_back(inst); + if (inst->Opcode() != IrOpcode::Ballot) continue; + for (const IrUse& use : inst->OperandUses()) { + if (!isBallotWord(use.user, 0u) && !isBallotWord(use.user, 1u)) return Rejection{"a ballot read other than as a lane mask", use.user}; + if (isMask.insert(use.user).second) masks.push_back(use.user); + } + } + } + for (std::size_t next = 0; next < masks.size(); ++next) { + for (const IrUse& use : masks[next]->OperandUses()) { + const IrValue* user = use.user; + if (isHalfWord(user) && use.operand != 0u) { + if (isHalf.insert(user).second) halfWords.push_back(user); + continue; + } + const bool combines = user->Opcode() == IrOpcode::Identity || (isLaneMaskOp(user->Opcode()) && (user->Opcode() != IrOpcode::SelectU32 || use.operand != 0u)); + if (!combines) return Rejection{"a lane mask read as a number", user}; + if (isMask.insert(user).second) masks.push_back(user); + } + } + for (const IrValue* lane : lanes) { + for (const IrUse& laneUse : lane->OperandUses()) { + const IrValue* user = laneUse.user; + if (isHalfSelector(user)) { + for (const IrUse& use : user->OperandUses()) { + if (!isHalfWord(use.user) || use.operand != 0u) return Rejection{"the lane id read as a number", use.user}; + if (isHalf.insert(use.user).second) halfWords.push_back(use.user); + } + continue; + } + if (!isBitSelector(user)) return Rejection{"the lane id read as a number", user}; + for (const IrUse& shift : user->OperandUses()) { + const IrValue* shifted = shift.user; + if (shifted->Opcode() != IrOpcode::ShiftRightLogical32 || shift.operand != 1u) return Rejection{"the lane id read as a number", shifted}; + const IrValue* word = shifted->Argument(0)->Resolve(); + if (!isHalfWord(word) && !word->HasImmediate()) return Rejection{"a lane's bit read from one half of a lane mask", shifted}; + for (const IrUse& bit : shifted->OperandUses()) { + if (bit.user->Opcode() != IrOpcode::BitwiseAnd32 || otherThan(bit.user, 1u) != shifted) return Rejection{"a lane mask shifted by the lane id read as a number", bit.user}; + } + } + } + } + PairSymmetry symmetry; + for (const IrValue* word : halfWords) { + for (const IrUse& use : word->OperandUses()) { + if (use.user->Opcode() != IrOpcode::ShiftRightLogical32 || use.operand != 0u || !isBitSelector(use.user->Argument(1))) return Rejection{"half of a lane mask read as a number", use.user}; + } + if (!symmetry.Symmetric(word->Argument(1), word->Argument(2))) return Rejection{"a lane mask whose halves differ", word}; + } + return std::nullopt; +} + +struct Affine { + std::array coefficients{}; + std::uint64_t constant = 0; +}; + +class AffineEvaluator { +public: + explicit AffineEvaluator(const std::array& sizes) : sizes(sizes) {} + + [[nodiscard]] std::uint64_t Max(const Affine& affine) const { + std::uint64_t result = affine.constant; + for (std::size_t axis = 0; axis < 3u; ++axis) result += affine.coefficients[axis] * (sizes[axis] - 1u); + return result; + } + + [[nodiscard]] std::optional Of(const IrValue* value, std::uint32_t depth = 0) const { + value = value->Resolve(); + if (value->HasImmediate()) return value->Type() == IrType::U32 ? std::optional(Affine{{}, value->ImmediateU32()}) : std::nullopt; + if (depth > DepthLimit) return std::nullopt; + const auto argument = [&](std::size_t index) { return Of(value->Argument(index), depth + 1u); }; + const auto immediate = [&](std::size_t index) -> std::optional { + const IrValue* operand = value->Argument(index)->Resolve(); + return operand->HasImmediate() && operand->Type() == IrType::U32 ? std::optional(operand->ImmediateU32()) : std::nullopt; + }; + std::optional result; + switch (value->Opcode()) { + case IrOpcode::GetBuiltin: { + const auto kind = immediate(0); + if (!kind) return std::nullopt; + if (static_cast(*kind) == StageInputKind::LocalInvocationIndex) return Affine{{1u, sizes[0], std::uint64_t{sizes[0]} * sizes[1]}, 0u}; + const auto axis = immediate(1); + if (static_cast(*kind) != StageInputKind::LocalInvocationId || !axis || *axis >= 3u) return std::nullopt; + Affine id; + id.coefficients[*axis] = 1u; + return id; + } + case IrOpcode::IAdd32: { + const auto a = argument(0), b = argument(1); + if (!a || !b) return std::nullopt; + result = sum(*a, *b); + break; + } + case IrOpcode::IMul32: { + auto a = argument(0), b = argument(1); + if (!a || !b) return std::nullopt; + if (!isConstant(*a)) std::swap(a, b); + if (!isConstant(*a)) return std::nullopt; + result = scaled(*b, a->constant); + break; + } + case IrOpcode::ShiftLeftLogical32: { + const auto a = argument(0); + const auto shift = immediate(1); + if (!a || !shift || *shift >= 32u) return std::nullopt; + result = scaled(*a, std::uint64_t{1} << *shift); + break; + } + case IrOpcode::BitFieldUExtract: { + const auto a = argument(0); + const auto offset = immediate(1), count = immediate(2); + if (!a || offset != 0u || !count || *count > 32u || (*count < 32u && Max(*a) >> *count != 0u)) return std::nullopt; + result = a; + break; + } + case IrOpcode::BitwiseAnd32: + case IrOpcode::BitwiseOr32: { + auto mask = immediate(1); + auto a = argument(0); + if (!mask) { + mask = immediate(0); + a = argument(1); + } + if (!a || !mask) return std::nullopt; + if (value->Opcode() == IrOpcode::BitwiseAnd32) { + if ((*mask & (*mask + 1u)) != 0u || Max(*a) > *mask) return std::nullopt; + result = a; + } else { + std::uint32_t zeros = 32u; + if (a->constant != 0u) zeros = std::min(zeros, static_cast(std::countr_zero(a->constant))); + for (const auto coefficient : a->coefficients) { + if (coefficient != 0u) zeros = std::min(zeros, static_cast(std::countr_zero(coefficient))); + } + if (zeros < 32u && (*mask >> zeros) != 0u) return std::nullopt; + result = sum(*a, Affine{{}, *mask}); + } + break; + } + default: + return std::nullopt; + } + if (!result || Max(*result) > 0xffffffffull) return std::nullopt; + return result; + } + +private: + static bool isConstant(const Affine& affine) { + return affine.coefficients == std::array{}; + } + + static Affine sum(const Affine& a, const Affine& b) { + Affine result; + for (std::size_t axis = 0; axis < 3u; ++axis) result.coefficients[axis] = a.coefficients[axis] + b.coefficients[axis]; + result.constant = a.constant + b.constant; + return result; + } + + std::optional scaled(const Affine& affine, std::uint64_t factor) const { + if (factor > 0xffffffffull) return std::nullopt; + Affine result; + for (std::size_t axis = 0; axis < 3u; ++axis) { + if (affine.coefficients[axis] > 0xffffffffull) return std::nullopt; + result.coefficients[axis] = affine.coefficients[axis] * factor; + } + if (affine.constant > 0xffffffffull) return std::nullopt; + result.constant = affine.constant * factor; + return result; + } + + std::array sizes; +}; + +std::optional checkLds(const IrProgram& program, const std::array& sizes, bool& writes) { + const AffineEvaluator evaluator(sizes); + std::optional> stride; + std::vector> footprint; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + const auto access = SharedAccessOf(inst->Opcode()); + if (access == SharedAccess::None) continue; + const auto index = inst->Flags().index; + if (index >= program.Resources().memoryInfo.size()) return Rejection{"an LDS access without memory metadata", inst}; + const auto& memory = program.Resources().memoryInfo[index]; + if (memory.kind != ResourceKind::Lds) return Rejection{"a GDS access", inst}; + const auto bytes = sharedBytes(inst->Opcode()); + if (bytes == 0u || inst->ArgumentCount() == 0u) return Rejection{"an LDS access that counts or locks across the wave", inst}; + writes |= access != SharedAccess::Read; + auto affine = evaluator.Of(inst->Argument(0)); + if (!affine) return Rejection{"an LDS address that is not affine in the local invocation id", inst}; + for (std::size_t axis = 0; axis < 3u; ++axis) { + if (sizes[axis] == 1u) affine->coefficients[axis] = 0u; + } + if (stride && *stride != affine->coefficients) return Rejection{"LDS accesses at different strides", inst}; + stride = affine->coefficients; + const auto begin = affine->constant + memory.offset; + if (evaluator.Max(*affine) + memory.offset + bytes > 0x100000000ull) return Rejection{"an LDS address that wraps", inst}; + footprint.emplace_back(begin, begin + bytes); + } + } + if (!stride) return std::nullopt; + std::sort(footprint.begin(), footprint.end()); + std::vector> merged; + for (const auto& range : footprint) { + if (!merged.empty() && range.first <= merged.back().second) merged.back().second = std::max(merged.back().second, range.second); + else merged.push_back(range); + } + if (merged.size() > FootprintLimit) return Rejection{"an LDS footprint of too many ranges", nullptr}; + std::uint64_t offsets = 1; + for (const auto size : sizes) offsets *= 2u * size - 1u; + if (offsets > LaneOffsetLimit) return Rejection{"a workgroup too large to check LDS privacy", nullptr}; + std::array delta{}; + for (std::int64_t x = 1 - static_cast(sizes[0]); x < static_cast(sizes[0]); ++x) { + for (std::int64_t y = 1 - static_cast(sizes[1]); y < static_cast(sizes[1]); ++y) { + for (std::int64_t z = 1 - static_cast(sizes[2]); z < static_cast(sizes[2]); ++z) { + if (x == 0 && y == 0 && z == 0) continue; + delta = {x, y, z}; + std::int64_t shift = 0; + for (std::size_t axis = 0; axis < 3u; ++axis) shift += delta[axis] * static_cast((*stride)[axis]); + for (const auto& moved : merged) { + for (const auto& fixed : merged) { + const auto begin = static_cast(moved.first) + shift; + const auto end = static_cast(moved.second) + shift; + if (begin < static_cast(fixed.second) && static_cast(fixed.first) < end) return Rejection{"lanes that share LDS bytes", nullptr}; + } + } + } + } + } + return std::nullopt; +} + +bool sameWord(const IrValue* left, const IrValue* right) { + left = left->Resolve(); + right = right->Resolve(); + if (left == right) return true; + return left->HasImmediate() && right->HasImmediate() && left->Type() == IrType::U32 && right->Type() == IrType::U32 && left->ImmediateU32() == right->ImmediateU32(); +} + +bool isSelect(IrOpcode opcode) { + return opcode == IrOpcode::Select || opcode == IrOpcode::SelectU1 || opcode == IrOpcode::SelectF32 || opcode == IrOpcode::SelectU32; +} + +bool writesMemory(IrOpcode opcode) { + const auto buffer = BufferAccessOf(opcode); + const auto address = AddressOpcodeInfoOf(opcode).access; + const auto image = ImageOpcodeInfoOf(opcode).access; + const auto shared = SharedAccessOf(opcode); + return buffer == BufferAccess::Write || buffer == BufferAccess::Atomic || address == AddressAccess::Write || address == AddressAccess::Atomic || image == ImageAccess::Write || image == ImageAccess::Atomic || (shared != SharedAccess::None && shared != SharedAccess::Read); +} + +const IrValue* lanePredicate(const IrValue& inst) { + for (std::size_t index = inst.ArgumentCount(); index > 0u; --index) { + const IrValue* argument = inst.Argument(index - 1u); + if (argument != nullptr && argument->Resolve()->Type() == IrType::Bool) return argument->Resolve(); + } + return nullptr; +} + +const IrValue* branchExec(const BlockInfo& info) { + if (info.exec != nullptr) return info.exec->Resolve(); + const IrValue* condition = info.condition->Resolve(); + if (info.terminator.condition == BranchCondition::ExecNonZero) return condition; + if (info.terminator.condition == BranchCondition::ExecZero && condition->Opcode() == IrOpcode::LogicalNot) return condition->Argument(0)->Resolve(); + return nullptr; +} + +bool pureLaneOp(IrOpcode opcode) { + if (opcode == IrOpcode::ReadConst || opcode == IrOpcode::ReadConstBuffer) return true; + if (!isLaneLocal(opcode) || opcode == IrOpcode::Phi || opcode == IrOpcode::ShaderClock || opcode == IrOpcode::RealtimeClock || opcode == IrOpcode::LoadVectorWindow || opcode == IrOpcode::StoreVectorWindow) return false; + return BufferAccessOf(opcode) == BufferAccess::None && AddressOpcodeInfoOf(opcode).access == AddressAccess::None && ImageOpcodeInfoOf(opcode).access == ImageAccess::None && SharedAccessOf(opcode) == SharedAccess::None; +} + +bool halfDecided(const BlockInfo& info, const Uniformity& uniformity) { + if (info.terminator.kind != TerminatorKind::ConditionalBranch || info.condition == nullptr) return false; + switch (info.terminator.condition) { + case BranchCondition::ExecZero: + case BranchCondition::ExecNonZero: + return true; + case BranchCondition::VccZero: + case BranchCondition::VccNonZero: { + const IrValue* predicate = info.condition->Resolve(); + if (info.terminator.condition == BranchCondition::VccZero && predicate->Opcode() == IrOpcode::LogicalNot) predicate = predicate->Argument(0)->Resolve(); + std::vector terms; + bool never = false; + collectTerms(predicate, terms, never); + return std::ranges::any_of(terms, [&](const Term& term) { return term.high != nullptr || !uniformity.Uniform(term.value); }); + } + default: + return false; + } +} + +class SkippedRegions { +public: + SkippedRegions(const IrProgram& program, const Uniformity& uniformity) : program(program), uniformity(uniformity), order(program.BlockOrder()) { + const auto& infos = program.Metadata().blockInfo; + for (std::size_t index = 0; index < order.size(); ++index) { + indices.emplace(order[index], index); + if (index < infos.size()) ids.emplace(infos[index].id, index); + } + postDominate(); + observe(); + } + + [[nodiscard]] std::optional Check() { + const auto& infos = program.Metadata().blockInfo; + for (std::size_t branch = 0; branch < order.size() && branch < infos.size(); ++branch) { + if (!halfDecided(infos[branch], uniformity)) continue; + if (auto rejected = checkBranch(branch, infos[branch])) return rejected; + } + return std::nullopt; + } + +private: + using Bits = std::vector; + + static bool test(const Bits& bits, std::size_t index) { + return ((bits[index / 64u] >> (index % 64u)) & 1u) != 0u; + } + + static void set(Bits& bits, std::size_t index) { + bits[index / 64u] |= std::uint64_t{1} << (index % 64u); + } + + std::size_t blockIndex(const IrBlock* block) const { + const auto found = indices.find(block); + return found == indices.end() ? order.size() : found->second; + } + + std::size_t blockOf(std::uint32_t id) const { + const auto found = ids.find(id); + return found == ids.end() ? order.size() : found->second; + } + + bool inRegion(const IrBlock* block) const { + const auto index = blockIndex(block); + return index < order.size() && region[index] != 0; + } + + bool definedInRegion(const IrValue* value) const { + return !value->HasImmediate() && inRegion(value->Parent()); + } + + void postDominate() { + const std::size_t exit = order.size(); + const std::size_t words = (exit + 64u) / 64u; + const Bits all(words, ~std::uint64_t{0}); + dominators.assign(exit + 1u, all); + dominators[exit].assign(words, 0u); + set(dominators[exit], exit); + bool changed = true; + while (changed) { + changed = false; + for (std::size_t index = exit; index-- > 0u;) { + Bits next = order[index]->Successors().empty() ? dominators[exit] : all; + for (const IrBlock* successor : order[index]->Successors()) { + const auto& after = dominators[blockIndex(successor)]; + for (std::size_t word = 0; word < words; ++word) next[word] &= after[word]; + } + set(next, index); + if (next == dominators[index]) continue; + dominators[index] = std::move(next); + changed = true; + } + } + reachesEnd.assign(exit, 0); + std::vector pending; + for (std::size_t index = 0; index < exit; ++index) { + if (!order[index]->Successors().empty()) continue; + reachesEnd[index] = 1; + pending.push_back(index); + } + while (!pending.empty()) { + const auto current = pending.back(); + pending.pop_back(); + for (const IrBlock* predecessor : order[current]->Predecessors()) { + const auto index = blockIndex(predecessor); + if (index >= exit || reachesEnd[index]) continue; + reachesEnd[index] = 1; + pending.push_back(index); + } + } + } + + void observe() { + std::vector pending; + const auto mark = [&](const IrValue* value) { + if (value == nullptr) return; + value = value->Resolve(); + if (value->HasImmediate() || !observed.insert(value).second) return; + if (value->Opcode() == IrOpcode::Phi) pending.push_back(value); + }; + for (const IrBlock* block : order) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::Phi) continue; + for (std::size_t index = 0; index < inst->ArgumentCount(); ++index) mark(inst->Argument(index)); + } + } + while (!pending.empty()) { + const IrValue* phi = pending.back(); + pending.pop_back(); + for (std::size_t incoming = 0; incoming < phi->PhiBlockCount(); ++incoming) mark(phi->Argument(incoming)); + } + } + + std::size_t immediatePostDominator(std::size_t block) const { + std::size_t best = order.size(); + std::size_t bestCount = 0; + for (std::size_t candidate = 0; candidate < order.size(); ++candidate) { + if (candidate == block || !test(dominators[block], candidate)) continue; + std::size_t count = 0; + for (const auto word : dominators[candidate]) count += static_cast(std::popcount(word)); + if (count <= bestCount) continue; + best = candidate; + bestCount = count; + } + return best; + } + + std::vector reach(std::size_t from, std::size_t stop) const { + std::vector seen(order.size(), 0); + if (from >= order.size() || from == stop) return seen; + std::vector pending{from}; + seen[from] = 1; + while (!pending.empty()) { + const auto current = pending.back(); + pending.pop_back(); + for (const IrBlock* successor : order[current]->Successors()) { + const auto next = blockIndex(successor); + if (next >= order.size() || next == stop || seen[next]) continue; + seen[next] = 1; + pending.push_back(next); + } + } + return seen; + } + + std::optional checkBranch(std::size_t branch, const BlockInfo& info) { + const IrValue* condition = info.condition->Resolve(); + exec = branchExec(info); + if (exec == nullptr) return Rejection{"a branch one half can decide alone without its exec mask", condition}; + terms.clear(); + bool never = false; + collectTerms(exec, terms, never); + if (never || terms.empty()) return std::nullopt; + if (!reachesEnd[branch]) return Rejection{"a branch one half can decide alone that never reaches the end", condition}; + const bool skipOnFalse = info.terminator.condition == BranchCondition::ExecNonZero || info.terminator.condition == BranchCondition::VccNonZero; + const auto skip = blockOf(skipOnFalse ? info.terminator.falseBlock : info.terminator.trueBlock); + const auto wave = blockOf(skipOnFalse ? info.terminator.trueBlock : info.terminator.falseBlock); + const auto merge = immediatePostDominator(branch); + branchIndex = branch; + skipIndex = skip; + mergeIndex = merge; + region = reach(wave, merge); + skippedRegion = reach(skip, merge); + for (std::size_t index = 0; index < region.size(); ++index) region[index] = region[index] != 0 && skippedRegion[index] == 0 ? 1 : 0; + cyclic = region[branch] != 0; + stability.clear(); + skippedValues.clear(); + if (cyclic && !monotone()) return Rejection{"a loop left through a branch one half can decide alone whose exec can widen", condition}; + for (std::size_t index = 0; index < order.size(); ++index) { + if (region[index] != 0) { + for (const IrValue* inst : order[index]->Instructions()) { + if (auto rejected = checkValue(*inst)) return rejected; + } + continue; + } + if (std::ranges::none_of(order[index]->Predecessors(), [&](const IrBlock* predecessor) { return inRegion(predecessor); })) continue; + for (const IrValue* inst : order[index]->Instructions()) { + if (inst->Opcode() != IrOpcode::Phi || !observed.contains(inst)) continue; + if (auto rejected = checkExit(*inst)) return rejected; + } + } + return std::nullopt; + } + + std::optional checkValue(const IrValue& inst) { + const auto opcode = inst.Opcode(); + if (opcode == IrOpcode::StoreVectorWindow && !windowKept(inst.Argument(1), inst.Argument(0)->Resolve(), 0)) return Rejection{"a vector window store one half can skip", &inst}; + if (writesMemory(opcode)) { + const IrValue* predicate = lanePredicate(inst); + if (predicate == nullptr || !impliesExec(predicate)) return Rejection{"a memory write one half can skip outside the branch's exec", &inst}; + } + for (const IrUse& use : inst.OperandUses()) { + const IrValue* user = use.user; + if (user->Opcode() == IrOpcode::Phi || inRegion(user->Parent())) continue; + if (cyclic && stable(&inst, 0)) continue; + return Rejection{cyclic ? "a value that leaves a loop one half can leave early" : "a value that leaves a region one half can skip", &inst}; + } + return std::nullopt; + } + + std::optional checkExit(const IrValue& phi) { + if (!cyclic) return merges(phi) ? std::nullopt : std::optional(Rejection{"a value that leaves a region one half can skip", &phi}); + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (inRegion(phi.PhiBlock(incoming)) && !stable(phi.Argument(incoming), 0)) return Rejection{"a value that leaves a loop one half can leave early", &phi}; + } + return std::nullopt; + } + + bool impliesExec(const IrValue* predicate) { + Implication implication; + return std::ranges::all_of(terms, [&](const Term& term) { return implication.Implies(predicate, term); }); + } + + bool halvesImplyExec(const IrValue* low, const IrValue* high) { + Implication implication; + return std::ranges::all_of(terms, [&](const Term& term) { return implication.HalvesImply(low, high, term); }); + } + + bool monotone() { + if (!definedInRegion(exec)) return true; + for (const Term& term : terms) { + const IrValue* low = term.value->Resolve(); + const IrValue* high = term.high == nullptr ? nullptr : term.high->Resolve(); + if (low->Opcode() != IrOpcode::Phi || !definedInRegion(low)) continue; + if (high != nullptr && (high->Opcode() != IrOpcode::Phi || high->Parent() != low->Parent())) continue; + bool entered = false; + bool narrows = true; + for (std::size_t incoming = 0; narrows && incoming < low->PhiBlockCount(); ++incoming) { + const IrBlock* from = low->PhiBlock(incoming); + if (!inRegion(from)) { + entered = true; + continue; + } + if (high == nullptr) { + narrows = impliesExec(low->Argument(incoming)); + continue; + } + const IrValue* matching = nullptr; + for (std::size_t other = 0; other < high->PhiBlockCount(); ++other) { + if (high->PhiBlock(other) == from) matching = high->Argument(other); + } + narrows = matching != nullptr && halvesImplyExec(low->Argument(incoming), matching); + } + if (entered && narrows) return true; + } + return false; + } + + bool skipSide(const IrBlock* from) const { + const auto index = blockIndex(from); + return index < order.size() && (skippedRegion[index] != 0 || (index == branchIndex && skipIndex == mergeIndex)); + } + + bool merges(const IrValue& phi) { + bool skippedPath = false; + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (!skipSide(phi.PhiBlock(incoming))) continue; + skippedPath = true; + const IrValue* kept = phi.Argument(incoming)->Resolve(); + for (std::size_t other = 0; other < phi.PhiBlockCount(); ++other) { + if (!inRegion(phi.PhiBlock(other))) continue; + if (!sameWord(skippedValue(phi.Argument(other)), skippedValue(kept)) && !chainTo(phi.Argument(other), kept, 0)) return false; + } + } + return skippedPath; + } + + const IrValue* settled(const IrValue* value) const { + for (std::uint32_t step = 0; step <= DepthLimit; ++step) { + const auto found = skippedValues.find(value); + if (found == skippedValues.end() || found->second == value) return value; + value = found->second; + } + return value; + } + + const IrValue* skippedValue(const IrValue* value) { + value = value->Resolve(); + if (const auto found = skippedValues.find(value); found != skippedValues.end()) return settled(found->second); + const std::size_t depth = passing.size(); + if (const auto open = passing.find(value); open != passing.end()) { + lowestOpen = std::min(lowestOpen, open->second); + return value; + } + passing.emplace(value, depth); + const std::size_t outer = std::exchange(lowestOpen, NoOpenCycle); + const IrValue* result = value; + if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u) { + const IrValue* kept = skippedValue(value->Argument(2)); + if (impliesExec(value->Argument(0)) || sameWord(skippedValue(value->Argument(1)), kept)) result = kept; + } else if (value->Opcode() == IrOpcode::Phi && definedInRegion(value)) { + const IrValue* base = nullptr; + bool agree = true; + for (std::size_t incoming = 0; agree && incoming < value->PhiBlockCount(); ++incoming) { + const IrValue* candidate = skippedValue(value->Argument(incoming)); + if (candidate == value) continue; + if (base == nullptr) base = candidate; + else agree = sameWord(base, candidate); + } + if (agree && base != nullptr) result = base; + } + passing.erase(value); + const bool pending = lowestOpen < depth; + if (!pending || result != value) skippedValues.emplace(value, result); + lowestOpen = std::min(outer, pending ? lowestOpen : NoOpenCycle); + return settled(result); + } + + bool chainTo(const IrValue* value, const IrValue* kept, std::uint32_t depth) { + value = value->Resolve(); + if (value == kept) return true; + if (depth > DepthLimit) return false; + if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u && chainTo(value->Argument(2), kept, depth + 1u) && (impliesExec(value->Argument(0)) || chainTo(value->Argument(1), kept, depth + 1u))) return true; + if (value->Opcode() == IrOpcode::Phi && definedInRegion(value)) { + if (!visiting.insert(value).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < value->PhiBlockCount(); ++incoming) all = chainTo(value->Argument(incoming), kept, depth + 1u); + visiting.erase(value); + if (all) return true; + } + if (value->Type() == IrType::Bool && kept->Type() == IrType::Bool) return impliesExec(value) && impliesExec(kept); + if (value->Type() != IrType::U32 || kept->Type() != IrType::U32) return false; + if (value->ArgumentCount() == 2u) { + const IrValue* left = value->Argument(0); + const IrValue* right = value->Argument(1); + if (value->Opcode() == IrOpcode::BitwiseAnd32 && ((chainTo(left, kept, depth + 1u) && fullWord(right, 0)) || (chainTo(right, kept, depth + 1u) && fullWord(left, 0)))) return true; + if ((value->Opcode() == IrOpcode::BitwiseOr32 || value->Opcode() == IrOpcode::BitwiseXor32) && ((chainTo(left, kept, depth + 1u) && boundedWord(right, 0)) || (chainTo(right, kept, depth + 1u) && boundedWord(left, 0)))) return true; + } + return boundedWord(value, 0) && boundedWord(kept, 0); + } + + bool fullWord(const IrValue* word, std::uint32_t depth) { + word = word->Resolve(); + if (word->HasImmediate()) return word->Type() == IrType::U32 && word->ImmediateU32() == 0xffffffffu; + if (depth > DepthLimit) return false; + switch (word->Opcode()) { + case IrOpcode::BitwiseNot32: + return boundedWord(word->Argument(0), depth + 1u); + case IrOpcode::BitwiseOr32: + return fullWord(word->Argument(0), depth + 1u) || fullWord(word->Argument(1), depth + 1u); + case IrOpcode::BitwiseAnd32: + return fullWord(word->Argument(0), depth + 1u) && fullWord(word->Argument(1), depth + 1u); + default: + return false; + } + } + + bool windowKept(const IrValue* value, const IrValue* index, std::uint32_t depth) { + value = value->Resolve(); + if (value->Opcode() == IrOpcode::LoadVectorWindow) return sameWord(value->Argument(0)->Resolve(), index); + if (depth > DepthLimit || !isSelect(value->Opcode()) || value->ArgumentCount() != 3u) return false; + if (!windowKept(value->Argument(2), index, depth + 1u)) return false; + return impliesExec(value->Argument(0)) || windowKept(value->Argument(1), index, depth + 1u); + } + + bool boundedWord(const IrValue* word, std::uint32_t depth) { + word = word->Resolve(); + if (word->HasImmediate()) return word->Type() == IrType::U32 && word->ImmediateU32() == 0u; + if (depth > DepthLimit) return false; + if (terms.size() == 1u && terms.front().high != nullptr && (terms.front().value->Resolve() == word || terms.front().high->Resolve() == word)) return true; + if (isBallotWord(word, 0u) || isBallotWord(word, 1u)) return impliesExec(word->Argument(0)->Resolve()->Argument(0)); + const auto argument = [&](std::size_t index) { return word->Argument(index); }; + switch (word->Opcode()) { + case IrOpcode::BitwiseAnd32: + return boundedWord(argument(0), depth + 1u) || boundedWord(argument(1), depth + 1u); + case IrOpcode::BitwiseOr32: + case IrOpcode::BitwiseXor32: + return boundedWord(argument(0), depth + 1u) && boundedWord(argument(1), depth + 1u); + case IrOpcode::SelectU32: + return boundedWord(argument(1), depth + 1u) && boundedWord(argument(2), depth + 1u); + case IrOpcode::Phi: { + if (!visiting.insert(word).second) return true; + bool all = true; + for (std::size_t incoming = 0; all && incoming < word->PhiBlockCount(); ++incoming) all = boundedWord(argument(incoming), depth + 1u); + visiting.erase(word); + return all; + } + default: + return false; + } + } + + bool stable(const IrValue* value, std::uint32_t depth) { + value = value->Resolve(); + if (!definedInRegion(value)) return true; + if (depth > DepthLimit) return false; + if (const auto found = stability.find(value); found != stability.end()) return found->second; + stability.emplace(value, true); + const auto arguments = [&] { + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { + if (!stable(value->Argument(index), depth + 1u)) return false; + } + return true; + }; + bool result = false; + if (value->Opcode() == IrOpcode::Phi) { + result = stablePhi(*value, depth); + } else if (isSelect(value->Opcode()) && value->ArgumentCount() == 3u) { + result = (impliesExec(value->Argument(0)) && stable(value->Argument(2), depth + 1u)) || arguments(); + } else if (value->Opcode() == IrOpcode::LoadVectorWindow) { + result = stable(value->Argument(0), depth + 1u); + } else if (value->Type() == IrType::Bool && impliesExec(value)) { + result = true; + } else if (pureLaneOp(value->Opcode())) { + result = arguments(); + } else { + result = boundedWord(value, 0); + } + stability[value] = result; + return result; + } + + bool stablePhi(const IrValue& phi, std::uint32_t depth) { + const bool entered = std::ranges::any_of(phi.Parent()->Predecessors(), [&](const IrBlock* predecessor) { return !inRegion(predecessor); }); + if (entered) { + for (std::size_t incoming = 0; incoming < phi.PhiBlockCount(); ++incoming) { + if (inRegion(phi.PhiBlock(incoming)) && !chainTo(phi.Argument(incoming), &phi, 0)) return false; + } + return true; + } + for (std::size_t base = 0; base < phi.PhiBlockCount(); ++base) { + const IrValue* kept = phi.Argument(base)->Resolve(); + bool all = true; + for (std::size_t incoming = 0; all && incoming < phi.PhiBlockCount(); ++incoming) all = incoming == base || chainTo(phi.Argument(incoming), kept, 0); + if (all && stable(kept, depth + 1u)) return true; + } + return false; + } + + const IrProgram& program; + const Uniformity& uniformity; + const std::vector& order; + std::unordered_map indices; + std::unordered_map ids; + std::vector dominators; + std::vector reachesEnd; + std::vector region; + std::vector skippedRegion; + std::size_t branchIndex = 0; + std::size_t skipIndex = 0; + std::size_t mergeIndex = 0; + bool cyclic = false; + const IrValue* exec = nullptr; + std::vector terms; + std::unordered_set visiting; + std::unordered_map stability; + static constexpr std::size_t NoOpenCycle = ~std::size_t{0}; + + std::unordered_map skippedValues; + std::unordered_map passing; + std::size_t lowestOpen = NoOpenCycle; + std::unordered_set observed; +}; + +std::optional analyze(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup) { + if (program.Resources().stage != IrShaderStage::Compute || program.WaveSize() != 64u) return Rejection{"not a wave64 compute program", nullptr}; + const std::array sizes{std::max(workgroup.threadsNum[0], 1u), std::max(workgroup.threadsNum[1], 1u), std::max(workgroup.threadsNum[2], 1u)}; + bool reads = false; + bool barrier = false; + const auto& memory = program.Resources().memoryInfo; + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + const auto opcode = inst->Opcode(); + reads |= opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::ReadLane; + barrier |= opcode == IrOpcode::Barrier; + if (opcode == IrOpcode::GetBuiltin) { + const IrValue* kind = inst->Argument(0)->Resolve(); + const auto input = kind->HasImmediate() ? static_cast(kind->ImmediateU32()) : StageInputKind::Parameter; + if (input != StageInputKind::WorkgroupId && input != StageInputKind::LocalInvocationId && input != StageInputKind::LocalInvocationIndex && input != StageInputKind::GlobalInvocationId && input != StageInputKind::DispatchThreadLimit) return Rejection{"a builtin outside compute dispatch ids", inst}; + continue; + } + const bool wave = opcode == IrOpcode::LaneId || opcode == IrOpcode::Ballot || opcode == IrOpcode::ReadFirstLane || opcode == IrOpcode::ReadLane; + if (!wave && !isLaneLocal(opcode) && SharedAccessOf(opcode) == SharedAccess::None) return Rejection{"an operation across lanes", inst}; + if (opcode == IrOpcode::ReadConstBuffer || (BufferAccessOf(opcode) == BufferAccess::None && AddressOpcodeInfoOf(opcode).access == AddressAccess::None && ImageOpcodeInfoOf(opcode).access == ImageAccess::None)) continue; + const auto index = inst->Flags().index; + if (index >= memory.size()) continue; + if (memory[index].gpuDescriptor) return Rejection{"a GPU-selected buffer descriptor", inst}; + } + } + if (auto rejected = checkLaneMasks(program)) return rejected; + bool writes = false; + if (auto rejected = checkLds(program, sizes, writes)) return rejected; + if (writes && sizes[0] * sizes[1] * sizes[2] <= program.WaveSize()) return Rejection{"LDS writes in a one-wave workgroup", nullptr}; + std::optional uniformity; + const auto uniform = [&]() -> const Uniformity& { + if (!uniformity) uniformity.emplace(program); + return *uniformity; + }; + if (reads) { + for (const IrBlock* block : program.BlockOrder()) { + for (const IrValue* inst : block->Instructions()) { + if ((inst->Opcode() == IrOpcode::ReadFirstLane || inst->Opcode() == IrOpcode::ReadLane) && !uniform().Uniform(inst->Argument(0))) return Rejection{"a lane read of a lane-varying value", inst}; + } + } + } + bool execBranches = false; + for (const auto& info : program.Metadata().blockInfo) { + if (info.indirectTarget != nullptr && !uniform().Uniform(info.indirectTarget)) return Rejection{"an indirect branch on a lane-varying value", info.indirectTarget}; + if (info.terminator.kind != TerminatorKind::ConditionalBranch || info.condition == nullptr) continue; + switch (info.terminator.condition) { + case BranchCondition::Always: + break; + case BranchCondition::ExecZero: + case BranchCondition::ExecNonZero: + execBranches = true; + break; + case BranchCondition::VccZero: + case BranchCondition::VccNonZero: + if (auto rejected = checkVccBranch(info, uniform())) return rejected; + execBranches |= halfDecided(info, uniform()); + break; + default: + if (!uniform().Uniform(info.condition)) return Rejection{"a scalar branch on a lane-varying value", info.condition->Resolve()}; + break; + } + } + if (barrier && execBranches) return Rejection{"a barrier in a program with exec branches", nullptr}; + const auto& metadata = program.Metadata(); + if (metadata.skippedRegionsChecked) return metadata.skippedRegionRejection.empty() ? std::nullopt : std::optional(Rejection{metadata.skippedRegionRejection.c_str(), nullptr}); + if (execBranches) return SkippedRegions(program, uniform()).Check(); + return std::nullopt; +} + +} + +SingleLaneVerdict AnalyzeSingleLane(const IrProgram& program, const ShaderWorkgroupInputInfo& workgroup) { + const auto rejected = analyze(program, workgroup); + if (!rejected) return {true, nullptr, nullptr}; + return {false, rejected->reason, rejected->at}; +} + +void RecordSkippedRegions(IrProgram& program) { + auto& metadata = program.Metadata(); + if (metadata.skippedRegionsChecked || program.Resources().stage != IrShaderStage::Compute || program.WaveSize() != 64u) return; + const Uniformity uniformity(program); + const auto rejected = SkippedRegions(program, uniformity).Check(); + metadata.skippedRegionsChecked = true; + if (rejected) metadata.skippedRegionRejection = std::string(rejected->reason) + (rejected->at == nullptr ? std::string() : " at " + DescribeValue(rejected->at, 3)); +} + +} diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 74739dfae658..32700dc3f814 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -21,20 +21,6 @@ std::string& failureReason() { return HostThreadLocal(); } -std::string DescribeValue(const IrValue* value, std::uint32_t depth) { - if (value == nullptr) return "null"; - value = value->Resolve(); - std::string text(IrOpcodeName(value->Opcode())); - if (value->HasImmediate() && value->Type() == IrType::U32) return text + "(" + std::to_string(value->ImmediateU32()) + ")"; - if (depth == 0 || value->ArgumentCount() == 0) return text; - text += "("; - for (std::size_t index = 0; index < value->ArgumentCount(); ++index) { - if (index != 0) text += ", "; - text += DescribeValue(value->Argument(index), depth - 1); - } - return text + ")"; -} - bool Fail(std::string reason) { failureReason() = std::move(reason); return false; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 437d284e5e5e..b38bbff8c453 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -42,6 +42,7 @@ #include "Optimization/include/Optimization/ResourceTracker.hpp" #include "Optimization/include/Optimization/VectorWindowStoreEliminator.hpp" #include "Optimization/include/Optimization/ShaderInfoCollector.hpp" +#include "Optimization/include/Optimization/SingleLaneAnalysis.hpp" #include "Optimization/include/Optimization/SrtWalker.hpp" #include "Optimization/include/Optimization/SsaBuilder.hpp" #include "Optimization/include/Optimization/TessellationLowering.hpp" @@ -108,6 +109,15 @@ std::uint32_t HostSubgroupSize(const RecompileRequest& request) { return request.target.subgroupSize; } +bool SingleLaneCandidate(const RecompileRequest& request) { + static const bool enabled = std::getenv("APS5_NO_SINGLE_LANE_RULE") == nullptr; + return enabled && request.shader.stage == ShaderStage::Compute && request.context.waveSize == 64u && HostSubgroupSize(request) == 32u; +} + +std::uint32_t CacheLayout(const RecompileRequest& request) { + return HostSubgroupSize(request) | (SingleLaneCandidate(request) ? 0x100u : 0u); +} + bool CapturesGeometryInput(const RecompileRequest& request) { return request.shader.stage == ShaderStage::TessellationEvaluation && request.graphics && request.graphics->mesh; } @@ -308,6 +318,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat deadCodeEliminator.Eliminate(program); } + if (SingleLaneCandidate(request)) RecordSkippedRegions(program); if (vectorWindowStoreEliminator.Eliminate(program).removedStores != 0u) { deadCodeEliminator.Eliminate(program); } @@ -597,11 +608,9 @@ void prepareInterface(const RecompileRequest& request, IrProgram& program) { resourceMaterializer.ApplyStaticInterface(program, nativeSampleOffsets); constexpr ConstantFolder constantFolder; - if (request.shader.stage == ShaderStage::Vertex && constantFolder.FoldVertexLaneBounds(program, HostSubgroupSize(request), request.target.spirvVersion)) { - constantFolder.Fold(program); - ResolveControlFlowIdentities(program); - } + if (request.shader.stage == ShaderStage::Vertex && constantFolder.FoldVertexLaneBounds(program, HostSubgroupSize(request), request.target.spirvVersion)) constantFolder.Fold(program); + ResolveControlFlowIdentities(program); deadCodeEliminator.RemoveIdentities(program); constexpr RedundantValueEliminator redundantValueEliminator; static_cast(redundantValueEliminator.Eliminate(program)); @@ -619,6 +628,25 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) const bool stencilReplay = request.shader.stage == ShaderStage::Fragment && std::ranges::any_of(program.Info().outputs, [](const StageOutput& output) { return output.kind == StageOutputKind::StencilRef; }) && std::ranges::find(request.target.supportedCapabilities, static_cast(spv::CapabilityStencilExportEXT)) == request.target.supportedCapabilities.end(); auto bindings = bindingAllocator.Allocate(program, request.layout, registration, stencilReplay); + auto emitInfo = inputInfo; + ShaderComputeInputInfo singleLane; + auto hostSubgroupSize = HostSubgroupSize(request); + if (SingleLaneCandidate(request) && inputInfo.compute != nullptr) { + const auto verdict = AnalyzeSingleLane(program, *inputInfo.compute); + static const bool trace = std::getenv("APS5_TRACE_SINGLE_LANE") != nullptr; + if (trace) { + const auto& threads = inputInfo.compute->threadsNum; + const std::string at = verdict.at == nullptr ? std::string() : " at " + DescribeValue(verdict.at, 3); + std::fprintf(stderr, "[single-lane] program 0x%llx (%ux%ux%u threads): %s%s%s\n", static_cast(request.shader.codeAddress), threads[0], threads[1], threads[2], verdict.eligible ? "one lane per invocation" : "two lanes per invocation: ", verdict.eligible ? "" : verdict.reason, at.c_str()); + } + if (verdict.eligible) { + singleLane = *inputInfo.compute; + singleLane.hostSubgroupSize = 64u; + emitInfo.compute = &singleLane; + hostSubgroupSize = 64u; + } + } + SpirvTargetOptions targetOptions {}; targetOptions.vulkanVersion = request.target.vulkanVersion; targetOptions.spirvVersion = request.target.spirvVersion; @@ -683,18 +711,18 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.variantId = nextVariantId(); SpirvEmitInfo emitted; try { - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } catch (const SpannedWaveUnsupported& error) { std::fprintf(stderr, "[spanned-wave] 0x%llx falls back to two lanes per invocation: %s\n", static_cast(request.shader.codeAddress), error.what()); targetOptions.spannedWaves = false; - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } if (registration && result.spirv.size() > request.target.registrationWordLimit) { registration = false; program.Metadata().bindingLayoutComplete = false; bindings = bindingAllocator.Allocate(program, request.layout, false, stencilReplay); targetOptions.registrationModules = false; - result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); + result.spirv = spirvEmitter.Emit(program, emitInfo, bindings, targetOptions, &emitted); } result.registrationModule = registration; if (request.shader.stage == ShaderStage::Mesh && request.graphics) result.meshBarycentrics = request.graphics->meshBarycentrics; @@ -713,7 +741,7 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.shaderDataDwords = bindings.layout.ShaderDataDwords(); result.imageMetadataDword = bindings.layout.ImageMetadataDword(); result.runtimeImageCount = bindings.layout.runtimeImageCount; - result.hostSubgroupSize = HostSubgroupSize(request); + result.hostSubgroupSize = hostSubgroupSize; result.fullSubgroups = emitted.fullSubgroups; result.functionLdsBuffer = emitted.functionLdsBuffer; result.vertexOffsetSgpr = program.Info().vertexOffsetSgpr; @@ -1381,7 +1409,7 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, std::vector diskKey; std::shared_ptr variant; if (disk) { - ShaderDiskCache::BuildKey(request, HostSubgroupSize(request), diskKey); + ShaderDiskCache::BuildKey(request, CacheLayout(request), diskKey); CompiledVariant loaded; if (ShaderDiskCache::Load(diskKey, loaded)) { loaded.layout = request.layout; diff --git a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp index 5a87cd02e408..71a400eaef2d 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp @@ -214,6 +214,11 @@ void TranslationContext::AddBranchCondition(const BasicBlock& source, BlockInfo& } info.condition = &condition.Value(); (void)ir.Emit(IrOpcode::Reference, IrType::Void, {info.condition}); + const auto kind = source.terminator.condition; + if (kind == BranchCondition::VccZero || kind == BranchCondition::VccNonZero || kind == BranchCondition::ExecZero || kind == BranchCondition::ExecNonZero) { + info.exec = &ir.GetExec(); + (void)ir.Emit(IrOpcode::Reference, IrType::Void, {info.exec}); + } } } diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 05678c738d62..6fd5fd3841c4 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -48,6 +48,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PERF_DOM_INTERSECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | computes dominator intersections into a fresh vector instead of intersecting in place | | `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them | | `APS5_NO_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes a fresh result for every recompile instead of reusing the result memo | +| `APS5_NO_SINGLE_LANE_RULE` | presence | `core/shader/recompiler/Recompiler.cpp` | | keeps every wave64 compute program on a 32-lane host at two guest lanes per invocation, even where the single-lane analysis proves one lane gives the same results | | `APS5_NO_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | runs the runtime-source walk of a plan in the IR evaluator instead of the compiled tape | | `APS5_NO_TABLE_RESOLUTION_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | stops a result memo miss from reusing an unchanged image table's resolution, so the table is resolved again | | `APS5_NO_TABLE_SNAPSHOT_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | turns off the image table snapshot reuse, so every capture reads and compares each table column | @@ -62,6 +63,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_TRACE_BDA` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | prints the first few accesses that make a program take the address-based path | | `APS5_TRACE_BUFFER_WRITTEN` | presence | `core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp` | | prints how many guest buffer elements a program may store to and how many it only loads | | `APS5_TRACE_IMAGE_TABLE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | traces image-table resolution | +| `APS5_TRACE_SINGLE_LANE` | presence | `core/shader/recompiler/Recompiler.cpp` | | prints each wave64 compute program's single-lane verdict and, for a program kept at two lanes, the reason and the value it was found at | | `APS5_VERIFY_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | also materializes every patched result memo hit, prints the first twenty whose patched result differs and counts them in the result memo report of `APS5_PROFILE_DRAW` | | `APS5_VERIFY_SRT_COMPACT` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp` | | walks every SRT descriptor a second time through the IR evaluator and throws when the two walks differ | | `APS5_VERIFY_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | walks a plan's runtime sources a second time through the evaluator, adopts its answers where the two differ and prints the first twenty differences |