From 9c00ef9d5edfa9fc86981df749083238911d7b34 Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:22:00 +0200 Subject: [PATCH 1/3] fix(recompiler): give wave64 vertex programs on 32-lane hosts a 32-lane s3 A wave64 vertex program runs at one guest lane per invocation, so on a 32-lane host subgroup its lanes 32-63 have no invocation. The vertex entry still set s3 = 64 | 64 << 8, telling the program that its wave holds 64 vertices and 64 primitives. An NGG culling program then keys a full-wave OR on lane < s3's primitive count, lanes 32-63 hold real cull results, and its v_readlane_b32 of lane 63 cannot be built (#2783). The entry now sets s3 = 32 | 32 << 8 for a wave64 program on a 32-lane host, the wave the host runs; the console launches waves with fewer vertices and primitives. The host subgroup width reaches the vertex input the way it reaches the compute and mesh inputs (BuildShaderStageInputInfo from HostSubgroupSize), and the source, prepared-shader and disk cache keys already carry it for every stage. Programs that do not read s3 are unchanged; wave32 programs and other host widths keep the full wave. The keys are then 0 in lanes 32-63, so the read is sound. On a 32-lane host, when the existing half-wave scan rules fail, the scan may also: - run under s_mov_b64 exec, -1 (EXEC constant true); - use bound_ctrl:1 row_shr steps when 0, the fill, is the identity (OR, add, unsigned max), also into another register, whose old value never shows since such a step writes every lane; - take keys shown to be 0 in lanes 32-63: built from 0 through selects under a lane id or v_mbcnt of -1 below at most 32, v_cmp_ne_u32 against 0, ORs, DPP and permlanes, in stages whose entry EXEC is every lane. Lane 63 is then the identity and lane 31 the subgroup reduction of the host's invocations. A shuffle from lane 31 is not enough: an RTX 5070 runs the vertex warps of a non-indexed triangle list with 30 of their 32 invocations. A v_readlane_b32 outside the host subgroup of another value that is 0 in lanes 32-63 reads 0. Only reads that used to throw change, besides lane 31 of such scans. Co-Authored-By: Claude Opus 5.5 --- .../libSceAgcDriver/tests/ShaderMemory.cpp | 90 ++++++++++ .../Optimization/ShaderStageInputInfo.hpp | 1 + .../src/SpirvAlu/SpirvAluEmitterMath.cpp | 154 +++++++++++++++--- .../Translation/src/InstructionTranslator.cpp | 3 +- .../src/ShaderInputInfoBuilder.cpp | 1 + docs/dev/TechnicalDebt.md | 4 +- 6 files changed, 229 insertions(+), 24 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 8ecb49655590..e761077a564f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1210,6 +1210,95 @@ void verifyHalfWaveReduction() { expectFailure([&] { static_cast(pixel(umin, 8u, true)); }, "v_permlanex16_b32 from lanes 16-31 is outside the 8-lane host subgroup", "half-wave reduction: the scan built on an 8-lane subgroup"); } +void verifyNarrowVertexWave() { + using namespace ShaderRecompiler; + static constexpr std::array capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic}; + static constexpr std::array userData{0u, 0u, 0u, 0x2020u}; + const auto vertex = [](std::span code, std::uint32_t waveSize, std::uint32_t subgroupSize, std::uint32_t userDataBase) { + RecompileRequest request{}; + request.shader = {ShaderStage::Vertex, 0x50000u, code, 0, {}}; + request.context.waveSize = waveSize; + request.context.userDataBaseRegister = userDataBase; + if (userDataBase == 0u) request.context.userData = userData; + request.context.vertex = ShaderVertexStageInfo{}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = subgroupSize; + request.target.supportedCapabilities = capabilities; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + return request; + }; + const auto prepared = [&vertex](std::span code, std::uint32_t subgroupSize, std::uint32_t userDataBase = 8u) { + return GetPreparedArtifact(*PrepareShader(vertex(code, 64u, subgroupSize, userDataBase))).spirv; + }; + const auto prepares = [&prepared](std::span code, std::uint32_t subgroupSize, std::uint32_t userDataBase = 8u) { + return !prepared(code, subgroupSize, userDataBase).empty(); + }; + const auto reductions = [](const std::vector& spirv) { + std::uint32_t count = 0; + for (std::size_t cursor = 5; cursor < spirv.size() && (spirv[cursor] >> 16u) != 0u; cursor += spirv[cursor] >> 16u) { + if ((spirv[cursor] & 0xffffu) == spv::OpGroupNonUniformBitwiseOr && (spirv[cursor] >> 16u) == 6u && spirv[cursor + 4u] == spv::GroupOperationReduce) ++count; + } + return count; + }; + const std::vector culling{ + 0xd7650015u, 0x000100c1u, 0xd7660015u, 0x00022ac1u, 0x7e0e0280u, 0x7da206f9u, 0x81060015u, 0x360e0a81u, + 0xbefe04c1u, 0xd4c50002u, 0x00020e80u, 0xd763000fu, 0x00022a81u, 0xd5010000u, 0x000a1e80u, + 0x380000fau, 0xff091100u, 0x380000fau, 0xff091200u, 0x380000fau, 0xff091400u, 0x380000fau, 0xff091800u, + 0xd778000fu, 0x03058300u, 0x38001f00u, 0xd7600002u, 0x00013f00u, 0xd760006bu, 0x00017f00u, 0x88026b02u, + 0x7e020202u, 0xf80008cfu, 0x01010101u, 0xbf810000u, + }; + const auto patched = [&culling](std::initializer_list> words) { + auto code = culling; + for (const auto& [index, word] : words) code[index] = word; + return code; + }; + const char* lane63 = "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup"; + require(reductions(prepared(culling, 32u)) == 1u, "narrow vertex wave: the NGG culling scan, keyed on lanes below the primitive count in s3, did not read lane 31 as a subgroup OR on 32 lanes"); + const auto wide64 = prepared(culling, 64u); + require(!wide64.empty() && reductions(wide64) == 0u, "narrow vertex wave: the NGG culling scan, keyed on lanes below 64 on 64 lanes, did not keep its DPP scan"); + require(prepares(patched({{6, 0x80060015u}}), 32u), "narrow vertex wave: the scan keyed on lanes below the vertex count in s3 did not prepare on 32 lanes"); + require(prepares(patched({{5, 0x7da82aa0u}, {6, 0xbf800000u}}), 32u), "narrow vertex wave: the scan keyed on lanes below 32 did not prepare on 32 lanes"); + expectFailure([&] { static_cast(prepares(patched({{5, 0x7da82aa1u}, {6, 0xbf800000u}}), 32u)); }, lane63, "narrow vertex wave: a scan keyed on lanes below 33 read lane 63"); + expectFailure([&] { static_cast(prepares(culling, 32u, 0u)); }, lane63, "narrow vertex wave: a scan keyed on a count from user data read lane 63"); + expectFailure([&] { static_cast(prepares(patched({{6, 0x81060005u}}), 32u)); }, lane63, "narrow vertex wave: a scan keyed on the vertex index instead of the lane read lane 63"); + auto otherRegister = patched({{21, 0x380200fau}, {24, 0x03058301u}, {25, 0x38001f01u}}); + otherRegister.insert(otherRegister.begin(), 0x7e020305u); + require(reductions(prepared(otherRegister, 32u)) == 1u, "narrow vertex wave: a scan whose bound_ctrl:1 step is written to another register did not read lane 31 as a subgroup OR on 32 lanes"); + otherRegister[23] = 0xff011800u; + expectFailure([&] { static_cast(prepares(otherRegister, 32u)); }, lane63, "narrow vertex wave: a scan step written to another register without bound_ctrl:1, which keeps that register's lanes outside the row, read lane 63"); + auto threeSteps = culling; + threeSteps.erase(threeSteps.begin() + 21, threeSteps.begin() + 23); + const auto partialScan = prepared(threeSteps, 32u); + require(!partialScan.empty() && reductions(partialScan) == 0u, "narrow vertex wave: lane 63 of row_shr 1/2/4 steps over keys of 0 in lanes 32-63 did not read 0, or lane 31 became a subgroup OR"); + threeSteps[5] = 0x7da82aa1u; + threeSteps[6] = 0xbf800000u; + expectFailure([&] { static_cast(prepares(threeSteps, 32u)); }, lane63, "narrow vertex wave: lane 63 of row_shr 1/2/4 steps over keys of lanes below 33 read 0"); + + std::array, 2> narrow; + std::array, 2> wide; + for (const bool narrowFirst : {false, true}) { + const std::vector code{0xbf800001u + static_cast(narrowFirst), 0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + const auto narrowRequest = vertex(code, 64u, 32u, 8u); + const auto wideRequest = vertex(code, 64u, 64u, 8u); + ShaderPreparationContext preparation; + const auto first = PrepareShader(narrowFirst ? narrowRequest : wideRequest, &preparation); + const auto second = PrepareShader(narrowFirst ? wideRequest : narrowRequest, &preparation); + const auto& narrowHandle = narrowFirst ? *first : *second; + const auto& wideHandle = narrowFirst ? *second : *first; + require(first->source != second->source, "narrow vertex wave: the program prepared for 32 and for 64 lanes shares one translated source"); + require(MatchesPreparedShader(narrowRequest, narrowHandle) && MatchesPreparedShader(wideRequest, wideHandle), "narrow vertex wave: a prepared program did not match its own request"); + require(!MatchesPreparedShader(narrowRequest, wideHandle) && !MatchesPreparedShader(wideRequest, narrowHandle), "narrow vertex wave: a program prepared for one subgroup width matched a request for the other"); + narrow[narrowFirst ? 1u : 0u] = GetPreparedArtifact(narrowHandle).spirv; + wide[narrowFirst ? 1u : 0u] = GetPreparedArtifact(wideHandle).spirv; + } + require(narrow[0] == narrow[1] && wide[0] == wide[1], "narrow vertex wave: preparing for one subgroup width after the other gave another program than preparing it first"); + require(narrow[0] != wide[0], "narrow vertex wave: s3 of a wave64 vertex program was the same on 32 and 64 lanes"); + const std::vector wave32{0x7e020203u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + require(GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 32u, 8u))).spirv == GetPreparedArtifact(*PrepareShader(vertex(wave32, 32u, 64u, 8u))).spirv, "narrow vertex wave: s3 of a wave32 vertex program depended on the subgroup width"); +} + void verifyMeshConfiguration() { using namespace ShaderRecompiler; ShaderMeshInputInfo list; @@ -2583,6 +2672,7 @@ int main(int argc, char** argv) { verifyProgramCounterRelativeData(); verifyLanesOutsideHostSubgroup(); verifyHalfWaveReduction(); + verifyNarrowVertexWave(); verifyMeshConfiguration(); verifyPixelInputs(); verifyPixelRequestSerialization(); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index 8f72d9821bb8..b4c41cfd5085 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -129,6 +129,7 @@ struct ShaderVertexInputInfo { int fetchBufferReg = 0; int buffersNum = 0; std::uint32_t scratchSizeDwords = 0; + std::uint32_t hostSubgroupSize = 64; std::uint32_t paClVsOutCntl = 0; ShaderClipSpaceTransform clipSpace; ShaderMeshInputInfo mesh; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index 6a78209dc1eb..c471874aa31b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -301,9 +301,13 @@ bool AllLanesBit(const IrValue* bit) { return words && AllOnesWord((*words)[0]) && AllOnesWord((*words)[1]); } +bool EveryInvocationEnters(IrShaderStage stage) { + return stage == IrShaderStage::Vertex || stage == IrShaderStage::Local || stage == IrShaderStage::TessellationControl || stage == IrShaderStage::TessellationEvaluation; +} + class EntryLaneWalk { public: - explicit EntryLaneWalk(IrShaderStage stage) : everyInvocationEnters(stage == IrShaderStage::Vertex || stage == IrShaderStage::Local || stage == IrShaderStage::TessellationControl || stage == IrShaderStage::TessellationEvaluation) {} + explicit EntryLaneWalk(IrShaderStage stage) : everyInvocationEnters(EveryInvocationEnters(stage)) {} bool ZeroBit(const IrValue* bit) { bit = Resolved(bit); @@ -376,9 +380,107 @@ class EntryLaneWalk { std::uint32_t budget = 512u; }; -bool RowShiftFlags(const IrValue* value, std::uint32_t control) { +bool EveryLaneBit(const IrValue* bit, bool constantTrue) { + const auto* resolved = Resolved(bit); + if (constantTrue && resolved != nullptr && resolved->HasImmediate()) return resolved->Type() == IrType::Bool && resolved->ImmediateBool(); + return AllLanesBit(resolved); +} + +bool WritesEveryLane(const IrValue& update) { + const auto flags = update.Flags(); + return flags.boundControl && flags.rowMask == 0xfu && flags.bankMask == 0xfu && EveryLaneBit(update.Argument(2), true); +} + +bool KeepsNoOldValue(const IrValue& permlane) { + const auto flags = permlane.Flags(); + return flags.fetchInactive || flags.boundControl || EveryLaneBit(permlane.Argument(3), true); +} + +bool UpperLaneTest(const IrValue* bit) { + const auto* compare = Resolved(bit); + return Is(compare, IrOpcode::UGreaterThanEqual32) && Is(Resolved(compare->Argument(0)), IrOpcode::LaneId) && IsU32(compare->Argument(1), 32u); +} + +class UpperLaneWalk { +public: + explicit UpperLaneWalk(IrShaderStage stage) : enters(EveryInvocationEnters(stage)), laneIndexed(enters && stage != IrShaderStage::TessellationControl) {} + + bool ZeroValue(const IrValue* value) { + value = Resolved(value); + if (!enters || value == nullptr || !spend()) return false; + if (value->HasImmediate()) return IsU32(value, 0u); + if (zeroValues.contains(value)) return true; + bool zero = false; + switch (value->Opcode()) { + case IrOpcode::SelectU32: zero = ZeroValue(value->Argument(2)) && (zeroBit(value->Argument(0)) || ZeroValue(value->Argument(1))); break; + case IrOpcode::BitwiseOr32: zero = ZeroValue(value->Argument(0)) && ZeroValue(value->Argument(1)); break; + case IrOpcode::DppMoveU32: zero = ZeroValue(value->Argument(0)); break; + case IrOpcode::DppUpdateU32: zero = ZeroValue(value->Argument(0)) && (WritesEveryLane(*value) || ZeroValue(value->Argument(1))); break; + case IrOpcode::Permlane16U32: zero = ZeroValue(value->Argument(0)) && (KeepsNoOldValue(*value) || ZeroValue(value->Argument(4))); break; + case IrOpcode::Phi: zero = incoming(value, [this](const IrValue* source) { return ZeroValue(source); }); break; + default: break; + } + if (zero && phis.empty()) zeroValues.insert(value); + return zero; + } + +private: + bool spend() { + if (budget == 0u) return false; + --budget; + return true; + } + + bool zeroBit(const IrValue* bit) { + bit = Resolved(bit); + if (bit == nullptr || !spend()) return false; + if (bit->HasImmediate()) return bit->Type() == IrType::Bool && !bit->ImmediateBool(); + switch (bit->Opcode()) { + case IrOpcode::LogicalAnd: return zeroBit(bit->Argument(0)) || zeroBit(bit->Argument(1)); + case IrOpcode::SelectU1: return zeroBit(bit->Argument(2)) && (zeroBit(bit->Argument(0)) || zeroBit(bit->Argument(1))); + case IrOpcode::Phi: return incoming(bit, [this](const IrValue* source) { return zeroBit(source); }); + case IrOpcode::INotEqual32: return EitherOrder(bit, [this](const IrValue* value, const IrValue* zero) { return IsU32(zero, 0u) && ZeroValue(value); }); + case IrOpcode::ULessThan32: return belowUpperLanes(bit->Argument(0), bit->Argument(1)); + case IrOpcode::UGreaterThan32: return belowUpperLanes(bit->Argument(1), bit->Argument(0)); + default: return false; + } + } + + bool belowUpperLanes(const IrValue* index, const IrValue* bound) { + const auto* limit = Resolved(bound); + return limit != nullptr && limit->HasImmediate() && limit->Type() == IrType::U32 && limit->ImmediateU32() <= 32u && upperLaneIndex(index); + } + + bool upperLaneIndex(const IrValue* index) { + index = Resolved(index); + if (!laneIndexed || index == nullptr || !spend()) return false; + if (Is(index, IrOpcode::LaneId)) return true; + if (Is(index, IrOpcode::BitCount32)) { + const auto* mask = Resolved(index->Argument(0)); + return Is(mask, IrOpcode::SelectU32) && IsU32(mask->Argument(1), 0xffffffffu) && UpperLaneTest(mask->Argument(0)); + } + return Is(index, IrOpcode::IAdd32) && EitherOrder(index, [this](const IrValue* counted, const IrValue* added) { return Is(added, IrOpcode::BitCount32) && upperLaneIndex(counted); }); + } + + template + bool incoming(const IrValue* phi, TCheck&& check) { + if (!phis.insert(phi).second) return true; + bool result = phi->ArgumentCount() != 0u; + for (std::size_t index = 0; result && index < phi->ArgumentCount(); ++index) result = check(phi->Argument(index)); + phis.erase(phi); + return result; + } + + bool enters; + bool laneIndexed; + std::unordered_set zeroValues; + std::unordered_set phis; + std::uint32_t budget = 1024u; +}; + +bool RowShiftFlags(const IrValue* value, std::uint32_t control, bool zeroFill) { const auto flags = value->Flags(); - return flags.control == control && flags.rowMask == 0xfu && flags.bankMask == 0xfu && !flags.boundControl && !flags.fetchInactive; + return flags.control == control && flags.rowMask == 0xfu && flags.bankMask == 0xfu && (!flags.boundControl || zeroFill) && !flags.fetchInactive; } const IrValue* Unflushed(const IrValue* value) { @@ -408,47 +510,54 @@ std::optional> ReductionOperands(const IrValue* co return operands; } -const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control) { - if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control) || !AllLanesBit(step->Argument(2))) return nullptr; - const auto* previous = Resolved(step->Argument(1)); +const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control, bool narrow) { + const bool zeroFill = narrow && reduction.identity == 0u && !reduction.floating; + if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control, zeroFill) || !EveryLaneBit(step->Argument(2), narrow)) return nullptr; + const auto* previous = zeroFill && WritesEveryLane(*step) ? nullptr : Resolved(step->Argument(1)); + const IrValue* input = nullptr; const auto operands = ReductionOperands(Resolved(step->Argument(0)), reduction); const bool matched = operands && EitherOrder(*operands, [&](const IrValue* moved, const IrValue* own) { - return own == previous && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control) && Resolved(moved->Argument(0)) == previous && AllLanesBit(moved->Argument(1)); + input = own; + return (previous == nullptr || own == previous) && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control, zeroFill) && Resolved(moved->Argument(0)) == own && EveryLaneBit(moved->Argument(1), narrow); }); - return matched ? previous : nullptr; + return matched ? input : nullptr; } -const IrValue* Unmasked(const IrValue* value) { - while (Is(value, IrOpcode::SelectU32) && AllLanesBit(value->Argument(0))) value = Resolved(value->Argument(1)); +const IrValue* Unmasked(const IrValue* value, bool narrow) { + while (Is(value, IrOpcode::SelectU32) && EveryLaneBit(value->Argument(0), narrow)) value = Resolved(value->Argument(1)); return value; } -bool CrossesRowsOf(const IrValue* value, const IrValue* scan) { - const auto* permlane = Unmasked(value); +bool CrossesRowsOf(const IrValue* value, const IrValue* scan, bool narrow) { + const auto* permlane = Unmasked(value, narrow); if (!Is(permlane, IrOpcode::Permlane16U32)) return false; const auto flags = permlane->Flags(); - return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && AllLanesBit(permlane->Argument(3)); + return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && EveryLaneBit(permlane->Argument(3), narrow); } -std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStage stage) { - const auto* combined = Unmasked(Resolved(value)); +std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStage stage, bool narrow) { + const auto* combined = Unmasked(Resolved(value), narrow); std::optional> operands; const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return (operands = ReductionOperands(combined, candidate)).has_value(); }); const IrValue* scan = nullptr; - if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; + if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own, narrow); })) return std::nullopt; for (const auto control : {0x118u, 0x114u, 0x112u, 0x111u}) { - scan = RowScanStepInput(scan, *reduction, control); + scan = RowScanStepInput(scan, *reduction, control, narrow); if (scan == nullptr) return std::nullopt; } - const auto* source = Unmasked(scan); - if (!Is(source, IrOpcode::SelectU32) || !IsU32(source->Argument(2), reduction->identity) || !EntryLaneWalk{stage}.ZeroBit(source->Argument(0))) return std::nullopt; + const auto* source = Unmasked(scan, narrow); + const bool entryKeys = Is(source, IrOpcode::SelectU32) && IsU32(source->Argument(2), reduction->identity) && EntryLaneWalk{stage}.ZeroBit(source->Argument(0)); + const bool upperKeys = narrow && reduction->identity == 0u && !reduction->floating && UpperLaneWalk{stage}.ZeroValue(source); + if (!entryKeys && !upperKeys) return std::nullopt; return HalfWaveScan{&*reduction, source}; } std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t lane) { auto& state = ctx.state; if (state.program.WaveSize() != 64u || (lane & 31u) != 31u) return std::nullopt; - const auto scan = MatchHalfWaveScan(inst.Argument(0), state.program.Resources().stage); + const auto stage = state.program.Resources().stage; + auto scan = MatchHalfWaveScan(inst.Argument(0), stage, false); + if (!scan && HostSubgroupNarrowerThanWave(state) && state.hostSubgroupSize == 32u) scan = MatchHalfWaveScan(inst.Argument(0), stage, true); if (!scan) return std::nullopt; if (scan->reduction->floating) { if (lane < 32u || state.hostSubgroupSize > 32u) return std::nullopt; @@ -836,7 +945,10 @@ std::uint32_t EmitReadLane(SpirvValueEmitContext& ctx, const IrValue& inst) { if (selector != nullptr && selector->HasImmediate() && state.program.Resources().stage != IrShaderStage::Compute && state.laneCount == 1u) { const auto index = selector->ImmediateU32() & (state.program.WaveSize() - 1u); if (const auto reduced = EmitHalfWaveReduction(ctx, inst, index)) return *reduced; - if (HostSubgroupNarrowerThanWave(state) && index >= state.hostSubgroupSize) FailOutsideHostSubgroup(ctx, inst, "v_readlane_b32 of lane " + std::to_string(index)); + if (HostSubgroupNarrowerThanWave(state) && index >= state.hostSubgroupSize) { + if (index >= 32u && UpperLaneWalk{state.program.Resources().stage}.ZeroValue(inst.Argument(0))) return ConstantU32(state, 0u); + FailOutsideHostSubgroup(ctx, inst, "v_readlane_b32 of lane " + std::to_string(index)); + } } const auto lane = Binary(state, spv::OpBitwiseAnd, TypeU32(state), ctx.Arg(inst, 1), ConstantU32(state, state.program.WaveSize() - 1u)); return ctx.Shuffle(inst, 0, lane); diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index b764b46af612..51156b729c3d 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -351,7 +351,8 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO } } else if (options.stage == ShaderStageKind::Vertex) { if (options.userDataBaseRegister >= 8u) { - entryIr.SetScalarReg(static_cast(3), entryIr.Constant(options.waveSize | (options.waveSize << 8u))); + const std::uint32_t lanes = options.waveSize == 64u && options.inputInfo.vertex->hostSubgroupSize == 32u ? 32u : options.waveSize; + entryIr.SetScalarReg(static_cast(3), entryIr.Constant(lanes | (lanes << 8u))); } entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::VertexIndex)); entryIr.SetVectorReg(static_cast(8), drawIndex(StageInputKind::InstanceIndex)); diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index c14e244be7cb..de64e9f67d75 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -159,6 +159,7 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues auto& vertexStorage = HostThreadLocal(); vertexStorage = ShaderVertexInputInfo{}; vertexStorage.logicalStage = _toIrShaderStage(stage); + vertexStorage.hostSubgroupSize = hostSubgroupSize; vertexStorage.fetchEmbedded = vertex.fetchEmbedded; vertexStorage.fetchExternal = false; vertexStorage.fetchAttribReg = static_cast(vertex.fetchAttribReg); diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index f47df2886e6c..94616ff7149c 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -119,7 +119,7 @@ Throughout the project, every function at every stage either **does exactly what - [User data registers](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp) (AGC driver) - a `SPI_SHADER_USER_DATA_*` or `COMPUTE_USER_DATA_*` register the title never wrote reads as 0 when a stage's RSRC2 count covers it, as Kyty (zeroed `UserSgprInfo`) and SharpEmu (missing registers read 0) do, and as AMD PAL and Mesa initialize them; the console's value is not confirmed - [SET_PREDICATION](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (AGC driver) - boolean predication follows the AMD PM4 encoding [Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/sid.h) uses: operation 3 reads 64 bits and 4 reads 32 bits, and bit 8 runs predicated packets when the value is non-zero (clear: when it is zero); the bit 12 hint is ignored. The address must have its low 4 bits clear, as `sceAgcDcbSetPredication` writes it. Query predication (Z-pass, primitive count), a predicated COND_EXEC, predicated command buffer chains and flips in predicated command buffers throw - [sceAgcGetGsOversubscription](../../core/libs/prx/libSceAgc/Shader/src/GsOversubscription.cpp) (libSceAgc) - signature and occupancy formula (base and expanded vertex/export capacities, budget shift, `GE_PC_ALLOC` / `SPI_SHADER_PGM_RSRC4_GS` encoding) follow KytyPS5 only, not confirmed on hardware or a title; a negative or NaN factor that gives a negative target, a missing GS context register or a zero `GE_MAX_OUTPUT_PER_SUBGROUP` throws -- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0). Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave. +- [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0), or to `32 | 32 << 8` for a wave64 program on a 32-lane host subgroup, which runs it at one lane per invocation and holds no lanes 32-63. Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave, and so does a host wave with idle invocations (an RTX 5070 ran every vertex warp of a non-indexed triangle list with 30 of its 32 invocations). - [s_barrier in vertex-path programs](../../core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp) (recompiler) - emits no instruction when the program has no LDS or GDS access and no memory write or atomic, so no wave of the subgroup can observe another; the `GS_ALLOC_REQ` such a barrier orders on NGG passthrough programs is already dropped on the vertex path. The Vertex execution model has no workgroup barrier. Programs with LDS, GDS or memory writes throw. Not measured on hardware - [Zero-stride vertex fetch range](../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - for a stride-0 V# with OOB_SELECT 2 the whole fetch reads memory unless NUM_RECORDS is 0; with OOB_SELECT 3 it reads zero when NUM_RECORDS is 0 and throws when the fetch only partly fits. Rules taken from the GFX10 OOB_SELECT table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c), not measured on hardware. Whether OOB_SELECT 3 checks each dword of a multi-dword fetch separately is unknown. OOB_SELECT 0 and 1 keep the structured record checks. - [sceAgcCbCondWrite](../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) (libSceAgc) - argument order and the 9-dword COND_WRITE layout follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs), which agree on them; SharpProspero lists another order. The write space is accepted only as 1 (memory), the one value both encode the same way, and other values throw. @@ -370,7 +370,7 @@ Throughout the project, every function at every stage either **does exactly what - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR. - Without fetch-inactive, [`v_permlane16_b32` and `v_permlanex16_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take a source lane that the host subgroup does not hold as inactive and keep the old value (0 with bound_ctrl); the console reads that lane's VGPR. -- On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1); other such reads throw. +- On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1) or, in a program whose entry EXEC is every lane (vertex, local and tessellation stages), as 0 when the value is 0 in lanes 32-63; other such reads throw. A value is 0 there when it is built from 0 through selects under a lane index (lane id or `v_mbcnt` of -1) below at most 32, `v_cmp_ne_u32` against 0, OR, DPP and permlanes; the counts in `s3` give such a bound on a 32-lane host. On a 32-lane host a half-wave scan may also run under `s_mov_b64 exec, -1`, use bound_ctrl:1 steps for an OR, add or unsigned max, and take keys that are 0 in lanes 32-63; its lane 31 is then the subgroup reduction of the host's invocations: idle invocations add the identity, where the console would add the keys of the lanes `s3` claims. - [NGG mesh entry](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) uses the guest wave size (32 or 64) for wave indices, per-wave vertex and primitive counts, and the workgroup wave count in `s3`. The four-bit wave fields follow Mesa's [get_wave_id_in_tg, get_tgsize and get_thread_id_in_tg](https://chromium.googlesource.com/chromiumos/third_party/mesa/+/refs/heads/stabilize-13982.70.B-chromeos-amd/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c); partial waves and groups exceeding 15 waves throw. Indexed and non-indexed wave32 mesh rendering with 32, 64 and 128 threads was checked by pixel readback on Windows with an NVIDIA RTX 4070 Laptop GPU. The guest register values have not been compared with console hardware. - [Shader recompilation](../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../core/shader/recompiler) was written completely independently from [libs/prx](../../core/libs/prx). - [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64. From 559910e307066da006e00a370d0e7b19e59f9a00 Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:22:01 +0200 Subject: [PATCH 2/3] test(agc): run a wave64 NGG full-wave OR on a 32-lane device A wave64 vertex program with the culling scan of #2783 (lane < s3's primitive count, row_shr bound_ctrl:1 OR scan, v_permlanex16_b32, v_readlane_b32 of lanes 31 and 63) compares the wave OR with the ballot of its keys and collapses its triangle when they differ. On a 32-lane device every pixel of the grid must be covered. Before the s3 change the program does not build on 32 lanes; with a shuffle from lane 31 instead of the subgroup reduction no pixel is covered, as the device leaves lanes of each vertex warp idle. Other subgroup widths skip. Co-Authored-By: Claude Opus 5.5 --- .../execution/NggVertexWaveReduction.cpp | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveReduction.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveReduction.cpp new file mode 100644 index 000000000000..e5822da0d9b1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveReduction.cpp @@ -0,0 +1,149 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::uint32_t Columns = 7; +constexpr std::uint32_t Rows = 5; +alignas(256) std::array Pixels{}; + +alignas(256) constexpr std::array VertexCode{ + 0xd7650015, 0x000100c1, 0xd7660015, 0x00022ac1, 0x7e0e0280, 0x7da206f9, 0x81060015, 0x360e0a81, + 0xbefe04c1, 0xd4c50002, 0x00020e80, 0xd763000f, 0x00022a81, 0xd5010000, 0x000a1e80, 0x380000fa, + 0xff091100, 0x380000fa, 0xff091200, 0x380000fa, 0xff091400, 0x380000fa, 0xff091800, 0xd778000f, + 0x03058300, 0x38001f00, 0x88040302, 0xd7600002, 0x00013f00, 0xd760006b, 0x00017f00, 0x88026b02, + 0xbf060402, 0x850580f2, 0xe0382000, 0x80020005, 0xbf8c3f70, 0x10000005, 0x10020205, 0x10040405, + 0x10060605, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::vector> Triangles() { + std::vector> vertices; + for (std::uint32_t cell = 0; cell < Columns * Rows; ++cell) { + const float left = -1.0f + 2.0f * static_cast(cell % Columns) / Columns; + const float right = -1.0f + 2.0f * static_cast(cell % Columns + 1u) / Columns; + const float bottom = -1.0f + 2.0f * static_cast(cell / Columns) / Rows; + const float top = -1.0f + 2.0f * static_cast(cell / Columns + 1u) / Rows; + for (const auto& corner : {std::array{left, bottom}, std::array{right, bottom}, std::array{right, top}, std::array{left, bottom}, std::array{right, top}, std::array{left, top}}) { + vertices.push_back({corner[0], corner[1], 0.5f, 1.0f}); + } + } + return vertices; +} + +void Draw(AgcDriver::VulkanDevice& device) { + static const auto triangles = Triangles(); + Pixels.fill(std::byte{0}); + const auto target = device.Target(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(triangles.data(), 16u, static_cast(triangles.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64u, 8, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(triangles.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + const auto actual = std::to_integer(Pixels[pixel * 4u]); + Require(actual == 255u, "NGG vertex wave reduction: pixel " + std::to_string(pixel) + " is " + std::to_string(actual) + ", expected 255 (a vertex whose wave OR of its lanes' bits, read from lanes 31 and 63, differs from its wave's ballot collapses its triangle)"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto subgroupSize = device->Target().subgroupSize; + if (subgroupSize != 32u) { + std::printf("skipped, s3 describes a 32-lane wave only on 32-lane subgroups, and the device has %u lanes\n", subgroupSize); + return VulkanTestSkipped; + } + if ((device->SubgroupStages() & VK_SHADER_STAGE_VERTEX_BIT) == 0u || !TargetHasCapability(device->Target(), spv::CapabilityGroupNonUniformArithmetic)) { + std::puts("skipped, the device has no subgroup arithmetic in vertex shaders"); + return VulkanTestSkipped; + } + for (std::uint32_t pass = 0; pass < 3; ++pass) { + Draw(*device); + Check(); + } + std::puts("NGG vertex wave reduction tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} From c9fd4cc4e8aeb8f5e44e7a8ed33c85d2087f92aa Mon Sep 17 00:00:00 2001 From: AlxcBTW <187763517+AlxcBTW@users.noreply.github.com> Date: Sun, 11 Oct 2026 00:26:45 +0200 Subject: [PATCH 3/3] fix(recompiler): keep lane 31 a shuffle without subgroup arithmetic The half-wave scan rules for 32-lane hosts made lane 31 of a matched scan a subgroup reduction and threw on a device without subgroup arithmetic, where the read used to build as a shuffle of the scan. Such a device now keeps the shuffle for lane 31; lane 63 still reads the identity, which needs no arithmetic. Co-Authored-By: Claude Opus 5.5 --- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 6 ++++++ .../SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp | 8 ++++++-- docs/dev/TechnicalDebt.md | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index e761077a564f..d66100f0f2c3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1259,6 +1259,12 @@ void verifyNarrowVertexWave() { const auto wide64 = prepared(culling, 64u); require(!wide64.empty() && reductions(wide64) == 0u, "narrow vertex wave: the NGG culling scan, keyed on lanes below 64 on 64 lanes, did not keep its DPP scan"); require(prepares(patched({{6, 0x80060015u}}), 32u), "narrow vertex wave: the scan keyed on lanes below the vertex count in s3 did not prepare on 32 lanes"); + static constexpr std::array noArithmetic{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle}; + const auto otherKeys = patched({{7, 0x360e0a82u}}); + auto withoutArithmetic = vertex(otherKeys, 64u, 32u, 8u); + withoutArithmetic.target.supportedCapabilities = noArithmetic; + const auto shuffled = GetPreparedArtifact(*PrepareShader(withoutArithmetic)).spirv; + require(!shuffled.empty() && reductions(shuffled) == 0u, "narrow vertex wave: without subgroup arithmetic, the scan did not keep lane 31 as a shuffle with lane 63 read as 0"); require(prepares(patched({{5, 0x7da82aa0u}, {6, 0xbf800000u}}), 32u), "narrow vertex wave: the scan keyed on lanes below 32 did not prepare on 32 lanes"); expectFailure([&] { static_cast(prepares(patched({{5, 0x7da82aa1u}, {6, 0xbf800000u}}), 32u)); }, lane63, "narrow vertex wave: a scan keyed on lanes below 33 read lane 63"); expectFailure([&] { static_cast(prepares(culling, 32u, 0u)); }, lane63, "narrow vertex wave: a scan keyed on a count from user data read lane 63"); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index c471874aa31b..c3b0d5b33617 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -557,7 +557,8 @@ std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, c if (state.program.WaveSize() != 64u || (lane & 31u) != 31u) return std::nullopt; const auto stage = state.program.Resources().stage; auto scan = MatchHalfWaveScan(inst.Argument(0), stage, false); - if (!scan && HostSubgroupNarrowerThanWave(state) && state.hostSubgroupSize == 32u) scan = MatchHalfWaveScan(inst.Argument(0), stage, true); + const bool narrow = !scan && HostSubgroupNarrowerThanWave(state) && state.hostSubgroupSize == 32u; + if (narrow) scan = MatchHalfWaveScan(inst.Argument(0), stage, true); if (!scan) return std::nullopt; if (scan->reduction->floating) { if (lane < 32u || state.hostSubgroupSize > 32u) return std::nullopt; @@ -565,7 +566,10 @@ std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, c } const auto read = "v_readlane_b32 of lane " + std::to_string(lane) + " of a wave64 half-wave reduction scan"; const auto& capabilities = state.supportedCapabilities; - if (std::find(capabilities.begin(), capabilities.end(), static_cast(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) ctx.Fail(inst, (read + " needs subgroup arithmetic, which the device lacks").c_str()); + if (std::find(capabilities.begin(), capabilities.end(), static_cast(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) { + if (narrow) return lane >= 32u ? std::optional(ConstantU32(state, scan->reduction->identity)) : std::nullopt; + ctx.Fail(inst, (read + " needs subgroup arithmetic, which the device lacks").c_str()); + } if (state.hostSubgroupSize > 64u) ctx.Fail(inst, (read + " on a " + std::to_string(state.hostSubgroupSize) + "-lane host subgroup, which is wider than the wave").c_str()); const auto identity = ConstantU32(state, scan->reduction->identity); if (lane >= 32u && state.hostSubgroupSize <= 32u) return identity; diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 94616ff7149c..65d14f9c50c0 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -370,7 +370,7 @@ Throughout the project, every function at every stage either **does exactly what - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR. - Without fetch-inactive, [`v_permlane16_b32` and `v_permlanex16_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take a source lane that the host subgroup does not hold as inactive and keep the old value (0 with bound_ctrl); the console reads that lane's VGPR. -- On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1) or, in a program whose entry EXEC is every lane (vertex, local and tessellation stages), as 0 when the value is 0 in lanes 32-63; other such reads throw. A value is 0 there when it is built from 0 through selects under a lane index (lane id or `v_mbcnt` of -1) below at most 32, `v_cmp_ne_u32` against 0, OR, DPP and permlanes; the counts in `s3` give such a bound on a 32-lane host. On a 32-lane host a half-wave scan may also run under `s_mov_b64 exec, -1`, use bound_ctrl:1 steps for an OR, add or unsigned max, and take keys that are 0 in lanes 32-63; its lane 31 is then the subgroup reduction of the host's invocations: idle invocations add the identity, where the console would add the keys of the lanes `s3` claims. +- On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1) or, in a program whose entry EXEC is every lane (vertex, local and tessellation stages), as 0 when the value is 0 in lanes 32-63; other such reads throw. A value is 0 there when it is built from 0 through selects under a lane index (lane id or `v_mbcnt` of -1) below at most 32, `v_cmp_ne_u32` against 0, OR, DPP and permlanes; the counts in `s3` give such a bound on a 32-lane host. On a 32-lane host a half-wave scan may also run under `s_mov_b64 exec, -1`, use bound_ctrl:1 steps for an OR, add or unsigned max, and take keys that are 0 in lanes 32-63; its lane 31 is then the subgroup reduction of the host's invocations (a shuffle of the scan on a device without subgroup arithmetic): idle invocations add the identity, where the console would add the keys of the lanes `s3` claims. - [NGG mesh entry](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) uses the guest wave size (32 or 64) for wave indices, per-wave vertex and primitive counts, and the workgroup wave count in `s3`. The four-bit wave fields follow Mesa's [get_wave_id_in_tg, get_tgsize and get_thread_id_in_tg](https://chromium.googlesource.com/chromiumos/third_party/mesa/+/refs/heads/stabilize-13982.70.B-chromeos-amd/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c); partial waves and groups exceeding 15 waves throw. Indexed and non-indexed wave32 mesh rendering with 32, 64 and 128 threads was checked by pixel readback on Windows with an NVIDIA RTX 4070 Laptop GPU. The guest register values have not been compared with console hardware. - [Shader recompilation](../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../core/shader/recompiler) was written completely independently from [libs/prx](../../core/libs/prx). - [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64.