From 731575c8026ac44a794d8930dadde85ff00757eb Mon Sep 17 00:00:00 2001 From: billythemenace2 Date: Wed, 14 Oct 2026 18:28:26 +0100 Subject: [PATCH 1/2] fix(shader): flush ds_add_f32 denormals and NaNs as the hardware does ds_add_f32 and ds_add_rtn_f32 now follow the measured hardware: both operands are flushed to a signed zero when bit 0 of the f32 denormal field is clear and the result when bit 1 is clear, the sum is rounded to nearest even, a NaN data operand is stored quieted, else a NaN old value is stored quieted, and +inf + -inf stores 0xffc00000. Measured on gfx1010 and, in #5492, on gfx1031. The new MemoryInfo field is stored in the shader disk cache (field count 31, minimum record 79 bytes, FormatVersion 34). The parts that depend only on the data operand are computed once, before the CAS loop. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../tests/execution/LdsAddF32.cpp | 184 ++++++++++++++++++ .../IrMetadata/ResourceInfo.hpp | 1 + core/shader/recompiler/ShaderDiskCache.cpp | 6 +- .../SpirvMemory/SpirvMemoryInstructions.cpp | 16 +- .../SharedMemoryInstructions.cpp | 2 + docs/dev/debt/shader.md | 2 +- 6 files changed, 206 insertions(+), 5 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/LdsAddF32.cpp diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddF32.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddF32.cpp new file mode 100644 index 000000000000..abd04c9da155 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAddF32.cpp @@ -0,0 +1,184 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Results = 4; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020083, 0xe0341000, 0x80000201, 0x34080082, 0xbf8c3f70, 0xd8340000, 0x00000204, 0xd8340400, + 0x00000204, 0xbf8cc07f, 0xd9540000, 0x05000304, 0xd8540400, 0x00000304, 0xbf8cc07f, 0xd8d80000, + 0x06000004, 0xd8d80400, 0x07000004, 0x34020084, 0xbf8cc07f, 0xe07c1000, 0x80010501, 0xbf810000, +}; + +constexpr std::array, Threads> Operands{{ + {0x7f800000, 0xff800000}, {0xff800000, 0x7f800000}, {0x3f800000, 0xbf800000}, {0x80000000, 0x80000000}, + {0x00000000, 0x80000000}, {0x7f7fffff, 0x7f7fffff}, {0x7fc00000, 0xffc00001}, {0x7f800001, 0x3f800000}, + {0x3f800000, 0xff812345}, {0xffc00001, 0x7f800001}, {0x00000001, 0x80000001}, {0x00800000, 0x80000001}, + {0x8110c65e, 0x8074b225}, {0x00d42774, 0x80d0391a}, {0x80800000, 0x00800001}, {0x80431a89, 0x01bcb83b}, + {0x01bec9b9, 0x81bab7f2}, {0x007fffff, 0x00400000}, {0x80000000, 0x80000001}, {0x80f9f2ca, 0x00cd2d23}, + {0x005921e2, 0x8187bc63}, {0x014e8a46, 0x0060b9ff}, {0x81cf89be, 0x805706c5}, {0x00e62d65, 0x80e62d68}, + {0x00a66021, 0x800b6944}, {0x802976a1, 0x0156a967}, {0x0197eca0, 0x803c270f}, {0x8027bc33, 0x00f88e02}, + {0x0104a791, 0x80c6fa02}, {0x001f9fb3, 0x01e94ee7}, {0x80000000, 0x007fffff}, {0x007fffff, 0x007fffff}, + {0x010d380e, 0x0076291d}, {0x80000001, 0x80000001}, {0x81dd408a, 0x8023bed7}, {0x00000001, 0x00400000}, + {0x8033deaa, 0x001a010b}, {0x003da089, 0x01a8d6f4}, {0x002eafb3, 0x01f74c53}, {0x01fd86b4, 0x00195d7b}, + {0x80476063, 0x00b49438}, {0x00800001, 0x80000001}, {0xe34217a6, 0x5b100f9a}, {0xff7fffff, 0x00800000}, + {0x00400000, 0xcb000001}, {0x4b000000, 0x807fffff}, {0xa63b9449, 0x2d91f2c3}, {0x819c564f, 0x802c240f}, + {0x81b5b8a2, 0x81a1681d}, {0xe640b291, 0x5766d41e}, {0x334bd7d6, 0x3d01a33b}, {0x01eeda5d, 0x80e7c8f6}, + {0x7f800001, 0x80000001}, {0xff812345, 0x80000000}, {0xffc00001, 0x7f7fffff}, {0xff812345, 0x7f7fffff}, + {0xffc00001, 0x00800001}, {0xff812345, 0x80800000}, {0x6ef2f0aa, 0xeef2feaa}, {0x019a87f3, 0x80efc545}, + {0x71223030, 0xf122303c}, {0xff7fffff, 0xff7fffff}, {0x80ec8133, 0x005f7c8c}, {0xe914c8da, 0x6914c8db}, +}}; + +constexpr std::array, 5> Measured{{ + { + 0xffc00000, 0xffc00000, 0x00000000, 0x80000000, 0x00000000, 0x7f800000, 0xffc00001, 0x7fc00001, + 0xffc12345, 0x7fc00001, 0x00000000, 0x00800000, 0x8110c65e, 0x00000000, 0x00000000, 0x01bcb83b, + 0x00000000, 0x00000000, 0x80000000, 0x80000000, 0x8187bc63, 0x014e8a46, 0x81cf89be, 0x80000000, + 0x00a66021, 0x0156a967, 0x0197eca0, 0x00f88e02, 0x00000000, 0x01e94ee7, 0x00000000, 0x00000000, + 0x010d380e, 0x80000000, 0x81dd408a, 0x00000000, 0x00000000, 0x01a8d6f4, 0x01f74c53, 0x01fd86b4, + 0x00b49438, 0x00800001, 0xe3421716, 0xff7fffff, 0xcb000001, 0x4b000000, 0x2d91f14c, 0x819c564f, + 0x822b9060, 0xe640b291, 0x3d01a348, 0x01b4e820, 0x7fc00001, 0xffc12345, 0xffc00001, 0xffc12345, + 0xffc00001, 0xffc12345, 0xe8e00000, 0x013d2d44, 0xe7400000, 0xff800000, 0x80ec8133, 0x5d800000, + }, + { + 0xffc00000, 0xffc00000, 0x00000000, 0x80000000, 0x00000000, 0x7f800000, 0xffc00001, 0x7fc00001, + 0xffc12345, 0x7fc00001, 0x00000000, 0x00000000, 0x814b1f70, 0x00000000, 0x00000000, 0x01abf199, + 0x00000000, 0x00bfffff, 0x80000000, 0x80000000, 0x8162e7d5, 0x017ee746, 0x81e54b6f, 0x80000000, + 0x009af6dd, 0x0141ee16, 0x0188e2dc, 0x00d0d1cf, 0x00000000, 0x01f136d4, 0x00000000, 0x00fffffe, + 0x01484c9c, 0x80000000, 0x81e63040, 0x00000000, 0x80000000, 0x01b83f16, 0x02017c20, 0x0201ef09, + 0x00000000, 0x00800000, 0xe3421716, 0xff7fffff, 0xcb000001, 0x4b000000, 0x2d91f14c, 0x81a75f53, + 0x822b9060, 0xe640b291, 0x3d01a348, 0x01b4e820, 0x7fc00001, 0xffc12345, 0xffc00001, 0xffc12345, + 0xffc00001, 0xffc12345, 0xe8e00000, 0x013d2d44, 0xe7400000, 0xff800000, 0x808d04a7, 0x5d800000, + }, + { + 0xffc00000, 0xffc00000, 0x00000000, 0x80000000, 0x00000000, 0x7f800000, 0xffc00001, 0x7fc00001, + 0xffc12345, 0x7fc00001, 0x00000000, 0x00800000, 0x8110c65e, 0x0003ee5a, 0x00000001, 0x01bcb83b, + 0x0010471c, 0x00000000, 0x80000000, 0x802cc5a7, 0x8187bc63, 0x014e8a46, 0x81cf89be, 0x80000003, + 0x00a66021, 0x0156a967, 0x0197eca0, 0x00f88e02, 0x00425520, 0x01e94ee7, 0x00000000, 0x00000000, + 0x010d380e, 0x80000000, 0x81dd408a, 0x00000000, 0x00000000, 0x01a8d6f4, 0x01f74c53, 0x01fd86b4, + 0x00b49438, 0x00800001, 0xe3421716, 0xff7fffff, 0xcb000001, 0x4b000000, 0x2d91f14c, 0x819c564f, + 0x822b9060, 0xe640b291, 0x3d01a348, 0x01b4e820, 0x7fc00001, 0xffc12345, 0xffc00001, 0xffc12345, + 0xffc00001, 0xffc12345, 0xe8e00000, 0x013d2d44, 0xe7400000, 0xff800000, 0x80ec8133, 0x5d800000, + }, + { + 0xffc00000, 0xffc00000, 0x00000000, 0x80000000, 0x00000000, 0x7f800000, 0xffc00001, 0x7fc00001, + 0xffc12345, 0x7fc00001, 0x00000000, 0x007fffff, 0x814b1f70, 0x0003ee5a, 0x00000001, 0x01abf199, + 0x0010471c, 0x00bfffff, 0x80000001, 0x802cc5a7, 0x8162e7d5, 0x017ee746, 0x81e54b6f, 0x80000003, + 0x009af6dd, 0x0141ee16, 0x0188e2dc, 0x00d0d1cf, 0x00425520, 0x01f136d4, 0x007fffff, 0x00fffffe, + 0x01484c9c, 0x80000002, 0x81e63040, 0x00400001, 0x8019dd9f, 0x01b83f16, 0x02017c20, 0x0201ef09, + 0x006d33d5, 0x00800000, 0xe3421716, 0xff7fffff, 0xcb000001, 0x4b000000, 0x2d91f14c, 0x81a75f53, + 0x822b9060, 0xe640b291, 0x3d01a348, 0x01b4e820, 0x7fc00001, 0xffc12345, 0xffc00001, 0xffc12345, + 0xffc00001, 0xffc12345, 0xe8e00000, 0x013d2d44, 0xe7400000, 0xff800000, 0x808d04a7, 0x5d800000, + }, + { + 0xffc00000, 0xffc00000, 0x00000000, 0x80000000, 0x00000000, 0x7f800000, 0xffc00001, 0x7fc00001, + 0xffc12345, 0x7fc00001, 0x00000000, 0x007fffff, 0x814b1f70, 0x0003ee5a, 0x00000001, 0x01abf199, + 0x0010471c, 0x00bfffff, 0x80000001, 0x802cc5a7, 0x8162e7d5, 0x017ee746, 0x81e54b6f, 0x80000003, + 0x009af6dd, 0x0141ee16, 0x0188e2dc, 0x00d0d1cf, 0x00425520, 0x01f136d4, 0x007fffff, 0x00fffffe, + 0x01484c9c, 0x80000002, 0x81e63040, 0x00400001, 0x8019dd9f, 0x01b83f16, 0x02017c20, 0x0201ef09, + 0x006d33d5, 0x00800000, 0xe3421716, 0xff7fffff, 0xcb000001, 0x4b000000, 0x2d91f14c, 0x81a75f53, + 0x822b9060, 0xe640b291, 0x3d01a348, 0x01b4e820, 0x7fc00001, 0xffc12345, 0xffc00001, 0xffc12345, + 0xffc00001, 0xffc12345, 0xe8e00000, 0x013d2d44, 0xe7400000, 0xff800000, 0x808d04a7, 0x5d800000, + }, +}}; + +constexpr std::array FloatModes{0x00u | (3u << 6u), 0x10u | (3u << 6u), 0x20u | (3u << 6u), 0x30u | (3u << 6u), 0x33u | (3u << 6u)}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Input[tid * 2u] = Operands[tid][0]; + Input[tid * 2u + 1u] = Operands[tid][1]; + } + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(Code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 2048u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(Code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{floatMode, true, false, false}; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + device.WaitIdle(); +} + +bool Denormal(std::uint32_t value) { + return (value & 0x7f800000u) == 0u && (value & 0x007fffffu) != 0u; +} + +bool HostDependent(std::uint32_t mode, std::uint32_t tid) { + const bool keepInputs = ((FloatModes[mode] >> 4u) & 1u) != 0u; + const bool keepResult = ((FloatModes[mode] >> 5u) & 1u) != 0u; + return (keepInputs && (Denormal(Operands[tid][0]) || Denormal(Operands[tid][1]))) || (keepResult && Denormal(Measured[mode][tid])); +} + +std::uint32_t Check(std::uint32_t mode) { + std::uint32_t checked = 0; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + if (HostDependent(mode, tid)) continue; + ++checked; + const std::uint32_t* out = &Output[tid * Results]; + const std::string row = "FLOAT_MODE " + Hex(FloatModes[mode]) + " lane " + std::to_string(tid) + " (" + Hex(Operands[tid][0]) + " + " + Hex(Operands[tid][1]) + ")"; + Require(out[0] == Operands[tid][0], "lds add f32: ds_add_rtn_f32 " + row + " returned " + Hex(out[0])); + Require(out[1] == Measured[mode][tid], "lds add f32: ds_add_rtn_f32 " + row + " stored " + Hex(out[1]) + ", expected " + Hex(Measured[mode][tid])); + Require(out[2] == Measured[mode][tid], "lds add f32: ds_add_f32 " + row + " stored " + Hex(out[2]) + ", expected " + Hex(Measured[mode][tid])); + } + return checked; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + std::uint32_t checked = 0; + for (std::uint32_t mode = 0; mode < FloatModes.size(); ++mode) { + Run(*device, FloatModes[mode]); + checked += Check(mode); + } + Require(checked >= 200u, "lds add f32: only " + std::to_string(checked) + " rows were independent of host denormal preservation"); + std::puts("lds add f32 tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp index 7c8d5d26d1ac..e340873ac7e8 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp @@ -51,6 +51,7 @@ struct MemoryInfo { bool coherent = false; bool gpuDescriptor = false; bool flushDenormals = false; + bool flushDenormalResults = false; bool operator==(const MemoryInfo& other) const = default; }; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index e47ee0f68b20..a50275935ba0 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -147,7 +147,7 @@ static_assert(fieldCount() == 5, "StageInput changed: update the inf static_assert(fieldCount() == 4, "StageOutput changed: update the info encoder"); static_assert(fieldCount() == 21, "IrResourcePlan changed: update EncodePlan and DecodePlan"); static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); -static_assert(fieldCount() == 30, "MemoryInfo changed: update the plan encoder"); +static_assert(fieldCount() == 31, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); static_assert(fieldCount() == 15, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); @@ -918,6 +918,7 @@ void encodeMemoryInfo(Writer& out, const MemoryInfo& memory) { out.Value(memory.coherent); out.Value(memory.gpuDescriptor); out.Value(memory.flushDenormals); + out.Value(memory.flushDenormalResults); } void decodeMemoryInfo(Reader& in, MemoryInfo& memory) { @@ -951,6 +952,7 @@ void decodeMemoryInfo(Reader& in, MemoryInfo& memory) { in.Value(memory.coherent); in.Value(memory.gpuDescriptor); in.Value(memory.flushDenormals); + in.Value(memory.flushDenormalResults); } bool encodePlan(Writer& writer, const IrResourcePlan& plan) { @@ -1156,7 +1158,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { } } } - reader.List(plan.memoryInfo, 78, decodeMemoryInfo); + reader.List(plan.memoryInfo, 79, decodeMemoryInfo); reader.List(plan.descriptorSources, 72, [&](Reader& in, DescriptorSource& source) { for (auto& dword : source.dwords) dword = valueAt(in.Get()); in.Value(source.dwordCount); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index c454eee2c532..ae5cfc72fbb5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1847,10 +1847,22 @@ std::uint32_t EmitSharedAtomicRsub32(SpirvValueEmitContext& ctx, const IrValue& std::uint32_t EmitSharedAtomicFAdd32(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto data = ctx.Arg(inst, 1); + const auto& mem = SharedMemory(ctx, inst); + const bool flushInputs = mem.flushDenormals; + const bool flushResult = mem.flushDenormalResults; + // The parts that depend only on data are computed once, outside the CAS loop, to keep its body short. + const AtomicFloatBits dataBits{ctx.state, false}; + const auto dataOperand = Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), dataBits.flush(data, flushInputs)); + const auto dataNan = dataBits.nan(data); + const auto quietData = dataBits.op(spv::OpBitwiseOr, data, dataBits.constant(dataBits.quiet())); return SharedAtomicUpdate(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old) { - const auto sum = Binary(state, spv::OpFAdd, TypeF32(state), Unary(state, spv::OpBitcast, TypeF32(state), old), Unary(state, spv::OpBitcast, TypeF32(state), data)); + const AtomicFloatBits bits{state, false}; + const auto sum = Binary(state, spv::OpFAdd, TypeF32(state), Unary(state, spv::OpBitcast, TypeF32(state), bits.flush(old, flushInputs)), dataOperand); state.module.AddAnnotation(spv::OpDecorate, sum, spv::DecorationNoContraction); - return Unary(state, spv::OpBitcast, TypeU32(state), sum); + auto result = bits.flush(Unary(state, spv::OpBitcast, TypeU32(state), sum), flushResult); + result = bits.select(bits.nan(result), bits.constant(0xffc00000u), result); + result = bits.select(bits.nan(old), bits.op(spv::OpBitwiseOr, old, bits.constant(bits.quiet())), result); + return bits.select(dataNan, quietData, result); }); } diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp index 04115cbb32e8..353a5c4e64c8 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp @@ -33,6 +33,7 @@ MemoryInfo sharedMemoryInfoFromInstruction(const RdnaInstruction& inst) { bool TranslationContext::dsAtomic(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue) { MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); memory.flushDenormals = sharedFloatFlush(opcode); + memory.flushDenormalResults = opcode == IrOpcode::SharedAtomicFAdd32 && (!floatMode.has_value() || ((floatMode->floatMode >> 5u) & 1u) == 0u); const IrU32 address = readU32(inst.source0); const IrU32 value = readU32(inst.source1); IrValue& active = ir.GetExec(); @@ -217,6 +218,7 @@ bool TranslationContext::sharedFloatFlush(IrOpcode opcode) const { case IrOpcode::SharedAtomicFMin32: case IrOpcode::SharedAtomicFMax32: case IrOpcode::SharedAtomicCmpstF32: + case IrOpcode::SharedAtomicFAdd32: return !floatMode.has_value() || ((floatMode->floatMode >> 4u) & 1u) == 0u; case IrOpcode::SharedAtomicFMin64: case IrOpcode::SharedAtomicFMax64: diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 69770884ad2b..98163c2730df 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -119,7 +119,7 @@ - [Converted unorm images](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view): samples, gathers, LOD queries, 16-bit data and a DST_SEL of the missing W channel throw. - [Converted unorm image loads](../../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) divide by 2^n - 1 with the buffer format path's `OpFDiv`, which Vulkan allows to be up to 2.5 ULP off, so some drivers are not bit-exact. - [64-bit LDS atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. On RDNA2 (gfx1036) the LDS serialises a 32-bit atomic on a slot's high dword and a 64-bit atomic on that slot: 128 lanes of `ds_add_u32` and 128 lanes of `ds_add_u64` on one slot sum exactly. Only that pair was measured. Every 32-bit LDS atomic of a program that contains a 64-bit LDS atomic therefore takes the same lock, including a 32-bit `flat_atomic_*` whose address lands in the LDS aperture (a 64-bit one takes it already); the flat forms were not measured against a 64-bit atomic either (the other 32-bit LDS atomics, `ds_min/max`, `ds_cmpst`, the float forms, `ds_inc/dec`, `ds_mskor` and `ds_wrxchg`, are extrapolated from `ds_add_u32` and were not measured against a 64-bit atomic); a program without a 64-bit LDS atomic keeps its native 32-bit atomics. `ds_condxchg32_rtn_b64` is two `ds_mskor` halves, each under the lock but not atomic as a pair, so a 64-bit atomic between the two halves is not excluded. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms) and plain `ds_read_*`, `ds_write_*`, `ds_append_u32` and `ds_consume_u32` are not taken under the lock, so a plain access racing a 64-bit atomic on the same dwords can see or overwrite one half of it; whether the hardware orders them was not measured. The lock serialises the workgroup's LDS atomics of such a program; the throughput cost was not measured. `VK_KHR_workgroup_memory_explicit_layout` with `shaderSharedInt64Atomics` would make both widths native and is not used. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035. -- [LDS float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_min/max/cmpst_f32`, `ds_min/max/cmpst_f64` and their `_rtn` forms) follow the hardware measured on RDNA2 (gfx1036) over 14 x 14 operand sets (zeros, ones, infinities, denormals, quiet and signalling NaNs of both signs and with payloads) in IEEE 0 and 1 with every value 0-3 of the type's denormal field: both operands are flushed to a signed zero when bit 0 of the field is clear (0 and 2; 1 and 3 keep them; the f32 field for the f32 forms, the f16/f64 field for the f64 forms; a shader without a known float mode flushes f32 and keeps f64), then a signalling-NaN data operand is stored quieted, else a signalling-NaN old value is stored quieted, else a quiet-NaN old value is replaced by the data, else a quiet-NaN data operand loses, else the ordered compare with -0 below +0 decides, and the flushed value is what is written, also when the old value wins. `cmpst` compares the flushed bits (both zeros equal, NaN never equal) and leaves memory unchanged when they differ. The returned value is always the old value as it was in memory. `IEEE_MODE` has no effect. The same bit rule serves the buffer float atomics (`AtomicFloatBits`). Not measured: a denormal or signalling-NaN `data1` of `cmpst` (stored as given), and `ds_add_f32`, which keeps the host's `OpFAdd`. +- [LDS float atomics](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_min/max/cmpst_f32`, `ds_min/max/cmpst_f64` and their `_rtn` forms) follow the hardware measured on RDNA2 (gfx1036) over 14 x 14 operand sets (zeros, ones, infinities, denormals, quiet and signalling NaNs of both signs and with payloads) in IEEE 0 and 1 with every value 0-3 of the type's denormal field: both operands are flushed to a signed zero when bit 0 of the field is clear (0 and 2; 1 and 3 keep them; the f32 field for the f32 forms, the f16/f64 field for the f64 forms; a shader without a known float mode flushes f32 and keeps f64), then a signalling-NaN data operand is stored quieted, else a signalling-NaN old value is stored quieted, else a quiet-NaN old value is replaced by the data, else a quiet-NaN data operand loses, else the ordered compare with -0 below +0 decides, and the flushed value is what is written, also when the old value wins. `cmpst` compares the flushed bits (both zeros equal, NaN never equal) and leaves memory unchanged when they differ. The returned value is always the old value as it was in memory. `IEEE_MODE` has no effect. The same bit rule serves the buffer float atomics (`AtomicFloatBits`). Not measured: a denormal or signalling-NaN `data1` of `cmpst` (stored as given). `ds_add_f32` and `ds_add_rtn_f32` follow RDNA1 (gfx1010, RX 5700 XT), measured with the [hardware oracle](../HW_ORACLE.md) template run through the Windows driver's comgr and HIP runtime over 1574 operand pairs (signed zeros, denormals, the smallest normals, infinities, quiet and signalling NaNs with payloads, the largest normals, random pairs near the denormal range and pairs that nearly cancel) in all 16 combinations of the f32 denormal field and the f32 round mode, and at IEEE 1 and DX10 clamp 0: both operands are flushed to a signed zero when bit 0 of the field is clear and the result when bit 1 is clear, the sum is rounded to nearest even whatever the round mode, a NaN data operand is stored quieted, else a NaN old value is stored quieted, `+inf + -inf` stores 0xffc00000, and `IEEE_MODE` and `DX10_CLAMP` have no effect. The sum is the host's `OpFAdd` between those flushes, so where the field keeps f32 denormals the result also depends on the host preserving them, which the module does not request. thuanquang measured the same rule on RDNA2 (gfx1031, RX 6700 XT) in #5492, with the same operand classes and 18 modes in wave32 and wave64; the PS5's gfx1013 is not measured. - [LDS in pixel, vertex-path and fetch programs](../../../core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp) (shader recompiler) - is an array private to each host invocation, in `Function` storage, as only compute and mesh programs get workgroup memory; on the console one LDS region is shared by the wave (RDNA 2 ISA sections 3.6.6 and 10.4), so a lane does not see another lane's LDS writes. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms), the 32-bit LDS atomics, `ds_append` and `ds_consume` load, update and store the invocation's own dword without an atomic, as Vulkan allows atomics only on `Uniform`, `Workgroup`, `Image`, `StorageBuffer`, `PhysicalStorageBuffer` and `TaskPayloadWorkgroupEXT` pointers (VUID-StandaloneSpirv-None-04686). `ds_append` and `ds_consume` update only the first active lane's copy and return its old value to every lane, so a second append after the first active lane changes, or a read of the counter by another lane, differs from the hardware. Not measured on hardware. - [V# bases off a DWORD boundary](../../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - raw accesses use the exact bytes and `s_buffer_load` reads `(base & ~3) + (offset & ~3)`, as measured on gfx1035 (Linux, alignment mode UNALIGNED; the console's mode is unknown). Formatted accesses off their element's alignment and atomics throw. - [32-bit bitfield masks](../../../core/shader/recompiler/Translation/src/IntegerInstructions.cpp) (`S_BFM_B32`, `V_BFM_B32`) use the truncating 32-bit shift specified by the [RDNA 1.0 ISA](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/rdna-shader-instruction-set-architecture.pdf), sections 12.1 (opcode 36) and 12.12 (opcode 867). Width and offset use their low five bits. This matches gfx1035 (Linux) for every width/offset pair, with and without upper operand bits set, and both leave SCC unchanged; not measured on the console. From 3d4845966fd55025ada22bb11fdad4f0b76eafef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Franch?= Date: Wed, 14 Oct 2026 22:48:40 +0200 Subject: [PATCH 2/2] style(shader): drop the code comment in EmitSharedAtomicFAdd32 Co-Authored-By: Claude --- .../SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index ae5cfc72fbb5..2d09066d1b22 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1850,7 +1850,6 @@ std::uint32_t EmitSharedAtomicFAdd32(SpirvValueEmitContext& ctx, const IrValue& const auto& mem = SharedMemory(ctx, inst); const bool flushInputs = mem.flushDenormals; const bool flushResult = mem.flushDenormalResults; - // The parts that depend only on data are computed once, outside the CAS loop, to keep its body short. const AtomicFloatBits dataBits{ctx.state, false}; const auto dataOperand = Unary(ctx.state, spv::OpBitcast, TypeF32(ctx.state), dataBits.flush(data, flushInputs)); const auto dataNan = dataBits.nan(data);