From 665cf974d86f4358a8ff862fa970b6a0e7014bbc Mon Sep 17 00:00:00 2001 From: Nesyth <18372752+Nesyth@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:36:41 +0100 Subject: [PATCH 1/2] perf(shader): keep the device's f16 NaN results with IEEE_MODE clear when a startup probe matches RDNA --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 1 + .../Execution/include/Float16NanProbe.hpp | 21 +++ .../Execution/src/Float16NanProbe.cpp | 155 ++++++++++++++++++ .../Execution/src/VulkanDevice.cpp | 13 ++ .../agc_driver_float16_nan_probe_tests.cmake | 7 + .../tests/execution/Float16NanProbe.cpp | 145 ++++++++++++++++ .../libSceAgcDriver/tools/ShaderReplay.cpp | 8 +- core/shader/recompiler/CacheKey.hpp | 1 + core/shader/recompiler/Recompiler.cpp | 1 + core/shader/recompiler/Recompiler.hpp | 1 + .../Translation/InstructionTranslator.hpp | 1 + .../Translation/TranslationContext.hpp | 4 +- .../Translation/src/FloatInstructions.cpp | 36 +++- .../Translation/src/InstructionTranslator.cpp | 2 +- docs/dev/ARCHITECTURE.md | 4 + docs/dev/BUILD.md | 2 +- docs/dev/ENVIRONMENT.md | 1 + docs/dev/debt/shader.md | 1 + 18 files changed, 393 insertions(+), 11 deletions(-) create mode 100644 core/libs/prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp create mode 100644 core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp create mode 100644 core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_float16_nan_probe_tests.cmake create mode 100644 core/libs/prx/libSceAgcDriver/tests/execution/Float16NanProbe.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 240a9c3342f2..e8605b0c3835 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -411,6 +411,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/FrameCapture.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/FrameRecorder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/VulkanDevice.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Float16NanProbe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/FloatFlushProbe.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/FloatProbeTolerance.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Float16ConversionProbe.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp new file mode 100644 index 000000000000..1d7feff33ca3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp @@ -0,0 +1,21 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FLOAT16NANPROBE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FLOAT16NANPROBE_HPP + +#include "Recompiler.hpp" + +#include + +namespace AgcDriver { + +class VulkanDevice; + +struct Float16NanProbeResult { + std::uint32_t mismatches = 0; + std::uint32_t compared = 0; +}; + +Float16NanProbeResult ProbeFloat16HostNans(VulkanDevice& device, const ShaderRecompiler::SpirvTarget& target); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp new file mode 100644 index 000000000000..74420d717539 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp @@ -0,0 +1,155 @@ +#include "prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" + +#include +#include +#include +#include +#include + +namespace AgcDriver { +namespace { + +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 256; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Combinations = 4096; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array PackedCode{ + 0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0xcc0f400au, 0x18020b04u, + 0xcc10400bu, 0x18020b04u, + 0xcc0e400cu, 0x1c1a0b04u, + 0xcc0f4a0du, 0x30020b04u, + 0xcc10c00eu, 0x18020d04u, + 0xcc0e330fu, 0x8c1a0b04u, + 0x7e200306u, 0x78200b04u, + 0xcc0ec011u, 0x3c1a0f04u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xbf810000u, +}; +constexpr std::uint32_t PackedResults = 8; + +alignas(256) constexpr std::array ScalarCode{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0x64140b04u, + 0x66160b04u, + 0x68180b04u, + 0x6a1a0b04u, + 0xd74b000eu, 0x041a0b04u, + 0xd74b980fu, 0x041a0b04u, + 0xd74b3110u, 0xc4160d04u, + 0x7e220306u, 0x6c220b04u, + 0x6e240b04u, 0x00003d55u, + 0x70260b04u, 0x00000001u, + 0xd74ba214u, 0x241a0b04u, + 0x662a08f2u, + 0xd5328116u, 0x20020b04u, + 0xd5350217u, 0x40020d04u, + 0xd5338018u, 0x00020b04u, + 0xd5340019u, 0x20020b04u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; +constexpr std::uint32_t ScalarFmaFirst = 4; +constexpr std::uint32_t ScalarFmaLast = 10; + +constexpr std::array Halves{ + 0x7e00u, 0xfe00u, 0xff2au, 0x7d55u, 0xfc01u, 0x7c01u, 0x7c00u, 0xfc00u, + 0x0000u, 0x8000u, 0x0001u, 0x0400u, 0x3c00u, 0xbc00u, 0x3555u, 0x7bffu, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +bool IsNan16(std::uint32_t half) { + return (half & 0x7fffu) > 0x7c00u; +} + +std::uint32_t Half(std::uint32_t combination, std::uint32_t operand) { + return Halves[(combination >> (operand * 4u)) & 15u]; +} + +void FillRows(std::uint32_t first) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t low = (first + tid) % Combinations; + const std::uint32_t high = Combinations - 1u - low; + for (std::uint32_t operand = 0; operand < Inputs; ++operand) { + const std::uint32_t lowOperand = operand < 3u ? operand : 1u; + const std::uint32_t highOperand = operand < 3u ? operand : 2u; + Input[tid * Inputs + operand] = Half(low, lowOperand) | (Half(high, highOperand) << 16u); + } + } +} + +std::vector Run(VulkanDevice& device, const ShaderRecompiler::RecompileResult& result, std::span code, std::uint32_t first) { + FillRows(first); + Output.fill(0xdeadbeefu); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + return {Output.begin(), Output.end()}; +} + +ShaderRecompiler::RecompileResult Compile(ShaderRecompiler::SpirvTarget target, bool hostNans, const std::optional& floatMode, std::span code) { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + target.float16HostNans = hostNans; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + return ShaderRecompiler::Recompile(request); +} + +} + +Float16NanProbeResult ProbeFloat16HostNans(VulkanDevice& device, const ShaderRecompiler::SpirvTarget& target) { + Float16NanProbeResult result; + const std::array, 3> modes{std::nullopt, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, true}, ShaderRecompiler::ShaderFloatMode{0xc0u, false, false, true}}; + for (const auto& mode : modes) { + const auto packedSelected = Compile(target, false, mode, PackedCode); + const auto packedHost = Compile(target, true, mode, PackedCode); + const auto scalarSelected = Compile(target, false, mode, ScalarCode); + const auto scalarHost = Compile(target, true, mode, ScalarCode); + for (std::uint32_t first = 0; first < Combinations; first += Threads) { + const auto selected = Run(device, packedSelected, PackedCode, first); + const auto host = Run(device, packedHost, PackedCode, first); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < PackedResults; ++index) { + ++result.compared; + if (selected[tid * PackedResults + index] != host[tid * PackedResults + index]) ++result.mismatches; + } + } + const auto scalarExact = Run(device, scalarSelected, ScalarCode, first); + const auto scalar = Run(device, scalarHost, ScalarCode, first); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = ScalarFmaFirst; index <= ScalarFmaLast; ++index) { + const std::uint32_t expected = scalarExact[tid * Results + index]; + const std::uint32_t actual = scalar[tid * Results + index]; + for (const std::uint32_t shift : {0u, 16u}) { + const std::uint32_t expectedHalf = (expected >> shift) & 0xffffu; + const std::uint32_t actualHalf = (actual >> shift) & 0xffffu; + ++result.compared; + if (expectedHalf != actualHalf && !(IsNan16(expectedHalf) && IsNan16(actualHalf) && expectedHalf != 0xfe00u)) ++result.mismatches; + } + } + } + } + } + return result; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index c79d7ee0e3ad..19e1f7804768 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -13,6 +13,7 @@ #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Execution/include/Float16ConversionProbe.hpp" +#include "prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp" #include "prx/libSceAgcDriver/Execution/include/FloatFlushProbe.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -576,6 +577,8 @@ struct VulkanDevice::State { bool float16WidenNans = false; bool float16NarrowNans = false; bool float32DenormPreserve = false; + bool float16HostNanCandidate = false; + bool float16HostNans = false; bool probingFloatControls = false; bool demoteToHelperInvocation = false; bool earlyFragmentSampleMaskTestBeforeSampleCounting = false; @@ -1770,6 +1773,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : floatControls.shaderDenormPreserveFloat16 == VK_TRUE && floatControls.shaderSignedZeroInfNanPreserveFloat16 == VK_TRUE; float16Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR}; float16Features.shaderFloat16 = VK_TRUE; + state->float16HostNanCandidate = float16Rte && std::getenv("APS5_NO_HOST_F16_NANS") == nullptr; if (float16Rte) { deviceExtensions.push_back(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME); state->capabilities.push_back(spv::CapabilityFloat16); @@ -2722,6 +2726,14 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : }; std::fprintf(stderr, "[float-controls] f16 to f32 NaN results: %s; f32 to f16 NaN results: %s (%.0f ms)\n", describe(probe.widen, state->float16WidenNans).c_str(), describe(probe.narrow, state->float16NarrowNans).c_str(), std::chrono::duration(std::chrono::steady_clock::now() - probeStart).count()); } + if (state->float16HostNanCandidate) { + const auto probeStart = std::chrono::steady_clock::now(); + state->probingFloatControls = true; + const auto probe = ProbeFloat16HostNans(*this, Target()); + state->probingFloatControls = false; + state->float16HostNans = probe.mismatches == 0u; + std::fprintf(stderr, "[float-controls] f16 NaN results with IEEE_MODE clear: %s (%u of %u probe results differ from the selected NaN, %.0f ms)\n", state->float16HostNans ? "host" : "selected", probe.mismatches, probe.compared, std::chrono::duration(std::chrono::steady_clock::now() - probeStart).count()); + } // A device without a window is usually replaced by the window's device before the title draws // (Driver::AttachWindow): its replay starts at its first draw or dispatch instead, so the // journal is replayed once, on the device the title draws with. @@ -4556,6 +4568,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { require(state->shaderProfile != nullptr, "shader device profile is unavailable"); auto target = state->shaderProfile->Target(); target.float32FlushToZero = state->float32FlushToZero; + target.float16HostNans = state->float16HostNans; target.fusedExactArithmetic = state->fusedExactArithmetic; target.hostInterpolationFirst = state->hostInterpolationFirst; target.float32DenormPreserve = state->float32DenormPreserve; diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_float16_nan_probe_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_float16_nan_probe_tests.cmake new file mode 100644 index 000000000000..f2e51030dd79 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_float16_nan_probe_tests.cmake @@ -0,0 +1,7 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_float16_nan_probe_spirv COMMAND agc_driver_float16_nan_probe_tests --verify-spirv) + set_tests_properties(agc_driver_float16_nan_probe_spirv PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_float16_nan_probe_spirv PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16NanProbe.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16NanProbe.cpp new file mode 100644 index 000000000000..fd84ee285daf --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16NanProbe.cpp @@ -0,0 +1,145 @@ +#include "prx/libSceAgcDriver/Execution/include/Float16NanProbe.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; + +alignas(256) constexpr std::array PackedCode{ + 0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0xcc0f400au, 0x18020b04u, + 0xcc10400bu, 0x18020b04u, + 0xcc0e400cu, 0x1c1a0b04u, + 0xcc0f4a0du, 0x30020b04u, + 0xcc10c00eu, 0x18020d04u, + 0xcc0e330fu, 0x8c1a0b04u, + 0x7e200306u, 0x78200b04u, + 0xcc0ec011u, 0x3c1a0f04u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ScalarFmaCode{ + 0x34020084u, 0x34060085u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0xd74b000au, 0x041a0b04u, + 0xd74b980bu, 0x041a0b04u, + 0x7e180306u, 0x6c180b04u, + 0x6e1a0b04u, 0x00003d55u, + 0x701c0b04u, 0x00000001u, + 0xe0781000u, 0x80010a03u, 0xe0701010u, 0x80010e03u, 0xbf810000u, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::uint32_t Selects(std::span words) { + std::uint32_t selects = 0; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + Require(count != 0 && cursor + count <= words.size(), "f16 NaN probe: malformed SPIR-V"); + if ((words[cursor] & 0xffffu) == spv::OpSelect) ++selects; + cursor += count; + } + return selects; +} + +void VerifySpirv() { + constexpr std::array capabilities{ + spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + spv::CapabilityDenormPreserve, spv::CapabilitySignedZeroInfNanPreserve, spv::CapabilityFloat16, + spv::CapabilityRoundingModeRTE, spv::CapabilityInt16, spv::CapabilityFMAKHR, + }; + constexpr std::array extensions{ + "SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", + "SPV_KHR_storage_buffer_storage_class", "SPV_KHR_16bit_storage", "SPV_KHR_fma", + }; + std::vector input(Threads * 4u), output(Threads * 8u); + std::vector userData(8, 0u); + const auto inputDescriptor = BufferDescriptor(input.data(), static_cast(input.size() * 4u)); + const auto outputDescriptor = BufferDescriptor(output.data(), static_cast(output.size() * 4u)); + std::copy(inputDescriptor.begin(), inputDescriptor.end(), userData.begin()); + std::copy(outputDescriptor.begin(), outputDescriptor.end(), userData.begin() + 4); + ShaderRecompiler::SpirvTarget target{}; + target.vulkanVersion = 0x00403000u; + target.spirvVersion = 0x00010600u; + target.subgroupSize = 32; + target.bdaAbiVersion = 1; + target.supportedCapabilities = capabilities; + target.supportedExtensions = extensions; + target.maxWorkgroupSize = {1024, 1024, 64}; + target.maxWorkgroupInvocations = 1024; + target.maxWorkgroupSharedMemoryBytes = 49152; + target.float16DenormPreserve = true; + const auto compile = [&](std::span code, bool hostNans, bool ieee) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + target.float16HostNans = hostNans; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xc0u, false, ieee, true}; + return ShaderRecompiler::Recompile(request).spirv.Copy(); + }; + for (const auto& [code, name] : {std::pair{std::span(PackedCode), "packed add, mul, fma and fmac"}, std::pair{std::span(ScalarFmaCode), "scalar fma forms"}}) { + const auto selected = compile(code, false, false); + const auto host = compile(code, true, false); + Require(Selects(host) < Selects(selected), std::string("f16 NaN probe: the ") + name + " with float16HostNans and IEEE_MODE clear have " + std::to_string(Selects(host)) + " selects, without " + std::to_string(Selects(selected))); + Require(compile(code, true, true) == compile(code, false, true), std::string("f16 NaN probe: float16HostNans changed the ") + name + " with IEEE_MODE set"); + } +} + +} + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string_view(argv[1]) == "--verify-spirv") { + VerifySpirv(); + std::puts("f16 NaN results drop their selection only when the target says the device returns RDNA's NaN with IEEE_MODE clear"); + return 0; + } + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto target = device->Target(); + const auto supports = [&](spv::Capability capability) { + return std::ranges::find(target.supportedCapabilities, static_cast(capability)) != target.supportedCapabilities.end(); + }; + if (!supports(spv::CapabilityFloat16) || !supports(spv::CapabilityRoundingModeRTE) || !supports(spv::CapabilityDenormPreserve)) { + std::puts("f16 NaN probe: skipped, the device has no native f16 with RTE rounding and kept denormals"); + return VulkanTestSkipped; + } + const auto probe = AgcDriver::ProbeFloat16HostNans(*device, target); + std::printf("f16 NaN probe: %u of %u results differ from the selected NaN; the device %s its own NaN results\n", probe.mismatches, probe.compared, target.float16HostNans ? "keeps" : "does not keep"); + Require(probe.compared != 0u, "f16 NaN probe: nothing was compared"); + Require(!target.float16HostNans || probe.mismatches == 0u, "f16 NaN probe: the device keeps its NaN results although they differ from RDNA's"); + Require(target.float16HostNans || probe.mismatches != 0u || std::getenv("APS5_NO_HOST_F16_NANS") != nullptr, "f16 NaN probe: the device matches RDNA but keeps the selection"); + std::puts("f16 NaN probe tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp index 14f34c2cfac6..0c55fa064cb7 100644 --- a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -54,6 +54,7 @@ bool g_waveLdsMemoryBarriers = false; bool g_deviceScopeLdsMemoryBarriers = false; bool g_ldsLockFences = false; bool g_shaderFma = false; +bool g_f16HostNans = false; bool g_code = false; bool g_float16WidenNans = false; bool g_float16NarrowNans = false; @@ -65,6 +66,7 @@ bool Replay(const char* path) { if (g_waveLdsMemoryBarriers) request.request.target.waveLdsMemoryBarriers = true; if (g_deviceScopeLdsMemoryBarriers) request.request.target.deviceScopeLdsMemoryBarriers = true; if (g_ldsLockFences) request.request.target.ldsLockFences = true; + if (g_f16HostNans) request.request.target.float16HostNans = true; if (g_shaderFma) { request.supportedCapabilities.push_back(spv::CapabilityFMAKHR); request.supportedExtensions.push_back("SPV_KHR_fma"); @@ -223,7 +225,7 @@ bool Replay(const char* path) { int main(int argc, char** argv) { if (argc < 2) { - std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] [--robust2] [--wave-lds-memory-barriers] [--device-scope-lds-memory-barriers] [--lds-lock-fences] [--shader-fma] [--f16-widen-nans] [--f16-narrow-nans] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set; shader_
.prepare.req holds a request whose preparation failed before any resource capture, so replay it with --prepare\n"); + std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] [--robust2] [--wave-lds-memory-barriers] [--device-scope-lds-memory-barriers] [--lds-lock-fences] [--shader-fma] [--f16-widen-nans] [--f16-narrow-nans] [--f16-host-nans] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set; shader_
.prepare.req holds a request whose preparation failed before any resource capture, so replay it with --prepare\n"); return 2; } int failures = 0; @@ -276,6 +278,10 @@ int main(int argc, char** argv) { g_float16NarrowNans = true; continue; } + if (std::string(argv[i]) == "--f16-host-nans") { + g_f16HostNans = true; + continue; + } if (std::string(argv[i]) == "--mem") { g_memory = true; continue; diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 2901cae7ce0c..e2a433253776 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -381,6 +381,7 @@ class RecompileCacheKey { append(key, value.shaderImageMinLod); append(key, value.float16WidenNans); append(key, value.float16NarrowNans); + append(key, value.float16HostNans); } }; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index ad91c93b142a..7b3a77929289 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -264,6 +264,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat translateOptions.nativeFloat16 = supports(spv::CapabilityFloat16) && supports(spv::CapabilityRoundingModeRTE) && supports(spv::CapabilityDenormPreserve); translateOptions.interpolationFunctions = supports(spv::CapabilityInterpolationFunction); translateOptions.nativeFloat16Fma = translateOptions.nativeFloat16 && supports(spv::CapabilityFMAKHR); + translateOptions.float16HostNans = translateOptions.nativeFloat16 && request.target.float16HostNans; translateOptions.floatMode = request.context.floatMode; translateOptions.float32FlushToZero = HardwareFloat32Flush(request); translateOptions.float32FlushDeclared = supports(spv::CapabilityDenormFlushToZero); diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 13d647b0d6ef..4f0fcefceaf5 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -364,6 +364,7 @@ struct SpirvTarget { bool shaderImageMinLod = false; bool float16WidenNans = false; bool float16NarrowNans = false; + bool float16HostNans = false; }; struct BindingLayout { diff --git a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp index 0f977ecd503e..586c024fd6e9 100644 --- a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp +++ b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp @@ -41,6 +41,7 @@ struct TranslateOptions { bool float64 = false; bool gcnLegacyVertexInputs = false; bool nativeFloat16Fma = false; + bool float16HostNans = false; std::optional floatMode; ShaderStageInputInfo inputInfo; const EmbeddedFetchPlan* embeddedFetch = nullptr; diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 9630be712685..353b6ab076ac 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -24,7 +24,7 @@ class TranslationContext { rounding = RoundingState{static_cast(mode.has_value() ? mode->floatMode & ModeRoundBits : 0u), 0u}; } void SetEntryRounding(const RoundingState& state) { setRounding(state); } - void SetNativeFloat16(bool enabled, bool fma) { nativeFloat16 = enabled; nativeFloat16Fma = enabled && fma; } + void SetNativeFloat16(bool enabled, bool fma, bool hostNans) { nativeFloat16 = enabled; nativeFloat16Fma = enabled && fma; float16HostNans = enabled && hostNans; } void SetFloat32DenormPreserve(bool enabled) { float32DenormPreserve = enabled; } void SetFloat32FlushToZero(bool enabled) { float32FlushToZero = enabled; } void SetFloat32FlushDeclared(bool enabled) { float32FlushDeclared = enabled; } @@ -67,6 +67,7 @@ class TranslationContext { bool dx10Clamp() const { return !floatMode.has_value() || floatMode->dx10Clamp; } bool fp16Overflow() const { return floatMode.has_value() && floatMode->fp16Overflow; } bool nativeFloat16Mode() const { return nativeFloat16 && (!floatMode.has_value() || (floatMode->floatMode & 0xccu) == 0xc0u); } + bool hostFloat16Nans(const RdnaInstruction& inst, std::uint32_t count, bool accumulator); IrF32 clampF16Overflow(IrF32 value, std::initializer_list sources); void rejectHalfOrDoubleOutputModifier(const RdnaOperand& operand) const; IrU32 clampF16Bits(const RdnaOperand& operand, IrU32 bits); @@ -387,6 +388,7 @@ class TranslationContext { bool ieeeMode = false; bool nativeFloat16 = false; bool nativeFloat16Fma = false; + bool float16HostNans = false; IrBuilder ir; IrBlock& block; IrU1 instructionBranchCondition; diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 7124d59a6d0f..ad5320fb0b5b 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -60,9 +60,16 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc }; const auto minMaxLane = [&](bool high) { return minMaxF16Bits(opcode == IrOpcode::FPMax32, sourceLaneBits(0u, high), sourceLaneBits(1u, high)); }; const bool native = nativeFloat16 && !flushResults && inst.destination.omod == 0u && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32 || (nativeFloat16Fma && opcode == IrOpcode::FPFma32)); + const bool hostNans = native && hostFloat16Nans(inst, accumulator ? 3u : inst.sourceCount, accumulator); + const auto nativeSecond = [&](bool high) { + const IrU32 first = sourceLaneBits(0u, high); + const IrU32 second = sourceLaneBits(1u, high); + if (!hostNans) return second; + return IrU32(ir.Select(ir.UGreaterThan(ir.BitwiseAnd(first.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u)), first.Value(), second.Value())); + }; const auto nativeResult = [&]() { const IrU32 lhs = packU16Lanes(sourceLaneBits(0u, false), sourceLaneBits(0u, true)); - const IrU32 rhs = packU16Lanes(sourceLaneBits(1u, false), sourceLaneBits(1u, true)); + const IrU32 rhs = packU16Lanes(nativeSecond(false), nativeSecond(true)); if (opcode == IrOpcode::FPFma32) { const IrU32 addend = packU16Lanes(sourceLaneBits(2u, false), sourceLaneBits(2u, true)); return IrU32(ir.Emit(IrOpcode::FPFma16x2, IrType::U32, {&lhs.Value(), &rhs.Value(), &addend.Value()})); @@ -100,6 +107,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc } lane = IrU32(ir.Select(overflow.Value(), ir.BitwiseOr(ir.BitwiseAnd(lane.Value(), ir.Constant(0x8000u)), ir.Constant(0x7bffu)), lane.Value())); } + if (hostNans) return lane; if (quietSnan) { lane = IrU32(ir.Select(isNan(bits[0]).Value(), bits[1].Value(), lane.Value())); lane = IrU32(ir.Select(isNan(bits[1]).Value(), bits[0].Value(), lane.Value())); @@ -308,25 +316,39 @@ bool TranslationContext::float16Native(const RdnaInstruction& inst, IrOpcode opc return true; } +bool TranslationContext::hostFloat16Nans(const RdnaInstruction& inst, std::uint32_t count, bool accumulator) { + if (!float16HostNans || ieeeMode) return false; + const auto nan = [](std::uint32_t half) { return (half & 0x7fffu) > 0x7c00u; }; + for (std::uint32_t index = 0u; index < count; ++index) { + const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index); + if (operand.kind == RdnaOperandKind::LiteralConstant && (nan(operand.value & 0xffffu) || nan(operand.value >> 16u))) return false; + } + return true; +} + bool TranslationContext::float16NativeFma(const RdnaInstruction& inst, bool accumulator) { std::array sources{}; for (std::uint32_t index = 0u; index < sources.size(); ++index) { sources[index] = readF16Bits(accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index)); } - IrU32 result(ir.BitwiseAnd(ir.Emit(IrOpcode::FPFma16x2, IrType::U32, {&sources[0].Value(), &sources[1].Value(), &sources[2].Value()}), ir.Constant(0xffffu))); const auto magnitude = [&](const IrU32& value) { return IrU32(ir.BitwiseAnd(value.Value(), ir.Constant(0x7fffu))); }; const auto infinite = [&](const IrU32& value) { return IrU1(ir.IEqual(magnitude(value).Value(), ir.Constant(0x7c00u))); }; const auto nan = [&](const IrU32& value) { return IrU1(ir.UGreaterThan(magnitude(value).Value(), ir.Constant(0x7c00u))); }; + const bool hostNans = hostFloat16Nans(inst, 3u, accumulator); + const IrU32 second = hostNans ? IrU32(ir.Select(nan(sources[0]).Value(), sources[0].Value(), sources[1].Value())) : sources[1]; + IrU32 result(ir.BitwiseAnd(ir.Emit(IrOpcode::FPFma16x2, IrType::U32, {&sources[0].Value(), &second.Value(), &sources[2].Value()}), ir.Constant(0xffffu))); if (fp16Overflow()) { const IrU1 infiniteSource(ir.LogicalOr(ir.LogicalOr(infinite(sources[0]).Value(), infinite(sources[1]).Value()), infinite(sources[2]).Value())); const IrU1 overflow(ir.LogicalAnd(infinite(result).Value(), ir.LogicalNot(infiniteSource.Value()))); result = IrU32(ir.Select(overflow.Value(), ir.BitwiseOr(ir.BitwiseAnd(result.Value(), ir.Constant(0x8000u)), ir.Constant(0x7bffu)), result.Value())); } - const auto infZero = [&](const IrU32& inf, const IrU32& zero) { return IrU1(ir.LogicalAnd(infinite(inf).Value(), ir.IEqual(magnitude(zero).Value(), ir.Constant(0u)))); }; - const IrU1 invalidProduct(ir.LogicalOr(infZero(sources[0], sources[1]).Value(), infZero(sources[1], sources[0]).Value())); - const IrU1 nanSource(ir.LogicalOr(ir.LogicalOr(nan(sources[0]).Value(), nan(sources[1]).Value()), nan(sources[2]).Value())); - const IrU1 invalid(ir.LogicalAnd(nan(result).Value(), ir.LogicalOr(invalidProduct.Value(), ir.LogicalNot(nanSource.Value())))); - result = IrU32(ir.Select(invalid.Value(), ir.Constant(0xfe00u), result.Value())); + if (!hostNans) { + const auto infZero = [&](const IrU32& inf, const IrU32& zero) { return IrU1(ir.LogicalAnd(infinite(inf).Value(), ir.IEqual(magnitude(zero).Value(), ir.Constant(0u)))); }; + const IrU1 invalidProduct(ir.LogicalOr(infZero(sources[0], sources[1]).Value(), infZero(sources[1], sources[0]).Value())); + const IrU1 nanSource(ir.LogicalOr(ir.LogicalOr(nan(sources[0]).Value(), nan(sources[1]).Value()), nan(sources[2]).Value())); + const IrU1 invalid(ir.LogicalAnd(nan(result).Value(), ir.LogicalOr(invalidProduct.Value(), ir.LogicalNot(nanSource.Value())))); + result = IrU32(ir.Select(invalid.Value(), ir.Constant(0xfe00u), result.Value())); + } write16Bits(inst.destination, clampF16Bits(inst.destination, result)); return true; } diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index de03c21cfe88..aa384019e61e 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -634,7 +634,7 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con TranslationContext context(program, *blocks[typedIndex], vectorLimit); context.SetPixelInput(options.inputInfo.pixel, options.fragmentShaderBarycentricEnabled); context.SetFloatMode(options.floatMode); - context.SetNativeFloat16(options.nativeFloat16, options.nativeFloat16Fma); + context.SetNativeFloat16(options.nativeFloat16, options.nativeFloat16Fma, options.float16HostNans); context.SetFloat32DenormPreserve(options.float32DenormPreserve); context.SetFloat32FlushToZero(options.float32FlushToZero); context.SetFloat32FlushDeclared(options.float32FlushDeclared); diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 9911629a8dad..97042beb940e 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -238,3 +238,7 @@ A device with `shaderDenormPreserveFloat32` and an f32 denormal behaviour indepe ### Native f16 conversion NaN results A device with native f16 conversions (`Float16`, `RoundingModeRTE` and `DenormPreserve` for 16-bit floats) also runs a probe when it is created (`ProbeFloat16Conversions`). `OpFConvert` and `UnpackHalf2x16` round and handle denormals as RDNA does under those modes, but Vulkan leaves the NaN they return to the driver, so the emitter rebuilds RDNA's NaN: sign and payload kept from the f16 input by `v_cvt_f32_f16`, quiet bit set and payload truncated by `v_cvt_f16_f32`. The probe compiles `v_cvt_f32_f16` over the 65,536 halves, and `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32` and a `v_cvt_f16_f32` then `v_cvt_f32_f16` round trip over every f32 NaN payload class, signalling NaNs, infinities, denormals, ties and overflow, with no float mode, with every denormal kept, with every denormal flushed and with a directed f16 rounding mode, each twice: with the rebuilt NaN and with the native result. Inputs come from a buffer, so the driver cannot fold them. `SpirvTarget::float16WidenNans` and `float16NarrowNans` are set per direction when every result matches, and the emitter then returns the native result. Both are part of the cache key and of the serialized request (version 54), so a saved request replays with the conversions of the device that wrote it. `APS5_NO_NATIVE_F16_NANS` skips the probe. + +### Device f16 NaN results with IEEE_MODE clear + +A device with native f16 math (`Float16`, `RoundingModeRTE` and `DenormPreserve` for 16-bit floats) also runs `ProbeFloat16HostNans` when it is created. The native packed `v_pk_add_f16`, `v_pk_mul_f16`, `v_pk_fma_f16` and `v_pk_fmac_f16` pick their NaN result on the bits after the native instruction (the first NaN source, quieted only with `IEEE_MODE` set, the addend skipped after an inf * 0 product, else the default NaN 0xfe00), and the native scalar fma forms test for the default NaN of an invalid operation. The probe compiles the packed instructions of `agc_driver_packed_float16_native` and the scalar fma forms of `agc_driver_float16_native` with that selection and with the device's own result, whose second operand is replaced by the first when the first is a NaN so that a driver swapping the operands cannot change which NaN comes back, over 4096 operand triples of special halves in both lanes, with no float mode and with `IEEE_MODE` clear and `FP16_OVFL` set, with and without `DX10_CLAMP`. Inputs come from a buffer. When every packed result matches bit for bit and every scalar fma result matches or is a NaN whose payload is already the driver's, `SpirvTarget::float16HostNans` is set and shaders with `IEEE_MODE` clear keep the device's result; `IEEE_MODE` set, and an instruction with a literal NaN source, keep the selection. The flag is part of the cache key; recompile requests saved with `APS5_DUMP_SHADERS` do not carry it, and `agc_shader_replay --f16-host-nans` sets it on a replay. `APS5_NO_HOST_F16_NANS` skips the probe. diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 0ab140348846..d8cbfda487dd 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -219,7 +219,7 @@ On Windows MinGW frees `thread_local` objects before their destructors run, so t ## Compute pipeline replay -Build `agc_compute_pipeline_replay` in the full build with `-DANYPS5_ENABLE_SPIRV_TOOLS=ON`; replay validates the captured module before calling Vulkan. Set `APS5_DUMP_COMPUTE_PIPELINES` to a dedicated empty directory before starting a title to save each uncached compute pipeline's final SPIR-V, descriptor layouts, push constant ranges, specialization values, subgroup size and creation flags before compilation. Files are named `compute-/capture.aps5cp`; identical records are deduplicated. The pipelines of the start-up f32 denormal flush and f16 conversion probes are not captured. At 128 records or 256 MiB in total, capture turns itself off for the rest of the run after a `[compute-capture] optional capture disabled` line and the title keeps running; a single record over 36 MiB (at most 8,388,608 SPIR-V words, or 32 MiB of shader code) is an error. Capture files contain shader code and device metadata, but no guest buffers or textures; review them before sharing. +Build `agc_compute_pipeline_replay` in the full build with `-DANYPS5_ENABLE_SPIRV_TOOLS=ON`; replay validates the captured module before calling Vulkan. Set `APS5_DUMP_COMPUTE_PIPELINES` to a dedicated empty directory before starting a title to save each uncached compute pipeline's final SPIR-V, descriptor layouts, push constant ranges, specialization values, subgroup size and creation flags before compilation. Files are named `compute-/capture.aps5cp`; identical records are deduplicated. The pipelines of the start-up f32 denormal flush, f16 conversion and f16 NaN probes are not captured. At 128 records or 256 MiB in total, capture turns itself off for the rest of the run after a `[compute-capture] optional capture disabled` line and the title keeps running; a single record over 36 MiB (at most 8,388,608 SPIR-V words, or 32 MiB of shader code) is an error. Capture files contain shader code and device metadata, but no guest buffers or textures; review them before sharing. Set `APS5_DUMP_COMPUTE_MIN_WORDS` to capture only modules at or above that SPIR-V word count, for example `100000` when investigating large NVIDIA compilations. The default is `0` (all modules); values outside `0` through `8388608` fail. The filter applies only when capture is enabled. diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index e8679db60b09..ef5a900ff9fd 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -301,6 +301,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_HOOKSYNC_PROFILE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | leaves only the plain hook-sync counts under APS5_PROFILE_DRAW | | `APS5_NO_HOOK_COMPLETION_GUARD` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | lets the flush hook record the accessing worker's queued labels even from inside a completion action | | `APS5_NO_HOOK_SNAPSHOT` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | takes the GpuMutex on every access instead of using the pending-write snapshot | +| `APS5_NO_HOST_F16_NANS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | skips the startup f16 NaN probe and keeps the selected NaN results of the native f16 fma, add and mul forms in every shader | | `APS5_NO_HOST_IMPORT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | disables importing guest memory, so address-based shaders copy instead of using it in place | | `APS5_NO_HOST_INTERPOLATION_FIRST` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | on RADV, reads pixel shader inputs through VK_KHR_fragment_shader_barycentric again instead of trying host interpolation first | | `APS5_NO_HW_F32_FLUSH` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | skips the startup f32 denormal flush probe and keeps the emulated flush in every shader | diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index f3a640e5829b..59eecc67149a 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -149,6 +149,7 @@ - [v_mad_f32, v_mac_f32, v_madak_f32 and v_madmk_f32](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (gfx10.1 only) leave f32 denormals and NaN bits to the driver, unlike the console. Measured on gfx1010 (RX 5700 XT, #5782): the product is rounded before the add; f32 sources, product and result are flushed to a zero of the same sign in every f32 denormal mode; the product is flushed when tiny after rounding to 24 bits with an unbounded exponent; NaN results follow the fma rule, an invalid product beating a NaN src2; in round mode 2 a sum of zeros of opposite signs is -0. Translating this cost 15× `main` on a `v_mac_f32` chain on the RX 5700 XT and 23× on an RTX 3070, and a never-taken branch around it 6×, so it waits for the SPIR-V float controls of #981. - [v_fma_mixlo_f16 and v_fma_mixhi_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) round the exact product-sum once to f16 with the arithmetic of v_interp_p2_f16, in f64 or, without the Float64 capability, in 64-bit integers. On RDNA2 (gfx1036) they follow the `FLOAT_MODE` f16 round and denormal fields, not the f32 round field: -1.5 * 2^-24 + 2^-126 from f32 sources (0xbfc00000, 0x33800000, 0x00800000) gives 0x8001 with f16 denormals kept and round to nearest even, 0x8000 with f16 denormals flushed and 0x8002 with f16 round toward -infinity. With f16 denormals kept, every f16 rounding mode is translated: the exact product-sum is rounded once in the mode's direction. All four modes match RDNA2 (gfx1036) on the 2,233 rows of the #2631 sweep, including `FLOAT_MODE` 0xcc (toward zero, used by ps5-opengl) and the exact cancellations toward -infinity; the PS5's gfx1013 is not measured. A result beyond 65504 becomes infinity or 65504 as IEEE 754 gives for the mode. Flushed f16 denormals still throw, and packed f16 throws with any f16 rounding but nearest even. `FP16_OVFL` set is not measured. - [Signaling NaN inputs](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes, and so are those of the f64 forms (`v_sqrt/rcp/rsq/fract/floor/ceil/trunc/rndne/frexp_mant/ldexp/div_fixup_f64`) and `v_cvt_f64_f32`, measured on RDNA2 (gfx1036) in both modes; shaders without a known float mode follow IEEE=0. `v_pack_b32_f16` quiets each signaling NaN half by the same rule, measured on RDNA2 (gfx1036) in both modes. A signaling NaN result of `v_cubesc/cubetc/cubema_f32` is quieted by the same rule and keeps the sign the instruction gives it, measured on RDNA2 (gfx1036) in both modes. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 still wins). `v_mad_legacy_f32` and `v_mac_legacy_f32` follow the same rule with their separately rounded product, as measured on gfx1010 (#1281, #1290): a NaN source quieted only with IEEE=1, inf - inf giving 0xffc00000, the zero rule hiding a NaN factor and a NaN src2 winning. The f16 and f64 arithmetic, `v_fma_mix_f32` with f16 sources and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver, except that an invalid operation of `v_fma_f16`, `v_fmac_f16`, `v_fmamk_f16`, `v_fmaak_f16` and `v_fma_mix_f32` with f16 sources (an inf * 0 product, also with a NaN src2, or a NaN from sources that are not NaN) gives the default NaN 0xfe00 or 0xffc00000, which NVIDIA does not return on its own. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with f16 and f32 denormals kept and flushed, over six inf * 0 products with eight NaN src2 values in every f16 fma form (#7074); the AMD Windows driver 32.0.21042.62 on an RDNA2 iGPU (Ryzen 5 7600) instead returns src2 quieted through Vulkan in the f16 fma forms. `v_cvt_f16_f32`, `v_cvt_f32_f64`, `v_dot2_f32_f16`, `v_fma_mixlo_f16` and `v_fma_mixhi_f16` always quiet and `v_div_scale_f32` never does: the f32 signaling NaN 0x7f812345 as src0 or src2 gives 0x7e09 from mixlo and mixhi at IEEE=0 and IEEE=1 on RDNA2 (gfx1036), where `v_fma_mix_f32` gives 0x7f812345 at IEEE=0 and 0x7fc12345 at IEEE=1. The payload of an f16 NaN source of mixlo and mixhi is left to the driver's f16 to f32 conversion, which NVIDIA replaces. +- [f16 NaN results with IEEE_MODE clear](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are the device's when the startup probe (`ProbeFloat16HostNans`) finds them equal to the selected ones above: those of the native `v_pk_add_f16`, `v_pk_mul_f16`, `v_pk_fma_f16` and `v_pk_fmac_f16`, whose second operand is replaced by the first when the first is a NaN (a driver may swap the operands), and the default NaN of the native `v_fma_f16`, `v_fmac_f16`, `v_fmamk_f16` and `v_fmaak_f16`. The probe compares 4096 operand triples of 16 special halves in both lanes, with and without `op_sel`, `neg` and clamp, without FLOAT_MODE and in two FP16_OVFL modes, through a compute shader; the graphics stages rely on the same device behaviour. IEEE_MODE=1 keeps the selection (a Radeon RX 7900 XTX returns a signalling NaN source unchanged there, where RDNA quiets it), and so does a literal NaN source, which a driver may fold. `APS5_NO_HOST_F16_NANS` skips the probe. - [Scalar f16 add, sub, subrev and mul denormals](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field at nearest-even: bit 0 preserves inputs and bit 1 preserves the result; otherwise a denormal is flushed to its signed zero. Measured on RDNA2 (gfx1036) with IEEE=0/1 and FP16_OVFL=0/1, over 32 operand pairs in all four denormal modes, including signed zeros, denormals, minimum normals, infinities and NaNs. Multiply detects tininess after rounding to 11 significant bits with an unbounded exponent, before gradual half packing: `0x3bff * 0x0400` flushes, while `0x3c01 * 0x03ff` rounds to `0x0400` and is kept. `v_fma_f16`, `v_fmac_f16`, `v_fmamk_f16` and `v_fmaak_f16` follow the same field at nearest-even: inputs are flushed when bit 0 is clear, and the result is flushed when bit 1 is clear if the exact product-sum rounded to 11 significant bits with an unbounded exponent is below 2^-14, so `0x0400 * 0x0400 + 0x8400` rounds to `0x8400` and is kept. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) at IEEE=0 and IEEE=1 with denorm16 0..3 over 5075 operand triples of `v_fma_f16` and `v_fmac_f16` (4000 from 20 halves around the denormal range, 75 whose product-sum lies just below 2^-14 and 1000 random), and over the first 4075 of them for `v_fmamk_f16` and `v_fmaak_f16`, FP16_OVFL=1, the clamp, neg and abs modifiers and op_sel. Directed rounding and `v_fma_mixlo/mixhi_f16` keep their previous denormal lowering. Shaders without a known float mode keep the previous lowering. Scalar `v_sub_f16`/`v_subrev_f16` share `float16Binary` with add/mul and were measured on gfx1036 over the same operand set as add (1156 non-NaN-input rows per mode); NaN result bits still depend on the driver as recorded above. - [f16 unary, frexp and conversion denormals](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follow the `FLOAT_MODE` f16 denormal field: `v_ceil_f16`, `v_floor_f16`, `v_fract_f16`, `v_rcp_f16`, `v_rsq_f16`, `v_sqrt_f16`, `v_log_f16`, `v_exp_f16`, `v_sin_f16`, `v_frexp_mant_f16`, `v_frexp_exp_i16_f16`, `v_cvt_norm_i16_f16`, `v_cvt_norm_u16_f16` and `v_cvt_f32_f16` flush a denormal half input to its signed zero when bit 0 is clear (0 and 2), and the f16-result ones flush a denormal half result to its signed zero when bit 1 is clear (0 and 1), after the host rounds it to half. Measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with denorm16 0..3 over every non-NaN f16 input (63,490 rows per mode and instruction): the only mismatches are the 1-ULP driver differences of `v_exp_f16` (2 inputs) and `v_sin_f16` (14), which are the same with denormals kept. NaN inputs are not compared. `v_ldexp_f16` shares the input rule. `v_cvt_pknorm_i16_f16` and `v_cvt_pknorm_u16_f16` flush both half inputs by the same bit, measured on RDNA2 (gfx1030) over every f16 input (each half also negated in the other source) at IEEE=0 and IEEE=1 with denorm16 0..3 and at f16 rounding 1..3: every row matches. Shaders without a known float mode keep the previous lowering. - [v_div_fixup_f16](../../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) flushes its src1 and src2 to a zero of the same sign when bit 0 of the `FLOAT_MODE` f16 denormal field is clear (0 and 2) and keeps a denormal src0 and result in every mode, as `v_div_fixup_f32` does with the f32 field. Measured on RDNA2 (gfx1031, Radeon RX 6700 XT, AMD Windows driver 32.0.21045.5002 through HIP) over 6859 triples of 19 halves (zeros, denormals, minimum normals, ±0.5, ±1, the largest finite value, infinities and quiet and signaling NaNs) at IEEE=0 and IEEE=1 with denorm16 0..3 and FP16_OVFL 0 and 1, also with neg and abs source modifiers and with op_sel high halves. Shaders without a known float mode keep the previous lowering. The PS5's gfx1013 is not measured. From c51af50be587dcd2572a54025300c8fd0e419f41 Mon Sep 17 00:00:00 2001 From: Nesyth <18372752+Nesyth@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:28:39 +0100 Subject: [PATCH 2/2] perf(agc): stop the f16 NaN probe at the first batch that differs --- .../prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp | 8 ++++++-- docs/dev/ARCHITECTURE.md | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp index 74420d717539..1eb7d34ce3fb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Float16NanProbe.cpp @@ -122,8 +122,6 @@ Float16NanProbeResult ProbeFloat16HostNans(VulkanDevice& device, const ShaderRec for (const auto& mode : modes) { const auto packedSelected = Compile(target, false, mode, PackedCode); const auto packedHost = Compile(target, true, mode, PackedCode); - const auto scalarSelected = Compile(target, false, mode, ScalarCode); - const auto scalarHost = Compile(target, true, mode, ScalarCode); for (std::uint32_t first = 0; first < Combinations; first += Threads) { const auto selected = Run(device, packedSelected, PackedCode, first); const auto host = Run(device, packedHost, PackedCode, first); @@ -133,6 +131,11 @@ Float16NanProbeResult ProbeFloat16HostNans(VulkanDevice& device, const ShaderRec if (selected[tid * PackedResults + index] != host[tid * PackedResults + index]) ++result.mismatches; } } + if (result.mismatches != 0u) return result; + } + const auto scalarSelected = Compile(target, false, mode, ScalarCode); + const auto scalarHost = Compile(target, true, mode, ScalarCode); + for (std::uint32_t first = 0; first < Combinations; first += Threads) { const auto scalarExact = Run(device, scalarSelected, ScalarCode, first); const auto scalar = Run(device, scalarHost, ScalarCode, first); for (std::uint32_t tid = 0; tid < Threads; ++tid) { @@ -147,6 +150,7 @@ Float16NanProbeResult ProbeFloat16HostNans(VulkanDevice& device, const ShaderRec } } } + if (result.mismatches != 0u) return result; } } return result; diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 97042beb940e..cc6e4895594d 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -241,4 +241,4 @@ A device with native f16 conversions (`Float16`, `RoundingModeRTE` and `DenormPr ### Device f16 NaN results with IEEE_MODE clear -A device with native f16 math (`Float16`, `RoundingModeRTE` and `DenormPreserve` for 16-bit floats) also runs `ProbeFloat16HostNans` when it is created. The native packed `v_pk_add_f16`, `v_pk_mul_f16`, `v_pk_fma_f16` and `v_pk_fmac_f16` pick their NaN result on the bits after the native instruction (the first NaN source, quieted only with `IEEE_MODE` set, the addend skipped after an inf * 0 product, else the default NaN 0xfe00), and the native scalar fma forms test for the default NaN of an invalid operation. The probe compiles the packed instructions of `agc_driver_packed_float16_native` and the scalar fma forms of `agc_driver_float16_native` with that selection and with the device's own result, whose second operand is replaced by the first when the first is a NaN so that a driver swapping the operands cannot change which NaN comes back, over 4096 operand triples of special halves in both lanes, with no float mode and with `IEEE_MODE` clear and `FP16_OVFL` set, with and without `DX10_CLAMP`. Inputs come from a buffer. When every packed result matches bit for bit and every scalar fma result matches or is a NaN whose payload is already the driver's, `SpirvTarget::float16HostNans` is set and shaders with `IEEE_MODE` clear keep the device's result; `IEEE_MODE` set, and an instruction with a literal NaN source, keep the selection. The flag is part of the cache key; recompile requests saved with `APS5_DUMP_SHADERS` do not carry it, and `agc_shader_replay --f16-host-nans` sets it on a replay. `APS5_NO_HOST_F16_NANS` skips the probe. +A device with native f16 math (`Float16`, `RoundingModeRTE` and `DenormPreserve` for 16-bit floats) also runs `ProbeFloat16HostNans` when it is created. The native packed `v_pk_add_f16`, `v_pk_mul_f16`, `v_pk_fma_f16` and `v_pk_fmac_f16` pick their NaN result on the bits after the native instruction (the first NaN source, quieted only with `IEEE_MODE` set, the addend skipped after an inf * 0 product, else the default NaN 0xfe00), and the native scalar fma forms test for the default NaN of an invalid operation. The probe compiles the packed instructions of `agc_driver_packed_float16_native` and the scalar fma forms of `agc_driver_float16_native` with that selection and with the device's own result, whose second operand is replaced by the first when the first is a NaN so that a driver swapping the operands cannot change which NaN comes back, over 4096 operand triples of special halves in both lanes, with no float mode and with `IEEE_MODE` clear and `FP16_OVFL` set, with and without `DX10_CLAMP`. Inputs come from a buffer. The probe stops at the first batch of 256 triples with a mismatch and compiles the scalar programs of a float mode only after its packed results matched, so a device that does not keep the selected NaNs pays for two compilations and two dispatches. When every packed result matches bit for bit and every scalar fma result matches or is a NaN whose payload is already the driver's, `SpirvTarget::float16HostNans` is set and shaders with `IEEE_MODE` clear keep the device's result; `IEEE_MODE` set, and an instruction with a literal NaN source, keep the selection. The flag is part of the cache key; recompile requests saved with `APS5_DUMP_SHADERS` do not carry it, and `agc_shader_replay --f16-host-nans` sets it on a replay. `APS5_NO_HOST_F16_NANS` skips the probe.