diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1ad0a76d95d7..464718b711f0 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -616,6 +616,7 @@ target_include_directories(agc_driver_prepared_graphics_tests PRIVATE $ ) +target_link_libraries(agc_driver_interpolation_arithmetic_tests PRIVATE shader_recompiler_agc_driver) if(BUILD_TESTING) add_test(NAME agc_shader_bindless COMMAND agc_shader_memory_tests --bindless) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 767af1e1411e..484f8cd26062 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -1102,7 +1102,7 @@ void verifyPixelInputs() { } } -ShaderRecompiler::RecompileResult recompileSlots(std::initializer_list controls, std::span code) { +ShaderRecompiler::RecompileResult recompileSlots(std::initializer_list controls, std::span code, bool float64 = true, bool barycentric = true) { using namespace ShaderRecompiler; ShaderPixelStageInfo pixel{}; pixel.interpolatorCount = static_cast(controls.size()); @@ -1119,7 +1119,9 @@ ShaderRecompiler::RecompileResult recompileSlots(std::initializer_list capabilities{spv::CapabilityFloat64}; + request.target.supportedCapabilities = std::span(capabilities).first(float64 ? 1u : 0u); request.layout.pushConstantSizeBytes = 128; request.useCache = false; return Recompile(request); @@ -1172,6 +1174,30 @@ void verifyPixelParameterSlots() { expectFailure([] { static_cast(recompileSlots({0x423u, 0x3u}, shared)); }, "passes its vertices through unchanged", "an interpolated pass-through input was accepted"); } +void verifyF16InterpolationCapabilities() { + static constexpr std::array p1ll{0xd7420002u, 0x00020000u, 0xf800180fu, 0x02020202u, 0xbf810000u}; + static constexpr std::array p1lv{0x7e0e02ffu, 0x4500bc00u, 0xd7430008u, 0x041e0000u, 0xf800180fu, 0x08080808u, 0xbf810000u}; + static constexpr std::array p2{0x7e040280u, 0xd75a0003u, 0x040a0200u, 0xf800180fu, 0x03030303u, 0xbf810000u}; + for (const auto code : {std::span(p1ll), std::span(p1lv), std::span(p2)}) { + require(!recompileSlots({0x03080000u}, code).spirv.empty(), "16-bit interpolation did not compile with its required capabilities"); + expectFailure([&] { static_cast(recompileSlots({0x03080000u}, code, false)); }, "Float64 capability", "16-bit interpolation compiled without Float64"); + expectFailure([&] { static_cast(recompileSlots({0x03080000u}, code, true, false)); }, "requires fragmentShaderBarycentric", "16-bit interpolation compiled without barycentrics"); + } +} + +void verifyF16PixelParameterSlots() { + static constexpr std::array high{0xd7420002u, 0x00020100u, 0xd75a0003u, 0x040a0300u, 0xf800180fu, 0x03030303u, 0xbf810000u}; + static constexpr std::array low{0xd7420002u, 0x00020000u, 0xd75a0003u, 0x040a0200u, 0xf800180fu, 0x03030303u, 0xbf810000u}; + auto inputs = slotInputs({0x03080003u}, high); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "a 16-bit interpolated input was not read per vertex at its slot"); + inputs = slotInputs({0x03080023u}, high); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "an input with a defaulted low half was not read per vertex for its high half"); + require(slotInputs({0x03180023u}, high).empty(), "an input whose high half is defaulted was declared for a high-half read"); + require(slotInputs({0x03080023u}, low).empty(), "an input whose low half is defaulted was declared for a low-half read"); + inputs = slotInputs({0x03180003u}, low); + require(inputs.size() == 1u && inputs[0].first == 3u && inputs[0].second, "an input with a defaulted high half was not read per vertex for its low half"); +} + } void verifyComputedTexelOffsets() { @@ -1769,6 +1795,8 @@ int main(int argc, char** argv) { verifyLegacyPixelRequests(); verifyPixelExportReplay(); verifyPixelParameterSlots(); + verifyF16PixelParameterSlots(); + verifyF16InterpolationCapabilities(); verifyComputedTexelOffsets(); verifyShaderClockScopes(); verifyUnnormalizedSamplers(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationArithmetic.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationArithmetic.cpp new file mode 100644 index 000000000000..98d21bf3d08e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationArithmetic.cpp @@ -0,0 +1,125 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "SpirvBackend/SpirvEmitterInstructions.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; +using AgcDriver::Graphics::Require; + +struct Vector { + std::array input; + std::uint32_t expected; +}; + +constexpr std::array Vectors{{ + {{0x00000000u, 0x7f800000u, 0x477fe000u}, 0xffc00000u}, + {{0xff800000u, 0x00000000u, 0xffc02000u}, 0xffc00000u}, + {{0x7f800000u, 0x7f7fffffu, 0xff800000u}, 0xffc00000u}, + {{0x00000000u, 0x7f800000u, 0x4f000000u}, 0xfe00u}, + {{0x80000000u, 0xff800000u, 0x7fc00000u}, 0xfe00u}, + {{0xffc02000u, 0x3f800000u, 0x47800000u}, 0xfe01u}, +}}; +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +alignas(256) std::array Data{}; + +RecompileResult Compile() { + IrProgram program; + const ShaderStageInputInfo inputs{}; + SpirvEmitterState state(program, inputs); + auto& module = state.module; + module.EmitCapability(spv::CapabilityShader); + module.EmitCapability(spv::CapabilityFloat64); + module.EmitCapability(spv::CapabilitySignedZeroInfNanPreserve); + module.EmitExtension("SPV_KHR_float_controls"); + module.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); + const auto u32 = TypeU32(state); + const auto array = module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 4u}}}, u32); + const auto structure = module.Type(spv::OpTypeStruct, array); + module.AddAnnotation(spv::OpDecorate, structure, spv::DecorationBlock); + module.AddAnnotation(spv::OpMemberDecorate, structure, 0u, spv::DecorationOffset, 0u); + const auto buffer = module.DefineGlobalVariable(module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, structure), spv::StorageClassStorageBuffer); + module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationDescriptorSet, 0u); + module.AddAnnotation(spv::OpDecorate, buffer, spv::DecorationBinding, 0u); + const auto element = module.Type(spv::OpTypePointer, spv::StorageClassStorageBuffer, u32); + const auto main = module.AllocateId(); + const auto voidType = module.Type(spv::OpTypeVoid); + module.EmitEntryPoint(spv::ExecutionModelGLCompute, main, "main", {}); + module.AddExecutionMode(main, spv::ExecutionModeLocalSize, 1u, 1u, 1u); + for (const auto width : {32u, 64u}) module.AddExecutionMode(main, spv::ExecutionModeSignedZeroInfNanPreserve, width); + module.AddFunction(spv::OpFunction, voidType, main, spv::FunctionControlMaskNone, module.Type(spv::OpTypeFunction, voidType)); + module.AddFunction(spv::OpLabel, module.AllocateId()); + const auto address = [&](std::uint32_t index) { + const auto pointer = module.AllocateId(); + module.AddFunction(spv::OpAccessChain, element, pointer, buffer, ConstantU32(state, 0u), ConstantU32(state, index)); + return pointer; + }; + const auto load = [&](std::uint32_t index) { + const auto word = module.AllocateId(); + module.AddFunction(spv::OpLoad, u32, word, address(index)); + return Unary(state, spv::OpBitcast, TypeF32(state), word); + }; + for (std::uint32_t row = 0; row < Vectors.size(); ++row) { + const auto delta = load(row * 4u); + const auto coordinate = load(row * 4u + 1u); + const auto base = load(row * 4u + 2u); + const auto value = row < 3u ? EmitFPInterpolateF32(state, delta, coordinate, base) : EmitFPInterpolateF16(state, delta, coordinate, base); + const auto result = row < 3u ? Unary(state, spv::OpBitcast, u32, value) : EmitConvertF16F32(state, value); + module.AddFunction(spv::OpStore, address(row * 4u + 3u), result); + } + module.AddFunction(spv::OpReturn); + module.AddFunction(spv::OpFunctionEnd); + RecompileResult result{}; + result.spirv = module.Finalize(); + const auto base = reinterpret_cast(Data.data()); + result.bindings.push_back({DescriptorKind::StorageBuffer, DescriptorRole::GuestBuffers, 0u, 0u, 1u, + {static_cast(base), static_cast((base >> 32u) & 0xffffu), static_cast(sizeof(Data)), 0x01016facu}, false}); + return result; +} + +void Run(AgcDriver::VulkanDevice& device, const RecompileResult& shader, std::uint32_t permutation) { + Data.fill(Sentinel); + std::array indices{}; + for (std::uint32_t row = 0; row < Vectors.size(); ++row) { + indices[row] = (row / 3u) * 3u + (row + permutation) % 3u; + std::copy(Vectors[indices[row]].input.begin(), Vectors[indices[row]].input.end(), Data.begin() + row * 4u); + } + device.Dispatch(shader, 1u, 1u, 1u); + device.WaitIdle(); + for (std::uint32_t row = 0; row < Vectors.size(); ++row) { + const auto& vector = Vectors[indices[row]]; + for (std::uint32_t input = 0; input < vector.input.size(); ++input) Require(Data[row * 4u + input] == vector.input[input], "interpolation arithmetic changed an input"); + char message[160]; + std::snprintf(message, sizeof(message), "interpolation arithmetic permutation %u, native row %u: got 0x%08x, expected 0x%08x", permutation, indices[row], Data[row * 4u + 3u], vector.expected); + Require(Data[row * 4u + 3u] == vector.expected, message); + } + for (std::size_t index = Vectors.size() * 4u; index < Data.size(); ++index) Require(Data[index] == Sentinel, "interpolation arithmetic wrote beyond the result rows"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto target = device->Target(); + if (std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityFloat64) == target.supportedCapabilities.end()) { + std::puts("skipped, interpolation arithmetic requires Float64"); + return VulkanTestSkipped; + } + const auto shader = Compile(); + for (const auto permutation : {0u, 1u}) Run(*device, shader, permutation); + std::puts("interpolation arithmetic passed: 6 published native rows, 2 dispatches, 12 output comparisons"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp new file mode 100644 index 000000000000..a06e875b9cfc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp @@ -0,0 +1,307 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::PixelInput; +using ShaderRecompiler::PixelInputBit; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Sentinel = 0xdeadbeefu; +constexpr std::uint32_t Width = 192; +constexpr std::uint32_t Height = 128; +constexpr std::uint32_t ResultDwords = 12; +constexpr std::uint32_t Fp16Input = 0x03080000u; +constexpr std::uint32_t Fp16DefaultHighInput = 0x01580000u; +alignas(256) std::array Pixels{}; +alignas(256) std::array Results{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000u, 0x80000005u, 0xe0382010u, 0x80000605u, 0xbf8c3f70u, 0xf800020fu, 0x09080706u, 0xf80008cfu, 0x03020100u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e080f02u, 0x7e0a0f03u, 0xd5430006u, 0x0411ff05u, 0x000000c0u, 0x7e300300u, 0x7e320301u, 0xd742001au, + 0x00020000u, 0x7e3602ffu, 0xabcd0000u, 0xd75a001bu, 0x046a0200u, 0xd7420008u, 0x00020100u, 0x7e3802ffu, + 0x12340000u, 0xd75a001cu, 0x04220300u, 0x7e1e02ffu, 0x4500bc00u, 0xd743001du, 0x043e0140u, 0x7e3c0280u, + 0xd75a001eu, 0x04760340u, 0x7e3e0280u, 0xd75a801fu, 0x046a0200u, 0xd7420020u, 0x00020181u, 0xd7420021u, + 0x000201c1u, 0xd7420022u, 0x40020000u, 0xd7430023u, 0x043e0000u, 0xe0782000u, 0x80011806u, 0xe0782010u, + 0x80011c06u, 0xe0782020u, 0x80012006u, 0x7e0e02f2u, 0xf800180fu, 0x07070707u, 0xbf810000u, +}; + +constexpr std::array LowX{0x3400u, 0x3f00u, 0xb800u}; +constexpr std::array HighX{0x4200u, 0xc100u, 0x4700u}; +constexpr std::array LowY{0x3800u, 0x3a00u, 0x3000u}; +constexpr std::array HighY{0x6c00u, 0x3c00u, 0xc800u}; + +struct Vertex { + std::array position; + std::array parameter; +}; + +float HalfToFloat(std::uint32_t half) { + const auto sign = (half & 0x8000u) != 0u ? -1.0f : 1.0f; + const auto exponent = static_cast((half >> 10u) & 0x1fu); + const auto mantissa = static_cast(half & 0x3ffu); + if (exponent == 0) return sign * std::ldexp(mantissa, -24); + if (exponent == 31) return mantissa == 0.0f ? sign * INFINITY : NAN; + return sign * std::ldexp(1024.0f + mantissa, exponent - 25); +} + +std::uint32_t FloatToHalf(float value) { + const auto bits = std::bit_cast(value); + const auto sign = (bits >> 16u) & 0x8000u; + const auto magnitude = bits & 0x7fffffffu; + if (magnitude > 0x7f800000u) return sign | 0x7e00u; + if (magnitude >= 0x477ff000u) return sign | 0x7c00u; + if (magnitude < 0x38800000u) { + const auto scaled = std::fabs(value) * 16777216.0f; + return sign | static_cast(std::nearbyint(scaled)); + } + const auto rounded = magnitude + 0xfffu + ((magnitude >> 13u) & 1u); + return sign | ((rounded - 0x38000000u) >> 13u); +} + +float RoundToHalf(float value) { + return HalfToFloat(FloatToHalf(value)); +} + +std::uint32_t Pack(std::uint32_t low, std::uint32_t high) { + return low | (high << 16u); +} + +std::vector Triangles() { + std::vector vertices; + constexpr std::uint32_t columns = 6; + constexpr std::uint32_t rows = 4; + for (std::uint32_t cell = 0; cell < columns * rows; ++cell) { + const float width = 2.0f / columns; + const float height = 2.0f / rows; + const float x = -1.0f + width * static_cast(cell % columns); + const float y = -1.0f + height * static_cast(cell / columns); + const float skew = 0.07f * static_cast(cell % 5u); + const std::array, 3> corners{{ + {x + (0.05f + skew) * width, y + 0.05f * height}, + {x + 0.93f * width, y + (0.11f + skew) * height}, + {x + (0.37f - skew * 0.5f) * width, y + 0.91f * height}, + }}; + for (std::uint32_t vertex = 0; vertex < 3u; ++vertex) { + vertices.push_back({{corners[vertex][0], corners[vertex][1], 0.5f, 1.0f}, {Pack(LowX[vertex], HighX[vertex]), Pack(LowY[vertex], HighY[vertex]), 0x3c003c00u, 0x3c003c00u}}); + } + } + return vertices; +} + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +ShaderRecompiler::ShaderPixelStageInfo PixelStage(std::uint32_t waveSize, std::uint32_t input0) { + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.wave32 = waveSize == 32u; + pixel.interpolatorCount = 2; + pixel.interpolatorSettings[0] = input0; + pixel.interpolatorSettings[1] = Fp16DefaultHighInput; + pixel.inputAddr = PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::PositionX) | PixelInputBit(PixelInput::PositionY); + pixel.hasPerspectiveCenterVgpr = true; + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + return pixel; +} + +ShaderRecompiler::RecompileResult RecompilePixel(std::span code, std::uint32_t waveSize, std::uint32_t input0, const ShaderRecompiler::SpirvTarget& target, std::uint32_t pushOffset) { + std::vector pixelUserData(8, 0u); + const auto results = BufferDescriptor(Results.data(), ResultDwords * 4u, Width * Height); + std::copy(results.begin(), results.end(), pixelUserData.begin() + 4); + const std::array pixelMemory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, PixelStage(waveSize, input0), std::nullopt, pixelMemory}, + target, + {0, 0, pushOffset, 128 - pushOffset} + }; + fragment.useCache = false; + return ShaderRecompiler::Recompile(fragment); +} + +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { + static const auto triangles = Triangles(); + const auto target = device.Target(); + Pixels.fill(std::byte{0}); + Results.fill(Sentinel); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(triangles.data(), sizeof(Vertex), static_cast(triangles.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + const auto pixelResult = RecompilePixel(PixelCode, waveSize, Fp16Input, target, vertexPush); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, vertexPush} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(triangles.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +float Delta(const std::array& values, std::uint32_t vertex) { + return RoundToHalf(HalfToFloat(values[vertex]) - HalfToFloat(values[0])); +} + +float FirstStep(float delta, float coordinate, float origin) { + return std::fma(delta, coordinate, origin); +} + +double SecondStep(float delta, float coordinate, float firstStep) { + const double product = static_cast(delta) * static_cast(coordinate); + const double sum = product + firstStep; + const double productPart = sum - firstStep; + const double error = (product - productPart) + (firstStep - (sum - productPart)); + if (error == 0.0 || (std::bit_cast(sum) & 1u) != 0u) return sum; + return std::nextafter(sum, error > 0.0 ? INFINITY : -INFINITY); +} + +double Saturate(double value) { + return value > 0.0 ? std::min(value, 1.0) : 0.0; +} + +std::uint32_t HalfOf(double value) { + if (value == 0.0) return std::signbit(value) ? 0x8000u : 0u; + const int step = std::max(std::ilogb(value) - 10, -24); + const double rounded = std::ldexp(std::nearbyint(std::ldexp(value, -step)), step); + if (std::fabs(rounded) > 65504.0) return std::signbit(value) ? 0xfc00u : 0x7c00u; + if (rounded == 0.0) return std::signbit(value) ? 0x8000u : 0u; + return FloatToHalf(static_cast(rounded)); +} + +void Check(std::uint32_t waveSize) { + const auto what = "16-bit interpolation wave" + std::to_string(waveSize); + std::uint32_t covered = 0; + for (std::uint32_t pixel = 0; pixel < Width * Height; ++pixel) { + const auto* result = &Results[pixel * ResultDwords]; + if (std::to_integer(Pixels[pixel * 4u]) != 255u) { + Require(result[0] == Sentinel, what + ": uncovered pixel " + std::to_string(pixel) + " stored a result"); + continue; + } + ++covered; + const float i = std::bit_cast(result[0]); + const float j = std::bit_cast(result[1]); + const auto lowX = FirstStep(Delta(LowX, 1), i, HalfToFloat(LowX[0])); + const auto highX = FirstStep(Delta(HighX, 1), i, HalfToFloat(HighX[0])); + const auto highY = FirstStep(Delta(HighY, 1), i, HalfToFloat(0x4500u)); + const std::array expected{ + result[0], + result[1], + std::bit_cast(lowX), + 0xabcd0000u | HalfOf(SecondStep(Delta(LowX, 2), j, lowX)), + 0x12340000u | HalfOf(SecondStep(Delta(HighX, 2), j, highX)), + std::bit_cast(highY), + HalfOf(SecondStep(Delta(HighY, 2), j, highY)), + HalfOf(Saturate(SecondStep(Delta(LowX, 2), j, lowX))), + std::bit_cast(FirstStep(0.0f, i, 1.0f)), + std::bit_cast(FirstStep(0.0f, i, 0.0f)), + std::bit_cast(FirstStep(Delta(LowX, 1), -i, HalfToFloat(LowX[0]))), + std::bit_cast(FirstStep(Delta(LowX, 1), i, -1.0f)), + }; + for (std::uint32_t dword = 2; dword < ResultDwords; ++dword) { + char text[160]; + std::snprintf(text, sizeof(text), ": pixel %u dword %u is 0x%08x, expected 0x%08x (I %a, J %a)", pixel, dword, result[dword], expected[dword], static_cast(i), static_cast(j)); + Require(result[dword] == expected[dword], what + text); + } + } + Require(covered > 1000, what + ": the triangles covered only " + std::to_string(covered) + " pixels"); +} + +ShaderRecompiler::SpirvTarget OfflineTarget(bool barycentric) { + ShaderRecompiler::SpirvTarget target{}; + target.vulkanVersion = 0x00401000u; + target.spirvVersion = 0x00010300u; + target.subgroupSize = 64; + target.fragmentShaderBarycentricEnabled = barycentric; + return target; +} + +void ExpectFailure(std::span code, std::uint32_t input0, bool barycentric, const char* expected, const char* message) { + try { + static_cast(RecompilePixel(code, 64u, input0, OfflineTarget(barycentric), 0u)); + } catch (const std::exception& error) { + Require(std::string(error.what()).find(expected) != std::string::npos, std::string(message) + ": " + error.what()); + return; + } + throw std::runtime_error(message); +} + +void ExpectRejected(std::uint32_t word0, std::uint32_t word1, const char* expected, const char* message) { + alignas(256) const std::array code{word0, word1, 0xbf810000u}; + ExpectFailure(code, Fp16Input, true, expected, message); +} + +} + +int main() { + try { + ExpectFailure(PixelCode, Fp16Input, false, "requires fragmentShaderBarycentric", "16-bit interpolation was accepted without barycentrics"); + ExpectFailure(PixelCode, 0x03000000u, true, "without FP16_INTERP_MODE", "16-bit interpolation of a 32-bit input was accepted"); + ExpectFailure(PixelCode, 0x03080420u, true, "passes its vertices through unchanged", "16-bit interpolation of a pass-through input was accepted"); + ExpectRejected(0xd75a001bu, 0x0c6a0200u, "interpolation modifiers", "v_interp_p2_f16 accepted an output modifier"); + ExpectRejected(0xd75a081bu, 0x046a0200u, "interpolation modifiers", "v_interp_p2_f16 accepted op_sel"); + ExpectRejected(0xd742001au, 0x20020000u, "interpolation modifiers", "v_interp_p1ll_f16 accepted a negated attribute"); + ExpectRejected(0xd742001au, 0x00000000u, "not a vector register", "v_interp_p1ll_f16 accepted a scalar I coordinate"); + ExpectRejected(0xd743001du, 0x003e0140u, "not a vector register", "v_interp_p1lv_f16 accepted a scalar P0"); + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->Target().fragmentShaderBarycentricEnabled) { + std::puts("skipped, the device has no fragmentShaderBarycentric"); + return VulkanTestSkipped; + } + for (const auto waveSize : {64u, 32u}) { + Draw(*device, waveSize); + Check(waveSize); + } + std::puts("16-bit interpolation tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 484c345197ee..d223ca69606c 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -262,6 +262,8 @@ enum class IrOpcode : std::uint16_t { FPSqrt64, FPTrigPreop64, FPDot2F32F16, + FPInterpolateF32, + FPInterpolateF16, ConvertF32F64, ConvertF64F32, ConvertF64S32, @@ -457,6 +459,7 @@ enum class IrOpcode : std::uint16_t { ImageAtomicCmpSwap64, GetAttribute, GetInterpolationParameter, + GetInterpolationParameterF16, SetAttribute, ControlNop, Waitcnt, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index 5919c08e60f4..a41d9fe9f27b 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -303,6 +303,8 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("FPSqrt64", U64, U64), makeMeta("FPTrigPreop64", U64, U64, U32), makeMeta("FPDot2F32F16", U32, U32, U32, U32), + makeMeta("FPInterpolateF32", F32, F32, F32, F32), + makeMeta("FPInterpolateF16", F32, F32, F32, F32), makeMeta("ConvertF32F64", F32, U64), makeMeta("ConvertF64F32", U64, F32), makeMeta("ConvertF64S32", U64, U32), @@ -498,6 +500,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("ImageAtomicCmpSwap64", U64, ImageResource, ImageAddress, U64, U64, U1), makeMeta("GetAttribute", U32, U32, U32), makeMeta("GetInterpolationParameter", U32, U32, U32, U32), + makeMeta("GetInterpolationParameterF16", F32, U32, U32, U32, U32), makeMeta("SetAttribute", Void, U32x4, U1), makeMeta("ControlNop", Void), makeMeta("Waitcnt", Void), diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index 45aed32d9812..996730b67e4c 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -201,6 +201,26 @@ struct ShaderPixelInputInfo { return one ? 0x3f800000u : 0u; } + [[nodiscard]] bool InputIsFp16(std::uint32_t input) const { + return input < inputNum && input < 32u && (interpolatorSettings[input] & 0x80000u) != 0u; + } + + [[nodiscard]] bool InputHalfIsDefault(std::uint32_t input, bool high) const { + if (!high) { + return InputIsDefault(input); + } + return input < inputNum && input < 32u && (interpolatorSettings[input] & 0x100000u) != 0u; + } + + [[nodiscard]] std::uint32_t InputHalfDefaultBits(std::uint32_t input, std::uint32_t component, bool high) const { + if (!high) { + return InputDefaultBits(input, component); + } + const auto value = input < 32u ? (interpolatorSettings[input] >> 21u) & 0x3u : 0u; + const bool one = component == 3u ? (value & 0x1u) != 0u : (value & 0x2u) != 0u; + return one ? 0x3f800000u : 0u; + } + [[nodiscard]] bool InputIsLinear(std::uint32_t input, std::uint32_t linearInputs, std::uint32_t perspectiveInputs) const { const auto bit = input < 32u ? 1u << input : 0u; if ((linearInputs & perspectiveInputs & bit) != 0u) { diff --git a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp index c5d5a3aa5435..93f66f1c3c9a 100644 --- a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp +++ b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp @@ -106,6 +106,20 @@ void ValidateValueReferences(const IrProgram& program, ShaderStageInputInfo inpu } break; } + case IrOpcode::GetInterpolationParameterF16: { + const IrValue* input = inst->Argument(0)->Resolve(); + const IrValue* component = inst->Argument(1)->Resolve(); + const IrValue* mode = inst->Argument(2)->Resolve(); + const IrValue* high = inst->Argument(3)->Resolve(); + if (program.Resources().stage != IrShaderStage::Pixel || !input->HasImmediate() || input->Type() != IrType::U32 || !component->HasImmediate() || component->Type() != IrType::U32 || + !mode->HasImmediate() || mode->Type() != IrType::U32 || !high->HasImmediate() || high->Type() != IrType::U32) { + return Fail("16-bit interpolation parameter reference is invalid"); + } + if (input->ImmediateU32() >= inputInfo.pixel->inputNum || component->ImmediateU32() >= 4u || mode->ImmediateU32() >= 3u || high->ImmediateU32() >= 2u) { + return Fail("16-bit interpolation parameter reference is out of range"); + } + break; + } case IrOpcode::GetBuiltin: { const IrValue* kindValue = inst->Argument(0)->Resolve(); const IrValue* componentValue = inst->Argument(1)->Resolve(); @@ -219,6 +233,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi std::array read {}; std::array perVertex {}; std::array interpolated {}; + std::array halfRead {}; for (const auto& block : program.Blocks()) { for (const IrValue* inst : block->Instructions()) { if (inst->Opcode() == IrOpcode::GetAttribute) { @@ -233,6 +248,14 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi const auto mode = inst->Argument(2)->Resolve()->ImmediateU32(); read[input] = true; perVertex[input] = perVertex[input] || mode < 2u || !IsPixelParameterFlat(*pixel, input); + } else if (inst->Opcode() == IrOpcode::GetInterpolationParameterF16) { + const auto input = inst->Argument(0)->Resolve()->ImmediateU32(); + const bool high = inst->Argument(3)->Resolve()->ImmediateU32() != 0u; + if (!pixel->InputHalfIsDefault(input, high)) { + read[input] = true; + perVertex[input] = true; + halfRead[input] = true; + } } } } @@ -248,7 +271,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi slotInterpolation.fill(unassigned); std::array slotPerVertex {}; for (std::uint32_t input = 0; input < pixel->inputNum; input++) { - if (!read[input] || pixel->InputIsDefault(input)) { + if (!read[input] || (pixel->InputIsDefault(input) && !halfRead[input])) { continue; } const auto slot = pixel->InputSlot(input); @@ -262,7 +285,7 @@ void CollectPixelInputs(const IrProgram& program, const ShaderPixelInputInfo* pi bool smooth = false; bool noPerspective = false; for (std::uint32_t input = 0; input < pixel->inputNum; input++) { - if (!read[input] || pixel->InputIsDefault(input)) { + if (!read[input] || (pixel->InputIsDefault(input) && !halfRead[input])) { continue; } const bool vertexInput = slotPerVertex[pixel->InputSlot(input)]; diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp index bd54285a7239..c71a14a0f3af 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp @@ -1009,6 +1009,9 @@ enum class RdnaOpcode : std::uint16_t { VInterpP1F32, VInterpP2F32, VInterpMovF32, + VInterpP1llF16, + VInterpP1lvF16, + VInterpP2F16, SWaitcntDepctr, SSendmsg, SGetregB32, diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp index 187c31d13d3f..3e72a7ad1516 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp @@ -623,6 +623,9 @@ bool IsVectorAluOpcode(RdnaOpcode opcode) { case RdnaOpcode::VInterpP1F32: case RdnaOpcode::VInterpP2F32: case RdnaOpcode::VInterpMovF32: + case RdnaOpcode::VInterpP1llF16: + case RdnaOpcode::VInterpP1lvF16: + case RdnaOpcode::VInterpP2F16: return true; default: return false; diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp index 880f083830c8..326343f03877 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp @@ -493,6 +493,7 @@ constexpr VectorOpcodeInfo vop3Opcodes[] = { {0x357u, RdnaOpcode::VMed3F16}, {0x358u, RdnaOpcode::VMed3I16}, {0x359u, RdnaOpcode::VMed3U16}, + {0x35au, RdnaOpcode::VInterpP2F16}, {0x35fu, RdnaOpcode::VDivFixupF16}, {0x15au, RdnaOpcode::VSadU8}, {0x15bu, RdnaOpcode::VSadHiU8}, @@ -534,6 +535,8 @@ constexpr VectorOpcodeInfo vop3Opcodes[] = { {0x311u, RdnaOpcode::VPackB32F16}, {0x314u, RdnaOpcode::VLshlrevB16}, {0x319u, RdnaOpcode::VSubrevI32}, + {0x342u, RdnaOpcode::VInterpP1llF16}, + {0x343u, RdnaOpcode::VInterpP1lvF16}, {0x344u, RdnaOpcode::VPermB32}, {0x14du, RdnaOpcode::VLerpU8}, {0x345u, RdnaOpcode::VXadU32}, @@ -2073,6 +2076,43 @@ void checkNativeVop3Modifiers(RdnaOpcode opcode, bool permlane, bool carryInOut, } } +bool isVop3F16InterpolationOpcode(RdnaOpcode opcode) { + return opcode == RdnaOpcode::VInterpP1llF16 || opcode == RdnaOpcode::VInterpP1lvF16 || opcode == RdnaOpcode::VInterpP2F16; +} + +void decodeVop3F16Interpolation(RdnaInstruction& instruction, std::uint32_t vdst, std::uint32_t src0, std::uint32_t src1, std::uint32_t src2, std::uint32_t abs, std::uint32_t opSel, std::uint32_t clamp, std::uint32_t omod, std::uint32_t neg) { + const bool hasSource2 = instruction.op != RdnaOpcode::VInterpP1llF16; + const bool f16Result = instruction.op == RdnaOpcode::VInterpP2F16; + if ((abs & 0x1u) != 0u || (neg & 0x1u) != 0u || opSel != 0u || (f16Result && omod != 0u) || (!hasSource2 && ((abs | neg) & 0x4u) != 0u)) { + throw std::invalid_argument("VOP3 interpolation modifiers are not implemented"); + } + if (src1 < 256u || (hasSource2 && src2 < 256u)) { + throw std::invalid_argument("VOP3 interpolation source is not a vector register"); + } + const bool high = ((src0 >> 8u) & 0x1u) != 0u; + instruction.destination = DecodeRdnaVectorGpr(vdst); + instruction.destination.clamp = clamp != 0u; + instruction.destination.omod = omod; + instruction.source0 = DecodeRdnaVectorGpr(src1 - 256u); + instruction.source0.absolute = (abs & 0x2u) != 0u; + instruction.source0.negate = (neg & 0x2u) != 0u; + instruction.source1.kind = RdnaOperandKind::IntegerInlineConstant; + instruction.source1.value = src0 & 0x3fu; + instruction.source1.signedVal = static_cast(instruction.source1.value); + instruction.source1.opSel = high; + instruction.source2.kind = RdnaOperandKind::IntegerInlineConstant; + instruction.source2.value = (src0 >> 6u) & 0x3u; + instruction.source2.signedVal = static_cast(instruction.source2.value); + instruction.sourceCount = 3; + if (hasSource2) { + instruction.source3 = DecodeRdnaVectorGpr(src2 - 256u); + instruction.source3.absolute = (abs & 0x4u) != 0u; + instruction.source3.negate = (neg & 0x4u) != 0u; + instruction.source3.opSel = high && !f16Result; + instruction.sourceCount = 4; + } +} + void applyNativeVop3SourceModifiers(RdnaInstruction& instruction, std::uint32_t abs, std::uint32_t neg) { RdnaOperand* sources[] = {&instruction.source0, &instruction.source1, &instruction.source2}; for (std::uint32_t i = 0; i < instruction.sourceCount && i < 3u; ++i) { @@ -2233,6 +2273,11 @@ RdnaInstruction DecodeRdnaVop3(std::uint32_t programCounter, std::span(fractionBits))), ConstantI32(state, static_cast(smallestStep))); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto magicExponent = Binary(state, spv::OpShiftLeftLogical, u32, Unary(state, spv::OpBitcast, u32, Binary(state, spv::OpIAdd, TypeI32(state), step, ConstantI32(state, 52))), ConstantU32(state, 20u)); + const auto magic = ToF64(state, Join(state, {ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, u32, sign, magicExponent)})); + const auto rounded = Split(state, FromF64(state, Exact(state, spv::OpFSub, Exact(state, spv::OpFAdd, value, magic), magic))); + return ToF64(state, Join(state, {rounded.low, Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpBitwiseAnd, u32, rounded.high, ConstantU32(state, 0x7fffffffu)), sign)})); +} + +std::uint32_t F32BitsOfRounded(SpirvEmitterState& state, std::uint32_t value) { + const auto u32 = TypeU32(state); + const auto bits = Split(state, FromF64(state, value)); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto exponent = ExponentField(state, bits.high); + const auto fraction = Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftLeftLogical, u32, Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x000fffffu)), ConstantU32(state, 3u)), Binary(state, spv::OpShiftRightLogical, u32, bits.low, ConstantU32(state, 29u))); + const auto normal = Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftLeftLogical, u32, Binary(state, spv::OpISub, u32, exponent, ConstantU32(state, 896u)), ConstantU32(state, 23u)), fraction); + const auto magnitude = ToF64(state, Join(state, {bits.low, Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x7fffffffu))})); + const auto subnormal = Unary(state, spv::OpConvertFToU, u32, Exact(state, spv::OpFMul, magnitude, ConstantF64(state, 0x4940000000000000ull))); + auto result = Select(state, u32, Binary(state, spv::OpULessThan, TypeBool(state), exponent, ConstantU32(state, 897u)), subnormal, normal); + result = Select(state, u32, Binary(state, spv::OpUGreaterThanEqual, TypeBool(state), exponent, ConstantU32(state, 1151u)), ConstantU32(state, 0x7f800000u), result); + return Binary(state, spv::OpBitwiseOr, u32, result, sign); +} + +std::uint32_t NonFinite32(SpirvEmitterState& state, std::uint32_t bits) { + const auto exponent = Binary(state, spv::OpBitwiseAnd, TypeU32(state), bits, ConstantU32(state, 0x7f800000u)); + return Binary(state, spv::OpIEqual, TypeBool(state), exponent, ConstantU32(state, 0x7f800000u)); +} + +std::uint32_t Nan32(SpirvEmitterState& state, std::uint32_t bits) { + return Binary(state, spv::OpUGreaterThan, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x7f800000u)); +} + +std::uint32_t Interpolate(SpirvEmitterState& state, std::uint32_t delta, std::uint32_t coordinate, std::uint32_t base, bool halfResult) { + const auto u32 = TypeU32(state); + const auto product = Exact(state, spv::OpFMul, FromF32Exact(state, delta), FromF32Exact(state, coordinate)); + const auto sum = AddRoundToOdd(state, product, FromF32Exact(state, base)); + auto rounded = halfResult ? RoundToPrecision(state, sum, 10u, 999u) : RoundToPrecision(state, sum, 23u, 874u); + if (halfResult) { + const auto bits = Split(state, FromF64(state, rounded)); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x80000000u)); + const auto overflow = Binary(state, spv::OpUGreaterThan, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits.high, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x40f00000u)); + rounded = ToF64(state, Join(state, SelectBits(state, overflow, {ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, u32, sign, ConstantU32(state, 0x40f00000u))}, bits))); + } + const auto finite = F32BitsOfRounded(state, rounded); + const auto inputs = std::array{Unary(state, spv::OpBitcast, u32, delta), Unary(state, spv::OpBitcast, u32, coordinate), Unary(state, spv::OpBitcast, u32, base)}; + const auto unbounded = Split(state, FromF64(state, Exact(state, spv::OpFAdd, product, FromF32Exact(state, base)))); + auto special = Select(state, u32, Classify(state, Join(state, unbounded)).nan, ConstantU32(state, 0xffc00000u), Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpBitwiseAnd, u32, unbounded.high, ConstantU32(state, 0x80000000u)), ConstantU32(state, 0x7f800000u))); + auto nonFinite = NonFinite32(state, inputs[0]); + for (const auto input : {inputs[1], inputs[2]}) nonFinite = Binary(state, spv::OpLogicalOr, TypeBool(state), nonFinite, NonFinite32(state, input)); + const auto zero = [&](std::uint32_t bits) { return Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0u)); }; + const auto infinite = [&](std::uint32_t bits) { return Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffffffu)), ConstantU32(state, 0x7f800000u)); }; + const auto invalidProduct = Binary(state, spv::OpLogicalOr, TypeBool(state), Binary(state, spv::OpLogicalAnd, TypeBool(state), infinite(inputs[0]), zero(inputs[1])), Binary(state, spv::OpLogicalAnd, TypeBool(state), zero(inputs[0]), infinite(inputs[1]))); + const auto quiet = [&](std::uint32_t bits) { return Binary(state, spv::OpBitwiseOr, u32, bits, ConstantU32(state, 0x00400000u)); }; + special = Select(state, u32, Binary(state, spv::OpLogicalAnd, TypeBool(state), Nan32(state, inputs[2]), Unary(state, spv::OpLogicalNot, TypeBool(state), invalidProduct)), quiet(inputs[2]), special); + special = Select(state, u32, Nan32(state, inputs[1]), quiet(inputs[1]), special); + special = Select(state, u32, Nan32(state, inputs[0]), quiet(inputs[0]), special); + return Unary(state, spv::OpBitcast, TypeF32(state), Select(state, u32, nonFinite, special, finite)); +} + std::uint32_t FusedMultiplyAdd(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, std::uint32_t scale) { const auto a = Classify(state, arg0); const auto b = Classify(state, arg1); @@ -532,6 +608,14 @@ std::uint32_t EmitFPDot2F32F16(SpirvEmitterState& state, std::uint32_t arg0, std return pick(aLow.nan, quietHalf(aLow), result); } +std::uint32_t EmitFPInterpolateF32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2) { + return Interpolate(state, arg0, arg1, arg2, false); +} + +std::uint32_t EmitFPInterpolateF16(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2) { + return Interpolate(state, arg0, arg1, arg2, true); +} + std::uint32_t EmitConvertF32F64(SpirvEmitterState& state, std::uint32_t arg0) { const auto converted = Unary(state, spv::OpBitcast, TypeU32(state), Unary(state, spv::OpFConvert, TypeF32(state), ToF64(state, arg0))); const auto cls = Classify(state, arg0); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index 0288e18748a4..91b74f5ece44 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -37,6 +37,7 @@ bool LaneSource(const IrProgram& program, const IrValue& value) { case IrOpcode::LaneId: case IrOpcode::GetAttribute: case IrOpcode::GetInterpolationParameter: + case IrOpcode::GetInterpolationParameterF16: case IrOpcode::GetTessellationAttribute: case IrOpcode::DppMoveU32: case IrOpcode::DppUpdateU32: diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 2a6b3e1bc67a..c08132cb32c9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -476,6 +476,8 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::FPSqrt64: return Invoke(EmitFPSqrt64, ctx, inst); case IrOpcode::FPTrigPreop64: return Invoke(EmitFPTrigPreop64, ctx, inst); case IrOpcode::FPDot2F32F16: return Invoke(EmitFPDot2F32F16, ctx, inst); + case IrOpcode::FPInterpolateF32: return Invoke(EmitFPInterpolateF32, ctx, inst); + case IrOpcode::FPInterpolateF16: return Invoke(EmitFPInterpolateF16, ctx, inst); case IrOpcode::ConvertF32F64: return Invoke(EmitConvertF32F64, ctx, inst); case IrOpcode::ConvertF64F32: return Invoke(EmitConvertF64F32, ctx, inst); case IrOpcode::ConvertF64S32: return Invoke(EmitConvertF64S32, ctx, inst); @@ -672,6 +674,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::ImageAtomicCmpSwap64: return Invoke(EmitImageAtomicCmpSwap64, ctx, inst); case IrOpcode::GetAttribute: return Invoke(EmitGetAttribute, ctx, inst); case IrOpcode::GetInterpolationParameter: return Invoke(EmitGetInterpolationParameter, ctx, inst); + case IrOpcode::GetInterpolationParameterF16: return Invoke(EmitGetInterpolationParameterF16, ctx, inst); case IrOpcode::SetAttribute: return Invoke(EmitSetAttribute, ctx, inst); case IrOpcode::ControlNop: return Invoke(EmitControlNop, ctx, inst); case IrOpcode::Waitcnt: return Invoke(EmitWaitcnt, ctx, inst); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp index ff0c2def5924..2c3eb07f548a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvInputOutput.cpp @@ -28,7 +28,7 @@ namespace ShaderRecompiler if (input >= PixelParameterLimit || input >= info.inputNum) { FailEmit("pixel interpolator index is out of range"); } - if (info.InputIsDefault(input)) { + if (info.InputIsDefault(input) && (!info.InputIsFp16(input) || info.InputHalfIsDefault(input, true))) { FailEmit("pixel input " + std::to_string(input) + " reads a default value, not a parameter"); } return info.InputSlot(input); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 3e4478aa9edd..4fb0e39aaf9d 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -200,6 +200,38 @@ std::uint32_t EmitInterpolationParameterValue(SpirvEmitterState& state, std::uin return bits; } +std::uint32_t EmitInterpolationParameterF16Value(SpirvEmitterState& state, std::uint32_t attr, std::uint32_t chan, std::uint32_t mode, bool high) { + const auto& pixel = *state.inputInfo.pixel; + if (pixel.InputHalfIsDefault(attr, high)) { + return ConstantF32(state, mode == 2u ? pixel.InputHalfDefaultBits(attr, chan & 3u, high) : 0u); + } + const auto* input = SpirvInputBindingForParameter(state, attr); + if (input == nullptr || !input->perVertex) { + throw std::runtime_error("16-bit interpolation parameter is not read per vertex"); + } + const auto loadVertex = [&](std::uint32_t vertex) { + const auto pointer = state.module.AllocateId(); + const auto value = state.module.AllocateId(); + auto bits = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeF32(state)), pointer, input->variableId, ConstantU32(state, vertex), ConstantU32(state, chan & 3u)); + state.module.AddFunction(spv::OpLoad, TypeF32(state), value, pointer); + state.module.AddFunction(spv::OpBitcast, TypeU32(state), bits, value); + if (high) { + const auto shifted = state.module.AllocateId(); + state.module.AddFunction(spv::OpShiftRightLogical, TypeU32(state), shifted, bits, ConstantU32(state, 16u)); + bits = shifted; + } + return EmitF16BitsToF32(state, bits); + }; + const auto value = loadVertex((mode + 1u) % 3u); + if (mode == 2u) { + return value; + } + const auto delta = state.module.AllocateId(); + state.module.AddFunction(spv::OpFSub, TypeF32(state), delta, value, loadVertex(0u)); + return EmitF16BitsToF32(state, EmitConvertF16F32(state, delta)); +} + std::uint32_t MrtOutputMode(const SpirvEmitterState& state, const ExportInfo& exp) { if (state.program.Resources().stage != IrShaderStage::Pixel || exp.kind != ExportTargetKind::Mrt) { return 0u; @@ -1085,6 +1117,10 @@ std::uint32_t EmitGetInterpolationParameter(SpirvValueEmitContext& ctx, const Ir return EmitInterpolationParameterValue(ctx.state, inst.Argument(0)->ImmediateU32(), inst.Argument(1)->ImmediateU32(), inst.Argument(2)->ImmediateU32()); } +std::uint32_t EmitGetInterpolationParameterF16(SpirvValueEmitContext& ctx, const IrValue& inst) { + return EmitInterpolationParameterF16Value(ctx.state, inst.Argument(0)->ImmediateU32(), inst.Argument(1)->ImmediateU32(), inst.Argument(2)->ImmediateU32(), inst.Argument(3)->ImmediateU32() != 0u); +} + void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; const auto& exp = ctx.Export(inst); diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 4a1cbbae7c62..be6f8ffc7951 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -295,6 +295,9 @@ class TranslationContext { void vInterpP1F32(const RdnaInstruction& inst); void vInterpP2F32(const RdnaInstruction& inst); void vInterpMovF32(const RdnaInstruction& inst); + IrF32 interpolationParameterF16(const RdnaInstruction& inst, std::uint32_t mode); + void vInterpP1F16(const RdnaInstruction& inst); + void vInterpP2F16(const RdnaInstruction& inst); void eXP(const RdnaInstruction& inst); bool emitScalar(const RdnaInstruction& inst); bool emitVector(const RdnaInstruction& inst); diff --git a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp index 32a84d1b4665..2576b9770217 100644 --- a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp +++ b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp @@ -93,6 +93,27 @@ void TranslationContext::vInterpMovF32(const RdnaInstruction& inst) { writeOperand(inst.destination, &value); } +IrF32 TranslationContext::interpolationParameterF16(const RdnaInstruction& inst, std::uint32_t mode) { + if (!fragmentShaderBarycentricEnabled) { + throw std::runtime_error("16-bit interpolation of pixel input " + std::to_string(inst.source1.value) + " requires fragmentShaderBarycentric"); + } + if (pixelInput == nullptr || !pixelInput->InputIsFp16(inst.source1.value)) { + throw std::runtime_error("pixel input " + std::to_string(inst.source1.value) + " is read with 16-bit interpolation without FP16_INTERP_MODE"); + } + return IrF32(ir.Emit(IrOpcode::GetInterpolationParameterF16, IrType::F32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), &ir.Constant(mode), &ir.Constant(inst.source1.opSel ? 1u : 0u)})); +} + +void TranslationContext::vInterpP1F16(const RdnaInstruction& inst) { + const IrF32 delta = interpolationParameterF16(inst, 0u); + const IrF32 origin = inst.op == RdnaOpcode::VInterpP1lvF16 ? readF16AsF32(inst.source3) : interpolationParameterF16(inst, 2u); + writeOperand(inst.destination, &ir.Emit(IrOpcode::FPInterpolateF32, IrType::F32, {&delta.Value(), readOperand(inst.source0, IrType::F32), &origin.Value()})); +} + +void TranslationContext::vInterpP2F16(const RdnaInstruction& inst) { + const IrF32 delta = interpolationParameterF16(inst, 1u); + writeF16(inst.destination, IrF32(ir.Emit(IrOpcode::FPInterpolateF16, IrType::F32, {&delta.Value(), readOperand(inst.source0, IrType::F32), readOperand(inst.source3, IrType::F32)}))); +} + void TranslationContext::eXP(const RdnaInstruction& inst) { std::uint32_t index = 0u; if (exportTargetKindFromTarget(inst.exportTarget, index) == ExportTargetKind::Unknown) { @@ -109,7 +130,8 @@ void TranslationContext::eXP(const RdnaInstruction& inst) { } bool TranslationContext::emitInterpolation(const RdnaInstruction& inst) { - if ((inst.op == RdnaOpcode::VInterpP1F32 || inst.op == RdnaOpcode::VInterpP2F32) && pixelInput != nullptr && pixelInput->InputIsCustom(inst.source1.value)) { + const bool interpolates = inst.op == RdnaOpcode::VInterpP1F32 || inst.op == RdnaOpcode::VInterpP2F32 || inst.op == RdnaOpcode::VInterpP1llF16 || inst.op == RdnaOpcode::VInterpP1lvF16 || inst.op == RdnaOpcode::VInterpP2F16; + if (interpolates && pixelInput != nullptr && pixelInput->InputIsCustom(inst.source1.value)) { throw std::runtime_error("pixel input " + std::to_string(inst.source1.value) + " passes its vertices through unchanged but is read with v_interp_p1/p2"); } switch (inst.op) { @@ -122,6 +144,13 @@ bool TranslationContext::emitInterpolation(const RdnaInstruction& inst) { case RdnaOpcode::VInterpMovF32: vInterpMovF32(inst); return true; + case RdnaOpcode::VInterpP1llF16: + case RdnaOpcode::VInterpP1lvF16: + vInterpP1F16(inst); + return true; + case RdnaOpcode::VInterpP2F16: + vInterpP2F16(inst); + return true; default: return false; } diff --git a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp index e2d3e4322ad8..c77fff4adf53 100644 --- a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp +++ b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp @@ -38,7 +38,7 @@ void TranslationContext::TranslateInstruction(const RdnaInstruction& decoded) { case RdnaInstructionFamily::VOP3: case RdnaInstructionFamily::VOP3P: case RdnaInstructionFamily::VOPC: - translated = emitVector(instruction); + translated = emitVector(instruction) || emitInterpolation(instruction); break; case RdnaInstructionFamily::SMEM: case RdnaInstructionFamily::MUBUF: diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 25b90199fd41..400ee0317dd9 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -274,6 +274,7 @@ Throughout the project, every function at every stage either **does exactly what - Without fetch-inactive, [`v_permlane16_b32` and `v_permlanex16_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take a source lane that the host subgroup does not hold as inactive and keep the old value (0 with bound_ctrl); the console reads that lane's VGPR. - On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1); other such reads throw. - [Shader recompilation](../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../core/shader/recompiler) was written completely independently from [libs/prx](../../core/libs/prx). +- [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. Lavapipe has no barycentrics, so the raster execution test is skipped in CI. The standalone arithmetic execution test checks six published gfx1035 rows without barycentrics; it does not verify SPI coefficients or guest floating-point modes. - [v_fma_f64](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) is fused exactly only for normal inputs: with a subnormal input it is a separate multiply and add, which rounds twice. - [SINT storage images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. - [`StoreTexel`](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944.