fix(libSceAgcDriver): export FP16_ABGR into UINT color targets

core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp

@@ -1010,12 +1010,22 @@ struct DecodedColorFormat {
     ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None;
 };
 
+bool UintPacked(ShaderRecompiler::ColorExportPacking packing) {
+    return packing == ShaderRecompiler::ColorExportPacking::Uint || packing == ShaderRecompiler::ColorExportPacking::UintClamp8 || packing == ShaderRecompiler::ColorExportPacking::UintClamp16;
+}
+
 const char* PackedColorTargetName(ShaderRecompiler::ColorExportPacking packing) {
     if (packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4) return "4_4_4_4 unorm";
-    if (packing == ShaderRecompiler::ColorExportPacking::Uint) return "unsigned integer";
+    if (UintPacked(packing)) return "unsigned integer";
     return packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "11_11_10 float" : "10_11_11 unorm";
 }
 
+ShaderRecompiler::ColorExportPacking HalfExportPacking(VkFormat format) {
+    if (format == VK_FORMAT_R8_UINT || format == VK_FORMAT_R8G8_UINT || format == VK_FORMAT_R8G8B8A8_UINT) return ShaderRecompiler::ColorExportPacking::UintClamp8;
+    if (format == VK_FORMAT_R16_UINT || format == VK_FORMAT_R16G16_UINT || format == VK_FORMAT_R16G16B16A16_UINT) return ShaderRecompiler::ColorExportPacking::UintClamp16;
+    return ShaderRecompiler::ColorExportPacking::Uint;
+}
+
 bool WordPacked(ShaderRecompiler::ColorExportPacking packing) {
     return packing == ShaderRecompiler::ColorExportPacking::Unorm10_11_11 || packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 || packing == ShaderRecompiler::ColorExportPacking::Unorm4_4_4_4;
 }
@@ -1829,7 +1839,7 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
                 if ((weighted(state.colorBlendOp) && (constantColor(state.srcColorBlendFactor) || constantColor(state.dstColorBlendFactor))) || (weighted(state.alphaBlendOp) && (constantColor(state.srcAlphaBlendFactor) || constantColor(state.dstAlphaBlendFactor)))) throw std::runtime_error("AGC graphics: blending into an 8_8 color target with the alternate component order with a constant color factor, which Vulkan applies to the second channel with the green blend constant instead of the alpha one, is not implemented");
             }
         }
-        if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) state.blendEnable = VK_FALSE;
+        if (UintPacked(color.packing)) state.blendEnable = VK_FALSE;
         if (WordPacked(color.packing)) {
             const std::string packed = std::string(color.packing == ShaderRecompiler::ColorExportPacking::Float11_11_10 ? "an " : "a ") + PackedColorTargetName(color.packing);
             const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
@@ -1857,7 +1867,8 @@ State DecodeState(const QueueState& queue, bool vertexWorkgroup) {
         }
         if (color.packing == ShaderRecompiler::ColorExportPacking::Uint) {
             const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu;
-            if (slotExport != 1 && slotExport != 2 && slotExport != 3 && slotExport != 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into an unsigned integer target is unsupported");
+            if (slotExport == 4) color.packing = HalfExportPacking(color.format);
+            else if (slotExport != 1 && slotExport != 2 && slotExport != 3 && slotExport != 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into an unsigned integer target is unsupported");
         }
         if (state.blendEnable && !reversed && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) {
             const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; };

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -10635,7 +10635,11 @@ void uintTargetTests() {
     Require(uint16.color.packing == ColorExportPacking::None && uint16.color.uintExport, "a UINT16_ABGR export into a UINT target was retyped twice");
     auto halves = queue;
     halves.context[0x1c5] = 4u;
-    expectFailure([&] { static_cast<void>(AgcDriver::Graphics::DecodeState(halves)); }, "color export format 4 into an unsigned integer target is unsupported");
+    for (const auto& [format, vkFormat, packing] : std::array<std::tuple<std::uint32_t, VkFormat, ColorExportPacking>, 5>{{{4u, VK_FORMAT_R32_UINT, ColorExportPacking::Uint}, {1u, VK_FORMAT_R8_UINT, ColorExportPacking::UintClamp8}, {10u, VK_FORMAT_R8G8B8A8_UINT, ColorExportPacking::UintClamp8}, {2u, VK_FORMAT_R16_UINT, ColorExportPacking::UintClamp16}, {12u, VK_FORMAT_R16G16B16A16_UINT, ColorExportPacking::UintClamp16}}}) {
+        halves.context[0x31c] = (queue.context.at(0x31c) & ~0x1f7cu) | (format << 2u) | (4u << 8u);
+        const auto half = AgcDriver::Graphics::DecodeState(halves);
+        Require(half.color.format == vkFormat && half.color.packing == packing && !half.color.uintExport && AgcDriver::Graphics::ExportPackings(half)[0] == packing && !half.blends.at(0).blendEnable, "an FP16_ABGR export into a UINT target did not get the packing that clamps to its channel width");
+    }
     auto unorm = queue;
     unorm.context[0x31c] = (unorm.context.at(0x31c) & ~0x1f7cu) | (4u << 2u) | (7u << 8u);
     Require(AgcDriver::Graphics::DecodeState(unorm).color.packing == ColorExportPacking::None, "a 32-bit float target got the unsigned integer export packing");
@@ -11118,7 +11122,7 @@ void float11_11_10TargetTests() {
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &packed, 0}}};
     Require(AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false) == std::set<std::uint32_t>{0u}, "the packed float export did not reach attachment 0");
     auto invalid = request;
-    invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(5);
+    invalid.context.pixel->targetExportPacking[0] = static_cast<ColorExportPacking>(7);
     expectFailure([&] { static_cast<void>(ShaderRecompiler::Recompile(invalid)); }, "invalid prepared fragment export packing");
 }
 

core/libs/prx/libSceAgcDriver/tests/execution/UintColorTarget.cpp

@@ -20,6 +20,7 @@
 #else
 #include <sys/mman.h>
 #endif
+#include <algorithm>
 #include <array>
 #include <bit>
 #include <cstdint>
@@ -58,6 +59,9 @@ constexpr TargetFormat R32{"R32_UINT", 0x410u, VK_FORMAT_R32_UINT, 1, 4};
 constexpr TargetFormat R32G32{"R32G32_UINT", 0x42cu, VK_FORMAT_R32G32_UINT, 2, 4};
 constexpr TargetFormat R32G32B32A32{"R32G32B32A32_UINT", 0x438u, VK_FORMAT_R32G32B32A32_UINT, 4, 4};
 constexpr TargetFormat R8{"R8_UINT", 0x404u, VK_FORMAT_R8_UINT, 1, 1};
+constexpr TargetFormat R16{"R16_UINT", 0x408u, VK_FORMAT_R16_UINT, 1, 2};
+constexpr TargetFormat R8G8B8A8{"R8G8B8A8_UINT", 0x428u, VK_FORMAT_R8G8B8A8_UINT, 4, 1};
+constexpr TargetFormat R16G16B16A16{"R16G16B16A16_UINT", 0x430u, VK_FORMAT_R16G16B16A16_UINT, 4, 2};
 
 struct ExportCase {
     const char* name;
@@ -69,6 +73,7 @@ constexpr ExportCase Export32R{"32_R", 1, VK_COLOR_COMPONENT_R_BIT};
 constexpr ExportCase Export32GR{"32_GR", 2, VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT};
 constexpr ExportCase Export32AR{"32_AR", 3, VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_A_BIT};
 constexpr ExportCase Export32ABGR{"32_ABGR", 9, VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT};
+constexpr ExportCase ExportFp16ABGR{"FP16_ABGR", 4, VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT};
 
 alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
     0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
@@ -99,6 +104,10 @@ std::span<const std::uint32_t> WordProgram(const std::array<std::uint32_t, 4>& w
     return Program(words, false);
 }
 
+alignas(256) constexpr std::array<std::uint32_t, 5> UserDataHalvesProgram{
+    0x7e080200u, 0x7e0a0201u, 0xf8001c0fu, 0x00000504u, 0xbf810000u,
+};
+
 struct Block {
     explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) {
 #ifdef _WIN32
@@ -157,7 +166,7 @@ struct Block {
     std::uint8_t* data = nullptr;
 };
 
-AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::size_t bytes, const TargetFormat& target, std::uint32_t attrib3) {
+AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::size_t bytes, const TargetFormat& target, std::uint32_t attrib3, ShaderRecompiler::ColorExportPacking packing) {
     AgcDriver::Registers cx;
     cx[0x318] = static_cast<std::uint32_t>(block.Address() >> 8u);
     cx[0x31b] = 0;
@@ -168,10 +177,11 @@ AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::size_t by
     cx[0x390] = static_cast<std::uint32_t>(block.Address() >> 40u);
     auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0);
     Require(color.format == target.format && color.elementBytes == target.channels * target.channelBytes && color.componentMapping == 0xe4u && color.packing == ShaderRecompiler::ColorExportPacking::Uint && color.bytes == bytes, std::string("the UINT target did not decode as an ") + target.name + " surface with the unsigned integer packing");
+    color.packing = packing;
     return color;
 }
 
-void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, const ExportCase& exported, std::span<const std::uint32_t> pixelCode) {
+std::uint64_t Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, const ExportCase& exported, std::span<const std::uint32_t> pixelCode, std::span<const std::uint32_t> pixelUserData = {}) {
     const auto exportFormat = exported.format;
     const auto target = device.Target();
     constexpr std::uint32_t waveSize = 64;
@@ -213,7 +223,7 @@ void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarge
     const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}};
     ShaderRecompiler::RecompileRequest fragment{
         {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}},
-        {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory},
+        {waveSize, 0, pixelUserData, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory},
         target,
         PixelPushLayout(vertexPush, target)
     };
@@ -223,7 +233,7 @@ void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarge
     fragment.shader.code = snapshot.code;
     fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state));
     fragment.context.pixel->targetExportPacking = AgcDriver::Graphics::ExportPackings(state);
-    const ShaderRecompiler::SrtRuntime runtime{};
+    const ShaderRecompiler::SrtRuntime runtime{pixelUserData};
     const auto materialize = [&](const ShaderRecompiler::RecompileRequest& request) {
         const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request);
         return invocation.Materialize(*invocation.Capture(runtime));
@@ -234,12 +244,18 @@ void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarge
     const auto plain = materialize(unpacked);
     Require(pixelResult->variantId == ShaderRecompiler::GetPreparedArtifact(*handle).variantId && plain->variantId == pixelResult->variantId, "the draw did not use the registered pixel shader");
     Require(plain->PipelineVariantId() != pixelResult->PipelineVariantId() && materialize(fragment)->PipelineVariantId() == pixelResult->PipelineVariantId(), "the unsigned integer packing did not key its own specialization");
+    if (color.packing != ShaderRecompiler::ColorExportPacking::Uint) {
+        auto unclamped = fragment;
+        unclamped.context.pixel->targetExportPacking[color.exportIndex] = ShaderRecompiler::ColorExportPacking::Uint;
+        Require(materialize(unclamped)->PipelineVariantId() != pixelResult->PipelineVariantId(), "the clamping unsigned integer packing did not key its own specialization");
+    }
     const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
         {ShaderStage::Vertex, &vertexResult, 0},
         {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)}
     }};
     const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(Triangle.size()), 0, 1, 0, false};
     device.Draw(state, draw, shaders);
+    return pixelResult->PipelineVariantId();
 }
 
 std::vector<std::uint8_t> ReadBack(AgcDriver::VulkanDevice& device, const Block& block) {
@@ -286,24 +302,51 @@ std::array<std::uint32_t, 4> Expected(const ExportCase& exported, const std::arr
     return vgprs;
 }
 
-void TargetTests(AgcDriver::VulkanDevice& device, const TargetFormat& target, const ExportCase& exported, std::span<const std::array<std::uint32_t, 4>> values, ColorTileMode mode, const char* kind) {
+std::uint32_t WidenedHalf(std::uint32_t half) {
+    const auto sign = (half & 0x8000u) << 16u;
+    const auto exponent = (half >> 10u) & 0x1fu;
+    auto mantissa = half & 0x3ffu;
+    if (exponent == 0x1fu) return sign | 0x7f800000u | (mantissa << 13u) | (mantissa != 0u ? 0x400000u : 0u);
+    if (exponent != 0u) return sign | ((exponent + 112u) << 23u) | (mantissa << 13u);
+    if (mantissa == 0u) return sign;
+    std::uint32_t shift = 0;
+    while ((mantissa & 0x400u) == 0u) {
+        mantissa <<= 1u;
+        ++shift;
+    }
+    return sign | ((113u - shift) << 23u) | ((mantissa & 0x3ffu) << 13u);
+}
+
+std::array<std::uint32_t, 4> HalfExpected(ShaderRecompiler::ColorExportPacking packing, const std::array<std::uint32_t, 4>& vgprs) {
+    const auto limit = packing == ShaderRecompiler::ColorExportPacking::UintClamp8 ? 0xffu : packing == ShaderRecompiler::ColorExportPacking::UintClamp16 ? 0xffffu : 0xffffffffu;
+    std::array<std::uint32_t, 4> expected{};
+    for (std::uint32_t channel = 0; channel < expected.size(); ++channel) expected[channel] = std::min(WidenedHalf((vgprs[channel / 2u] >> (16u * (channel % 2u))) & 0xffffu), limit);
+    return expected;
+}
+
+void TargetTests(AgcDriver::VulkanDevice& device, const TargetFormat& target, const ExportCase& exported, std::span<const std::array<std::uint32_t, 4>> values, ColorTileMode mode, const char* kind, ShaderRecompiler::ColorExportPacking packing) {
     const auto elementBytes = target.channels * target.channelBytes;
     const ColorTargetLayout layout(Width, Height, mode, elementBytes);
     Block block((layout.Bytes() + BlockBytes - 1) / BlockBytes * BlockBytes);
-    const auto color = DecodeTarget(block, layout.Bytes(), target, mode == ColorTileMode::Linear ? LinearAttrib3 : TiledAttrib3);
+    const auto color = DecodeTarget(block, layout.Bytes(), target, mode == ColorTileMode::Linear ? LinearAttrib3 : TiledAttrib3, packing);
+    const bool halves = exported.format == ExportFp16ABGR.format;
     auto before = Fill(block);
+    std::uint64_t variant = 0;
     for (const auto& vgprs : values) {
-        Draw(device, color, exported, WordProgram(vgprs));
-        char what[160];
-        std::snprintf(what, sizeof(what), "a %s export of 0x%08x 0x%08x 0x%08x 0x%08x into a %s %s target", exported.name, vgprs[0], vgprs[1], vgprs[2], vgprs[3], kind, target.name);
-        ExpectSurface(device, block, target, layout, before, exported, Expected(exported, vgprs), what);
+        const std::array<std::uint32_t, 2> userData{vgprs[0], vgprs[1]};
+        const auto drawn = halves ? Draw(device, color, exported, UserDataHalvesProgram, userData) : Draw(device, color, exported, WordProgram(vgprs));
+        Require(!halves || variant == 0 || drawn == variant, "the exported halves were folded into the pixel shader instead of read at run time");
+        variant = drawn;
+        char what[192];
+        std::snprintf(what, sizeof(what), "a %s export of 0x%08x 0x%08x 0x%08x 0x%08x into a %s %s target with packing %u", exported.name, vgprs[0], vgprs[1], vgprs[2], vgprs[3], kind, target.name, static_cast<unsigned>(packing));
+        ExpectSurface(device, block, target, layout, before, exported, halves ? HalfExpected(packing, vgprs) : Expected(exported, vgprs), what);
         before = ReadBack(device, block);
     }
 }
 
-void FormatTests(AgcDriver::VulkanDevice& device, const TargetFormat& target, const ExportCase& exported, std::span<const std::array<std::uint32_t, 4>> values) {
-    TargetTests(device, target, exported, values, ColorTileMode::RenderTarget, "tiled");
-    TargetTests(device, target, exported, values, ColorTileMode::Linear, "linear");
+void FormatTests(AgcDriver::VulkanDevice& device, const TargetFormat& target, const ExportCase& exported, std::span<const std::array<std::uint32_t, 4>> values, ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::Uint) {
+    TargetTests(device, target, exported, values, ColorTileMode::RenderTarget, "tiled", packing);
+    TargetTests(device, target, exported, values, ColorTileMode::Linear, "linear", packing);
 }
 
 }
@@ -323,6 +366,21 @@ int main() {
         FormatTests(*device, R32G32B32A32, Export32ABGR, wide);
         const std::vector<std::array<std::uint32_t, 4>> narrow{{0u, 0x11111111u, 0x22222222u, 0x33333333u}, {1u, 0u, 0u, 0u}, {0x7fu, 0u, 0u, 0u}, {0xa5u, 0u, 0u, 0u}, {0xffu, 0u, 0u, 0u}};
         FormatTests(*device, R8, Export32R, narrow);
+        constexpr std::array<std::uint32_t, 13> halves{{0x0000u, 0x8000u, 0x3c00u, 0xbc00u, 0x3800u, 0x0001u, 0x83ffu, 0x03ffu, 0x7bffu, 0x7c00u, 0xfc00u, 0x7c01u, 0xfe00u}};
+        std::vector<std::array<std::uint32_t, 4>> packed;
+        for (std::size_t index = 0; index < halves.size(); ++index) {
+            const auto half = [&](std::size_t offset) { return halves[(index + offset) % halves.size()]; };
+            packed.push_back({half(0) | (half(3) << 16u), half(5) | (half(7) << 16u), 0u, 0u});
+        }
+        using ShaderRecompiler::ColorExportPacking;
+        FormatTests(*device, R32, ExportFp16ABGR, packed);
+        FormatTests(*device, R32, ExportFp16ABGR, packed, ColorExportPacking::UintClamp8);
+        FormatTests(*device, R32, ExportFp16ABGR, packed, ColorExportPacking::UintClamp16);
+        FormatTests(*device, R32G32B32A32, ExportFp16ABGR, packed);
+        FormatTests(*device, R16, ExportFp16ABGR, packed, ColorExportPacking::UintClamp16);
+        FormatTests(*device, R16G16B16A16, ExportFp16ABGR, packed, ColorExportPacking::UintClamp16);
+        FormatTests(*device, R8, ExportFp16ABGR, packed, ColorExportPacking::UintClamp8);
+        FormatTests(*device, R8G8B8A8, ExportFp16ABGR, packed, ColorExportPacking::UintClamp8);
         std::puts("UINT color target tests passed");
         return 0;
     } catch (const std::exception& error) {

core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp

@@ -244,7 +244,7 @@ ConservativeZExport readConservativeZExport(Reader& reader) {
 
 ColorExportPacking readColorExportPacking(Reader& reader) {
     const auto value = reader.ReadU8();
-    if (value > static_cast<std::uint8_t>(ColorExportPacking::Uint)) {
+    if (value > static_cast<std::uint8_t>(ColorExportPacking::UintClamp16)) {
         throw std::runtime_error("RequestSerializer: invalid ColorExportPacking value");
     }
     return static_cast<ColorExportPacking>(value);

core/shader/recompiler/FragmentOutputSpecialization.hpp

@@ -32,7 +32,7 @@ inline std::vector<std::uint32_t> SpecializeFragmentOutputs(std::vector<std::uin
             reversed = constant.value;
         }
         if (constant.id < PipelineSpecialization::ExportPackingBase || constant.id >= PipelineSpecialization::ExportPackingBase + 8u) continue;
-        if (constant.value > static_cast<std::uint32_t>(ColorExportPacking::Uint)) throw std::runtime_error("invalid prepared fragment export packing");
+        if (constant.value > static_cast<std::uint32_t>(ColorExportPacking::UintClamp16)) throw std::runtime_error("invalid prepared fragment export packing");
         if (constant.value != static_cast<std::uint32_t>(ColorExportPacking::None)) packed.insert(constant.id - PipelineSpecialization::ExportPackingBase);
     }
     if (packed.empty() && !dualSource && reversed == 0u) return words;