fix(shader): read quad_perm DPP lanes in pixel shaders through quad operations

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -1158,6 +1158,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_BALLOT_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformBallot);
         if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_SHUFFLE_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformShuffle);
         if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_ARITHMETIC_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformArithmetic);
+        if ((state->subgroup.supportedOperations & VK_SUBGROUP_FEATURE_QUAD_BIT) != 0) state->capabilities.push_back(spv::CapabilityGroupNonUniformQuad);
     }
     APS5_LOG_OUT("Selected GPU name=%s vendor=0x%x device=0x%x subgroup=%u", state->properties.deviceName, state->properties.vendorID, state->properties.deviceID, state->subgroup.subgroupSize);
     state->InstanceFunction<PFN_vkGetPhysicalDeviceMemoryProperties>("vkGetPhysicalDeviceMemoryProperties")(selected, &state->memoryProperties);

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp

@@ -219,6 +219,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi
                     case spv::CapabilityGroupNonUniformBallot: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_BALLOT_BIT; break;
                     case spv::CapabilityGroupNonUniformShuffle: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_SHUFFLE_BIT; break;
                     case spv::CapabilityGroupNonUniformArithmetic: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_ARITHMETIC_BIT; break;
+                    case spv::CapabilityGroupNonUniformQuad: subgroupOperations = VK_SUBGROUP_FEATURE_BASIC_BIT | VK_SUBGROUP_FEATURE_QUAD_BIT; break;
                     default: break;
                 }
                 const bool isSubgroupCapability = subgroupOperations != 0;

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -7484,6 +7484,51 @@ void verifyFmaskImageMode() {
     expectFailure([&] { static_cast<void>(ResourceMaterializer::RuntimeImageMode(image, descriptor, modes)); }, "FMASK requires a direct 32-bit image load", "an incompatible FMASK operation was accepted");
 }
 
+void verifyPixelQuadPermReads() {
+    using namespace ShaderRecompiler;
+    constexpr std::array<std::uint32_t, 6> shuffleCapabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic};
+    constexpr std::array<std::uint32_t, 7> quadCapabilities{spv::CapabilityShader, spv::CapabilityInt64, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityGroupNonUniformQuad};
+    constexpr std::array<std::uint32_t, 6> quadPerm{0xbefe0ac1u, 0x7e0202fau, 0xff00b100u, 0xf800180fu, 0x01010101u, 0xbf810000u};
+    constexpr std::array<std::uint32_t, 6> quadPermRowMask{0xbefe0ac1u, 0x7e0202fau, 0x5f00b100u, 0xf800180fu, 0x01010101u, 0xbf810000u};
+    constexpr std::array<std::uint32_t, 6> rowShift{0xbefe0ac1u, 0x7e0202fau, 0xff011100u, 0xf800180fu, 0x01010101u, 0xbf810000u};
+    const auto compile = [&](std::span<const std::uint32_t> code, bool quad) {
+        RecompileRequest request{};
+        request.shader = {ShaderStage::Fragment, 0x58000u, code, 0, {}};
+        request.context.waveSize = 32u;
+        request.context.pixel = ShaderPixelStageInfo{};
+        request.context.pixel->inputAddr = PixelInputBit(PixelInput::PositionX);
+        request.context.pixel->posX = true;
+        request.context.pixel->quadPixelMask = 0xfu;
+        request.context.pixel->targetOutputMode[0] = 9u;
+        request.context.pixel->targetExportMapping.fill(0xe4u);
+        request.target.vulkanVersion = 0x00401000u;
+        request.target.spirvVersion = 0x00010300u;
+        request.target.subgroupSize = 32u;
+        request.target.supportedCapabilities = quad ? std::span<const std::uint32_t>(quadCapabilities) : std::span<const std::uint32_t>(shuffleCapabilities);
+        request.layout.pushConstantSizeBytes = 128u;
+        request.useCache = false;
+        return Recompile(request);
+    };
+    const auto declares = [](const RecompileResult& result, spv::Op opcode) {
+        const auto& words = result.spirv.Words();
+        for (std::size_t cursor = 5; cursor < words.size();) {
+            const auto count = words[cursor] >> 16u;
+            require(count != 0 && count <= words.size() - cursor, "quad perm: truncated SPIR-V instruction");
+            if ((words[cursor] & 0xffffu) == static_cast<std::uint32_t>(opcode)) return true;
+            cursor += count;
+        }
+        return false;
+    };
+    for (const auto code : {std::span<const std::uint32_t>(quadPerm), std::span<const std::uint32_t>(quadPermRowMask)}) {
+        const auto quad = compile(code, true);
+        require(declares(quad, spv::OpGroupNonUniformQuadBroadcast) && !declares(quad, spv::OpGroupNonUniformShuffle), "a pixel quad_perm read did not go through quad operations");
+        const auto shuffle = compile(code, false);
+        require(declares(shuffle, spv::OpGroupNonUniformShuffle) && !declares(shuffle, spv::OpGroupNonUniformQuadBroadcast), "a pixel quad_perm read without quad operations did not keep its shuffle");
+    }
+    const auto shifted = compile(rowShift, true);
+    require(declares(shifted, spv::OpGroupNonUniformShuffle) && !declares(shifted, spv::OpGroupNonUniformQuadBroadcast), "a pixel row shift read went through quad operations");
+}
+
 void verifyCubeDepthCompareMode() {
     using namespace ShaderRecompiler;
     ImageResource image{};
@@ -7765,6 +7810,7 @@ int main(int argc, char** argv) {
         verifyDynamicStorageMipChain();
         verifySrgbDepthCompareModes();
         verifyQuadMaskNeighbourReads();
+        verifyPixelQuadPermReads();
         verifyCoherentGds();
 #if ANYPS5_ENABLE_SPIRV_TOOLS
         const std::vector<std::uint32_t> minimalSpirv{

core/libs/prx/libSceAgcDriver/tests/execution/PixelQuadPermHelpers.cpp

@@ -0,0 +1,182 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <algorithm>
+#include <array>
+#include <cstdio>
+#include <iostream>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Sentinel = 0xdeadbeefu;
+constexpr std::uint32_t Width = 192;
+constexpr std::uint32_t Height = 128;
+alignas(256) std::array<std::byte, Width * Height * 4> Pixels{};
+alignas(256) std::array<std::uint32_t, Width * Height> Results{};
+
+alignas(256) constexpr std::array<std::uint32_t, 6> VertexCode{
+    0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000,
+};
+
+std::vector<std::uint32_t> PixelCode(std::uint32_t waveSize, bool fetchInactive) {
+    const bool wave32 = waveSize == 32u;
+    const std::uint32_t fi = fetchInactive ? 0x00040000u : 0u;
+    return {
+        wave32 ? 0xbe8a037eu : 0xbe8a047eu, wave32 ? 0xbefe097eu : 0xbefe0a7eu, 0x7e040f00, 0x7e060f01, 0xd5430006, 0x0409ff03, 0x000000c0,
+        0x7e0802fa, 0xff00b106 | fi, 0x7e0a02fa, 0xff004e06 | fi, wave32 ? 0xbefe030au : 0xbefe040au, 0xd76f0005, 0x04112105,
+        0xe0702000, 0x80010506, 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000,
+    };
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu};
+}
+
+std::vector<std::array<float, 4>> Triangles() {
+    std::vector<std::array<float, 4>> vertices;
+    constexpr std::uint32_t columns = 6;
+    constexpr std::uint32_t rows = 4;
+    for (std::uint32_t cell = 0; cell < columns * rows; ++cell) {
+        const float width = 2.0f / columns;
+        const float height = 2.0f / rows;
+        const float x = -1.0f + width * static_cast<float>(cell % columns);
+        const float y = -1.0f + height * static_cast<float>(cell / columns);
+        const float skew = 0.07f * static_cast<float>(cell % 5u);
+        vertices.push_back({x + (0.05f + skew) * width, y + 0.05f * height, 0.5f, 1.0f});
+        vertices.push_back({x + 0.93f * width, y + (0.11f + skew) * height, 0.5f, 1.0f});
+        vertices.push_back({x + (0.37f - skew * 0.5f) * width, y + 0.91f * height, 0.5f, 1.0f});
+    }
+    return vertices;
+}
+
+void Draw(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> pixelCode, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target) {
+    static const auto triangles = Triangles();
+    Pixels.fill(std::byte{0});
+    Results.fill(Sentinel);
+
+    std::vector<std::uint32_t> vertexUserData(4, 0u);
+    const auto vertexBuffer = BufferDescriptor(triangles.data(), 16u, static_cast<std::uint32_t>(triangles.size()));
+    std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin());
+    const std::array<ShaderRecompiler::MemoryRegion, 1> vertexMemory{{{reinterpret_cast<std::uintptr_t>(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}};
+    ShaderRecompiler::RecompileRequest vertex{
+        {ShaderStage::Vertex, reinterpret_cast<std::uintptr_t>(VertexCode.data()), VertexCode, 0, {}},
+        {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory},
+        target,
+        {0, 0, 0, 64}
+    };
+    vertex.useCache = false;
+    const auto vertexResult = ShaderRecompiler::Recompile(vertex);
+    const auto vertexPush = static_cast<std::uint32_t>(vertexResult.pushConstants.size());
+
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = waveSize == 32u;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    std::vector<std::uint32_t> pixelUserData(8, 0u);
+    const auto results = BufferDescriptor(Results.data(), 4u, static_cast<std::uint32_t>(Results.size()));
+    std::copy(results.begin(), results.end(), pixelUserData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(pixelCode.data()), std::as_bytes(pixelCode)}}};
+    ShaderRecompiler::RecompileRequest fragment{
+        {ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(pixelCode.data()), pixelCode, 0, {}},
+        {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory},
+        target,
+        PixelPushLayout(vertexPush, target)
+    };
+    fragment.useCache = false;
+    const auto pixelResult = ShaderRecompiler::Recompile(fragment);
+    const std::array<AgcDriver::Graphics::CompiledShader, 2> shaders{{
+        {ShaderStage::Vertex, &vertexResult, 0},
+        {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}
+    }};
+
+    AgcDriver::Graphics::State state{};
+    state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt};
+    state.color = {reinterpret_cast<std::uintptr_t>(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u};
+    state.colors = {state.color};
+    state.hasColorTarget = true;
+    state.renderExtent = {Width, Height};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.viewport = {0, static_cast<float>(Height), static_cast<float>(Width), -static_cast<float>(Height), 0, 1};
+    state.negativeOneToOne = false;
+    state.scissor = {{0, 0}, {Width, Height}};
+    state.cullMode = VK_CULL_MODE_NONE;
+    state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE;
+    state.blend.colorWriteMask = 15;
+    state.blends = {state.blend};
+    state.blendConstants = {};
+    const AgcDriver::Pm4::DrawParameters draw{0, static_cast<std::uint32_t>(triangles.size()), 0, 1, 0, false};
+    device.Draw(state, draw, shaders);
+    device.WaitIdle();
+}
+
+bool Covered(std::uint32_t pixel) {
+    return std::to_integer<std::uint8_t>(Pixels[pixel * 4u]) == 255u;
+}
+
+std::uint32_t Check(const std::string& what) {
+    std::uint32_t covered = 0;
+    std::uint32_t helperReads = 0;
+    for (std::uint32_t pixel = 0; pixel < Results.size(); ++pixel) {
+        if (!Covered(pixel)) {
+            Require(Results[pixel] == Sentinel, what + ": uncovered pixel " + std::to_string(pixel) + " stored " + std::to_string(Results[pixel]));
+            continue;
+        }
+        ++covered;
+        const auto x = pixel % Width;
+        const auto y = pixel / Width;
+        const auto across = y * Width + (x ^ 1u);
+        const auto down = (y ^ 1u) * Width + x;
+        Require(Results[pixel] != Sentinel, what + ": covered pixel " + std::to_string(pixel) + " stored nothing");
+        Require((Results[pixel] & 0xffffu) == across, what + ": pixel " + std::to_string(pixel) + " read " + std::to_string(Results[pixel] & 0xffffu) + " from quad_perm [1,0,3,2], its " + (Covered(across) ? "covered" : "helper") + " neighbour is pixel " + std::to_string(across));
+        Require((Results[pixel] >> 16u) == down, what + ": pixel " + std::to_string(pixel) + " read " + std::to_string(Results[pixel] >> 16u) + " from quad_perm [2,3,0,1], its " + (Covered(down) ? "covered" : "helper") + " neighbour is pixel " + std::to_string(down));
+        helperReads += (Covered(across) ? 0u : 1u) + (Covered(down) ? 0u : 1u);
+    }
+    Require(covered > 1000, what + ": the triangles covered only " + std::to_string(covered) + " pixels");
+    Require(helperReads > 100, what + ": only " + std::to_string(helperReads) + " reads had a helper neighbour");
+    return helperReads;
+}
+
+}
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        const auto target = device->Target();
+        if (!TargetHasCapability(target, spv::CapabilityGroupNonUniformQuad)) {
+            std::puts("skipped, the device has no subgroup quad operations");
+            return VulkanTestSkipped;
+        }
+        if (target.subgroupSize < 32u) {
+            std::printf("skipped, subgroup size %u cannot hold a wave32\n", target.subgroupSize);
+            return VulkanTestSkipped;
+        }
+        for (const auto waveSize : {64u, 32u}) {
+            for (const bool fetchInactive : {false, true}) {
+                const auto code = PixelCode(waveSize, fetchInactive);
+                const auto what = "pixel quad_perm wave" + std::to_string(waveSize) + (fetchInactive ? " fi" : "");
+                std::uint32_t helperReads = 0;
+                for (std::uint32_t pass = 0; pass < 4; ++pass) {
+                    Draw(*device, code, waveSize, target);
+                    helperReads += Check(what);
+                }
+                std::printf("%s: %u reads from helper neighbours over 4 draws\n", what.c_str(), helperReads);
+            }
+        }
+        std::puts("pixel quad_perm helper tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvSubgroup.hpp

@@ -21,6 +21,8 @@ DppTargetLane EmitDppRowShiftTargetLane(SpirvEmitterState& state, std::uint32_t
 DppTargetLane EmitDppRowRotateRightTargetLane(SpirvEmitterState& state, std::uint32_t subid, std::uint32_t amount);
 DppTargetLane EmitDppMirrorTargetLane(SpirvEmitterState& state, std::uint32_t subid, bool halfRow);
 DppTargetLane EmitDppTargetLane(SpirvEmitterState& state, std::uint32_t control);
+bool DppQuadRead(const SpirvEmitterState& state, std::uint32_t control);
+std::uint32_t EmitDppQuadRead(SpirvEmitterState& state, std::uint32_t type, std::uint32_t value, std::uint32_t control);
 std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t ballot, std::uint32_t lane);
 std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane);
 

core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp

@@ -172,7 +172,7 @@ std::uint32_t EmitDppWriteCondition(SpirvValueEmitContext& ctx, const IrValue& i
         state.module.AddFunction(spv::OpLogicalAnd, TypeBool(state), bounded, writable, target.valid);
         writable = bounded;
         if (!flags.fetchInactive && (flags.control & DppMoveFlags::Lanes8) == 0u) {
-            const auto sourceActive = EmitBallotLaneActiveBool(state, ctx.Ballot(inst.Argument(2)), target.lane);
+            const auto sourceActive = DppQuadRead(state, flags.control) ? EmitDppQuadRead(state, TypeBool(state), ctx.Arg(inst, 2), flags.control) : EmitBallotLaneActiveBool(state, ctx.Ballot(inst.Argument(2)), target.lane);
             writable = Binary(state, spv::OpLogicalAnd, TypeBool(state), writable, sourceActive);
         }
     }
@@ -1030,6 +1030,11 @@ std::uint32_t EmitUndefU1(SpirvEmitterState& state, const IrValue& inst) {
 std::uint32_t EmitDppMoveU32(SpirvValueEmitContext& ctx, const IrValue& inst) {
     auto& state = ctx.state;
     const auto flags = inst.Flags<DppMoveFlags>();
+    if (DppQuadRead(state, flags.control)) {
+        const auto read = EmitDppQuadRead(state, TypeU32(state), ctx.Arg(inst, 0), flags.control);
+        if (flags.fetchInactive) return read;
+        return EmitNative<spv::OpSelect, IrType::U32>(state, EmitDppQuadRead(state, TypeBool(state), ctx.Arg(inst, 1), flags.control), read, ConstantU32(state, 0u));
+    }
     const auto target = EmitDppTargetLane(state, flags.control);
     const auto shuffled = ctx.ShuffleInHalf(inst, 0, target.lane);
     if (flags.fetchInactive) {

core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvSubgroup.cpp

@@ -1,6 +1,7 @@
 #include "SpirvBackend/SpirvEmitter.hpp"
 #include "SpirvBackend/SpirvEmitterHelpers.hpp"
 #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp"
+#include <algorithm>
 #include <spirv/unified1/GLSL.std.450.h>
 #include <spirv/unified1/spirv.hpp>
 #include <stdexcept>
@@ -284,6 +285,24 @@ namespace ShaderRecompiler
         FailEmit("DPP control " + std::to_string(control) + " is not supported");
     }
 
+    bool DppQuadRead(const SpirvEmitterState& state, std::uint32_t control) {
+        return control <= 0xffu && StageOf(state) == IrShaderStage::Pixel && !HostSubgroupSingleLane(state) && std::ranges::find(state.supportedCapabilities, static_cast<std::uint32_t>(spv::CapabilityGroupNonUniformQuad)) != state.supportedCapabilities.end();
+    }
+
+    std::uint32_t EmitDppQuadRead(SpirvEmitterState& state, std::uint32_t type, std::uint32_t value, std::uint32_t control) {
+        state.module.EmitCapability(spv::CapabilityGroupNonUniformQuad);
+        const auto quadLane = EmitBinaryU32(state, spv::OpBitwiseAnd, EmitSubgroupLocalInvocationId(state), ConstantU32(state, 3u));
+        const auto shift = EmitBinaryU32(state, spv::OpShiftLeftLogical, quadLane, ConstantU32(state, 1u));
+        const auto source = EmitBinaryU32(state, spv::OpBitwiseAnd, EmitBinaryU32(state, spv::OpShiftRightLogical, ConstantU32(state, control), shift), ConstantU32(state, 3u));
+        std::uint32_t result = 0;
+        for (std::uint32_t lane = 0; lane < 4u; ++lane) {
+            const auto broadcast = state.module.AllocateId();
+            state.module.AddFunction(spv::OpGroupNonUniformQuadBroadcast, type, broadcast, ConstantU32(state, spv::ScopeSubgroup), value, ConstantU32(state, lane));
+            result = lane == 0u ? broadcast : Select(state, type, Binary(state, spv::OpIEqual, TypeBool(state), source, ConstantU32(state, lane)), broadcast, result);
+        }
+        return result;
+    }
+
     std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t ballot, std::uint32_t lane) {
         const auto waveSize = state.program.WaveSize();
         const auto low = state.module.AllocateId();