feat(shader): decode and implement ds_condxchg32_rtn_b64

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -293,6 +293,11 @@ target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_ag
 set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
 configure_windows_unwind(agc_vop1_control_decode_tests)
 
+add_test_executable(agc_ds_condxchg32_decode_tests EXCLUDE_FROM_ALL tests/DsCondxchg32Decode.cpp)
+target_link_libraries(agc_ds_condxchg32_decode_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_ds_condxchg32_decode_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_ds_condxchg32_decode_tests)
+
 add_test_executable(agc_dcc_key_sharing_tests EXCLUDE_FROM_ALL tests/DccKeySharing.cpp)
 target_include_directories(agc_dcc_key_sharing_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include)
 target_link_libraries(agc_dcc_key_sharing_tests PRIVATE libc)
@@ -523,7 +528,7 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources)
 endforeach()
 
 if(BUILD_TESTING)
-    foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
+    foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_ds_condxchg32_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests ${agcExecutionTests})
         string(REGEX REPLACE "_tests$" "" agcTestName "${agcTest}")
         set_target_properties(${agcTest} PROPERTIES EXCLUDE_FROM_ALL OFF)
         add_test(NAME ${agcTestName} COMMAND ${agcTest})

core/libs/prx/libSceAgcDriver/tests/DsCondxchg32Decode.cpp

@@ -0,0 +1,72 @@
+#include "RdnaDecoder/RdnaInstructionDecoder.hpp"
+#include <array>
+#include <cstdio>
+#include <exception>
+#include <vector>
+
+using namespace ShaderRecompiler;
+
+namespace {
+
+struct Expected {
+    std::uint32_t destinationRegister;
+    std::uint32_t addressRegister;
+    std::uint32_t dataRegister;
+    std::uint32_t memoryOffset;
+    bool gds;
+};
+
+bool check(const RdnaInstruction& instruction, const Expected& expected, std::size_t index) {
+    if (instruction.op != RdnaOpcode::DsCondxchg32RtnB64 || instruction.wordCount != 2u) {
+        std::fprintf(stderr, "instruction %zu decodes to the wrong opcode or length\n", index);
+        return false;
+    }
+    if (instruction.destination.kind != RdnaOperandKind::VectorRegister || instruction.destination.reg != expected.destinationRegister) {
+        std::fprintf(stderr, "instruction %zu has a wrong destination\n", index);
+        return false;
+    }
+    if (instruction.sourceCount != 2u || instruction.source0.reg != expected.addressRegister || instruction.source1.reg != expected.dataRegister) {
+        std::fprintf(stderr, "instruction %zu has wrong operands\n", index);
+        return false;
+    }
+    if (instruction.memoryOffset != expected.memoryOffset || instruction.gds != expected.gds) {
+        std::fprintf(stderr, "instruction %zu has wrong offset or gds flag\n", index);
+        return false;
+    }
+    return true;
+}
+
+}  // namespace
+
+int main() {
+    // ds_condxchg32_rtn_b64 v[0:1], v2, v[4:5]
+    // ds_condxchg32_rtn_b64 v[8:9], v10, v[12:13] offset:255
+    // ds_condxchg32_rtn_b64 v[2:3], v4, v[6:7] gds
+    const std::vector<std::uint32_t> code{
+        0xD9F80000u, 0x00000402u,
+        0xD9F800FFu, 0x08000C0Au,
+        0xD9FA0000u, 0x02000604u,
+        0xBF810000u,
+    };
+    constexpr std::array<Expected, 3> expected{
+        Expected{0u, 2u, 4u, 0u, false},
+        Expected{8u, 10u, 12u, 255u, false},
+        Expected{2u, 4u, 6u, 0u, true},
+    };
+    try {
+        const auto decoded = RdnaInstructionDecoder{}.Decode(code);
+        if (decoded.instructions.size() != expected.size() + 1u) {
+            std::fprintf(stderr, "decoded %zu instructions, expected %zu\n", decoded.instructions.size(), expected.size() + 1u);
+            return 1;
+        }
+        for (std::size_t index = 0; index < expected.size(); ++index) {
+            if (!check(decoded.instructions[index], expected[index], index)) {
+                return 1;
+            }
+        }
+        return 0;
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s\n", error.what());
+    }
+    return 1;
+}

core/libs/prx/libSceAgcDriver/tests/execution/DsCondxchg32B64.cpp

@@ -0,0 +1,151 @@
+#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
+#include "Recompiler.hpp"
+#include "VulkanTestDevice.hpp"
+#include <array>
+#include <cstdint>
+#include <cstdio>
+#include <iostream>
+#include <span>
+#include <string>
+#include <vector>
+
+namespace {
+
+using AgcDriver::Graphics::Require;
+using ShaderRecompiler::ShaderStage;
+
+constexpr std::uint32_t Threads = 32;
+constexpr std::uint32_t Inputs = 8;
+constexpr std::uint32_t Results = 16;
+constexpr std::uint32_t LdsDwords = 3072;
+alignas(256) std::array<std::uint32_t, Threads * Inputs> Input{};
+alignas(256) std::array<std::uint32_t, Threads * Results> Output{};
+
+// v_lshlrev_b32 v8/v9/v10 lane byte offsets; load init+data pairs from input; seed LDS with the init pair;
+// ds_condxchg32_rtn_b64 twice (first from data1, then from data2), reading LDS back after each; store all.
+alignas(256) constexpr auto Code = std::to_array<std::uint32_t>({
+    0x34100084u, 0x34120085u, 0x34140086u,
+    0xe0381000u, 0x80000409u, 0xe0381000u, 0x90000c09u,
+    0xbf8c3f70u,
+    0xd8340000u, 0x00000408u, 0xd8340004u, 0x00000508u,
+    0xbf8cc07fu,
+    0xd9f80000u, 0x10000608u,
+    0xbf8cc07fu,
+    0xd8d80000u, 0x12000008u, 0xd8d80004u, 0x13000008u,
+    0xbf8cc07fu,
+    0xd9f80000u, 0x14000c08u,
+    0xbf8cc07fu,
+    0xd8d80000u, 0x16000008u, 0xd8d80004u, 0x17000008u,
+    0xbf8cc07fu,
+    0xe0781000u, 0x8001100au, 0xe0781000u, 0x9001140au,
+    0xbf8c3f70u,
+    0xbf810000u,
+});
+
+// ds_condxchg32_rtn_b64 with the GDS bit set: a 64-bit GDS atomic is not supported.
+alignas(256) constexpr auto GdsVariant = std::to_array<std::uint32_t>({0xd9fa0000u, 0x10000608u, 0xbf810000u});
+
+std::uint32_t ConditionalWrite(std::uint32_t data, std::uint32_t old) {
+    return (data & 0x80000000u) != 0u ? (data & 0x7fffffffu) : old;
+}
+
+void Fill(std::uint32_t tid, std::uint32_t* words) {
+    std::uint64_t seed = (tid + 1u) * 0x9e3779b97f4a7c15ull;
+    for (std::uint32_t i = 0; i < Inputs; ++i) {
+        seed = seed * 0xbf58476d1ce4e5b9ull + 0x94d049bb133111ebull;
+        auto value = static_cast<std::uint32_t>(seed >> 32u);
+        if (i >= 2u) {
+            // drive the bit31 write-enable through every combination across lanes
+            constexpr std::array flags{0x00000000u, 0x80000000u};
+            value = (value & 0x7fffffffu) | flags[(tid / 2u + i) % 2u];
+        }
+        words[i] = value;
+    }
+}
+
+std::array<std::uint32_t, 4> BufferDescriptor(const void* data, std::uint32_t bytes) {
+    const auto address = reinterpret_cast<std::uintptr_t>(data);
+    return {static_cast<std::uint32_t>(address), static_cast<std::uint32_t>((address >> 32u) & 0xffffu), bytes, 0x01016facu};
+}
+
+std::string Hex(std::uint32_t value) {
+    char text[16];
+    std::snprintf(text, sizeof(text), "0x%08x", value);
+    return text;
+}
+
+template<typename TUse>
+void Translate(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, TUse&& use) {
+    std::vector<std::uint32_t> userData(8, 0u);
+    const auto input = BufferDescriptor(Input.data(), static_cast<std::uint32_t>(Input.size() * 4u));
+    const auto output = BufferDescriptor(Output.data(), static_cast<std::uint32_t>(Output.size() * 4u));
+    std::copy(input.begin(), input.end(), userData.begin());
+    std::copy(output.begin(), output.end(), userData.begin() + 4);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> memory{{{reinterpret_cast<std::uintptr_t>(code.data()), std::as_bytes(code)}}};
+    const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, LdsDwords, {false, false, false}, false, 1};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderStage::Compute, reinterpret_cast<std::uintptr_t>(code.data()), code, 0, {}},
+        {32, 0, userData, compute, std::nullopt, std::nullopt, memory},
+        device.Target(),
+        {0, 0, 0, 128}
+    };
+    request.useCache = false;
+    use(ShaderRecompiler::Recompile(request));
+}
+
+void Run(AgcDriver::VulkanDevice& device) {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]);
+    Output.fill(0xdeadbeefu);
+    Translate(device, Code, [&](const auto& result) {
+        device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast<std::uintptr_t>(Code.data()));
+        device.WaitIdle();
+    });
+}
+
+void Reject(AgcDriver::VulkanDevice& device, std::span<const std::uint32_t> code, const std::string& reason) {
+    std::string error;
+    try {
+        Translate(device, code, [](const auto&) {});
+    } catch (const std::exception& exception) {
+        error = exception.what();
+    }
+    Require(error.find(reason) != std::string::npos, "ds condxchg32 b64: expected a rejection for \"" + reason + "\", got \"" + error + "\"");
+}
+
+void CheckRejections(AgcDriver::VulkanDevice& device) {
+    Reject(device, GdsVariant, "64-bit GDS atomics are not supported");
+}
+
+void Check() {
+    for (std::uint32_t tid = 0; tid < Threads; ++tid) {
+        const std::uint32_t* in = &Input[tid * Inputs];
+        const std::uint32_t* out = &Output[tid * Results];
+        const std::uint32_t seedLo = in[0], seedHi = in[1];
+        const std::uint32_t firstLo = ConditionalWrite(in[2], seedLo);
+        const std::uint32_t firstHi = ConditionalWrite(in[3], seedHi);
+        const std::uint32_t secondLo = ConditionalWrite(in[4], firstLo);
+        const std::uint32_t secondHi = ConditionalWrite(in[5], firstHi);
+        const std::array expected{seedLo, seedHi, firstLo, firstHi, firstLo, firstHi, secondLo, secondHi};
+        for (std::uint32_t i = 0; i < expected.size(); ++i) {
+            Require(out[i] == expected[i], "ds condxchg32 b64: lane " + std::to_string(tid) + " output dword " + std::to_string(i) + " is " + Hex(out[i]) + ", expected " + Hex(expected[i]));
+        }
+    }
+}
+
+}  // namespace
+
+int main() {
+    try {
+        const auto device = OpenVulkanTestDevice();
+        if (!device) return VulkanTestSkipped;
+        Run(*device);
+        Check();
+        CheckRejections(*device);
+        std::puts("ds condxchg32 b64 tests passed");
+        return 0;
+    } catch (const std::exception& error) {
+        std::cerr << error.what() << '\n';
+        return 1;
+    }
+}

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp

@@ -419,6 +419,7 @@ enum class IrOpcode : std::uint16_t {
     SharedAtomicCmpst64,
     SharedAtomicCmpstF64,
     SharedAtomicMskor64,
+    SharedAtomicCondxchg64,
     DataAppend,
     DataConsume,
     SwizzleU32,

core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp

@@ -460,6 +460,7 @@ constexpr std::array<OpcodeMeta, static_cast<std::size_t>(IrOpcode::Count)> Meta
     makeMeta("SharedAtomicCmpst64", U64, U32, U64, U64, U1),
     makeMeta("SharedAtomicCmpstF64", U64, U32, U64, U64, U1),
     makeMeta("SharedAtomicMskor64", U64, U32, U64, U64, U1),
+    makeMeta("SharedAtomicCondxchg64", U64, U32, U64, U1),
     makeMeta("DataAppend", U32, U32, U1, U32, U32),
     makeMeta("DataConsume", U32, U32, U1, U32, U32),
     makeMeta("SwizzleU32", U32, U32, U32, U1),
@@ -680,6 +681,7 @@ SharedAccess SharedAccessOf(IrOpcode opcode) {
         case IrOpcode::SharedAtomicCmpst64:
         case IrOpcode::SharedAtomicCmpstF64:
         case IrOpcode::SharedAtomicMskor64:
+        case IrOpcode::SharedAtomicCondxchg64:
             return SharedAccess::Atomic;
         case IrOpcode::DataAppend:
             return SharedAccess::Append;
@@ -712,6 +714,7 @@ std::uint32_t SharedComponentCount(IrOpcode opcode) {
         case IrOpcode::SharedAtomicCmpst64:
         case IrOpcode::SharedAtomicCmpstF64:
         case IrOpcode::SharedAtomicMskor64:
+        case IrOpcode::SharedAtomicCondxchg64:
             return 2u;
         case IrOpcode::LoadSharedU32x3:
         case IrOpcode::WriteSharedU32x3:

core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp

@@ -932,6 +932,7 @@ enum class RdnaOpcode : std::uint16_t {
     DsXorRtnB64,
     DsMskorRtnB64,
     DsWrxchgRtnB64,
+    DsCondxchg32RtnB64,
     DsCmpstRtnB64,
     DsCmpstRtnF64,
     DsMinRtnF64,

core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp

@@ -273,6 +273,7 @@ constexpr MemoryOpcodeInfo dsOpcodes[] = {
     {0x6bu, RdnaOpcode::DsXorRtnB64, 2, 32, false, false, false},
     {0x6cu, RdnaOpcode::DsMskorRtnB64, 2, 32, false, false, false},
     {0x6du, RdnaOpcode::DsWrxchgRtnB64, 2, 32, false, false, false},
+    {0x7Eu, RdnaOpcode::DsCondxchg32RtnB64, 2, 32, false, false, false},
     {0x70u, RdnaOpcode::DsCmpstRtnB64, 2, 32, false, false, false},
     {0x71u, RdnaOpcode::DsCmpstRtnF64, 2, 32, false, false, false},
     {0x72u, RdnaOpcode::DsMinRtnF64, 2, 32, false, false, false},
@@ -526,6 +527,7 @@ bool isDsAtomicOpcode(RdnaOpcode opcode) {
         case RdnaOpcode::DsOrRtnB64:
         case RdnaOpcode::DsXorRtnB64:
         case RdnaOpcode::DsWrxchgRtnB64:
+        case RdnaOpcode::DsCondxchg32RtnB64:
         case RdnaOpcode::DsMinRtnF64:
         case RdnaOpcode::DsMaxRtnF64:
             return true;

core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp

@@ -336,6 +336,7 @@ std::uint32_t EmitSharedAtomicFMax64(SpirvValueEmitContext& ctx, const IrValue&
 std::uint32_t EmitSharedAtomicCmpst64(SpirvValueEmitContext& ctx, const IrValue& inst);
 std::uint32_t EmitSharedAtomicCmpstF64(SpirvValueEmitContext& ctx, const IrValue& inst);
 std::uint32_t EmitSharedAtomicMskor64(SpirvValueEmitContext& ctx, const IrValue& inst);
+std::uint32_t EmitSharedAtomicCondxchg64(SpirvValueEmitContext& ctx, const IrValue& inst);
 std::uint32_t EmitDataAppend(SpirvValueEmitContext& ctx, const IrValue& inst);
 std::uint32_t EmitDataConsume(SpirvValueEmitContext& ctx, const IrValue& inst);
 void EmitImageQueryDimensions(SpirvValueEmitContext& ctx, const IrValue& inst);

core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp

@@ -630,6 +630,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) {
         case IrOpcode::SharedAtomicCmpst64: return Invoke(EmitSharedAtomicCmpst64, ctx, inst);
         case IrOpcode::SharedAtomicCmpstF64: return Invoke(EmitSharedAtomicCmpstF64, ctx, inst);
         case IrOpcode::SharedAtomicMskor64: return Invoke(EmitSharedAtomicMskor64, ctx, inst);
+        case IrOpcode::SharedAtomicCondxchg64: return Invoke(EmitSharedAtomicCondxchg64, ctx, inst);
         case IrOpcode::DataAppend: return Invoke(EmitDataAppend, ctx, inst);
         case IrOpcode::DataConsume: return Invoke(EmitDataConsume, ctx, inst);
         case IrOpcode::SwizzleU32: return Invoke(EmitSwizzleU32, ctx, inst);

core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp

@@ -1896,6 +1896,26 @@ std::uint32_t EmitSharedAtomicMskor64(SpirvValueEmitContext& ctx, const IrValue&
     });
 }
 
+std::uint32_t EmitSharedAtomicCondxchg64(SpirvValueEmitContext& ctx, const IrValue& inst) {
+    const auto data = ScalarU64Argument(ctx, inst, 2u);
+    return SharedAtomic64(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old) {
+        const auto lowMask = ScalarU64Constant(state, 0xffffffffu);
+        const auto bit31 = ScalarU64Constant(state, 0x80000000u);
+        const auto shift = ScalarU64Constant(state, 32u);
+        const auto oldLow = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), old, lowMask);
+        const auto oldHigh = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), old, shift);
+        const auto dataLow = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), data, lowMask);
+        const auto dataHigh = Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), data, shift);
+        const auto writeLow = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), dataLow, Unary(state, spv::OpNot, TypeScalarU64(state), bit31));
+        const auto writeHigh = Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), dataHigh, Unary(state, spv::OpNot, TypeScalarU64(state), bit31));
+        const auto lowEnabled = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), dataLow, ScalarU64Constant(state, 31u)), ScalarU64Constant(state, 0u));
+        const auto highEnabled = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), dataHigh, ScalarU64Constant(state, 31u)), ScalarU64Constant(state, 0u));
+        const auto nextLow = Select(state, TypeScalarU64(state), lowEnabled, writeLow, oldLow);
+        const auto nextHigh = Select(state, TypeScalarU64(state), highEnabled, writeHigh, oldHigh);
+        return Binary(state, spv::OpBitwiseOr, TypeScalarU64(state), nextLow, Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state), nextHigh, shift));
+    });
+}
+
 std::uint32_t EmitBufferAtomicInc32(SpirvValueEmitContext& ctx, const IrValue& inst) {
     return EmitAtomicUpdate(ctx, inst, BufferMemory(ctx, inst), AtomicIncrement);
 }

core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp

@@ -420,6 +420,8 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) {
         return dsAtomic64(inst, IrOpcode::SharedAtomicCmpstF64, false);
     case RdnaOpcode::DsCmpstRtnF64:
         return dsAtomic64(inst, IrOpcode::SharedAtomicCmpstF64, true);
+    case RdnaOpcode::DsCondxchg32RtnB64:
+        return dsAtomic64(inst, IrOpcode::SharedAtomicCondxchg64, true);
     case RdnaOpcode::DsMinF64:
         return dsAtomic64(inst, IrOpcode::SharedAtomicFMin64, false);
     case RdnaOpcode::DsMinRtnF64: