perf(shader): run wave64 compute at one lane where no lane can tell

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1210,6 +1210,35 @@ if(BUILD_TESTING)
     endif()
 endif()
 
+add_test_executable(agc_bitwise_identities_tests EXCLUDE_FROM_ALL tests/BitwiseIdentities.cpp)
+target_link_libraries(agc_bitwise_identities_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_bitwise_identities_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_bitwise_identities_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_bitwise_identities_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_bitwise_identities COMMAND agc_bitwise_identities_tests)
+    if(WIN32)
+        set_tests_properties(agc_bitwise_identities PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
+add_test_executable(agc_single_lane_tests EXCLUDE_FROM_ALL tests/SingleLane.cpp)
+target_link_libraries(agc_single_lane_tests PRIVATE shader_recompiler_agc_driver libc)
+set_target_properties(agc_single_lane_tests PROPERTIES CXX_EXTENSIONS OFF)
+configure_windows_unwind(agc_single_lane_tests)
+if(BUILD_TESTING)
+    set_target_properties(agc_single_lane_tests PROPERTIES EXCLUDE_FROM_ALL OFF)
+    add_test(NAME agc_single_lane COMMAND agc_single_lane_tests)
+    add_test(NAME agc_single_lane_off COMMAND agc_single_lane_tests)
+    set_tests_properties(agc_single_lane PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1")
+    set_tests_properties(agc_single_lane_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_SINGLE_LANE_RULE=1")
+    if(WIN32)
+        set_tests_properties(agc_single_lane agc_single_lane_off PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()
+
 add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp)
 target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc)
 set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF)

core/libs/prx/libSceAgcDriver/tests/BitwiseIdentities.cpp

@@ -0,0 +1,210 @@
+#include "IntermediateRepresentation/IrBuilder.hpp"
+#include "Optimization/ConstantFolder.hpp"
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <exception>
+#include <functional>
+#include <initializer_list>
+#include <vector>
+
+using namespace ShaderRecompiler;
+
+namespace {
+
+constexpr IrOpcode And = IrOpcode::BitwiseAnd32;
+constexpr IrOpcode Or = IrOpcode::BitwiseOr32;
+constexpr IrOpcode Xor = IrOpcode::BitwiseXor32;
+constexpr IrOpcode Shift = IrOpcode::ShiftRightLogical32;
+
+struct Builder {
+    IrProgram program;
+    IrBuilder ir;
+    IrBlock* block = nullptr;
+    std::vector<IrValue*> kept;
+
+    Builder() : ir(program) {
+        block = &program.CreateBlock();
+        program.SetEntryBlock(*block);
+        program.BlockOrder().push_back(block);
+        ir.SetInsertionPoint(*block);
+    }
+
+    IrValue& Emit(IrOpcode opcode, std::initializer_list<IrValue*> arguments) {
+        return ir.Emit(opcode, IrType::U32, arguments);
+    }
+
+    IrValue& Value() {
+        return Emit(IrOpcode::LaneId, {});
+    }
+
+    IrValue& Other(IrValue& value) {
+        return Emit(IrOpcode::IAdd32, {&value, &ir.Constant(1u)});
+    }
+
+    IrValue& Not(IrValue& value) {
+        return Emit(IrOpcode::BitwiseNot32, {&value});
+    }
+
+    IrValue& Same(IrValue& value) {
+        auto& identity = program.CreateValue(IrOpcode::Identity, IrType::U32);
+        identity.AddArgument(&value);
+        block->AppendInstruction(&identity);
+        return identity;
+    }
+
+    IrValue& Binary(IrOpcode opcode, IrValue& lhs, IrValue& rhs) {
+        return Emit(opcode, {&lhs, &rhs});
+    }
+
+    IrValue& Binary(IrOpcode opcode, IrValue& lhs, std::uint32_t rhs) {
+        return Binary(opcode, lhs, ir.Constant(rhs));
+    }
+
+    IrValue& Binary(IrOpcode opcode, std::uint32_t lhs, IrValue& rhs) {
+        return Binary(opcode, ir.Constant(lhs), rhs);
+    }
+
+    std::size_t Keep(IrValue& value) {
+        kept.push_back(&ir.Emit(IrOpcode::ReferenceU32, IrType::Void, {&value}));
+        return kept.size() - 1u;
+    }
+
+    void Fold() {
+        static_cast<void>(ir.Emit(IrOpcode::Return, IrType::Void, {}));
+        ConstantFolder().Fold(program);
+    }
+
+    const IrValue& Result(std::size_t index) const {
+        return *kept.at(index)->Argument(0)->Resolve();
+    }
+
+    bool Folds(std::size_t index, std::uint32_t expected) const {
+        const IrValue& value = Result(index);
+        return value.HasImmediate() && value.Type() == IrType::U32 && value.ImmediateU32() == expected;
+    }
+
+    bool Becomes(std::size_t index, const IrValue& expected) const {
+        return &Result(index) == &expected;
+    }
+
+    bool Keeps(std::size_t index, IrOpcode opcode) const {
+        return Result(index).Opcode() == opcode;
+    }
+};
+
+bool expect(const char* name, bool condition) {
+    if (!condition) std::fprintf(stderr, "%s\n", name);
+    return condition;
+}
+
+bool run(const char* name, const std::function<bool()>& test) {
+    try {
+        return expect(name, test());
+    } catch (const std::exception& error) {
+        std::fprintf(stderr, "%s: %s\n", name, error.what());
+        return false;
+    }
+}
+
+}
+
+int main() {
+    bool passed = true;
+
+    passed &= run("and and or of a value with itself give the value and xor gives zero", [] {
+        Builder b;
+        auto& value = b.Value();
+        const auto both = b.Keep(b.Binary(And, value, value));
+        const auto either = b.Keep(b.Binary(Or, value, value));
+        const auto differ = b.Keep(b.Binary(Xor, value, value));
+        b.Fold();
+        return b.Becomes(both, value) && b.Becomes(either, value) && b.Folds(differ, 0u);
+    });
+
+    passed &= run("a value and its complement give zero for and and all ones for or and xor in either order", [] {
+        Builder b;
+        auto& value = b.Value();
+        auto& inverse = b.Not(value);
+        const auto andLeft = b.Keep(b.Binary(And, value, inverse));
+        const auto andRight = b.Keep(b.Binary(And, inverse, value));
+        const auto orLeft = b.Keep(b.Binary(Or, value, inverse));
+        const auto orRight = b.Keep(b.Binary(Or, inverse, value));
+        const auto xorLeft = b.Keep(b.Binary(Xor, value, inverse));
+        const auto xorRight = b.Keep(b.Binary(Xor, inverse, value));
+        b.Fold();
+        return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu) && b.Folds(xorLeft, 0xffffffffu) && b.Folds(xorRight, 0xffffffffu);
+    });
+
+    passed &= run("and with zero and or with all ones absorb the other operand in either order", [] {
+        Builder b;
+        auto& value = b.Value();
+        const auto andLeft = b.Keep(b.Binary(And, value, 0u));
+        const auto andRight = b.Keep(b.Binary(And, 0u, value));
+        const auto orLeft = b.Keep(b.Binary(Or, value, 0xffffffffu));
+        const auto orRight = b.Keep(b.Binary(Or, 0xffffffffu, value));
+        b.Fold();
+        return b.Folds(andLeft, 0u) && b.Folds(andRight, 0u) && b.Folds(orLeft, 0xffffffffu) && b.Folds(orRight, 0xffffffffu);
+    });
+
+    passed &= run("operands match through identities", [] {
+        Builder b;
+        auto& value = b.Value();
+        auto& inverse = b.Same(b.Not(b.Same(value)));
+        const auto both = b.Keep(b.Binary(And, value, b.Same(value)));
+        const auto differ = b.Keep(b.Binary(Xor, b.Same(value), value));
+        const auto complement = b.Keep(b.Binary(Or, inverse, value));
+        b.Fold();
+        return b.Becomes(both, value) && b.Folds(differ, 0u) && b.Folds(complement, 0xffffffffu);
+    });
+
+    passed &= run("different values, the complement of another value and xor with all ones are kept", [] {
+        Builder b;
+        auto& value = b.Value();
+        auto& other = b.Other(value);
+        auto& inverse = b.Not(other);
+        const auto both = b.Keep(b.Binary(And, value, other));
+        const auto either = b.Keep(b.Binary(Or, value, other));
+        const auto differ = b.Keep(b.Binary(Xor, value, other));
+        const auto andInverse = b.Keep(b.Binary(And, value, inverse));
+        const auto orInverse = b.Keep(b.Binary(Or, inverse, value));
+        const auto xorInverse = b.Keep(b.Binary(Xor, value, inverse));
+        const auto flipped = b.Keep(b.Binary(Xor, value, 0xffffffffu));
+        b.Fold();
+        return b.Keeps(both, And) && b.Keeps(either, Or) && b.Keeps(differ, Xor) && b.Keeps(andInverse, And) && b.Keeps(orInverse, Or) && b.Keeps(xorInverse, Xor) && b.Keeps(flipped, Xor);
+    });
+
+    passed &= run("bit 0 of all ones shifted right by an amount masked to 31 is one in either order", [] {
+        Builder b;
+        auto& lane = b.Value();
+        auto& maskRight = b.Binary(And, lane, 31u);
+        auto& maskLeft = b.Binary(And, 31u, lane);
+        const auto bitRight = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, maskRight), 1u));
+        const auto bitLeft = b.Keep(b.Binary(And, 1u, b.Binary(Shift, 0xffffffffu, maskLeft)));
+        b.Fold();
+        return b.Folds(bitRight, 1u) && b.Folds(bitLeft, 1u);
+    });
+
+    passed &= run("bit 0 of an unmasked or wider shift, of another word, and bit 1 are kept", [] {
+        Builder b;
+        auto& lane = b.Value();
+        auto& masked = b.Binary(And, lane, 31u);
+        const auto unmasked = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, lane), 1u));
+        const auto wider = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, b.Binary(And, lane, 63u)), 1u));
+        const auto word = b.Keep(b.Binary(And, b.Binary(Shift, 0x7fffffffu, masked), 1u));
+        const auto second = b.Keep(b.Binary(And, b.Binary(Shift, 0xffffffffu, masked), 2u));
+        b.Fold();
+        return b.Keeps(unmasked, And) && b.Keeps(wider, And) && b.Keeps(word, And) && b.Keeps(second, And);
+    });
+
+    passed &= run("zero shifted right by any amount is zero and another word is kept", [] {
+        Builder b;
+        auto& lane = b.Value();
+        const auto zero = b.Keep(b.Binary(Shift, 0u, lane));
+        const auto word = b.Keep(b.Binary(Shift, b.Other(lane), lane));
+        b.Fold();
+        return b.Folds(zero, 0u) && b.Keeps(word, Shift);
+    });
+
+    return passed ? 0 : 1;
+}

core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp

@@ -5428,6 +5428,18 @@ void verifyHalfWaveReduction() {
     expectFailure([&] { static_cast<void>(pixel(upperHelperLoop, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: the low-word loop proof was reused for helper-inclusive upper lanes");
     require(reductions(pixel(loopScan, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: a loop-carried saved EXEC mask exhausted the entry-lane proof on 32 lanes");
     require(halves(pixel(loopScan, 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: a loop-carried saved EXEC mask did not preserve the two half-wave UMin reductions");
+    auto computeScan = scan(0xffffffffu, 0x26000000u, 0x83800000u);
+    const auto exported = std::find(computeScan.begin(), computeScan.end(), 0xf800180fu);
+    exported[0] = 0xe0700000u;
+    exported[1] = 0x80000100u;
+    static constexpr std::array<std::uint32_t, 4> computeBuffer{0x10000000u, 0u, 0x40u, 0x00027facu};
+    static constexpr std::array<std::uint32_t, 9> computeCapabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic};
+    auto computeRequest = FrontendRequest(computeScan, computeBuffer);
+    computeRequest.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
+    computeRequest.target.supportedCapabilities = computeCapabilities;
+    const auto twoLane = Recompile(computeRequest);
+    require(twoLane.hostSubgroupSize == 32u && reductions(twoLane.spirv, spv::OpGroupNonUniformUMin) == 0u, "half-wave reduction: a two-lane wave64 compute UMin scan under s_or_saveexec_b64 -1 reached the half-wave scan matcher");
+    require(halves(pixel(scan(0xffffffffu, 0x26000000u, 0x83800000u), 64u, true), spv::OpGroupNonUniformUMin, 0xffffffffu), "half-wave reduction: the wave64 UMin scan under s_or_saveexec_b64 -1 did not read lanes 31 and 63 as two half-wave UMin reductions on 64 lanes");
     auto comparedScan = scan(0u, 0x38000000u, 0x88000000u);
     *std::find(comparedScan.begin(), comparedScan.end(), 0xbe98047eu) = 0x8a986a7eu;
     comparedScan.insert(comparedScan.begin(), 0x7c080080u);
@@ -9315,8 +9327,10 @@ int main(int argc, char** argv) {
     try {
 #ifdef _WIN32
         _putenv_s("ANYPS5_NO_SHADER_CACHE", "1");
+        _putenv_s("APS5_NO_SINGLE_LANE_RULE", "1");
 #else
         setenv("ANYPS5_NO_SHADER_CACHE", "1", 1);
+        setenv("APS5_NO_SINGLE_LANE_RULE", "1", 1);
 #endif
         using namespace ShaderRecompiler;
         if (argc == 2 && std::string_view(argv[1]) == "--specialized-bda-abi") {