diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 121d80483eeb..225b0485450d 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -400,6 +400,16 @@ target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driv set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_masked_select_tests) +add_test_executable(agc_constant_bit_test_tests EXCLUDE_FROM_ALL tests/ConstantBitTest.cpp) +target_link_libraries(agc_constant_bit_test_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_constant_bit_test_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_constant_bit_test_tests) + +add_test_executable(agc_scalar_select_mask_tests EXCLUDE_FROM_ALL tests/ScalarSelectMask.cpp) +target_link_libraries(agc_scalar_select_mask_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_scalar_select_mask_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_scalar_select_mask_tests) + add_test_executable(agc_wave_control_decode_tests EXCLUDE_FROM_ALL tests/WaveControlDecode.cpp) target_include_directories(agc_wave_control_decode_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_wave_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) @@ -860,7 +870,7 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_shader_bindless agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) if(agcTest STREQUAL "agc_driver_constant_store_tests") continue() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp new file mode 100644 index 000000000000..92786750e41a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp @@ -0,0 +1,50 @@ +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "Optimization/ConstantFolder.hpp" +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +bool foldsTo(std::uint32_t word, bool expected, bool maskedShift) { + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + auto& shift = maskedShift ? builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&lane, &builder.Constant(31u)}) : lane; + auto& shifted = builder.Emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&builder.Constant(word), &shift}); + auto& bit = builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&shifted, &builder.Constant(1u)}); + auto& test = builder.Emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &builder.Constant(0u)}); + auto& user = builder.Emit(IrOpcode::Reference, IrType::Void, {&test}); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder().Fold(program); + const IrValue* result = user.Argument(0)->Resolve(); + if (!maskedShift || (word != 0u && word != 0xffffffffu)) return result->Opcode() == IrOpcode::INotEqual32; + return result->HasImmediate() && result->ImmediateBool() == expected; +} + +bool run(const char* name, bool passed) { + if (!passed) std::fprintf(stderr, "%s\n", name); + return passed; +} + +} + +int main() { + try { + bool passed = true; + passed &= run("an all-zero word must fold the lane bit test to false", foldsTo(0u, false, true)); + passed &= run("an all-one word must fold the lane bit test to true", foldsTo(0xffffffffu, true, true)); + passed &= run("a mixed word must keep the lane bit test", foldsTo(0x0000ffffu, false, true)); + passed &= run("an unmasked lane number must keep the lane bit test", foldsTo(0xffffffffu, true, false)); + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp b/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp new file mode 100644 index 000000000000..43258d0d96cc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp @@ -0,0 +1,352 @@ +#include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/Structurizer.hpp" +#include "IntermediateRepresentation/IrProgram.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Translation/InstructionTranslator.hpp" +#include "Translation/ShaderInputInfoBuilder.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +constexpr std::uint32_t Lanes = 64; + +using LaneWords = std::array; + +struct Value { + std::array component{}; +}; + +class Evaluator { +public: + Evaluator(const IrProgram& program, std::uint32_t waveSize) : lanes(waveSize), exec(waveSize >= 64u ? ~0ull : (1ull << waveSize) - 1ull) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) vector[0][lane] = lane; + for (const auto& block : program.Blocks()) { + for (const auto* inst : block->Instructions()) run(*inst); + } + } + + LaneWords output{}; + bool exported = false; + std::map maskTag; + std::map threadBitValue; + +private: + std::unordered_map values; + std::array vector{}; + std::array scalar{}; + std::map threadBit; + std::uint32_t lanes; + std::uint64_t exec; + std::uint64_t vcc = 0; + bool scc = false; + + static std::uint32_t registerIndex(const IrValue& inst) { + return inst.Argument(0)->Register().index; + } + + Value operand(const IrValue* argument) { + argument = argument->Resolve(); + const auto found = values.find(argument); + if (found != values.end()) return found->second; + if (!argument->HasImmediate()) throw std::runtime_error("the evaluator met a value that has not been computed"); + Value constant{}; + const std::uint32_t word = argument->Type() == IrType::Bool ? (argument->ImmediateBool() ? 1u : 0u) : argument->ImmediateU32(); + constant.component[0].fill(word); + return constant; + } + + Value fromBits(std::uint64_t bits) const { + Value result{}; + for (std::uint32_t lane = 0; lane < lanes; ++lane) result.component[0][lane] = static_cast((bits >> lane) & 1u); + return result; + } + + std::uint64_t toBits(const Value& value) const { + std::uint64_t bits = 0; + for (std::uint32_t lane = 0; lane < lanes; ++lane) bits |= static_cast(value.component[0][lane] & 1u) << lane; + return bits; + } + + static Value uniform(std::uint32_t word) { + Value result{}; + result.component[0].fill(word); + return result; + } + + std::uint32_t uniformWord(const Value& value) const { + for (std::uint32_t lane = 1; lane < lanes; ++lane) { + if (value.component[0][lane] != value.component[0][0]) throw std::runtime_error("a scalar register was written with a value that differs between lanes"); + } + return value.component[0][0]; + } + + template + Value lanewise(const IrValue& inst, TOperation operation) { + const Value left = operand(inst.Argument(0)); + const Value right = operand(inst.Argument(1)); + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = operation(left.component[0][lane], right.component[0][lane]); + return result; + } + + Value select(const IrValue& inst) { + const Value condition = operand(inst.Argument(0)); + const Value active = operand(inst.Argument(1)); + const Value inactive = operand(inst.Argument(2)); + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + result.component[0][lane] = (condition.component[0][lane] & 1u) != 0u ? active.component[0][lane] : inactive.component[0][lane]; + } + return result; + } + + void run(const IrValue& inst) { + switch (inst.Opcode()) { + case IrOpcode::SetExec: exec = toBits(operand(inst.Argument(0))); return; + case IrOpcode::SetExecLo: exec = (exec & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; + case IrOpcode::SetExecHi: exec = (exec & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; + case IrOpcode::GetExec: values[&inst] = fromBits(exec); return; + case IrOpcode::GetExecLo: values[&inst] = uniform(static_cast(exec)); return; + case IrOpcode::GetExecHi: values[&inst] = uniform(static_cast(exec >> 32)); return; + case IrOpcode::SetVcc: vcc = toBits(operand(inst.Argument(0))); return; + case IrOpcode::SetVccLo: vcc = (vcc & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; + case IrOpcode::SetVccHi: vcc = (vcc & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; + case IrOpcode::GetVcc: values[&inst] = fromBits(vcc); return; + case IrOpcode::GetVccLo: values[&inst] = uniform(static_cast(vcc)); return; + case IrOpcode::GetVccHi: values[&inst] = uniform(static_cast(vcc >> 32)); return; + case IrOpcode::SetScc: scc = (uniformWord(operand(inst.Argument(0))) & 1u) != 0u; return; + case IrOpcode::GetScc: values[&inst] = uniform(scc ? 1u : 0u); return; + case IrOpcode::SetScalarRegister: scalar.at(registerIndex(inst)) = uniformWord(operand(inst.Argument(1))); return; + case IrOpcode::GetScalarRegister: values[&inst] = uniform(scalar.at(registerIndex(inst))); return; + case IrOpcode::SetScalarMaskTag: maskTag[registerIndex(inst)] = (uniformWord(operand(inst.Argument(1))) & 1u) != 0u; return; + case IrOpcode::GetScalarMaskTag: { + const auto found = maskTag.find(registerIndex(inst)); + values[&inst] = uniform(found != maskTag.end() && found->second ? 1u : 0u); + return; + } + case IrOpcode::SetThreadBitScalarRegister: + threadBit[registerIndex(inst)] = operand(inst.Argument(1)).component[0]; + threadBitValue[registerIndex(inst)] = inst.Argument(1)->Resolve(); + return; + case IrOpcode::GetThreadBitScalarRegister: { + Value result{}; + const auto found = threadBit.find(registerIndex(inst)); + if (found != threadBit.end()) result.component[0] = found->second; + values[&inst] = result; + return; + } + case IrOpcode::SetVectorRegister: vector.at(registerIndex(inst)) = operand(inst.Argument(1)).component[0]; return; + case IrOpcode::GetVectorRegister: { + Value result{}; + result.component[0] = vector.at(registerIndex(inst)); + values[&inst] = result; + return; + } + case IrOpcode::GetBuiltin: values[&inst] = uniform(0u); return; + case IrOpcode::LaneId: { + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = lane; + values[&inst] = result; + return; + } + case IrOpcode::Ballot: { + const std::uint64_t bits = toBits(operand(inst.Argument(0))); + Value result{}; + result.component[0].fill(static_cast(bits)); + result.component[1].fill(static_cast(bits >> 32)); + values[&inst] = result; + return; + } + case IrOpcode::CompositeExtractU32x4: { + const Value source = operand(inst.Argument(0)); + Value result{}; + result.component[0] = source.component.at(operand(inst.Argument(1)).component[0][0]); + values[&inst] = result; + return; + } + case IrOpcode::CompositeConstructU32x4: { + Value result{}; + for (std::uint32_t index = 0; index < 4; ++index) result.component[index] = operand(inst.Argument(index)).component[0]; + values[&inst] = result; + return; + } + case IrOpcode::UGreaterThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a > b ? 1u : 0u; }); return; + case IrOpcode::ULessThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a < b ? 1u : 0u; }); return; + case IrOpcode::IEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a == b ? 1u : 0u; }); return; + case IrOpcode::INotEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a != b ? 1u : 0u; }); return; + case IrOpcode::LogicalAnd: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return (a & b) & 1u; }); return; + case IrOpcode::BitwiseAnd32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a & b; }); return; + case IrOpcode::ShiftRightLogical32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a >> (b & 31u); }); return; + case IrOpcode::SelectU1: + case IrOpcode::SelectU32: values[&inst] = select(inst); return; + case IrOpcode::SetAttribute: + output = operand(inst.Argument(0)).component[0]; + exported = true; + return; + default: throw std::runtime_error("the evaluator does not support IR opcode " + std::to_string(static_cast(inst.Opcode()))); + } + } +}; + +constexpr std::uint32_t Vcc = 0x6au; +constexpr std::uint32_t Exec = 0x7eu; +constexpr std::uint32_t Zero = 0x80u; +constexpr std::uint32_t MinusOne = 0xc1u; +constexpr std::uint32_t S4 = 0x04u; +constexpr std::uint32_t Literal = 0xffu; + +std::uint32_t vcmpLanesBelow(std::uint32_t count) { + return 0x7d880080u + count; +} + +std::uint32_t sCmpScc(bool value) { + return value ? 0xbf068080u : 0xbf068180u; +} + +std::uint32_t sCselectB64(std::uint32_t destination, std::uint32_t trueSource, std::uint32_t falseSource) { + return 0x85800000u | (destination << 16) | (falseSource << 8) | trueSource; +} + +std::uint32_t sMovB64(std::uint32_t destination, std::uint32_t source) { + return 0xbe800400u | (destination << 16) | source; +} + +std::array vCndmaskV1(std::uint32_t mask) { + return {0xd5010001u, 0x80u | (129u << 9) | (mask << 18)}; +} + +struct Outcome { + LaneWords lanes{}; + std::uint32_t waveSize = 64; + bool maskTagS4 = false; + bool constantSources = false; +}; + +Outcome run(const std::vector& body, std::uint32_t mask, std::uint32_t waveSize = 64) { + std::vector code = body; + const auto select = vCndmaskV1(mask); + code.insert(code.end(), {select[0], select[1], 0xf80008cfu, 0x01010101u, 0xbf810000u}); + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + auto cfg = GraphBuilder{}.Build(decoded); + Structurizer{}.Structurize(cfg); + GuestContext context{}; + context.waveSize = waveSize; + context.vertex = ShaderVertexStageInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Vertex; + options.waveSize = waveSize; + options.userDataCount = 0; + options.inputInfo = BuildShaderStageInputInfo(ShaderStageKind::Vertex, context, waveSize); + const auto program = InstructionTranslator{}.Translate(decoded, cfg, options); + const Evaluator evaluator(program, waveSize); + if (!evaluator.exported) throw std::runtime_error("the program exported nothing"); + Outcome outcome; + outcome.lanes = evaluator.output; + outcome.waveSize = waveSize; + const auto tag = evaluator.maskTag.find(S4); + outcome.maskTagS4 = tag != evaluator.maskTag.end() && tag->second; + const auto bit = evaluator.threadBitValue.find(S4); + if (bit != evaluator.threadBitValue.end() && bit->second->Opcode() == IrOpcode::SelectU1) { + outcome.constantSources = bit->second->Argument(1)->Resolve()->HasImmediate() && bit->second->Argument(2)->Resolve()->HasImmediate(); + } + return outcome; +} + +int failures = 0; + +void expectLanes(const char* name, const Outcome& outcome, std::uint64_t expectedBits) { + for (std::uint32_t lane = 0; lane < outcome.waveSize; ++lane) { + const std::uint32_t expected = static_cast((expectedBits >> lane) & 1u); + if (outcome.lanes[lane] != expected) { + std::fprintf(stderr, "%s: lane %u holds %u, expected %u (expected mask 0x%016llx)\n", name, lane, outcome.lanes[lane], expected, static_cast(expectedBits)); + ++failures; + return; + } + } +} + +void expectMaskPath(const char* name, const Outcome& outcome) { + if (!outcome.maskTagS4) { + std::fprintf(stderr, "%s: the destination pair was not marked as a mask\n", name); + ++failures; + } +} + +void expectConstantSources(const char* name, const Outcome& outcome, bool expected) { + if (outcome.constantSources != expected) { + std::fprintf(stderr, "%s: the sources of the select are %sconstants\n", name, expected ? "not " : ""); + ++failures; + } +} + +constexpr std::uint64_t AllLanes = ~0ull; + +std::uint64_t below(std::uint32_t count) { + return count >= 64u ? AllLanes : (1ull << count) - 1ull; +} + +void selectCase(const char* name, std::vector setup, std::uint32_t trueSource, std::uint32_t falseSource, std::uint64_t whenSccSet, std::uint64_t whenSccClear, bool maskPath, int constantSources = -1) { + for (const bool sccValue : {true, false}) { + auto body = setup; + body.push_back(sCmpScc(sccValue)); + body.push_back(sCselectB64(4u, trueSource, falseSource)); + const auto outcome = run(body, S4); + const std::string label = std::string(name) + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? whenSccSet : whenSccClear); + if (maskPath) expectMaskPath(label.c_str(), outcome); + if (constantSources >= 0) expectConstantSources(label.c_str(), outcome, constantSources != 0); + } +} + +} + +int main() { + try { + selectCase("vcc, exec", {vcmpLanesBelow(4)}, Vcc, Exec, below(4), AllLanes, true); + selectCase("exec, vcc", {vcmpLanesBelow(4)}, Exec, Vcc, AllLanes, below(4), true); + selectCase("wave64 halves, vcc, exec", {vcmpLanesBelow(40)}, Vcc, Exec, below(40), AllLanes, true); + selectCase("wave64 halves, exec, vcc", {vcmpLanesBelow(40)}, Exec, Vcc, AllLanes, below(40), true); + selectCase("exec, zero", {}, Exec, Zero, AllLanes, 0ull, true, 0); + selectCase("zero, exec", {}, Zero, Exec, 0ull, AllLanes, true, 0); + selectCase("zero, zero", {}, Zero, Zero, 0ull, 0ull, true, 1); + selectCase("minus one, zero", {}, MinusOne, Zero, AllLanes, 0ull, true, 1); + selectCase("minus one, exec", {}, MinusOne, Exec, AllLanes, AllLanes, true, 0); + selectCase("destination is the true source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, S4, Exec, below(40), AllLanes, false); + selectCase("destination is the false source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, Exec, S4, AllLanes, below(40), false); + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Exec), 0x0000ffffu}, S4); + const std::string label = std::string("literal 0xffff, exec") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(16) : AllLanes); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4); + const std::string label = std::string("wave64, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); + expectConstantSources(label.c_str(), outcome, false); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4, 32); + const std::string label = std::string("wave32, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); + expectConstantSources(label.c_str(), outcome, true); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({vcmpLanesBelow(40), sCmpScc(sccValue), sCselectB64(Vcc, Vcc, Exec)}, Vcc); + expectLanes(sccValue ? "destination vcc, scc = 1" : "destination vcc, scc = 0", outcome, sccValue ? below(40) : AllLanes); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } + return failures == 0 ? 0 : 1; +} diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index 7e1b3103445f..a37056fc7749 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -54,6 +54,28 @@ template bool foldU64Shift(IrBuilder& builder, IrValue& ins return true; } +bool foldConstantBitTest(IrBuilder& builder, IrValue& inst) { + auto& left = resolveArg(inst, 0); + auto& right = resolveArg(inst, 1); + if (!isImmediate(right, IrType::U32) || right.ImmediateU32() != 0u || left.Opcode() != IrOpcode::BitwiseAnd32) return false; + auto& first = resolveArg(left, 0); + auto& second = resolveArg(left, 1); + IrValue* shifted = nullptr; + if (isImmediate(second, IrType::U32) && second.ImmediateU32() == 1u) shifted = &first; + else if (isImmediate(first, IrType::U32) && first.ImmediateU32() == 1u) shifted = &second; + if (shifted == nullptr || shifted->Opcode() != IrOpcode::ShiftRightLogical32) return false; + auto& word = resolveArg(*shifted, 0); + auto& amount = resolveArg(*shifted, 1); + if (!isImmediate(word, IrType::U32) || (word.ImmediateU32() != 0u && word.ImmediateU32() != 0xffffffffu)) return false; + if (amount.Opcode() != IrOpcode::BitwiseAnd32) return false; + auto& amountLeft = resolveArg(amount, 0); + auto& amountRight = resolveArg(amount, 1); + const bool masked = (isImmediate(amountLeft, IrType::U32) && amountLeft.ImmediateU32() == 31u) || (isImmediate(amountRight, IrType::U32) && amountRight.ImmediateU32() == 31u); + if (!masked) return false; + replaceWith(inst, builder.ConstantBool(word.ImmediateU32() != 0u)); + return true; +} + template bool foldU32Compare(IrBuilder& builder, IrValue& inst, TFunction function) { auto& lhs = resolveArg(inst, 0); auto& rhs = resolveArg(inst, 1); @@ -617,7 +639,7 @@ bool ConstantFolder::tryFoldValue(IrProgram& program, IrValue& value) const { return foldU32(builder, value, [](std::uint32_t a, std::uint32_t b) { return std::max(a, b); }); } case IrOpcode::IEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a == b; }); - case IrOpcode::INotEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a != b; }); + case IrOpcode::INotEqual32: return foldConstantBitTest(builder, value) || foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a != b; }); case IrOpcode::ULessThan32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a < b; }); case IrOpcode::ULessThanEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a <= b; }); case IrOpcode::UGreaterThan32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a > b; }); diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 066c30d8dbe8..2dd73b25bf4a 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -302,6 +302,7 @@ class TranslationContext { void sSwappcB64(const RdnaInstruction& inst); void sCselectB32(const RdnaInstruction& inst); void scalarSelect64(const RdnaInstruction& inst, const RdnaOperand& falseSource); + void scalarSelectMask64(const RdnaInstruction& inst); void movB32(const RdnaInstruction& inst, bool applyFloatModifiers); void sMovB64(const RdnaInstruction& inst); void sWqm(const RdnaInstruction& inst, bool wide); diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 798eaf34c2e2..838346dc401b 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -244,6 +244,21 @@ void TranslationContext::scalarSelect64(const RdnaInstruction& inst, const RdnaO writeU32Pair(inst.destination, {low, high}); } +void TranslationContext::scalarSelectMask64(const RdnaInstruction& inst) { + if (inst.destination.kind != RdnaOperandKind::ScalarRegister) { + scalarSelect64(inst, sourceAt(inst, 1u)); + return; + } + const RdnaOperand& trueSource = sourceAt(inst, 0u); + const RdnaOperand& falseSource = sourceAt(inst, 1u); + IrValue& valid = ir.LogicalAnd(readMaskValid(trueSource).Value(), readMaskValid(falseSource).Value()); + IrValue& bit = ir.Emit(IrOpcode::SelectU1, IrType::U1, {&ir.GetScc(), &readMask(trueSource).Value(), &readMask(falseSource).Value()}); + scalarSelect64(inst, falseSource); + const auto dst = static_cast(inst.destination.reg); + ir.SetThreadBitScalarReg(dst, bit); + ir.SetScalarMaskTag(dst, valid); +} + void TranslationContext::movB32(const RdnaInstruction& inst, bool applyFloatModifiers) { if (applyFloatModifiers) { IrF32 value(*readOperand(sourceAt(inst, 0u), IrType::F32)); diff --git a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp index ada03c65b292..8484da20214e 100644 --- a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp @@ -43,7 +43,7 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { sCselectB32(inst); return true; case RdnaOpcode::SCselectB64: - scalarSelect64(inst, sourceAt(inst, 1u)); + scalarSelectMask64(inst); return true; case RdnaOpcode::SCmovB32: { const IrU32 source = readU32(sourceAt(inst, 0u)); diff --git a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp index 66b55249dced..6e94905dc9f7 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp @@ -19,6 +19,13 @@ IrU32 TranslationContext::hostExecWord(std::uint32_t half) { } IrU1 TranslationContext::threadBit(const std::array& mask) { + const bool wide = program.WaveSize() == 64u; + if (mask[0].Value().HasImmediate() && (!wide || mask[1].Value().HasImmediate())) { + const std::uint32_t low = mask[0].Value().ImmediateU32(); + const std::uint32_t high = wide ? mask[1].Value().ImmediateU32() : low; + if (low == 0u && high == 0u) return IrU1(ir.ConstantBool(false)); + if (low == 0xffffffffu && high == 0xffffffffu) return IrU1(ir.ConstantBool(true)); + } const IrU32 lane(ir.Emit(IrOpcode::LaneId, IrType::U32, {})); const IrU32 word = program.WaveSize() == 64u ? IrU32(ir.Select(ir.ULessThan(lane.Value(), ir.Constant(32u)), mask[0].Value(), mask[1].Value())) : mask[0]; const IrU32 bit(ir.BitwiseAnd(lane.Value(), ir.Constant(31u)));