diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e006f65ab76..ae437152c7ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,10 @@ if(BUILD_TESTING) add_test(NAME shader_ir_use_rewrite COMMAND shader_ir_use_rewrite_tests) set_tests_properties(shader_ir_use_rewrite PROPERTIES TIMEOUT 20) if(Python3_Interpreter_FOUND) + add_test(NAME shader_cache_native_paths COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/test_shader_cache_native_paths.py + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ShaderCacheVersion.cmake ${CMAKE_COMMAND}) + set_tests_properties(shader_cache_native_paths PROPERTIES TIMEOUT 20) add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) add_test(NAME pr_overlap_dependencies COMMAND ${Python3_EXECUTABLE} diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 9b4a46d7241e..5c788df25cba 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -697,6 +697,11 @@ target_link_libraries(agc_formatted_buffer_once_tests PRIVATE shader_recompiler_ set_target_properties(agc_formatted_buffer_once_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_formatted_buffer_once_tests) +add_test_executable(agc_fma_mix_once_tests EXCLUDE_FROM_ALL tests/FmaMixOnce.cpp) +target_link_libraries(agc_fma_mix_once_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_fma_mix_once_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_fma_mix_once_tests) + add_test_executable(agc_rect_list_position_invariant_tests EXCLUDE_FROM_ALL tests/RectListPositionInvariant.cpp) target_link_libraries(agc_rect_list_position_invariant_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_rect_list_position_invariant_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -1158,7 +1163,7 @@ if(BUILD_TESTING) if(WIN32) set_tests_properties(agc_shader_bindless agc_shader_type_zero_image_materialization agc_shader_frontend_pair agc_shader_frontend_pair_off agc_shader_layout_pair agc_shader_layout_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_spirv_bvh_intersect_function_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_spirv_bvh_intersect_function_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_integer_narrowing_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_placeholder_images_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests} agc_fma_mix_once_tests) if(agcTest STREQUAL "agc_driver_constant_store_tests" OR agcTest STREQUAL "agc_driver_flat_apertures_tests") continue() endif() diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 985f2d660eca..4caaf9d4102d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -24,9 +24,11 @@ #include #include #include +#include #include #include #include +#include "Optimization/ShaderStageInputInfo.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" namespace AgcDriver::DriverDetail { @@ -405,14 +407,48 @@ std::vector ReadUserDataCallee(std::uint64_t address, std::uint32 throw std::runtime_error("its callee at " + hexText(address) + " does not return within " + std::to_string(limitWords * 4u) + " bytes"); } -std::vector LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) { +std::vector ReadGuestBytes(std::uint64_t address, std::uint64_t bytes, const char* what) { + if (address > UINTPTR_MAX - bytes || !GuestMemory::Accessible(reinterpret_cast(static_cast(address)), static_cast(bytes))) { + throw std::runtime_error(std::string("its ") + what + " at " + hexText(address) + " + " + hexText(bytes) + " is not readable"); + } + std::vector data(static_cast(bytes)); + GuestMemory::Read(address, std::as_writable_bytes(std::span(data))); + return data; +} + +std::vector ReadTableCallees(const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::TableCall& call) { + constexpr std::uint64_t limitBytes = 64u << 20u; + constexpr std::size_t limitCallees = 64u; + const auto slot = call.userDataRegister - request.context.userDataBaseRegister; + const auto pointer = static_cast(request.context.userData[slot]) | static_cast(request.context.userData[slot + 1u]) << 32u; + const auto words = ReadGuestBytes(pointer + call.descriptorOffset, 16u, "call table descriptor"); + ShaderRecompiler::ShaderBufferResource descriptor{}; + std::memcpy(descriptor.fields.data(), words.data(), words.size()); + const auto bytes = descriptor.GetSize(); + if (bytes > limitBytes) throw std::runtime_error("its call table of " + std::to_string(bytes) + " bytes is larger than the " + std::to_string(limitBytes) + " bytes that are searched for callees"); + const auto table = bytes == 0u ? std::vector{} : ReadGuestBytes(descriptor.Base48(), bytes, "call table"); + std::set targets; + for (std::uint64_t offset = call.entryOffset; offset + 8u <= bytes; offset += call.entryStride) { + std::uint64_t target = 0; + std::memcpy(&target, table.data() + offset, sizeof(target)); + if (target != 0u) targets.insert(target); + if (call.entryStride == 0u) break; + } + if (targets.size() > limitCallees) throw std::runtime_error("its call table holds " + std::to_string(targets.size()) + " distinct targets; at most " + std::to_string(limitCallees) + " are linked"); + std::vector callees; + for (const auto target : targets) callees.push_back({target, ReadUserDataCallee(target, call.linkRegister)}); + return callees; +} + +ShaderRecompiler::LinkedCalls LinkUserDataCallees(const ShaderRecompiler::RecompileRequest& request) { if (request.shader.stage != ShaderRecompiler::ShaderStage::Compute) return {}; const auto code = request.shader.code; if (std::none_of(code.begin(), code.end(), [](std::uint32_t word) { return (word & 0xff80ff00u) == 0xbe802100u; })) return {}; const auto program = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); - const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const ShaderRecompiler::SwappcInfo swappc{false, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), request.context.waveSize}; const auto calls = ShaderRecompiler::FindUserDataCalls(program, swappc); - if (calls.empty()) return {}; + const auto tableCalls = ShaderRecompiler::FindTableCalls(program, swappc); + if (calls.empty() && tableCalls.empty()) return {}; std::vector> callees; for (const auto& call : calls) { const auto slot = call.userDataRegister - swappc.userDataBaseRegister; @@ -424,7 +460,15 @@ std::vector LinkUserDataCallees(const ShaderRecompiler::Recompile std::to_string(call.userDataRegister + 1u) + "] at program counter " + hexText(call.programCounter) + ", but " + error.what()); } } - return ShaderRecompiler::LinkUserDataCalls(program, calls, callees); + std::vector> tableCallees; + for (const auto& call : tableCalls) { + try { + tableCallees.push_back(ReadTableCallees(request, call)); + } catch (const std::exception& error) { + throw std::runtime_error("AGC driver: compute shader " + hexText(request.shader.codeAddress) + " calls a target loaded from a table at program counter " + hexText(call.programCounter) + ", but " + error.what()); + } + } + return ShaderRecompiler::LinkCalls(program, calls, callees, tableCalls, tableCallees); } } @@ -433,9 +477,12 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); const auto code = std::span(snapshot.code).subspan(codeOffset); require(request.shader.code.data() == code.data() && request.shader.code.size() == code.size(), "prepared invocation does not refer to registered code"); - const auto linked = LinkUserDataCallees(request); + auto linked = LinkUserDataCallees(request); auto source = request; - if (!linked.empty()) source.shader.code = linked; + if (!linked.code.empty()) { + source.shader.code = linked.code; + source.context.linkedCallFallbacks = linked.fallbacks; + } auto invocationRequest = source; struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); @@ -445,7 +492,7 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset != codeOffset) continue; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); - if (!linked.empty() && !std::ranges::equal(invocationRequest.shader.code, linked)) continue; + if (!linked.code.empty() && !std::ranges::equal(invocationRequest.shader.code, linked.code)) continue; if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation); } ReportPreparedAtUse(snapshot, source); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index ebb6da55f1a9..bb20cc9368fe 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -359,10 +359,10 @@ void Validate(std::span packet, std::uint32_t queue) { require((packet[8] & 3u) == 0 && packet[8] >= (opcode == 0x2c ? 16u : 20u), "invalid indirect draw stride"); require((packet[9] & ~0x20u) == (opcode == 0x2c ? 2u : 0u), "unsupported indirect draw initiator"); break; - case 0x15: size(5); if ((packet[4] & ~0xa066u) != 0x1u || (packet[4] & 0x22u) == 0x22u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; + case 0x15: size(5); if ((packet[4] & ~0xa06eu) != 0x1u || (packet[4] & 0x22u) == 0x22u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; case 0x16: require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size"); - require((packet.back() & ~0xa064u) == 0x1u, "indirect dispatch modifiers are not implemented"); + require((packet.back() & ~0xa06cu) == 0x1u, "indirect dispatch modifiers are not implemented"); break; case 0x22: size(5); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index 0b94d5e0c9f2..9eb9ce81ae89 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -254,6 +254,10 @@ void BdaResources::CheckFault() const { return; } std::ostringstream message; + if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::UnlinkedCall) { + message << "shader stage " << report.stage << " called address 0x" << std::hex << report.address << " at program counter 0x" << report.instruction << ", which no linked callee of that call matches"; + throw std::runtime_error(message.str()); + } message << "BDA access failed: address=0x" << std::hex << report.address << " instruction=0x" << report.instruction << std::dec << " bytes=" << report.bytes << " stage=" << report.stage << " reason=" << static_cast(report.reason); if (report.reason == ShaderRecompiler::BdaAbi::FaultReason::Permission && table != nullptr) { const auto bytes = table->Bytes(); diff --git a/core/libs/prx/libSceAgcDriver/tests/FmaMixOnce.cpp b/core/libs/prx/libSceAgcDriver/tests/FmaMixOnce.cpp new file mode 100644 index 000000000000..aa48f4f1e7ff --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FmaMixOnce.cpp @@ -0,0 +1,93 @@ +#include "BdaAbi.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void Check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +SpirvTarget Target() { + static constexpr std::array capabilities{ + spv::CapabilityInt64, spv::CapabilityFloat64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance, + }; + static constexpr std::array extensions{ + "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer", + }; + return {0x00401000u, 0x00010300u, 32, BdaAbi::Version, capabilities, extensions, false, + {1024, 1024, 64}, 1024, 32768, {}, {}}; +} + +struct Counts { + std::size_t words = 0; + std::size_t functions = 0; +}; + +Counts Count(std::span words) { + Check(words.size() >= 5 && words[0] == spv::MagicNumber, "fma mix once: missing SPIR-V module"); + Counts counts{words.size()}; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + Check(count != 0 && count <= words.size() - offset, "fma mix once: malformed SPIR-V instruction"); + if ((words[offset] & spv::OpCodeMask) == spv::OpFunction) ++counts.functions; + offset += count; + } + return counts; +} + +Counts Compile(std::uint32_t uses) { + std::vector code; + for (std::uint32_t load = 0; load < 3u; ++load) code.insert(code.end(), {0xe0300000u | (load * 4u), 0x80010000u | ((1u + load) << 8u)}); + code.push_back(0xbf8c3f70u); + std::uint32_t result = 1u; + for (std::uint32_t use = 0; use < uses; ++use) { + const std::uint32_t opcode = use % 2u == 0u ? 0x21u : 0x22u; + code.insert(code.end(), {0xcc000000u | (opcode << 16u) | 10u, (256u + result) | ((256u + 2u) << 9u) | ((256u + 3u) << 18u)}); + result = 10u; + } + code.insert(code.end(), {0xe0700000u, 0x80010000u | (result << 8u)}); + code.push_back(0xbf810000u); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const std::array userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}; + request.target = Target(); + request.layout = {0, 0, 0, 128}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.memory = memory; + request.useCache = false; + return Count(Recompile(request).spirv.Words()); +} + +} + +int main() { + try { + const auto none = Compile(0); + const auto one = Compile(1); + const auto three = Compile(3); + Check(three.words - one.words < one.words - none.words, "fma mix once: 2 more v_fma_mixlo/mixhi_f16 added " + std::to_string(three.words - one.words) + " SPIR-V words, as many as the first one with its shared emulation (" + std::to_string(one.words - none.words) + ")"); + Check(one.functions == none.functions + 1u, "fma mix once: one v_fma_mixlo_f16 emitted " + std::to_string(one.functions - none.functions) + " functions more than none, expected one shared emulation function"); + Check(three.functions == one.functions, "fma mix once: two more v_fma_mixlo/mixhi_f16 in the same float mode emitted " + std::to_string(three.functions - one.functions) + " more functions, expected calls to the shared one"); + std::printf("fma mix once tests passed (%zu words without v_fma_mix, %zu for 1, %zu for 3)\n", none.words, one.words, three.words); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index c9940f3080ea..f5b19424e951 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -214,7 +214,8 @@ void testContextAndBases() { expectFailure([] { static_cast(AgcDriver::Pm4::PartialGroupThreads({0x10002, 1, 1}, {0x0001ffff, 0x00010001, 0x00010001})); }, "exceeds 32 bits"); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x2063}, 2), 0); }, "dispatch modifiers"); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x41}, 4), 0); }, "header flags"); - expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x49}), 0); }, "dispatch modifiers"); + AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x49}), 0); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x51}), 0); }, "dispatch modifiers"); execute(state, makePacket(0x13, {32})); execute(state, makePacket(0x26, {0x1000, 1})); execute(state, makePacket(0x2a, {1})); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0c893691b256..586f7f01bb35 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2185,13 +2185,19 @@ void verifyPixelRequestSerialization() { minimal.context.pixel = ShaderPixelStageInfo{}; minimal.context.pixel->rasterPrimitive = RasterPrimitive::Rectangle; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQRgAAAA=", "new requests did not use serialization version 24"); + require(requestPrefix(encoded, 8u) == "NVNQQRkAAAA=", "new requests did not use serialization version 25"); + const std::array fallbacks{0x40u, 0x1ac4u}; + auto withFallbacks = minimal; + withFallbacks.context.linkedCallFallbacks = fallbacks; + const auto fallbackRequest = serializer.Deserialize(serializer.Serialize(withFallbacks)); + require(std::ranges::equal(fallbackRequest.request.context.linkedCallFallbacks, fallbacks), "the linked call fallbacks did not round-trip"); + require(serializer.Deserialize(encoded).request.context.linkedCallFallbacks.empty(), "a request without linked call fallbacks gained some"); require(serializer.Deserialize(encoded).request.context.pixel->rasterPrimitive == RasterPrimitive::Rectangle, "the pixel primitive class did not round-trip"); constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; for (std::size_t bytes = 0; bytes < 20u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag or primitive class was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRkAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRoAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp index 632181959326..534787fa7613 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace { @@ -77,14 +78,74 @@ void verifyReads() { } } +std::vector opcodeSequence(const IrProgram& program) { + std::vector sequence; + for (const auto& block : program.Blocks()) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() != IrOpcode::ControlNop) sequence.push_back(static_cast(instruction->Opcode())); + } + } + return sequence; +} + +constexpr std::uint32_t endProgram = 0xbf810000; +constexpr std::uint32_t getregMode = 0xb9141801; +constexpr std::uint32_t vAddF32 = 0x06000300; +constexpr std::uint32_t vCvtF32I32 = 0x7e000b00; +constexpr std::uint32_t vCvtF32U32 = 0x7e000d00; +constexpr std::uint32_t vCvtF16F32 = 0x7e001500; +constexpr std::uint32_t vAddF16 = 0x64000300; + +std::uint32_t roundMode(std::uint32_t value) { + return 0xbfa40000u | value; +} + +void requireOpcode(std::uint32_t word, RdnaOpcode expected) { + const std::array code{word, 0u, endProgram}; + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + require(!decoded.instructions.empty() && decoded.instructions.front().op == expected, "test encoding " + std::to_string(word) + " does not decode to the expected opcode"); +} + +void verifyEncodings() { + for (const auto& [word, opcode] : {std::pair{vAddF32, RdnaOpcode::VAddF32}, {vCvtF32I32, RdnaOpcode::VCvtF32I32}, {vCvtF32U32, RdnaOpcode::VCvtF32U32}, + {vCvtF16F32, RdnaOpcode::VCvtF16F32}, {vAddF16, RdnaOpcode::VAddF16}, {roundMode(8), RdnaOpcode::SRoundMode}}) { + const std::array code{word, endProgram}; + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + require(decoded.instructions.front().op == opcode, "test encoding " + std::to_string(word) + " does not decode to the expected opcode"); + } + requireOpcode(0xba801801, RdnaOpcode::SSetregImm32B32); + const std::array sgpr{0xb9800001, endProgram}; + require(RdnaInstructionDecoder{}.Decode(sgpr).instructions.front().op == RdnaOpcode::SSetregB32, "s_setreg_b32 encoding does not decode"); +} + void verifyDynamicWrites() { - for (const auto instruction : {0xb9800001u, 0xb9801801u, 0xb9800041u, 0xb9801881u, 0xb980f801u}) { - const std::array code{instruction, 0xbf810000}; + for (const auto instruction : {0xb9800001u, 0xb9801801u, 0xb9800041u}) { + const ShaderFloatMode mode{0xc0}; + const std::array unused{instruction, endProgram}; + static_cast(translate(unused, mode)); + expectRejected(unused, std::nullopt, "s_setreg_b32"); + const std::array read{instruction, getregMode, endProgram}; + expectRejected(read, mode, "s_getreg_b32", 4); + const std::array used{instruction, vAddF32, endProgram}; + expectRejected(used, mode, "reads MODE rounding bits", 4); + } + for (const auto instruction : {0xb9801881u, 0xb980f801u, 0xb9800101u}) { + const std::array code{instruction, endProgram}; expectRejected(code, std::nullopt, "s_setreg_b32"); expectRejected(code, ShaderFloatMode{0xc0}, "s_setreg_b32"); } } +void verifyDynamicWriteSurvivesPartialOverwrite() { + constexpr ShaderFloatMode mode{0xc0}; + const std::array code{0xb9801801, 0xba801801, 0, getregMode, vAddF32, endProgram}; + expectRegister(translate(code, mode), 20, 0); + const std::array partial{0xb9801801, 0xba800801, 0, getregMode, vAddF32, endProgram}; + expectRejected(partial, mode, "s_getreg_b32", 12); + const std::array split{0xb9801801, 0xba800801, 0, 0xba800881, 0, endProgram}; + static_cast(translate(split, mode)); +} + void verifyImmediateWrites() { constexpr std::array instructions{0xba801801, 0xba800801, 0xba800881, 0xba800841}; constexpr std::array masks{15, 3, 12, 6}; @@ -92,44 +153,41 @@ void verifyImmediateWrites() { for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { for (std::uint32_t value = 0; value < 16; ++value) { for (std::uint32_t field = 0; field < instructions.size(); ++field) { - const std::array code{instructions[field], 0xfffffff0u | value, 0xb9141801, 0xbf810000}; + const std::array code{instructions[field], 0xfffffff0u | value, getregMode, endProgram}; const ShaderFloatMode mode{0xc0u | rounding}; - if (((value << offsets[field]) & masks[field]) != (rounding & masks[field])) { - expectRejected(code, mode, "s_setreg_imm32_b32"); - } else { - expectRegister(translate(code, mode), 20, rounding); - } + const std::uint32_t expected = (rounding & ~masks[field]) | ((value << offsets[field]) & masks[field]); + expectRegister(translate(code, mode), 20, expected); } } } - constexpr std::array full{0xba801801, 0, 0xbf810000}; + constexpr std::array full{0xba801801, 0, endProgram}; static_cast(translate(full, std::nullopt)); - constexpr std::array nonzero{0xba801801, 15, 0xbf810000}; - constexpr std::array denorm{0xba801881, 0, 0xbf810000}; - expectRejected(nonzero, ShaderFloatMode{0xc0}, "s_setreg_imm32_b32"); + constexpr std::array nonzero{0xba801801, 15, endProgram}; + expectRejected(nonzero, std::nullopt, "s_setreg_imm32_b32"); + constexpr std::array denorm{0xba801881, 0, endProgram}; expectRejected(denorm, ShaderFloatMode{0xc0}, "s_setreg_imm32_b32"); + constexpr std::array otherRegister{0xba801802, 0, endProgram}; + expectRejected(otherRegister, ShaderFloatMode{0xc0}, "s_setreg_imm32_b32"); } void verifyRoundInstruction() { for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { for (std::uint32_t value = 0; value < 16; ++value) { for (const auto upper : {0u, 0x8000u}) { - const std::array code{0xbfa40000u | upper | value, 0xb9141801, 0xbf810000}; - const ShaderFloatMode mode{0xc0u | rounding}; - if (value != rounding) expectRejected(code, mode, "s_round_mode"); - else expectRegister(translate(code, mode), 20, rounding); + const std::array code{roundMode(upper | value), getregMode, endProgram}; + expectRegister(translate(code, ShaderFloatMode{0xc0u | rounding}), 20, value); } } } - constexpr std::array code{0xbfa40000, 0xbf810000}; + constexpr std::array code{0xbfa40000, endProgram}; static_cast(translate(code, std::nullopt)); - constexpr std::array nonzero{0xbfa40003, 0xbf810000}; - expectRejected(nonzero, ShaderFloatMode{0xc0}, "s_round_mode"); + constexpr std::array nonzero{0xbfa40003, endProgram}; + expectRejected(nonzero, std::nullopt, "s_round_mode"); } void verifyUnsupportedReads() { for (const auto instruction : {0xb9140002u, 0xb9140101u, 0xb91408c1u, 0xb914f801u}) { - const std::array code{instruction, 0xbf810000}; + const std::array code{instruction, endProgram}; expectRejected(code, ShaderFloatMode{0xc0}, "s_getreg_b32"); } } @@ -137,13 +195,85 @@ void verifyUnsupportedReads() { void verifyBranch() { for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { const ShaderFloatMode mode{0xc0u | rounding}; - std::array code{0xbf840002, 0xba801801, rounding, 0xb9141801, 0xbf810000}; + std::array code{0xbf840002, 0xba801801, rounding, getregMode, endProgram}; expectRegister(translate(code, mode), 20, rounding); code[2] ^= 1u; - expectRejected(code, mode, "s_setreg_imm32_b32", 4); + expectRejected(code, mode, "s_getreg_b32", 12); } } +void verifyDiamond() { + for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { + const ShaderFloatMode mode{0xc0u | rounding}; + const std::uint32_t other = rounding ^ 8u; + const std::array same{0xbf840003, 0xba801801, other, 0xbf820002, 0xba801801, other, getregMode, endProgram, endProgram}; + expectRegister(translate(same, mode), 20, other); + const std::array differing{0xbf840003, 0xba801801, other, 0xbf820002, 0xba801801, rounding, getregMode, endProgram, endProgram}; + expectRejected(differing, mode, "s_getreg_b32", 24); + const std::array differingUse{0xbf840003, 0xba801801, 3, 0xbf820002, 0xba801801, 0, vCvtF32I32, endProgram, endProgram}; + if (rounding == 0u) expectRejected(differingUse, mode, "reads MODE rounding bits", 24); + } +} + +void verifyRoundingBeforeDirected32Use() { + for (const auto instruction : {vCvtF32I32, vCvtF32U32}) { + for (std::uint32_t rounding = 1; rounding < 4; ++rounding) { + const std::array runtime{roundMode(rounding), instruction, endProgram}; + const std::array declared{instruction, endProgram}; + const auto converted = opcodeSequence(translate(runtime, ShaderFloatMode{0xc0})); + require(converted == opcodeSequence(translate(declared, ShaderFloatMode{0xc0u | rounding})), "runtime rounding does not match the declared rounding"); + require(converted != opcodeSequence(translate(declared, ShaderFloatMode{0xc0})), "runtime rounding did not change the conversion"); + } + } +} + +void verifyRoundingRestoredBeforeUse() { + for (const auto instruction : {vAddF32, vAddF16}) { + for (std::uint32_t directed = 1; directed < 16; ++directed) { + const std::array restored{roundMode(directed), roundMode(0), instruction, roundMode(directed), endProgram}; + const std::array plain{instruction, endProgram}; + require(opcodeSequence(translate(restored, ShaderFloatMode{0xc0})) == opcodeSequence(translate(plain, ShaderFloatMode{0xc0})), "restored rounding changed the translation"); + } + } +} + +void verifyUntouchedFieldsStayAccepted() { + const std::array f16Only{roundMode(8), vAddF32, roundMode(0), endProgram}; + const std::array plain{vAddF32, endProgram}; + require(opcodeSequence(translate(f16Only, ShaderFloatMode{0xc0})) == opcodeSequence(translate(plain, ShaderFloatMode{0xc0})), "f16 rounding changed an f32 operation"); + const std::array f32Only{roundMode(2), vAddF16, roundMode(0), endProgram}; + const std::array plain16{vAddF16, endProgram}; + require(opcodeSequence(translate(f32Only, ShaderFloatMode{0xc0})) == opcodeSequence(translate(plain16, ShaderFloatMode{0xc0})), "f32 rounding changed an f16 operation"); +} + +void verifyUnmodelledDirectedRounding() { + for (std::uint32_t directed = 1; directed < 4; ++directed) { + const std::array f32{roundMode(directed), vAddF32, endProgram}; + expectRejected(f32, ShaderFloatMode{0xc0}, "does not model directed rounding", 4); + const std::array f16{roundMode(directed << 2u), vAddF16, endProgram}; + expectRejected(f16, ShaderFloatMode{0xc0}, "does not model directed rounding", 4); + } +} + +void verifyDirectedF16Conversion() { + for (std::uint32_t rounding = 1; rounding < 4; ++rounding) { + const std::array runtime{roundMode(rounding << 2u), vCvtF16F32, endProgram}; + const std::array declared{vCvtF16F32, endProgram}; + const auto converted = opcodeSequence(translate(runtime, ShaderFloatMode{0xc0})); + require(converted == opcodeSequence(translate(declared, ShaderFloatMode{0xc0u | (rounding << 2u)})), "runtime f16 rounding does not match the declared rounding"); + expectRejected(runtime, ShaderFloatMode{0x00}, "f16 denormals not kept", 4); + } +} + +void verifyRoundingInLoop() { + const std::array stable{roundMode(0), vAddF32, 0xbf84fffd, roundMode(0), endProgram, endProgram}; + static_cast(translate(stable, ShaderFloatMode{0xc0})); + const std::array alternating{roundMode(0), vAddF32, 0xbf84fffd, roundMode(3), endProgram, endProgram}; + static_cast(translate(alternating, ShaderFloatMode{0xc0})); + const std::array carried{vAddF32, roundMode(3), 0xbf84fffd, getregMode, endProgram, endProgram}; + expectRejected(carried, ShaderFloatMode{0xc0}, "reads MODE rounding bits", 0); +} + void verifyRecompilePropagation() { constexpr std::array code{0xbfa40005, 0xbf810000}; constexpr std::array capabilities{1, 61}; @@ -175,12 +305,24 @@ void verifyRecompilePropagation() { } int main() { + struct Check { + const char* name; + void (*run)(); + }; + constexpr std::array checks{{ + {"encodings", verifyEncodings}, {"reads", verifyReads}, {"dynamicWrites", verifyDynamicWrites}, + {"partialOverwrite", verifyDynamicWriteSurvivesPartialOverwrite}, {"immediateWrites", verifyImmediateWrites}, + {"roundInstruction", verifyRoundInstruction}, {"unsupportedReads", verifyUnsupportedReads}, {"branch", verifyBranch}, + {"diamond", verifyDiamond}, {"directed32", verifyRoundingBeforeDirected32Use}, {"restored", verifyRoundingRestoredBeforeUse}, + {"untouchedFields", verifyUntouchedFieldsStayAccepted}, {"unmodelled", verifyUnmodelledDirectedRounding}, + {"directedF16", verifyDirectedF16Conversion}, {"loop", verifyRoundingInLoop}, {"propagation", verifyRecompilePropagation}, + }}; int failures = 0; - for (const auto check : {verifyReads, verifyDynamicWrites, verifyImmediateWrites, verifyRoundInstruction, verifyUnsupportedReads, verifyBranch, verifyRecompilePropagation}) { + for (const auto& check : checks) { try { - check(); + check.run(); } catch (const std::exception& error) { - std::fprintf(stderr, "%s\n", error.what()); + std::fprintf(stderr, "%s: %s\n", check.name, error.what()); ++failures; } } diff --git a/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp b/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp index e75cde7a9ffa..16faeba83fc6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WaveControlDecode.cpp @@ -102,6 +102,8 @@ void verifyTranslation() { require(recompile(withStore({0xbf8d0000u})).empty(), "s_sethalt 0 is refused: " + recompile(withStore({0xbf8d0000u}))); const std::vector trailingLoop{0x7e020281u, 0xbf820003u, 0xe0700000u, 0x80000100u, 0xbf810000u, 0xbf82fffcu, 0xbf9f0000u, 0xbf9f0000u, 0xffffffffu}; require(recompile(trailingLoop).empty(), "a program padded with s_code_end after its last branch is refused: " + recompile(trailingLoop)); + const auto orderedAdd = withStore({0xd8fe0000u, 0x05000001u}); + require(recompile(orderedAdd).empty(), "ds_ordered_count adding to a GDS counter is refused: " + recompile(orderedAdd)); const std::vector, std::string_view>> refused{ {"s_sethalt 1", withStore({0xbf8d0001u}), "s_sethalt 1 at pc 0 halts the wave"}, @@ -114,7 +116,8 @@ void verifyTranslation() { {"ds_gws_sema_p", withStore({0xd8720000u, 0x00000000u}), "ds_gws_sema_p at pc 0: global wave sync"}, {"ds_gws_sema_release_all", withStore({0xd8620000u, 0x00000000u}), "ds_gws_sema_release_all at pc 0: global wave sync"}, {"ds_gws_barrier", withStore({0xd8760000u, 0x00000003u}), "ds_gws_barrier at pc 0: global wave sync"}, - {"ds_ordered_count", withStore({0xd8fe0000u, 0x05000001u}), "ds_ordered_count at pc 0: GDS counters"}, + {"ds_ordered_count swap", withStore({0xd8fe1000u, 0x05000001u}), "ds_ordered_count at pc 0: ordered swap is not modeled"}, + {"ds_ordered_count of two dwords", withStore({0xd8fe4000u, 0x05000001u}), "ds_ordered_count at pc 0: counting more than one dword is not modeled"}, }; for (const auto& [name, code, message] : refused) { const auto error = recompile(code); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp new file mode 100644 index 000000000000..6b3bb066da31 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ComputeTableCalls.cpp @@ -0,0 +1,164 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "SceShaders.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Untouched = 0xcafebabeu; + +alignas(256) constexpr std::array Caller{ + 0x7e020280u, 0xf4080400u, 0xfa000000u, 0x8f148402u, 0xbf8cc07fu, 0xf4280208u, 0x28000000u, 0xbf8cc07fu, + 0xbe8e0308u, 0xbe8f0309u, 0xbf068103u, 0xbe8e210eu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, + 0xbf9f0000u, 0x30306c73u, 0x00000040u, 0x00000008u}; +alignas(256) constexpr std::array SelectSixteen{0x85159091u, 0x7e020215u, 0xbe80200eu}; +alignas(256) constexpr std::array SelectThirtyTwo{0x8515a0a1u, 0x7e020215u, 0xbe80200eu}; +alignas(16) std::array Table{}; +alignas(16) std::array Descriptors{}; +alignas(256) std::array Output{}; + +void Fill() { + const auto first = reinterpret_cast(SelectSixteen.data()); + const auto second = reinterpret_cast(SelectThirtyTwo.data()); + Table = {static_cast(first), static_cast(first >> 32u), 0u, 0u, static_cast(second), static_cast(second >> 32u), 0u, 0u}; + const auto table = reinterpret_cast(Table.data()); + Descriptors = {static_cast(table), static_cast((table >> 32u) & 0xffffu) | (16u << 16u), 2u, 0x31016facu}; +} + +std::array UserData(std::uint32_t index, bool scc) { + const auto descriptors = reinterpret_cast(Descriptors.data()); + const auto output = reinterpret_cast(Output.data()); + return {static_cast(descriptors), static_cast(descriptors >> 32u), index, scc ? 1u : 0u, + static_cast(output), static_cast((output >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +std::vector Commands(const std::array& data, bool wave32) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{Threads, 1, 1}; + registers(0x207, shape); + const auto address = reinterpret_cast(Caller.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array resource{static_cast(data.size()) << 1u}; + registers(0x213, resource); + registers(0x240, data); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x41u | (wave32 ? 0x8000u : 0u)}); + return words; +} + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +std::string Dispatch(std::uint32_t index, bool scc, bool wave32) { + Output.fill(Untouched); + const auto words = Commands(UserData(index, scc), wave32); + Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; + StderrCapture log; + std::string failure; + try { + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); + } catch (const std::exception& error) { + failure = error.what(); + } + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + return log.Finish() + failure; +} + +void Check(std::uint32_t index, bool scc, bool wave32, std::uint32_t expected, const std::string& what) { + const auto log = Dispatch(index, scc, wave32); + Require(log.find("no linked callee") == std::string::npos, what + ": a linked callee was reported missing: " + log); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Require(Output[lane] == expected, what + (wave32 ? " (wave32)" : " (wave64)") + ": lane " + std::to_string(lane) + " is " + std::to_string(Output[lane]) + ", expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + Fill(); + for (const bool wave32 : {false, true}) { + Check(0u, false, wave32, 16u, "the first record's callee with SCC 0"); + Check(0u, true, wave32, 17u, "the first record's callee with SCC 1"); + Check(1u, false, wave32, 32u, "the second record's callee with SCC 0"); + Check(1u, true, wave32, 33u, "the second record's callee with SCC 1"); + const auto log = Dispatch(2u, false, wave32); + Require(log.find("called address 0x0 at program counter") != std::string::npos && log.find("which no linked callee of that call matches") != std::string::npos, + std::string("a record past the table did not report an unlinked call") + (wave32 ? " (wave32)" : " (wave64)") + ": " + log); + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + Require(Output[lane] == Untouched, "a lane stored after the unlinked call: lane " + std::to_string(lane) + " is " + std::to_string(Output[lane])); + } + } + AgcDriverShutdown_nid_postfix(); + std::puts("compute table call tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception&) { + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp index 0ee7d0c78653..9ac9376530a6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp @@ -26,6 +26,8 @@ constexpr std::array Starts{1000, 2000, 3000, 4000}; constexpr std::uint32_t UntouchedAppend = 0x5eed0001u; constexpr std::uint32_t UntouchedConsume = 0x5eed0002u; constexpr std::size_t ActiveLanesLiteral = 12; +constexpr std::size_t OrderedM0Literal = 3; +constexpr std::array OrderedM0{0u, 7u}; alignas(256) std::array Output{}; alignas(16) std::array CounterReadback{}; @@ -44,6 +46,16 @@ alignas(256) auto Code = std::to_array({ 0xbf810000, }); +alignas(256) auto OrderedCode = std::to_array({ + 0x34060082, + 0x4a020083, + 0xbefc03ff, 0x0000ffff, + 0xd8fe0310, 0x05000001, + 0xbf8cc07f, + 0xe0702000, 0x80000503, + 0xbf810000, +}); + struct Workgroup { std::uint32_t threads; std::uint32_t waveSize; @@ -83,13 +95,11 @@ std::array ReadCounters(AgcDriver::QueueState& queue) { return CounterReadback; } -void Run(AgcDriver::VulkanDevice& device, const Workgroup& workgroup) { +void Run(AgcDriver::VulkanDevice& device, const Workgroup& workgroup, std::span code) { Output.fill(0xdeadbeefu); - Code[ActiveLanesLiteral] = workgroup.activeLanes; std::vector userData(4, 0u); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); std::copy(output.begin(), output.end(), userData.begin()); - const std::span code(Code); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{workgroup.threads, 1, 1}, 0, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -134,6 +144,28 @@ void CheckWaves(const Workgroup& workgroup, std::uint32_t result, std::uint32_t } } +void CheckOrdered(const Workgroup& workgroup, std::uint32_t m0, const std::array& counters) { + const auto waves = workgroup.threads / workgroup.waveSize; + const auto what = workgroup.Name() + " ds_ordered_count with M0 " + std::to_string(m0); + std::vector order(waves); + for (std::uint32_t wave = 0; wave < waves; ++wave) { + order[wave] = wave; + const auto first = wave * workgroup.waveSize; + for (std::uint32_t lane = first; lane < first + workgroup.waveSize; ++lane) { + const auto actual = Output[lane * Results]; + Require(actual == Output[first * Results], what + ": lane " + std::to_string(lane) + " returned " + std::to_string(actual) + ", its wave's first lane " + std::to_string(Output[first * Results])); + } + } + std::sort(order.begin(), order.end(), [&](std::uint32_t left, std::uint32_t right) { return Output[left * workgroup.waveSize * Results] < Output[right * workgroup.waveSize * Results]; }); + auto expected = Starts[0]; + for (const auto wave : order) { + const auto first = wave * workgroup.waveSize; + Require(Output[first * Results] == expected, what + ": wave " + std::to_string(wave) + " returned " + std::to_string(Output[first * Results]) + ", expected " + std::to_string(expected)); + expected += first + 3u; + } + Require(counters[0] == expected, what + ": GDS counter 0 is " + std::to_string(counters[0]) + ", expected " + std::to_string(expected)); +} + void Check(const Workgroup& workgroup, std::uint32_t pass, const std::array& counters) { const auto active = std::min(workgroup.activeLanes, workgroup.threads); const std::array steps{workgroup.threads, workgroup.threads, active, active}; @@ -164,10 +196,17 @@ int main() { for (const auto& workgroup : Workgroups) { SetCounters(queue); Require(ReadCounters(queue) == Starts, workgroup.Name() + ": the GDS counters did not read back as set"); + Code[ActiveLanesLiteral] = workgroup.activeLanes; for (std::uint32_t pass = 0; pass < 2; ++pass) { - Run(*device, workgroup); + Run(*device, workgroup, Code); Check(workgroup, pass, ReadCounters(queue)); } + for (const auto m0 : OrderedM0) { + SetCounters(queue); + OrderedCode[OrderedM0Literal] = m0; + Run(*device, workgroup, OrderedCode); + CheckOrdered(workgroup, m0, ReadCounters(queue)); + } } std::puts("gds append consume tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp index 4614998ce7d1..465df1cabf24 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp @@ -22,6 +22,7 @@ using ShaderRecompiler::ShaderStage; constexpr std::uint32_t Threads = 32; constexpr std::uint32_t Words = 32; constexpr std::uint32_t Size = 16; +constexpr std::uint32_t Mips = 5; constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type1D = 8; constexpr std::uint32_t Type2D = 9; @@ -30,6 +31,7 @@ constexpr std::uint32_t LoadResult = 12; alignas(256) std::array Buffer{}; alignas(256) std::array Line{}; alignas(256) std::array Plane{}; +alignas(256) std::array MippedLine{}; alignas(256) constexpr std::array Code{ 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, @@ -39,11 +41,43 @@ alignas(256) constexpr std::array Code{ 0x80001201, 0xe070103c, 0x80001301, 0xbf810000, }; -alignas(256) constexpr std::array ExplicitLodCode{ +alignas(256) constexpr std::array ExplicitLodCode{ 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, - 0x80000501, 0xbf8c3f70, 0xf0900f08, 0x00610c02, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xbf810000, + 0x80000501, 0xbf8c3f70, 0xf0900f08, 0x00610c02, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xe0701024, + 0x80000d01, 0xe0701028, 0x80000e01, 0xe070102c, 0x80000f01, 0xbf810000, }; +alignas(256) constexpr std::array LineLodCode{ + 0x34020087, 0xe0301000, 0x80000201, 0xe0301008, 0x80000301, 0xbf8c3f70, 0xf0900f00, 0x00610c02, 0xbf8c3f70, + 0xe0701020, 0x80000c01, 0xe0701024, 0x80000d01, 0xe0701028, 0x80000e01, 0xe070102c, 0x80000f01, 0xbf810000, +}; + +struct SamplerCase { + const char* name; + std::uint32_t clamp; + std::uint32_t filter; + std::uint32_t mipFilter; +}; + +constexpr std::uint32_t ClampWrap = 0; +constexpr std::uint32_t ClampEdge = 2; +constexpr std::uint32_t ClampBorder = 6; +constexpr std::uint32_t FilterPoint = 0; +constexpr std::uint32_t FilterLinear = 1; +constexpr std::uint32_t MipPoint = 1; +constexpr std::uint32_t MipLinear = 2; + +constexpr std::array SamplerCases{{ + {"point filtering, point mips, clamp to edge", ClampEdge, FilterPoint, MipPoint}, + {"point filtering, point mips, wrap", ClampWrap, FilterPoint, MipPoint}, + {"linear filtering, linear mips, clamp to edge", ClampEdge, FilterLinear, MipLinear}, + {"linear filtering, linear mips, wrap", ClampWrap, FilterLinear, MipLinear}, + {"linear filtering, linear mips, clamp to border", ClampBorder, FilterLinear, MipLinear}, +}}; + +constexpr std::array LodU{0.03125f, 0.33125f, 0.96875f, 0.0f, 1.0f, -0.2f, 1.3f, 0.6f}; +constexpr std::array Lods{0.0f, 1.0f, 2.0f, 1.25f, 2.75f, 3.6f, 6.0f, -1.0f}; + constexpr std::array LineV{0.0f, 0.5f, 1.0f, 1.5f, -0.25f, 3.0f, 0.25f, 0.75f}; constexpr std::array LineY{0u, 1u, 2u, 3u, 0xffffffffu, 7u, 0u, 5u}; @@ -81,6 +115,20 @@ std::array LineDescriptor() { }; } +std::array MippedLineDescriptor() { + const auto address = static_cast(reinterpret_cast(MippedLine.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + (Size - 1u) >> 2u, + 0xfacu | ((Mips - 1u) << 16u) | (Type1D << 28u), + 0u, + (Mips - 1u) << 4u, + 0u, + 0u, + }; +} + std::array PlaneDescriptor() { const auto address = static_cast(reinterpret_cast(Plane.data())); return { @@ -111,6 +159,25 @@ void FillTexture(std::span storage, const std::array MipTexel(std::uint32_t level, std::uint32_t x) { + return {10u + 40u * level, x * 12u + 3u, 200u - 30u * level, 255u - 7u * x - 11u * level}; +} + +void FillMippedLine() { + const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(MippedLineDescriptor())); + Require(geometry.guestBytes <= MippedLine.size(), "image plane address on a line: the mipped texture does not fit the texel storage"); + Require(geometry.mips.size() == Mips, "image plane address on a line: the mipped texture has " + std::to_string(geometry.mips.size()) + " mips"); + std::fill(MippedLine.begin(), MippedLine.end(), 0xeeu); + for (std::uint32_t level = 0; level < Mips; ++level) { + const auto& mip = geometry.mips[level]; + for (std::uint32_t x = 0; x < (Size >> level); ++x) { + const auto value = MipTexel(level, x); + auto* texel = &MippedLine[geometry.GuestLayerOffset(0) + mip.tiledOffset + x * 4u]; + for (std::uint32_t component = 0; component < 4u; ++component) texel[component] = static_cast(value[component]); + } + } +} + std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; @@ -120,10 +187,13 @@ std::array SamplerDescriptor() { return {0u, 0xfffu << 12u, 1u << 26u, 0u}; } -void Run(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture) { +std::array SamplerDescriptor(const SamplerCase& sampler) { + return {sampler.clamp | (sampler.clamp << 3u) | (sampler.clamp << 6u), 0xfffu << 12u, (sampler.filter << 20u) | (sampler.filter << 22u) | (sampler.mipFilter << 26u), 0u}; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture, const std::array& sampler = SamplerDescriptor()) { std::vector userData(16, 0u); const auto buffer = BufferDescriptor(Buffer.data(), static_cast(Buffer.size() * 4u)); - const auto sampler = SamplerDescriptor(); std::copy(buffer.begin(), buffer.end(), userData.begin()); std::copy(texture.begin(), texture.end(), userData.begin() + 4); std::copy(sampler.begin(), sampler.end(), userData.begin() + 12); @@ -164,16 +234,68 @@ void CheckPlane() { } } -void RequireRefused(AgcDriver::VulkanDevice& device) { - std::string refusal; - try { - Run(device, ExplicitLodCode, LineDescriptor()); - } catch (const std::exception& error) { - refusal = error.what(); +float LaneU(std::uint32_t tid) { + return LodU[(tid + tid / 8u) % LodU.size()]; +} + +float LaneLod(std::uint32_t tid) { + return Lods[tid / 4u]; +} + +void FillLodInput() { + Buffer.fill(0xdeadbeefu); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::array words{Bits(LaneU(tid)), Bits(LineV[(tid / 2u) % LineV.size()]), Bits(LaneLod(tid)), 0u}; + std::copy(words.begin(), words.end(), &Buffer[tid * Words]); } - Require(refusal.find("incompatible with the static runtime image interface") != std::string::npos, "image_sample_l 2d on a 1D texture was not refused: " + refusal); } +std::array SampleResults() { + std::array results{}; + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy_n(&Buffer[tid * Words + SampleResult], 4u, &results[tid * 4u]); + return results; +} + +std::uint32_t PointLevel(float lod) { + return static_cast(std::clamp(std::lround(lod), 0L, static_cast(Mips - 1u))); +} + +std::uint32_t PointTexel(float u, std::uint32_t level, std::uint32_t clamp) { + const auto width = static_cast(Size >> level); + const auto texel = static_cast(std::floor(u * static_cast(width))); + if (clamp == ClampWrap) return static_cast((texel % width + width) % width); + return static_cast(std::clamp(texel, 0, width - 1)); +} + +void CheckExplicitLod(AgcDriver::VulkanDevice& device) { + FillMippedLine(); + for (const auto& sampler : SamplerCases) { + const auto samplerWords = SamplerDescriptor(sampler); + FillLodInput(); + Run(device, LineLodCode, MippedLineDescriptor(), samplerWords); + const auto reference = SampleResults(); + FillLodInput(); + Run(device, ExplicitLodCode, MippedLineDescriptor(), samplerWords); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto u = LaneU(tid); + const auto lod = LaneLod(tid); + const auto what = std::string("image_sample_l 2d on a 1D texture with ") + sampler.name + ", u " + std::to_string(u) + ", v " + std::to_string(LineV[(tid / 2u) % LineV.size()]) + ", LOD " + std::to_string(lod); + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto value = Buffer[tid * Words + SampleResult + component]; + const auto expected = reference[tid * 4u + component]; + Require(value == expected, what + ": thread " + std::to_string(tid) + " component " + std::to_string(component) + " has bits " + std::to_string(value) + ", image_sample_l 1d returned bits " + std::to_string(expected)); + } + if (sampler.filter == FilterPoint) { + const auto level = PointLevel(lod); + Expect(tid, SampleResult, MipTexel(level, PointTexel(u, level, sampler.clamp)), what); + } else if (sampler.clamp != ClampBorder) { + const auto mark = std::bit_cast(Buffer[tid * Words + SampleResult]) * 255.0f; + const auto expected = 10.0f + 40.0f * std::clamp(lod, 0.0f, static_cast(Mips - 1u)); + Require(std::fabs(mark - expected) <= 1.0f, what + ": thread " + std::to_string(tid) + " blends the levels to " + std::to_string(mark) + ", expected " + std::to_string(expected)); + } + } + } +} } int main() { @@ -188,7 +310,7 @@ int main() { FillInput(false); Run(*device, Code, PlaneDescriptor()); CheckPlane(); - RequireRefused(*device); + CheckExplicitLod(*device); std::puts("image plane address on line tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/tests/ScalarCall.cpp b/core/libs/tests/ScalarCall.cpp index e797b7aeba84..dda7b2fea4ae 100644 --- a/core/libs/tests/ScalarCall.cpp +++ b/core/libs/tests/ScalarCall.cpp @@ -151,10 +151,8 @@ static void CheckUserDataCalls() { Require(std::equal(addFive.begin(), addFive.end(), linked.begin() + 12) && std::equal(addSixtyFour.begin(), addSixtyFour.end(), linked.begin() + 14)); auto readsPc = caller; readsPc[1] = 0xbe941f00u; - ExpectThrow("cannot have user-data callees linked", [&] { - const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); - static_cast(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees)); - }); + const auto readsPcProgram = RdnaInstructionDecoder{}.Decode(readsPc); + Require(LinkUserDataCalls(readsPcProgram, FindUserDataCalls(readsPcProgram, swappc), callees).size() == 17u); const auto linkedProgram = RdnaInstructionDecoder{}.Decode(linked); const auto cfg = GraphBuilder{}.Build(linkedProgram, &swappc); @@ -181,6 +179,101 @@ static void CheckUserDataCalls() { ExpectThrow("is shared with another call", [&] { static_cast(GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(sameCopy), &swappc)); }); } +static std::vector TableCaller() { + return {0x8f148414u, 0xf4080406u, 0xfa000270u, 0xbf8cc07fu, 0xf4280208u, 0x28000000u, 0xbf8cc07fu, 0xbe8e0308u, 0xbe8f0309u, 0xbe8e210eu, 0xbf810000u}; +} + +static ControlFlowGraph BuildCode(std::span code, const SwappcInfo& swappc) { + return GraphBuilder{}.Build(RdnaInstructionDecoder{}.Decode(code), &swappc); +} + +static IrProgram TranslateCode(std::span code, const SwappcInfo& swappc) { + const auto program = RdnaInstructionDecoder{}.Decode(code); + const auto cfg = GraphBuilder{}.Build(program, &swappc); + ShaderComputeInputInfo computeInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Compute; + options.inputInfo.compute = &computeInfo; + options.userDataCount = swappc.userDataCount; + return InstructionTranslator{}.Translate(program, cfg, options); +} + +static void CheckTableCalls() { + const SwappcInfo swappc{false, 0u, 16u}; + const auto caller = TableCaller(); + const auto program = RdnaInstructionDecoder{}.Decode(caller); + const auto calls = FindTableCalls(program, swappc); + Require(calls.size() == 1u); + Require(calls[0].programCounter == 0x24u && calls[0].linkRegister == 14u && calls[0].targetRegister == 14u && calls[0].userDataRegister == 12u); + Require(calls[0].descriptorOffset == 0x270u && calls[0].entryOffset == 0u && calls[0].entryStride == 16u); + Require(FindUserDataCalls(program, swappc).empty()); + const auto table = [&](std::size_t word, std::uint32_t value, std::uint32_t userDataCount = 16u) { + auto code = caller; + code[word] = value; + return FindTableCalls(RdnaInstructionDecoder{}.Decode(code), SwappcInfo{false, 0u, userDataCount}); + }; + Require(table(0u, 0x8f148114u).empty()); + Require(table(2u, 0x28000270u).empty()); + Require(table(8u, 0xbe8f030au).empty()); + Require(table(10u, 0xbf810000u, 12u).empty()); + Require(table(5u, 0x28000008u).size() == 1u && table(5u, 0x28000008u)[0].entryOffset == 8u); + Require(table(0u, 0xbe940380u).empty()); + auto readsLane = caller; + readsLane.insert(readsLane.begin(), {0xd4c2000bu, 0x00020300u}); + const auto readsLaneProgram = RdnaInstructionDecoder{}.Decode(readsLane); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 32u}).size() == 1u); + Require(FindTableCalls(readsLaneProgram, SwappcInfo{false, 0u, 16u, 64u}).empty()); + + ExpectThrow("not statically resolvable", [&] { static_cast(BuildCode(caller, swappc)); }); + const std::array fallback{0x24u}; + SwappcInfo listed = swappc; + listed.linkedCallFallbacks = fallback; + Require(BuildCode(caller, listed).unlinkedCallProgramCounters == std::vector{0x24u}); + const auto unlinked = TranslateCode(caller, listed); + Require(unlinked.Info().usesFaultBuffer && ProgramToString(unlinked).find("UnlinkedCall") != std::string::npos); + + const std::vector first{0xbf068000u, 0x4a020285u, 0xbe80200eu}; + const std::vector second{0x4a0202c0u, 0xbf068001u, 0xbe80200eu}; + const std::vector> callees{{{0x1122334455ull, first}, {0x66778899aaull, second}}}; + const auto linkedCalls = LinkCalls(program, {}, {}, calls, callees); + const auto& linked = linkedCalls.code; + Require(linkedCalls.fallbacks == std::vector{50u * 4u}); + ExpectThrow("is shared with another call", [&] { static_cast(BuildCode(linked, swappc)); }); + SwappcInfo linkedSwappc = swappc; + linkedSwappc.linkedCallFallbacks = linkedCalls.fallbacks; + Require(linked.size() == 59u && linked[9] == 0xbf820001u && linked[11] == 0xbf850010u && linked[58] == 0xbf9f0000u); + Require(linked[12] == 0xbf07ff0eu && linked[13] == 0x22334455u && linked[14] == 0xbf850004u && linked[15] == 0xbf07ff0fu && linked[16] == 0x11u && linked[17] == 0xbf850001u && linked[18] == 0xbf82001bu); + Require(linked[20] == 0x778899aau && linked[26] == 0xbf078080u && linked[27] == 0xbf820016u && linked[34] == 0xbf068080u && linked[35] == 0xbf82000au && linked[44] == 0xbf068080u && linked[45] == 0xbf820004u); + Require(linked[46] == 0xbb0e0005u && linked[47] == 0xbf82ffdau && linked[48] == 0xbb0e0006u && linked[49] == 0xbf82ffd8u && linked[50] == 0xbe8e210eu && linked[51] == 0xbf82ffd6u); + Require(std::equal(first.begin(), first.end(), linked.begin() + 52) && std::equal(second.begin(), second.end(), linked.begin() + 55)); + const auto cfg = BuildCode(linked, linkedSwappc); + const auto block = [&](std::uint32_t word) { return cfg.FindBlockByProgramCounter(word * 4u).id; }; + Require(cfg.FindBlockByProgramCounter(46u * 4u).terminator.trueBlock == block(52u)); + Require(cfg.FindBlockByProgramCounter(52u * 4u).terminator.trueBlock == block(47u)); + Require(cfg.FindBlockByProgramCounter(48u * 4u).terminator.trueBlock == block(55u)); + Require(cfg.FindBlockByProgramCounter(55u * 4u).terminator.trueBlock == block(49u)); + Require(cfg.unlinkedCallProgramCounters == std::vector{50u * 4u}); + static_cast(TranslateCode(linked, linkedSwappc)); + + std::vector large(0x8000u, 0xbf800000u); + const std::vector> far{{{0x10ull, large}, {0x20ull, first}}}; + ExpectThrow("out of s_call_b64 range", [&] { static_cast(LinkCalls(program, {}, {}, calls, far)); }); + + const std::vector jumpTable{ + 0xbe8c1f00u, 0x800cff0cu, 0x00000058u, 0x820d800du, 0x83928202u, 0xd4c40016u, 0x02000500u, 0x8f128212u, 0xf4000386u, 0x24000000u, + 0xbf8cc07fu, 0x7d820003u, 0x808c0e0cu, 0x828d800du, 0xbefd210cu, 0x7e020281u, 0xbf820003u, 0x7e020282u, 0xbf820001u, 0x7e020283u, + 0xe0700000u, 0x80000100u, 0xbf810000u, 0x00000020u, 0x00000018u, 0x00000010u}; + ExpectThrow("jumps through a table in its code", [&] { static_cast(LinkCalls(RdnaInstructionDecoder{}.Decode(jumpTable), {}, {}, {}, {})); }); + const std::array compare{0xbf068008u}; + const auto comparesLink = Build({DecodeRdnaSopc(0u, compare, 0u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(comparesLink.FindBlockByProgramCounter(0u).terminator.trueBlock == comparesLink.FindBlockByProgramCounter(12u).id); + const auto readsFirst = Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 1u), Sopp(8u, 1u), Sop1(12u, 0u, 0x20u, 8u)}); + Require(readsFirst.FindBlockByProgramCounter(0u).terminator.trueBlock == readsFirst.FindBlockByProgramCounter(12u).id); + ExpectThrow("escapes the constant-offset call/return model", [] { + Build({Sop1(0u, 0u, 3u, 8u), Call(4u, 8u, 2u), Sopp(8u, 2u, 0xfffdu), Sopp(12u, 1u), Sop1(16u, 0u, 0x20u, 8u)}); + }); +} + static void CheckRejections() { ExpectThrow("invalid instruction boundary", [] { Build({Call(0u, 8u, 16u), Sopp(4u, 1u)}); @@ -280,6 +373,7 @@ int main(int argc, char** argv) { CheckCallAfterEnd(); CheckRejections(); CheckUserDataCalls(); + CheckTableCalls(); CheckGuards(""); } else { Require(argc == 2); diff --git a/core/libs/tests/ScalarSwappc.cpp b/core/libs/tests/ScalarSwappc.cpp index 6ab8dc4f0512..7776183052a0 100644 --- a/core/libs/tests/ScalarSwappc.cpp +++ b/core/libs/tests/ScalarSwappc.cpp @@ -135,7 +135,7 @@ static void CheckThrows() { ExpectThrow("escapes the constant-offset call/return model", [] { Build(Program({ Sop1(0x00u, 4u, 0x1fu, 0u), - Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x18u), + Sop2Literal(0x04u, 4u, 0x00u, 4u, 0x20u), Sop2Literal(0x0cu, 5u, 0x04u, 5u, 0u), Sop1(0x14u, 8u, 0x21u, 4u), Sop1(0x18u, 8u, 0x03u, 8u), diff --git a/core/shader/recompiler/BdaAbi.hpp b/core/shader/recompiler/BdaAbi.hpp index f964d7fe9981..4afd13c058a5 100644 --- a/core/shader/recompiler/BdaAbi.hpp +++ b/core/shader/recompiler/BdaAbi.hpp @@ -28,7 +28,7 @@ struct Range { }; enum class FaultState : std::uint32_t { Empty, Writing, Ready }; -enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor }; +enum class FaultReason : std::uint32_t { Unmapped = 1, Permission, Overflow, InvalidTable, InvalidRectangle, LoopLimit, Unaligned, InvalidDescriptor, UnlinkedCall }; struct Fault { FaultState state; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index a90596796095..948e3576bf99 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -36,6 +36,7 @@ function(add_shader_recompiler target runtimeTarget) add_library(${target} STATIC EXCLUDE_FROM_ALL ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Recompiler.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderDiskCache.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCachePack.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/RectListShaders.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/SpirvBackend/src/BarycentricGeometryShader.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/RdnaDecoder/src/RdnaOpcode.cpp @@ -73,6 +74,7 @@ function(add_shader_recompiler target runtimeTarget) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/TranslationContext/SubwordPacking.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/TranslationContext/ExecMask.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/ScalarInstructions.cpp + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/RoundingMode.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/VectorInstructions.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/FloatInstructions.cpp ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Translation/src/Float64Instructions.cpp @@ -167,7 +169,7 @@ function(add_shader_recompiler target runtimeTarget) target_compile_features(${target} PUBLIC cxx_std_20) set(shaderCacheGeneratedDir "${CMAKE_CURRENT_BINARY_DIR}/${target}_generated") - set(shaderCacheVersionHeader "${shaderCacheGeneratedDir}/ShaderCacheVersion.hpp") + set(shaderCacheVersionSource "${shaderCacheGeneratedDir}/ShaderCacheVersion.cpp") file(GLOB_RECURSE shaderCacheRecompilerSources CONFIGURE_DEPENDS "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.cpp" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/*.hpp" @@ -255,6 +257,8 @@ function(add_shader_recompiler target runtimeTarget) FindUserDataCalls UserDataCalleeLength LinkUserDataCalls + FindTableCalls + LinkCalls WriteFileAtomically ) set(shaderCacheManifest) @@ -277,21 +281,20 @@ function(add_shader_recompiler target runtimeTarget) set(shaderCacheListFile "${shaderCacheGeneratedDir}/sources.txt") file(CONFIGURE OUTPUT "${shaderCacheListFile}" CONTENT "${shaderCacheManifest}" @ONLY) add_custom_command( - OUTPUT "${shaderCacheVersionHeader}" + OUTPUT "${shaderCacheVersionSource}" COMMAND ${CMAKE_COMMAND} "-DROOT=${PROJECT_SOURCE_DIR}" "-DLIST_FILE=${shaderCacheListFile}" "-DRECOMPILER_DIR=${CMAKE_CURRENT_FUNCTION_LIST_DIR}" "-DGIT=${GIT_EXECUTABLE}" "-DEXTRA=${shaderCacheExtra}" - "-DOUTPUT=${shaderCacheVersionHeader}" + "-DOUTPUT=${shaderCacheVersionSource}" -P "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCacheVersion.cmake" DEPENDS ${shaderCacheRecompilerSources} ${shaderCacheGuardSources} ${shaderCacheSubmoduleStamps} "${shaderCacheListFile}" "${CMAKE_CURRENT_FUNCTION_LIST_DIR}/ShaderCacheVersion.cmake" COMMENT "Hashing the shader recompiler sources for the shader disk cache" VERBATIM ) - target_sources(${target} PRIVATE "${shaderCacheVersionHeader}") - target_include_directories(${target} PRIVATE "${shaderCacheGeneratedDir}") + target_sources(${target} PRIVATE "${shaderCacheVersionSource}") set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 3ffc9cab3139..a6b3848c4d3a 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -52,6 +52,7 @@ class RecompileCacheKey { append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); std::uint64_t hash = 0xcbf29ce484222325ull; @@ -84,6 +85,7 @@ class RecompileCacheKey { append(key, request.context.pixel); append(key, request.context.vertex); append(key, request.context.floatMode); + appendFallbacks(key, request); appendMesh(key, request); appendTessellation(key, request); append(key, request.target); @@ -92,6 +94,11 @@ class RecompileCacheKey { append(key, RayTracingMiss()); } + static void appendFallbacks(std::vector& key, const RecompileRequest& request) { + append(key, request.context.linkedCallFallbacks.size()); + for (const auto programCounter : request.context.linkedCallFallbacks) append(key, programCounter); + } + static void appendMesh(std::vector& key, const RecompileRequest& request) { if (request.shader.stage != ShaderStage::Mesh) return; const auto* mesh = request.graphics && request.graphics->mesh ? &*request.graphics->mesh : nullptr; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp index d93905ba942d..3199579688c2 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/ControlFlowGraph.hpp @@ -107,6 +107,7 @@ struct ControlFlowGraph { std::vector components; std::vector codeTableLoadProgramCounters; std::vector codeTableLoads; + std::vector unlinkedCallProgramCounters; std::uint32_t entryBlock = InvalidControlFlowId; bool hasFetchCall = false; std::uint32_t fetchCallProgramCounter = 0; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index ada6f1084e29..629a428c78d4 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -14,6 +14,8 @@ struct SwappcInfo { bool fetchCallAllowed = false; std::uint32_t userDataBaseRegister = 0; std::uint32_t userDataCount = 0; + std::uint32_t waveSize = 0; + std::span linkedCallFallbacks; }; struct SwappcCall { @@ -32,10 +34,33 @@ struct UserDataCall { std::uint32_t userDataRegister = 0; }; +struct TableCall { + std::uint32_t programCounter = 0; + std::uint32_t linkRegister = 0; + std::uint32_t targetRegister = 0; + std::uint32_t userDataRegister = 0; + std::uint32_t descriptorOffset = 0; + std::uint32_t entryOffset = 0; + std::uint32_t entryStride = 0; +}; + +struct TableCallee { + std::uint64_t target = 0; + std::vector code; +}; + [[nodiscard]] std::optional UnresolvableSwappcTarget(const RdnaProgram& program, const SwappcInfo* swappc); [[nodiscard]] std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc); [[nodiscard]] std::optional UserDataCalleeLength(std::span callee, std::uint32_t linkRegister); [[nodiscard]] std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees); +[[nodiscard]] std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc); +struct LinkedCalls { + std::vector code; + std::vector fallbacks; +}; + +[[nodiscard]] LinkedCalls LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees); class GraphBuilder { public: diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp index 0f59c68558e0..bf76e576bdd5 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/RequestSerializer.hpp @@ -33,6 +33,7 @@ namespace ShaderRecompiler { std::optional vertex; std::vector> memoryBytesStorage; std::vector memory; + std::vector linkedCallFallbacks; std::vector supportedCapabilities; std::vector supportedExtensionStorage; std::vector supportedExtensions; diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 3b623565ebd0..b9cf4fb7953d 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -2,6 +2,7 @@ #include "ControlFlow/ControlFlowHelpers.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include +#include #include #include #include @@ -257,7 +258,7 @@ bool resolveJumpTable(const RdnaProgram& program, std::uint32_t index, BoundedJu return false; } -bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { +bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index, std::uint32_t waveSize = 0u) { std::uint32_t count = instruction.dataDwordCount; switch (instruction.family) { case RdnaInstructionFamily::SOP1: @@ -273,7 +274,7 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { case RdnaInstructionFamily::VOP3: case RdnaInstructionFamily::VOP3P: case RdnaInstructionFamily::VOPC: - count = 2u; + count = instruction.op == RdnaOpcode::VReadlaneB32 || instruction.op == RdnaOpcode::VReadfirstlaneB32 || waveSize == 32u ? 1u : 2u; break; case RdnaInstructionFamily::SOPC: case RdnaInstructionFamily::SOPP: @@ -293,11 +294,11 @@ bool writesScalar(const RdnaInstruction& instruction, std::uint32_t index) { return covers(instruction.destination) || covers(instruction.destination2); } -bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer) { +bool findLastWriter(const RdnaProgram& program, std::uint32_t end, std::initializer_list indices, std::uint32_t& writer, std::uint32_t waveSize = 0u) { for (auto position = end; position-- > 0u;) { const auto& instruction = program.instructions[position]; if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SSetpcB64 || instruction.op == RdnaOpcode::SEndpgm) return false; - if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index); })) { + if (std::ranges::any_of(indices, [&](std::uint32_t index) { return writesScalar(instruction, index, waveSize); })) { writer = position; return true; } @@ -475,7 +476,298 @@ bool isFetchCall(const RdnaProgram& program, std::uint32_t index, const SwappcIn return swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram); } -std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc) { +bool overwritesScalar(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + const auto first = scalarIndex(instruction.destination); + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + return isScalarLoad(instruction.op) && first != NoScalarRegister && reg >= first && reg - first < instruction.dataDwordCount; + case RdnaInstructionFamily::SOP1: + switch (instruction.op) { + case RdnaOpcode::SMovreldB32: + case RdnaOpcode::SMovreldB64: + case RdnaOpcode::SMovrelsd2B32: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SSwappcB64: + case RdnaOpcode::SSetpcB64: return false; + case RdnaOpcode::SMovB64: + case RdnaOpcode::SGetpcB64: return first != NoScalarRegister && reg >= first && reg - first < 2u; + default: return first != NoScalarRegister && reg == first; + } + case RdnaInstructionFamily::SOP2: return first != NoScalarRegister && reg == first; + case RdnaInstructionFamily::SOPK: return instruction.op == RdnaOpcode::SMovkI32 && reg == first; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: { + const std::uint32_t count = instruction.op != RdnaOpcode::VReadlaneB32 && instruction.op != RdnaOpcode::VReadfirstlaneB32 && waveSize == 64u ? 2u : 1u; + return std::ranges::any_of(std::array{&instruction.destination, &instruction.destination2}, [&](const RdnaOperand* operand) { + const auto slot = scalarIndex(*operand); + return slot != NoScalarRegister && reg >= slot && reg - slot < count; + }); + } + default: return false; + } +} + +bool readsVectorSources64(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::VMadU64U32: + case RdnaOpcode::VMadI64I32: + case RdnaOpcode::VFmaF64: + case RdnaOpcode::VAddF64: + case RdnaOpcode::VMulF64: + case RdnaOpcode::VMinF64: + case RdnaOpcode::VMaxF64: + case RdnaOpcode::VLdexpF64: + case RdnaOpcode::VCvtI32F64: + case RdnaOpcode::VCvtF64I32: + case RdnaOpcode::VCvtF32F64: + case RdnaOpcode::VCvtF64F32: + case RdnaOpcode::VCvtU32F64: + case RdnaOpcode::VCvtF64U32: + case RdnaOpcode::VTruncF64: + case RdnaOpcode::VCeilF64: + case RdnaOpcode::VRndneF64: + case RdnaOpcode::VFloorF64: + case RdnaOpcode::VFrexpExpI32F64: + case RdnaOpcode::VFrexpMantF64: + case RdnaOpcode::VFractF64: + case RdnaOpcode::VRcpF64: + case RdnaOpcode::VRsqF64: + case RdnaOpcode::VSqrtF64: + case RdnaOpcode::VTrigPreopF64: + case RdnaOpcode::VDivScaleF64: + case RdnaOpcode::VDivFmasF64: + case RdnaOpcode::VDivFixupF64: + case RdnaOpcode::VLshlrevB64: + case RdnaOpcode::VLshrrevB64: + case RdnaOpcode::VAshrrevI64: + case RdnaOpcode::VCmpEqI64: + case RdnaOpcode::VCmpLtU64: + case RdnaOpcode::VCmpEqU64: + case RdnaOpcode::VCmpGtU64: + case RdnaOpcode::VCmpNeU64: + case RdnaOpcode::VCmpxNeI64: + case RdnaOpcode::VCmpxNeU64: + case RdnaOpcode::VCmpFI64: + case RdnaOpcode::VCmpLtI64: + case RdnaOpcode::VCmpLeI64: + case RdnaOpcode::VCmpGtI64: + case RdnaOpcode::VCmpNeI64: + case RdnaOpcode::VCmpGeI64: + case RdnaOpcode::VCmpTI64: + case RdnaOpcode::VCmpxFI64: + case RdnaOpcode::VCmpxLtI64: + case RdnaOpcode::VCmpxEqI64: + case RdnaOpcode::VCmpxLeI64: + case RdnaOpcode::VCmpxGtI64: + case RdnaOpcode::VCmpxGeI64: + case RdnaOpcode::VCmpxTI64: + case RdnaOpcode::VCmpFU64: + case RdnaOpcode::VCmpLeU64: + case RdnaOpcode::VCmpGeU64: + case RdnaOpcode::VCmpTU64: + case RdnaOpcode::VCmpxFU64: + case RdnaOpcode::VCmpxLtU64: + case RdnaOpcode::VCmpxEqU64: + case RdnaOpcode::VCmpxLeU64: + case RdnaOpcode::VCmpxGtU64: + case RdnaOpcode::VCmpxGeU64: + case RdnaOpcode::VCmpxTU64: + case RdnaOpcode::VCmpFF64: + case RdnaOpcode::VCmpLtF64: + case RdnaOpcode::VCmpEqF64: + case RdnaOpcode::VCmpLeF64: + case RdnaOpcode::VCmpGtF64: + case RdnaOpcode::VCmpLgF64: + case RdnaOpcode::VCmpGeF64: + case RdnaOpcode::VCmpOF64: + case RdnaOpcode::VCmpUF64: + case RdnaOpcode::VCmpNgeF64: + case RdnaOpcode::VCmpNlgF64: + case RdnaOpcode::VCmpNgtF64: + case RdnaOpcode::VCmpNleF64: + case RdnaOpcode::VCmpNeqF64: + case RdnaOpcode::VCmpNltF64: + case RdnaOpcode::VCmpTruF64: + case RdnaOpcode::VCmpClassF64: + case RdnaOpcode::VCmpxFF64: + case RdnaOpcode::VCmpxLtF64: + case RdnaOpcode::VCmpxEqF64: + case RdnaOpcode::VCmpxLeF64: + case RdnaOpcode::VCmpxGtF64: + case RdnaOpcode::VCmpxLgF64: + case RdnaOpcode::VCmpxGeF64: + case RdnaOpcode::VCmpxOF64: + case RdnaOpcode::VCmpxUF64: + case RdnaOpcode::VCmpxNgeF64: + case RdnaOpcode::VCmpxNlgF64: + case RdnaOpcode::VCmpxNgtF64: + case RdnaOpcode::VCmpxNleF64: + case RdnaOpcode::VCmpxNeqF64: + case RdnaOpcode::VCmpxNltF64: + case RdnaOpcode::VCmpxTruF64: + case RdnaOpcode::VCmpxClassF64: + return true; + default: return false; + } +} + +bool readsScalarDword(const RdnaInstruction& instruction, std::uint32_t reg, std::uint32_t waveSize) { + if (instruction.op == RdnaOpcode::SMovrelsB32 || instruction.op == RdnaOpcode::SMovrelsB64) return true; + const auto overlaps = [&](const RdnaOperand& operand, std::uint32_t count) { + const auto slot = scalarIndex(operand); + return slot != NoScalarRegister && slot <= reg && reg - slot < count; + }; + std::uint32_t source0Count = 1u, source1Count = 1u, source2Count = 1u, source3Count = 1u; + switch (instruction.family) { + case RdnaInstructionFamily::SMEM: + switch (instruction.op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: source0Count = 4u; break; + default: source0Count = 2u; break; + } + break; + case RdnaInstructionFamily::MUBUF: + case RdnaInstructionFamily::MTBUF: source1Count = 4u; break; + case RdnaInstructionFamily::MIMG: + source1Count = instruction.imageR128 ? 4u : 8u; + source2Count = 4u; + break; + case RdnaInstructionFamily::FLAT: + source0Count = 2u; + source1Count = 2u; + break; + case RdnaInstructionFamily::SOPK: + if (instruction.op != RdnaOpcode::SMovkI32 && instruction.op != RdnaOpcode::SCallB64 && overlaps(instruction.destination, 1u)) return true; + break; + case RdnaInstructionFamily::SOP1: + case RdnaInstructionFamily::SOP2: + case RdnaInstructionFamily::SOPC: + switch (instruction.op) { + case RdnaOpcode::SBcnt0I32B64: + case RdnaOpcode::SBcnt1I32B64: + case RdnaOpcode::SFf0I32B64: + case RdnaOpcode::SFf1I32B64: + case RdnaOpcode::SFlbitI32B64: + case RdnaOpcode::SFlbitI32I64: + case RdnaOpcode::SCmpEqU64: + case RdnaOpcode::SCmpLgU64: + case RdnaOpcode::SBitcmp0B64: + case RdnaOpcode::SBitcmp1B64: source0Count = source1Count = 2u; break; + default: source0Count = source1Count = instruction.family == RdnaInstructionFamily::SOPC ? 1u : std::max(instruction.dataDwordCount, 1u); break; + } + break; + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: + source0Count = source1Count = source2Count = source3Count = readsVectorSources64(instruction.op) ? 2u : 1u; + switch (instruction.op) { + case RdnaOpcode::VCndmaskB32: + case RdnaOpcode::VAddcU32: + case RdnaOpcode::VSubCoCiU32: + case RdnaOpcode::VSubrevCoCiU32: + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFmasF64: source2Count = std::max(source2Count, waveSize == 32u ? 1u : 2u); break; + default: break; + } + break; + default: break; + } + switch (instruction.op) { + case RdnaOpcode::SBitset0B32: + case RdnaOpcode::SBitset0B64: + case RdnaOpcode::SBitset1B32: + case RdnaOpcode::SBitset1B64: + case RdnaOpcode::SCmovB32: + case RdnaOpcode::SCmovB64: + if (overlaps(instruction.destination, 2u)) return true; + break; + default: break; + } + return overlaps(instruction.source0, source0Count) || overlaps(instruction.source1, source1Count) || overlaps(instruction.source2, source2Count) || overlaps(instruction.source3, source3Count); +} + +template +void requireReturnAddressesStayInCalls(const RdnaProgram& program, const std::vector& calls, const std::vector& unlinked, std::uint32_t waveSize, const TError& escapeError) { + std::set links; + for (const auto& call : calls) { + if (!call.fetch) links.insert(call.linkRegister); + } + const auto size = static_cast(program.instructions.size()); + const auto successors = [&](std::uint32_t index) { + std::vector next; + const auto& instruction = program.instructions[index]; + if (instruction.op == RdnaOpcode::SEndpgm || instruction.op == RdnaOpcode::SCodeEnd) return next; + if (IsDirectBranchOpcode(instruction.op)) { + next.push_back(instructionIndexOfProgramCounter(program, instruction.branchTarget)); + if (IsConditionalBranchOpcode(instruction.op)) next.push_back(index + 1u); + } else if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { + const SwappcCall* call = findCallAt(calls, index); + next.push_back(call == nullptr || call->fetch ? index + 1u : call->targetIndex); + } else if (instruction.op == RdnaOpcode::SSetpcB64) { + std::uint32_t target = 0; + if (const SwappcCall* call = findReturnAt(calls, index); call != nullptr) { + next.push_back(call->callIndex + 1u); + } else if (resolveSetpcTarget(program, index, target)) { + next.push_back(instructionIndexOfProgramCounter(program, target)); + } + } else { + next.push_back(index + 1u); + } + std::erase_if(next, [&](std::uint32_t successor) { return successor >= size; }); + return next; + }; + for (const auto link : links) { + const auto writesLink = [&](std::uint32_t index) { + const SwappcCall* call = findCallAt(calls, index); + return (call != nullptr && call->linkRegister == link) || (std::ranges::find(unlinked, index) != unlinked.end() && scalarIndex(program.instructions[index].destination) == link); + }; + std::vector holds(size, 0u); + std::vector work; + for (std::uint32_t index = 0; index < size; ++index) { + if (writesLink(index)) work.push_back(index); + } + const auto after = [&](std::uint32_t index) -> std::uint8_t { + if (writesLink(index)) return 3u; + std::uint8_t value = holds[index]; + if (overwritesScalar(program.instructions[index], link, waveSize)) value &= 2u; + if (overwritesScalar(program.instructions[index], link + 1u, waveSize)) value &= 1u; + return value; + }; + while (!work.empty()) { + const auto index = work.back(); + work.pop_back(); + const auto value = after(index); + for (const auto successor : successors(index)) { + if ((holds[successor] | value) == holds[successor]) continue; + holds[successor] |= value; + work.push_back(successor); + } + } + for (std::uint32_t index = 0; index < size; ++index) { + const auto& instruction = program.instructions[index]; + if (holds[index] == 0u || writesLink(index) || instruction.family == RdnaInstructionFamily::SOPC || findReturnAt(calls, index) != nullptr) continue; + const bool inside = std::ranges::any_of(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link && index >= call.targetIndex && index <= call.returnIndex; }); + if (inside || !(((holds[index] & 1u) != 0u && readsScalarDword(instruction, link, waveSize)) || ((holds[index] & 2u) != 0u && readsScalarDword(instruction, link + 1u, waveSize)))) continue; + const auto owner = std::ranges::find_if(calls, [&](const SwappcCall& call) { return !call.fetch && call.linkRegister == link; }); + throw escapeError(*owner, instruction); + } + } +} + +std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc, std::vector& unlinked) { std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -502,6 +794,10 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } if (!staticTarget || call.targetIndex == InvalidControlFlowId) { + if (swappc != nullptr && std::ranges::find(swappc->linkedCallFallbacks, instruction.programCounter) != swappc->linkedCallFallbacks.end()) { + unlinked.push_back(index); + continue; + } if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister; })) { throw std::invalid_argument("unclosable scalar call/return pairing at program counter " + toHexString(instruction.programCounter) + ": link register s[" + std::to_string(call.linkRegister) + "] is shared with another call"); @@ -574,7 +870,8 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa continue; } const bool outsideRegion = regionEnd != InvalidControlFlowId && (index < call.targetIndex || index > regionEnd); - if (touchesLinkRegister(instruction, call.linkRegister) && !(outsideRegion && onlyWritesLinkRegister(instruction, call.linkRegister))) { + if (outsideRegion) continue; + if (touchesLinkRegister(instruction, call.linkRegister)) { throw escapeError(call, instruction); } } @@ -609,14 +906,12 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa if (std::ranges::any_of(calls, [&](const SwappcCall& other) { return other.callIndex == index && other.linkRegister == call.linkRegister; })) continue; const auto& instruction = program.instructions[index]; const auto region = std::ranges::find_if(calls, [&](const SwappcCall& other) { return other.linkRegister == call.linkRegister && index >= other.targetIndex && index <= other.returnIndex; }); - if (region != calls.end()) { - if (index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); - } else if (touchesLinkRegister(instruction, call.linkRegister) && !onlyWritesLinkRegister(instruction, call.linkRegister)) { - throw escapeError(call, instruction); - } + if (region != calls.end() && index != region->returnIndex && touchesLinkRegister(instruction, call.linkRegister)) throw escapeError(*region, instruction); } } + requireReturnAddressesStayInCalls(program, calls, unlinked, swappc != nullptr ? swappc->waveSize : 0u, escapeError); + for (std::size_t first = 0; first < calls.size(); ++first) { if (!calls[first].fetch && calls[first].callIndex >= calls[first].targetIndex && calls[first].callIndex <= calls[first].returnIndex) { throw std::invalid_argument("recursive scalar call at program counter " + toHexString(program.instructions[calls[first].callIndex].programCounter) + @@ -664,7 +959,7 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa } if (instruction.op == RdnaOpcode::SSwappcB64 || instruction.op == RdnaOpcode::SCallB64) { const SwappcCall* call = findCallAt(calls, index); - transfers.push_back(Transfer{index, call->fetch ? index + 1u : call->targetIndex}); + transfers.push_back(Transfer{index, call == nullptr || call->fetch ? index + 1u : call->targetIndex}); continue; } if (instruction.op != RdnaOpcode::SEndpgm && instruction.op != RdnaOpcode::SCodeEnd) transfers.push_back(Transfer{index, index + 1u}); @@ -958,23 +1253,75 @@ std::optional UnresolvableSwappcTarget(const RdnaProgram& program return std::nullopt; } -std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { +namespace { + +std::set branchLabels(const RdnaProgram& program) { std::set labels; for (const auto& instruction : program.instructions) { if (IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) labels.insert(instruction.branchTarget); } - const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; - const auto unchangedUntil = [&](std::uint32_t reg, std::uint32_t reader) { - for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { - if (!writesScalar(program.instructions[index], reg)) continue; - if (index <= reader) return false; - for (std::uint32_t later = index; later < program.instructions.size(); ++later) { - const auto& instruction = program.instructions[later]; - if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; - } + return labels; +} + +bool unchangedUntil(const RdnaProgram& program, std::uint32_t reg, std::uint32_t reader, std::uint32_t waveSize) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + if (!writesScalar(program.instructions[index], reg, waveSize)) continue; + if (index <= reader) return false; + for (std::uint32_t later = index; later < program.instructions.size(); ++later) { + const auto& instruction = program.instructions[later]; + if ((IsDirectBranchOpcode(instruction.op) || instruction.op == RdnaOpcode::SCallB64) && instruction.branchTarget <= program.instructions[reader].programCounter) return false; } - return true; - }; + } + return true; +} + +bool isScalarBufferLoad(RdnaOpcode op) { + switch (op) { + case RdnaOpcode::SBufferLoadDword: + case RdnaOpcode::SBufferLoadDwordx2: + case RdnaOpcode::SBufferLoadDwordx4: + case RdnaOpcode::SBufferLoadDwordx8: + case RdnaOpcode::SBufferLoadDwordx16: return true; + default: return false; + } +} + +bool loadedDword(const RdnaProgram& program, std::uint32_t reg, std::uint32_t before, std::uint32_t waveSize, std::uint32_t& load, std::uint32_t& dword) { + for (std::uint32_t copies = 0; copies < 4u; ++copies) { + std::uint32_t writer = 0; + if (!findLastWriter(program, before, {reg}, writer, waveSize)) return false; + const auto& instruction = program.instructions[writer]; + const auto destination = scalarIndex(instruction.destination); + const auto source = scalarIndex(instruction.source0); + if (instruction.op == RdnaOpcode::SMovB32 && destination == reg && source != NoScalarRegister) { + reg = source; + } else if (instruction.op == RdnaOpcode::SMovB64 && destination != NoScalarRegister && reg - destination < 2u && source != NoScalarRegister) { + reg = source + (reg - destination); + } else if (isScalarBufferLoad(instruction.op) && destination != NoScalarRegister && reg >= destination && reg - destination < instruction.dataDwordCount) { + load = writer; + dword = reg - destination; + return true; + } else { + return false; + } + before = writer; + } + return false; +} + +std::uint32_t branchWord(std::uint32_t opcode, std::size_t from, std::size_t to, const char* what) { + const auto offset = static_cast(to) - static_cast(from) - 1; + if (offset < -0x8000 || offset > 0x7fff) throw std::invalid_argument(std::string(what) + " lands out of the 16-bit branch range"); + return opcode | (static_cast(offset) & 0xffffu); +} + +} + + +std::vector FindUserDataCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto unchanged = [&](std::uint32_t reg, std::uint32_t reader) { return unchangedUntil(program, reg, reader, swappc.waveSize); }; std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { const auto& instruction = program.instructions[index]; @@ -985,7 +1332,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw const auto source = scalarIndex(instruction.source0); if (link == NoScalarRegister || source == NoScalarRegister || source % 2u != 0u) continue; std::uint32_t pair = NoScalarRegister; - if (source != link && userData(source) && userData(source + 1u) && unchangedUntil(source, index) && unchangedUntil(source + 1u, index)) { + if (source != link && userData(source) && userData(source + 1u) && unchanged(source, index) && unchanged(source + 1u, index)) { pair = source; } else { std::uint32_t low = 0, high = 0; @@ -1003,7 +1350,7 @@ std::vector FindUserDataCalls(const RdnaProgram& program, const Sw from = scalarIndex(lowWriter.source0); if (from != NoScalarRegister && !isScalar(highWriter.source0, from + 1u)) from = NoScalarRegister; } - if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchangedUntil(from, low) || !unchangedUntil(from + 1u, high)) continue; + if (!sameBlock || from == NoScalarRegister || from % 2u != 0u || !userData(from) || !userData(from + 1u) || !unchanged(from, low) || !unchanged(from + 1u, high)) continue; pair = from; } calls.push_back({instruction.programCounter, link, pair}); @@ -1040,32 +1387,132 @@ std::optional UserDataCalleeLength(std::span } std::vector LinkUserDataCalls(const RdnaProgram& program, std::span calls, std::span> callees) { - if (calls.size() != callees.size()) throw std::logic_error("user-data calls and callees differ in number"); - for (const auto& instruction : program.instructions) { - if (instruction.op == RdnaOpcode::SGetpcB64) { - throw std::invalid_argument("a program that reads its program counter (s_getpc_b64 at program counter " + toHexString(instruction.programCounter) + ") cannot have user-data callees linked after it"); + return LinkCalls(program, calls, callees, {}, {}).code; +} + +std::vector FindTableCalls(const RdnaProgram& program, const SwappcInfo& swappc) { + const auto labels = branchLabels(program); + const auto userData = [&](std::uint32_t reg) { return reg >= swappc.userDataBaseRegister && reg - swappc.userDataBaseRegister < swappc.userDataCount; }; + const auto sameBlock = [&](std::uint32_t first, std::uint32_t last) { + return std::none_of(labels.begin(), labels.end(), [&](std::uint32_t label) { + return label > program.instructions[first].programCounter && label <= program.instructions[last].programCounter; + }); + }; + std::vector calls; + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64) continue; + std::uint32_t ignored = 0; + if (resolveSetpcTarget(program, index, ignored)) continue; + const auto link = scalarIndex(instruction.destination); + const auto target = scalarIndex(instruction.source0); + if (link == NoScalarRegister || link % 2u != 0u || link > 104u || target == NoScalarRegister || target % 2u != 0u || target > 104u) continue; + std::uint32_t lowLoad = 0, lowDword = 0, highLoad = 0, highDword = 0; + if (!loadedDword(program, target, index, swappc.waveSize, lowLoad, lowDword) || !loadedDword(program, target + 1u, index, swappc.waveSize, highLoad, highDword)) continue; + if (lowLoad != highLoad || highDword != lowDword + 1u || !sameBlock(lowLoad, index)) continue; + const auto& load = program.instructions[lowLoad]; + const auto descriptor = scalarIndex(load.source0); + if (load.source0.kind != RdnaOperandKind::ScalarRegister || descriptor % 4u != 0u || load.memoryOffset >= 0x100000u || (load.memoryOffset & 3u) != 0u) continue; + std::uint32_t descriptorLoad = 0; + if (!findLastWriter(program, lowLoad, {descriptor, descriptor + 1u, descriptor + 2u, descriptor + 3u}, descriptorLoad, swappc.waveSize) || !sameBlock(descriptorLoad, index)) continue; + const auto& table = program.instructions[descriptorLoad]; + const auto pointer = scalarIndex(table.source0); + if (table.op != RdnaOpcode::SLoadDwordx4 || !isScalar(table.destination, descriptor) || table.source0.kind != RdnaOperandKind::ScalarRegister || table.source1.kind != RdnaOperandKind::Null || + table.memoryOffset >= 0x100000u || (table.memoryOffset & 3u) != 0u || !userData(pointer) || !userData(pointer + 1u) || + !unchangedUntil(program, pointer, descriptorLoad, swappc.waveSize) || !unchangedUntil(program, pointer + 1u, descriptorLoad, swappc.waveSize)) continue; + std::uint32_t stride = 0; + if (load.source1.kind != RdnaOperandKind::Null) { + const auto offset = scalarIndex(load.source1); + std::uint32_t shift = 0, shiftIndex = 0; + if (offset == NoScalarRegister || !findLastWriter(program, lowLoad, {offset}, shiftIndex, swappc.waveSize) || !sameBlock(shiftIndex, index)) continue; + const auto& shifted = program.instructions[shiftIndex]; + if (shifted.op != RdnaOpcode::SLshlB32 || !isScalar(shifted.destination, offset) || !isImmediate(shifted.source1, shift) || shift < 2u || shift > 24u) continue; + stride = 1u << shift; + } + calls.push_back({instruction.programCounter, link, target, pointer, table.memoryOffset, load.memoryOffset + lowDword * 4u, stride}); + } + return calls; +} + +LinkedCalls LinkCalls(const RdnaProgram& program, std::span userDataCalls, std::span> userDataCallees, + std::span tableCalls, std::span> tableCallees) { + if (userDataCalls.size() != userDataCallees.size()) throw std::logic_error("user-data calls and callees differ in number"); + if (tableCalls.size() != tableCallees.size()) throw std::logic_error("table calls and callees differ in number"); + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + BoundedJumpTable table; + if (program.instructions[index].op == RdnaOpcode::SSetpcB64 && resolveBoundedJumpTable(program, index, table)) { + throw std::invalid_argument("a program that jumps through a table in its code (s_setpc_b64 at program counter " + toHexString(program.instructions[index].programCounter) + ") cannot have callees linked over the words after its last instruction"); } } const auto end = instructionEndProgramCounter(program.instructions.back()) / 4u; std::vector linked(program.code.begin(), program.code.begin() + end); - for (std::size_t call = 0; call < calls.size(); ++call) { - const auto index = instructionIndexOfProgramCounter(program, calls[call].programCounter); + std::vector fallbacks; + const auto callWord = [&](std::uint32_t programCounter, const char* kind) { + const auto index = instructionIndexOfProgramCounter(program, programCounter); if (index == InvalidControlFlowId || program.instructions[index].op != RdnaOpcode::SSwappcB64 || program.instructions[index].wordCount != 1u) { - throw std::logic_error("user-data call does not name an s_swappc_b64"); + throw std::logic_error(std::string(kind) + " call does not name an s_swappc_b64"); } - const auto word = calls[call].programCounter / 4u; + return programCounter / 4u; + }; + for (std::size_t call = 0; call < userDataCalls.size(); ++call) { + const auto word = callWord(userDataCalls[call].programCounter, "user-data"); const auto offset = static_cast(linked.size()) - static_cast(word) - 1; - if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(calls[call].programCounter) + " lands out of s_call_b64 range"); - linked[word] = 0xbb000000u | (calls[call].linkRegister << 16u) | static_cast(offset); - linked.insert(linked.end(), callees[call].begin(), callees[call].end()); + if (offset > 0x7fff) throw std::invalid_argument("user-data callee of the call at program counter " + toHexString(userDataCalls[call].programCounter) + " lands out of s_call_b64 range"); + linked[word] = 0xbb000000u | (userDataCalls[call].linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), userDataCallees[call].begin(), userDataCallees[call].end()); + } + for (std::size_t call = 0; call < tableCalls.size(); ++call) { + const auto& table = tableCalls[call]; + const auto& callees = tableCallees[call]; + const auto word = callWord(table.programCounter, "table"); + const auto at = "the table callees of the call at program counter " + toHexString(table.programCounter); + const auto original = linked[word]; + const auto count = callees.size(); + const auto dispatch = linked.size(); + const auto zero = dispatch + 1u; + const auto one = zero + count * 7u + 2u; + const auto calls = one + count * 8u + 2u; + const auto fault = calls + count * 2u; + const auto branch = [&](std::uint32_t opcode, std::size_t target) { linked.push_back(branchWord(opcode, linked.size(), target, at.c_str())); }; + linked[word] = branchWord(0xbf820000u, word, dispatch, at.c_str()); + branch(0xbf850000u, one); + for (const bool sccOne : {false, true}) { + for (std::size_t callee = 0; callee < count; ++callee) { + const auto next = linked.size() + (sccOne ? 8u : 7u); + linked.insert(linked.end(), {0xbf07ff00u | table.targetRegister, static_cast(callees[callee].target)}); + branch(0xbf850000u, next); + linked.insert(linked.end(), {0xbf07ff00u | (table.targetRegister + 1u), static_cast(callees[callee].target >> 32u)}); + branch(0xbf850000u, next); + if (sccOne) linked.push_back(0xbf068080u); + branch(0xbf820000u, calls + callee * 2u); + } + linked.push_back(sccOne ? 0xbf068080u : 0xbf078080u); + branch(0xbf820000u, fault); + } + for (std::size_t callee = 0; callee < count; ++callee) { + linked.push_back(0u); + branch(0xbf820000u, word + 1u); + } + fallbacks.push_back(static_cast(linked.size() * 4u)); + linked.push_back(original); + branch(0xbf820000u, word + 1u); + for (std::size_t callee = 0; callee < count; ++callee) { + const auto site = calls + callee * 2u; + const auto offset = static_cast(linked.size()) - static_cast(site) - 1; + if (offset > 0x7fff) throw std::invalid_argument(at + " land out of s_call_b64 range"); + linked[site] = 0xbb000000u | (table.linkRegister << 16u) | static_cast(offset); + linked.insert(linked.end(), callees[callee].code.begin(), callees[callee].code.end()); + } } linked.push_back(0xbf9f0000u); - return linked; + return {std::move(linked), std::move(fallbacks)}; } ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { - const std::vector calls = analyzeSwappcCalls(program, swappc); + std::vector unlinked; + const std::vector calls = analyzeSwappcCalls(program, swappc, unlinked); ControlFlowGraph graph; + for (const auto index : unlinked) graph.unlinkedCallProgramCounters.push_back(program.instructions[index].programCounter); graph.blocks = splitIntoBlocks(program, calls); linkBlocks(graph.blocks, program, calls); for (const auto& call : calls) { diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 4ec8a1cd1089..68fafd123a6b 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -756,7 +756,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(24u); + writer.WriteU32(25u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -780,6 +780,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteU32(request.target.registrationWordLimit); writer.WriteBool(request.target.vertexSubgroups); writer.WriteBool(request.target.float16DenormPreserve); + writer.WriteU32Span(request.context.linkedCallFallbacks); return base64Encode(buffer); } @@ -788,7 +789,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 24u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 25u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -814,14 +815,17 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 18u) result.request.target.vertexSubgroups = reader.ReadBool(); if (version >= 21u) { if (version >= 22u) reader.Require(1u); - if (reader.Remaining() == 1u) { + if (reader.Remaining() != 0u) { const auto float16DenormPreserve = reader.ReadU8(); if (float16DenormPreserve > 1u) throw std::runtime_error("RequestSerializer: invalid float16DenormPreserve value"); result.request.target.float16DenormPreserve = float16DenormPreserve != 0u; - } else if (reader.Remaining() != 0u) { - throw std::runtime_error("RequestSerializer: unexpected trailing data"); } } + if (version >= 25u) { + result.linkedCallFallbacks = reader.ReadU32Vector(); + result.request.context.linkedCallFallbacks = result.linkedCallFallbacks; + } + if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data"); return result; } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index 63b0392ac96e..51a76133146f 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -73,6 +73,7 @@ enum class IrOpcode : std::uint16_t { Void, Reference, ReferenceU32, + UnlinkedCall, GetUserData, GetShaderBase, ShaderClock, @@ -433,6 +434,7 @@ enum class IrOpcode : std::uint16_t { SharedSrc2U64, DataAppend, DataConsume, + DataOrderedCount, SwizzleU32, ImageQueryDimensions, ImageQueryLod, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index bfc6572863af..5309722a677d 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -114,6 +114,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("Void", Void), makeMeta("Reference", Void, U1), makeMeta("ReferenceU32", Void, U32), + makeMeta("UnlinkedCall", Void, U32, U32, U32), makeMeta("GetUserData", U32, ScalarReg), makeMeta("GetShaderBase", U64), makeMeta("ShaderClock", U64), @@ -474,6 +475,7 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("SharedSrc2U64", Void, U32, U32, U1), makeMeta("DataAppend", U32, U32, U1, U32, U32), makeMeta("DataConsume", U32, U32, U1, U32, U32), + makeMeta("DataOrderedCount", U32, U32, U1, U32, U32, U32), makeMeta("SwizzleU32", U32, U32, U32, U1), makeMeta("ImageQueryDimensions", U32x4, ImageResource, ImageAddress), makeMeta("ImageQueryLod", U32x4, ImageResource, SamplerResource, ImageAddress), @@ -699,6 +701,7 @@ SharedAccess SharedAccessOf(IrOpcode opcode) { case IrOpcode::SharedAtomicMskor64: return SharedAccess::Atomic; case IrOpcode::DataAppend: + case IrOpcode::DataOrderedCount: return SharedAccess::Append; case IrOpcode::DataConsume: return SharedAccess::Consume; @@ -893,6 +896,7 @@ bool IrOpcodeHasSideEffects(IrOpcode opcode) { case IrOpcode::Discard: case IrOpcode::Reference: case IrOpcode::ReferenceU32: + case IrOpcode::UnlinkedCall: case IrOpcode::SetTessellationAttribute: case IrOpcode::InterpolateHostP2: case IrOpcode::SetThreadBitScalarRegister: diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 25951054f857..b18849b3e1a7 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -623,7 +623,12 @@ void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot, c snapshot.buffers.at(i).dwords[3] = 0u; } const auto decoded = decodeBufferDescriptor(snapshot.buffers.at(i)); - if (decoded.Type() != 0u) throw std::runtime_error("buffer descriptor uses an unsupported type"); + if (decoded.Type() != 0u) { + const auto& words = snapshot.buffers.at(i).dwords; + char detail[64]; + std::snprintf(detail, sizeof(detail), " (buffer %u V# %08x %08x %08x %08x)", i, words[0], words[1], words[2], words[3]); + throw std::runtime_error(std::string("buffer descriptor uses an unsupported type") + detail); + } if (plan.stage != IrShaderStage::Compute && decoded.AddTid()) throw std::runtime_error("buffer ADD_TID is only valid for compute shaders"); } if (snapshot.flattenedSrt.size() != plan.srtReads.size()) throw std::runtime_error("runtime SRT size differs from the static interface"); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 26a0c1a7b5dd..f465ca290405 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -1299,7 +1299,8 @@ class Tracker { const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u; const auto lineSampleFlags = memory.imageSampleFlags & ~RdnaImageSampleFlagCompare; const bool flatLineSample = op == IrOpcode::ImageSampleRaw && (lineSampleFlags == RdnaImageSampleFlagLevelZero || lineSampleFlags == 0u); - image.flatLineCompatible = image.flatLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u; + const bool flatLineExplicitLod = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLod; + image.flatLineCompatible = image.flatLineCompatible && (flatLineLoad || flatLineSample || flatLineExplicitLod) && !memory.imagePacked && memory.imageByElements == 0u; image.flatVolumeLineCompatible = image.flatVolumeLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u; if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) { constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index dcbaa36945ed..ac55461ecb7c 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -124,7 +124,7 @@ std::atomic frontEndBuilds{0}; struct PreparedControlFlow { PreparedControlFlow(const RecompileRequest& request, const SwappcInfo& swappcInfo) - : stage(request.shader.stage), swappc(swappcInfo), code(request.shader.code.begin(), request.shader.code.end()), + : stage(request.shader.stage), fallbacks(swappcInfo.linkedCallFallbacks.begin(), swappcInfo.linkedCallFallbacks.end()), swappc(WithFallbacks(swappcInfo, fallbacks)), code(request.shader.code.begin(), request.shader.code.end()), decoded(DecodeStageProgram(toShaderStageKind(request.shader.stage), code)), cfg(GraphBuilder{}.Build(decoded, &swappc)) { Structurizer{}.Structurize(cfg); } @@ -135,10 +135,16 @@ struct PreparedControlFlow { bool Matches(const RecompileRequest& request, const SwappcInfo& swappcInfo) const { return stage == request.shader.stage && swappc.fetchCallAllowed == swappcInfo.fetchCallAllowed && swappc.userDataBaseRegister == swappcInfo.userDataBaseRegister && swappc.userDataCount == swappcInfo.userDataCount && - std::ranges::equal(code, request.shader.code); + std::ranges::equal(fallbacks, swappcInfo.linkedCallFallbacks) && std::ranges::equal(code, request.shader.code); + } + + static SwappcInfo WithFallbacks(SwappcInfo info, const std::vector& fallbacks) { + info.linkedCallFallbacks = fallbacks; + return info; } ShaderStage stage; + std::vector fallbacks; SwappcInfo swappc; std::vector code; RdnaProgram decoded; @@ -147,7 +153,7 @@ struct PreparedControlFlow { std::shared_ptr ShaderPreparationContext::AcquireFrontend(const RecompileRequest& request) { const auto inputInfo = RequestInputInfo(request); - const SwappcInfo swappc{inputInfo.vertex != nullptr, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + const SwappcInfo swappc{inputInfo.vertex != nullptr, request.context.userDataBaseRegister, static_cast(request.context.userData.size()), 0u, request.context.linkedCallFallbacks}; static const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr; if (reuse && frontend != nullptr && frontend->Matches(request, swappc)) { return frontend; @@ -365,6 +371,7 @@ struct SourceEntry : std::enable_shared_from_this { // accepted only when its code matches word for word. Owned here because the request's span // points into a registration the driver may replace while the entry lives on. std::vector code; + std::vector linkedCallFallbacks; std::shared_ptr plan; // A plan build that threw (an unsupported resource chain or control flow) is remembered and // rethrown: the front end ran every pass before failing, ~13 ms per dispatch of a shader the @@ -463,6 +470,7 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr if (source == nullptr) { source = std::make_shared(); source->code.assign(request.shader.code.begin(), request.shader.code.end()); + source->linkedCallFallbacks.assign(request.context.linkedCallFallbacks.begin(), request.context.linkedCallFallbacks.end()); auto& bucket = sources[key]; if (!bucket.empty()) { // A second entry under one key is a code hash collision (or the unhashed key with @@ -1388,7 +1396,9 @@ std::span GetPreparedCode(const SourceHandle& handle) { return handle.source->code; } -PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) {} +PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) { + this->request.context.linkedCallFallbacks = handle->source->linkedCallFallbacks; +} std::optional PreparedShaderInvocation::TryCreate(const RecompileRequest& request, const std::shared_ptr& handle) { struct PreparedKeyStorage {}; diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index d6bf8dcba094..0197c4ffb259 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -200,6 +200,7 @@ struct GuestContext { std::optional vertex; std::span memory; std::optional floatMode; + std::span linkedCallFallbacks; }; struct MeshTargetLimits { diff --git a/core/shader/recompiler/ShaderCachePack.cpp b/core/shader/recompiler/ShaderCachePack.cpp new file mode 100644 index 000000000000..a9550483f541 --- /dev/null +++ b/core/shader/recompiler/ShaderCachePack.cpp @@ -0,0 +1,115 @@ +#include "ShaderCachePack.hpp" +#include "ShaderCacheDirectory.hpp" +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#include +#include +#include +#endif + +namespace ShaderRecompiler::ShaderDiskCache { +namespace { +using Clock = std::chrono::steady_clock; +std::uint64_t elapsed(Clock::time_point start) { + return std::chrono::duration_cast(Clock::now() - start).count(); +} +} + +std::array PackKeyHash(std::span key) { + return {HashBytes(key, 0x5eed0001ull), HashBytes(key, 0x5eed0002ull)}; +} + +std::vector PrebuiltCachePaths(const std::filesystem::path& cacheRoot) { + if (cacheRoot.empty()) return {}; + return {cacheRoot / "prebuilt"}; +} + +CachePack::CachePack(const std::filesystem::path& path) { + static_assert(std::endian::native == std::endian::little); +#ifdef _WIN32 + const auto handle = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (handle == INVALID_HANDLE_VALUE) { error = "cannot open"; return; } + LARGE_INTEGER size{}; + if (!GetFileSizeEx(handle, &size) || size.QuadPart < static_cast(sizeof(PackHeader)) || static_cast(size.QuadPart) > std::numeric_limits::max()) { + CloseHandle(handle); error = "invalid size"; return; + } + mapping = CreateFileMappingW(handle, nullptr, PAGE_READONLY, 0, 0, nullptr); + fileHandle = handle; + if (!mapping) { error = "cannot map"; return; } + const auto* data = static_cast(MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0)); + if (!data) { error = "cannot map view"; return; } + file = {data, static_cast(size.QuadPart)}; +#else + const auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (fd < 0) { error = "cannot open"; return; } + struct stat size{}; + if (fstat(fd, &size) != 0 || size.st_size < static_cast(sizeof(PackHeader)) || static_cast(size.st_size) > std::numeric_limits::max()) { + close(fd); error = "invalid size"; return; + } + auto* data = mmap(nullptr, size.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (data == MAP_FAILED) { error = "cannot map"; return; } + file = {static_cast(data), static_cast(size.st_size)}; +#endif + Validate(); +} + +CachePack::~CachePack() { +#ifdef _WIN32 + if (!file.empty()) UnmapViewOfFile(file.data()); + if (mapping) CloseHandle(mapping); + if (fileHandle) CloseHandle(fileHandle); +#else + if (!file.empty()) munmap(const_cast(file.data()), file.size()); +#endif +} + +void CachePack::Validate() { + PackHeader header; + std::memcpy(&header, file.data(), sizeof(header)); + if (header.magic != PackMagic) { error = "unknown pack format"; return; } + if (header.sourceVersion != SourceVersion()) { error = "different compiler source version"; return; } + if (header.fileBytes != file.size() || header.entryCount > (file.size() - sizeof(header)) / sizeof(PackRecord)) { error = "truncated index"; return; } + const auto indexBytes = static_cast(header.entryCount) * sizeof(PackRecord); + const auto index = file.subspan(sizeof(header), indexBytes); + if (HashBytes(index) != header.indexHash) { error = "index checksum mismatch"; return; } + const std::span checked(reinterpret_cast(index.data()), header.entryCount); + std::array previous{}; + for (const auto& record : checked) { + if (record.hash < previous || record.offset < sizeof(header) + indexBytes || record.offset > file.size() || record.size > file.size() - record.offset || record.size < 48) { error = "invalid index record"; return; } + previous = record.hash; + } + records = checked; +} + +LoadStatus CachePack::Load(std::span key, CompiledVariant& variant, PackLoadTimings& timings) const { + const auto start = Clock::now(); + const auto hash = PackKeyHash(key); + const auto first = std::lower_bound(records.begin(), records.end(), hash, [](const auto& record, const auto& value) { return record.hash < value; }); + timings.lookupNanoseconds += elapsed(start); + auto status = LoadStatus::Absent; + for (auto it = first; it != records.end() && it->hash == hash; ++it) { + const auto data = file.subspan(it->offset, it->size); + const auto decode = Clock::now(); + const auto result = DecodeEntry(data, key, variant); + timings.decodeNanoseconds += elapsed(decode); + timings.bytes += data.size(); + if (result == LoadStatus::Loaded) return result; + if (result == LoadStatus::Rejected) status = result; + } + return status; +} +} diff --git a/core/shader/recompiler/ShaderCachePack.hpp b/core/shader/recompiler/ShaderCachePack.hpp new file mode 100644 index 000000000000..24aff3361007 --- /dev/null +++ b/core/shader/recompiler/ShaderCachePack.hpp @@ -0,0 +1,51 @@ +#pragma once +#include "ShaderDiskCache.hpp" +#include + +namespace ShaderRecompiler::ShaderDiskCache { + +inline constexpr std::uint64_t PackMagic = 0x314b434150535041ull; +struct PackHeader { + std::uint64_t magic = PackMagic; + std::uint64_t sourceVersion = 0; + std::uint64_t fileBytes = 0; + std::uint64_t entryCount = 0; + std::uint64_t indexHash = 0; +}; +struct PackRecord { + std::array hash{}; + std::uint64_t offset = 0; + std::uint64_t size = 0; +}; +static_assert(sizeof(PackHeader) == 40 && sizeof(PackRecord) == 32); + +struct PackLoadTimings { + std::uint64_t lookupNanoseconds = 0; + std::uint64_t decodeNanoseconds = 0; + std::uint64_t bytes = 0; +}; + +[[nodiscard]] std::array PackKeyHash(std::span key); +[[nodiscard]] std::vector PrebuiltCachePaths(const std::filesystem::path& cacheRoot); + +class CachePack { +public: + explicit CachePack(const std::filesystem::path& path); + ~CachePack(); + CachePack(const CachePack&) = delete; + CachePack& operator=(const CachePack&) = delete; + [[nodiscard]] bool Valid() const { return error.empty(); } + [[nodiscard]] const std::string& Error() const { return error; } + [[nodiscard]] std::size_t Count() const { return records.size(); } + [[nodiscard]] LoadStatus Load(std::span key, CompiledVariant& variant, PackLoadTimings& timings) const; + +private: + void Validate(); + std::span file; + std::span records; + std::string error; + void* mapping = nullptr; + void* fileHandle = nullptr; +}; + +} diff --git a/core/shader/recompiler/ShaderCacheSourceVersion.hpp b/core/shader/recompiler/ShaderCacheSourceVersion.hpp new file mode 100644 index 000000000000..d73de7db7fa8 --- /dev/null +++ b/core/shader/recompiler/ShaderCacheSourceVersion.hpp @@ -0,0 +1,15 @@ +#ifndef CORE_SHADER_RECOMPILER_SHADERCACHESOURCEVERSION_HPP +#define CORE_SHADER_RECOMPILER_SHADERCACHESOURCEVERSION_HPP + +#include +#include +#include + +namespace ShaderRecompiler::ShaderDiskCache::Generated { + +extern const std::uint64_t SourceVersion; +extern const std::span RecompilerSwitches; + +} + +#endif diff --git a/core/shader/recompiler/ShaderCacheVersion.cmake b/core/shader/recompiler/ShaderCacheVersion.cmake index e952236aef86..81be546106a8 100644 --- a/core/shader/recompiler/ShaderCacheVersion.cmake +++ b/core/shader/recompiler/ShaderCacheVersion.cmake @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) -file(STRINGS "${LIST_FILE}" entries) +file(STRINGS "${LIST_FILE}" entries ENCODING UTF-8) set(sources) set(guard) set(submodules) @@ -46,7 +46,7 @@ foreach(source IN LISTS sources) if(NOT source MATCHES "\\.(cpp|hpp|h)$") continue() endif() - file(STRINGS "${source}" lines REGEX "\"APS5_[A-Z0-9_]+\"") + file(STRINGS "${source}" lines ENCODING UTF-8 REGEX "\"APS5_[A-Z0-9_]+\"") foreach(line IN LISTS lines) string(REGEX MATCHALL "\"APS5_[A-Z0-9_]+\"" names "${line}") list(APPEND switches ${names}) @@ -56,7 +56,7 @@ foreach(source IN LISTS sources) continue() endif() get_filename_component(sourceDirectory "${source}" DIRECTORY) - file(STRINGS "${source}" lines REGEX "^[ \t]*#[ \t]*include[ \t]*\"[^\"]+\"") + file(STRINGS "${source}" lines ENCODING UTF-8 REGEX "^[ \t]*#[ \t]*include[ \t]*\"[^\"]+\"") foreach(line IN LISTS lines) string(REGEX REPLACE "^[ \t]*#[ \t]*include[ \t]*\"([^\"]+)\".*$" "\\1" included "${line}") set(resolved) @@ -109,7 +109,7 @@ foreach(source IN LISTS guard) continue() endif() file(RELATIVE_PATH relative "${ROOT}" "${source}") - file(STRINGS "${source}" lines REGEX "ShaderRecompiler|setenv|putenv") + file(STRINGS "${source}" lines ENCODING UTF-8 REGEX "ShaderRecompiler|setenv|putenv") foreach(line IN LISTS lines) if(line MATCHES "using[ \t]+namespace[ \t]+ShaderRecompiler" OR line MATCHES "namespace[ \t]+[A-Za-z_][A-Za-z0-9_]*[ \t]*=[ \t]*(::)?ShaderRecompiler[ \t]*;") list(APPEND failures "${relative} brings the ShaderRecompiler namespace in whole, which hides its calls from the shader disk cache key review: qualify them") @@ -140,23 +140,22 @@ list(REMOVE_DUPLICATES switches) list(SORT switches) list(JOIN switches ",\n " switchList) -set(content "#ifndef CORE_SHADER_RECOMPILER_SHADERCACHEVERSION_HPP -#define CORE_SHADER_RECOMPILER_SHADERCACHEVERSION_HPP - -#include -#include - +set(content "#include \"ShaderCacheSourceVersion.hpp\"\n") +string(APPEND content " namespace ShaderRecompiler::ShaderDiskCache::Generated { -inline constexpr std::uint64_t SourceVersion = 0x${version}ull; +namespace { -inline constexpr std::string_view RecompilerSwitches[] = { +constexpr std::string_view recompilerSwitches[] = { ${switchList} }; } -#endif +constinit const std::uint64_t SourceVersion = 0x${version}ull; +constinit const std::span RecompilerSwitches = recompilerSwitches; + +} ") if(EXISTS "${OUTPUT}") diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index dd7dd0b54391..467992e1197b 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -1,5 +1,6 @@ #include "ShaderDiskCache.hpp" #include "CacheKey.hpp" +#include "ShaderCachePack.hpp" #include "ShaderCacheDirectory.hpp" #include "ShaderCacheEviction.hpp" #include "ThreadOwned.hpp" @@ -14,7 +15,7 @@ #endif #include #endif -#include "ShaderCacheVersion.hpp" +#include "ShaderCacheSourceVersion.hpp" #include #include #include @@ -1277,6 +1278,7 @@ class DiskStore { if (root.empty()) return; this->root = root; directory = root / hex(SourceVersion()); + discoverPrebuilt(); std::thread([this] { run(); }).detach(); } @@ -1284,6 +1286,7 @@ class DiskStore { [[nodiscard]] const std::filesystem::path& Directory() const { return directory; } bool Load(std::span key, CompiledVariant& variant) { + if (loadPrebuilt(key, variant)) return true; return load(key, [&](std::span file) { return DecodeEntry(file, key, variant); }, hits, misses, loadFailures); } @@ -1317,16 +1320,107 @@ class DiskStore { counters.planMisses = planMisses.load(std::memory_order_relaxed); counters.planWrites = planWrites.load(std::memory_order_relaxed); counters.planLoadFailures = planLoadFailures.load(std::memory_order_relaxed); + counters.prebuiltHits = prebuiltHits.load(std::memory_order_relaxed); + counters.prebuiltMisses = prebuiltMisses.load(std::memory_order_relaxed); + counters.prebuiltRejected = prebuiltRejected.load(std::memory_order_relaxed); + counters.lookupNanoseconds = lookupNanoseconds.load(std::memory_order_relaxed); + counters.readNanoseconds = readNanoseconds.load(std::memory_order_relaxed); + counters.decodeNanoseconds = decodeNanoseconds.load(std::memory_order_relaxed); + counters.loadNanoseconds = loadNanoseconds.load(std::memory_order_relaxed); + counters.maxLoadNanoseconds = maxLoadNanoseconds.load(std::memory_order_relaxed); + counters.prebuiltBytesRead = prebuiltBytesRead.load(std::memory_order_relaxed); + counters.prebuiltLoadNanoseconds = prebuiltLoadNanoseconds.load(std::memory_order_relaxed); return counters; } private: + void discoverPrebuilt() { + const auto start = std::chrono::steady_clock::now(); + std::size_t entries = 0; + const auto addPack = [&](const std::filesystem::path& path) { + auto pack = std::make_unique(path); + if (!pack->Valid()) { + std::fprintf(stderr, "[shader-prebuilt] ignored %s: %s\n", path.string().c_str(), pack->Error().c_str()); + return; + } + entries += pack->Count(); + packs.push_back(std::move(pack)); + }; + for (const auto& path : PrebuiltCachePaths(root)) { + std::error_code error; + if (std::filesystem::is_regular_file(path, error)) { addPack(path); continue; } + if (!std::filesystem::is_directory(path, error)) continue; + bool loose = false; + std::vector packPaths; + for (std::filesystem::directory_iterator it(path, error), end; !error && it != end; it.increment(error)) { + const auto extension = it->path().extension(); + if (extension == ".pack") packPaths.push_back(it->path()); + if (extension == ".bin" && it->path().stem().string().size() == 32) loose = true; + } + std::sort(packPaths.begin(), packPaths.end()); + for (const auto& item : packPaths) addPack(item); + if (loose) looseDirectories.push_back(path); + const auto versioned = path / hex(SourceVersion()); + if (std::filesystem::is_directory(versioned, error)) looseDirectories.push_back(versioned); + } + if (!packs.empty() || !looseDirectories.empty()) std::fprintf(stderr, "[shader-prebuilt] mapped %zu packs, %zu entries, %zu loose folders in %.3f ms; exact source/ABI/request keys required\n", packs.size(), entries, looseDirectories.size(), std::chrono::duration(std::chrono::steady_clock::now() - start).count()); + } + struct Job { std::vector key; std::shared_ptr variant; std::shared_ptr plan; }; + bool loadPrebuilt(std::span key, CompiledVariant& variant) { + if (packs.empty() && looseDirectories.empty()) return false; + const auto started = std::chrono::steady_clock::now(); + const auto elapsed = [](auto start) { return static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); }; + bool loaded = false; + PackLoadTimings timing; + for (const auto& pack : packs) { + const auto status = pack->Load(key, variant, timing); + if (status == LoadStatus::Loaded) { loaded = true; break; } + if (status == LoadStatus::Rejected) prebuiltRejected.fetch_add(1, std::memory_order_relaxed); + } + lookupNanoseconds.fetch_add(timing.lookupNanoseconds, std::memory_order_relaxed); + decodeNanoseconds.fetch_add(timing.decodeNanoseconds, std::memory_order_relaxed); + prebuiltBytesRead.fetch_add(timing.bytes, std::memory_order_relaxed); + if (!loaded) { + const auto name = EntryName(key); + thread_local std::vector* fileSlot = nullptr; + auto& file = ThreadOwned(fileSlot); + for (const auto& path : looseDirectories) { + const auto start = std::chrono::steady_clock::now(); + const auto read = ReadWholeFile(path / name, file); + readNanoseconds.fetch_add(elapsed(start), std::memory_order_relaxed); + if (!read) continue; + const auto decode = std::chrono::steady_clock::now(); + const auto status = DecodeEntry(file, key, variant); + decodeNanoseconds.fetch_add(elapsed(decode), std::memory_order_relaxed); + prebuiltBytesRead.fetch_add(file.size(), std::memory_order_relaxed); + if (status == LoadStatus::Rejected) { + const auto failures = prebuiltRejected.fetch_add(1, std::memory_order_relaxed); + if (failures < 8) std::fprintf(stderr, "[shader-prebuilt] rejected %s (%zu bytes): truncated, corrupt or another format; recompiling\n", name.c_str(), file.size()); + } + if (status == LoadStatus::Loaded) { loaded = true; break; } + } + if (file.capacity() > (4u << 20u)) std::vector().swap(file); + } + if (loaded) { + const auto previous = prebuiltHits.fetch_add(1, std::memory_order_relaxed); + if (previous == 0) std::fprintf(stderr, "[shader-prebuilt] first exact-key hit loaded in %.3f ms\n", elapsed(started) / 1e6); + } else prebuiltMisses.fetch_add(1, std::memory_order_relaxed); + if (loaded) { + const auto duration = elapsed(started); + prebuiltLoadNanoseconds.fetch_add(duration, std::memory_order_relaxed); + auto maximum = maxLoadNanoseconds.load(std::memory_order_relaxed); + while (maximum < duration && !maxLoadNanoseconds.compare_exchange_weak(maximum, duration, std::memory_order_relaxed)) {} + report(false); + } + return loaded; + } + template bool load(std::span key, TDecode&& decode, std::atomic& hitCount, std::atomic& missCount, std::atomic& failureCount) { const auto started = std::chrono::steady_clock::now(); @@ -1438,19 +1532,27 @@ class DiskStore { const auto totals = Totals(); const auto loadNs = loadNanoseconds.load(std::memory_order_relaxed); const auto delta = [](std::uint64_t after, std::uint64_t before) { return static_cast(after - before); }; - if (totals.hits == reported.hits && totals.misses == reported.misses && totals.writes == reported.writes && totals.loadFailures == reported.loadFailures && totals.writeFailures == reported.writeFailures && totals.planHits == reported.planHits && totals.planMisses == reported.planMisses && totals.planWrites == reported.planWrites && totals.planLoadFailures == reported.planLoadFailures) return; - std::fprintf(stderr, "[shader-disk-cache] (10 s): %llu hits, %llu misses, %llu writes (%.1f MiB), %llu load failures, %llu write failures; resource plans: %llu hits, %llu misses, %llu writes, %llu load failures; %.1f MiB read in %.1f ms (totals: %llu hits, %llu misses, %llu writes; plans %llu hits, %llu misses)\n", + const auto baseChanged = totals.hits != reported.hits || totals.misses != reported.misses || totals.writes != reported.writes || totals.loadFailures != reported.loadFailures || totals.writeFailures != reported.writeFailures || totals.planHits != reported.planHits || totals.planMisses != reported.planMisses || totals.planWrites != reported.planWrites || totals.planLoadFailures != reported.planLoadFailures; + const auto prebuiltChanged = totals.prebuiltHits != reported.prebuiltHits || totals.prebuiltMisses != reported.prebuiltMisses || totals.prebuiltRejected != reported.prebuiltRejected; + if (!baseChanged && !prebuiltChanged) return; + if (baseChanged) std::fprintf(stderr, "[shader-disk-cache] (10 s): %llu hits, %llu misses, %llu writes (%.1f MiB), %llu load failures, %llu write failures; resource plans: %llu hits, %llu misses, %llu writes, %llu load failures; %.1f MiB read in %.1f ms (totals: %llu hits, %llu misses, %llu writes; plans %llu hits, %llu misses)\n", delta(totals.hits, reported.hits), delta(totals.misses, reported.misses), delta(totals.writes, reported.writes), static_cast(totals.bytesWritten - reported.bytesWritten) / (1024.0 * 1024.0), delta(totals.loadFailures, reported.loadFailures), delta(totals.writeFailures, reported.writeFailures), delta(totals.planHits, reported.planHits), delta(totals.planMisses, reported.planMisses), delta(totals.planWrites, reported.planWrites), delta(totals.planLoadFailures, reported.planLoadFailures), static_cast(totals.bytesRead - reported.bytesRead) / (1024.0 * 1024.0), static_cast(loadNs - reportedLoadNs) / 1e6, static_cast(totals.hits), static_cast(totals.misses), static_cast(totals.writes), static_cast(totals.planHits), static_cast(totals.planMisses)); + if (prebuiltChanged && (!packs.empty() || !looseDirectories.empty())) std::fprintf(stderr, "[shader-prebuilt] interval: %llu hits, %llu misses, %llu rejected; lookup %.3f ms, file read %.3f ms, decode %.3f ms; total load %.3f ms, max %.3f ms (lifetime hits %llu)\n", + delta(totals.prebuiltHits, reported.prebuiltHits), delta(totals.prebuiltMisses, reported.prebuiltMisses), delta(totals.prebuiltRejected, reported.prebuiltRejected), + (totals.lookupNanoseconds - reported.lookupNanoseconds) / 1e6, (totals.readNanoseconds - reported.readNanoseconds) / 1e6, (totals.decodeNanoseconds - reported.decodeNanoseconds) / 1e6, + (totals.prebuiltLoadNanoseconds - reported.prebuiltLoadNanoseconds) / 1e6, totals.maxLoadNanoseconds / 1e6, static_cast(totals.prebuiltHits)); reported = totals; reportedLoadNs = loadNs; } std::filesystem::path root; std::filesystem::path directory; + std::vector> packs; + std::vector looseDirectories; std::mutex mutex; std::condition_variable wake; std::condition_variable idle; @@ -1458,6 +1560,7 @@ class DiskStore { bool writing = false; std::atomic hits{0}, misses{0}, writes{0}, loadFailures{0}, writeFailures{0}, bytesRead{0}, bytesWritten{0}, loadNanoseconds{0}; std::atomic planHits{0}, planMisses{0}, planWrites{0}, planLoadFailures{0}; + std::atomic prebuiltHits{0}, prebuiltMisses{0}, prebuiltRejected{0}, lookupNanoseconds{0}, readNanoseconds{0}, decodeNanoseconds{0}, maxLoadNanoseconds{0}, prebuiltBytesRead{0}, prebuiltLoadNanoseconds{0}; std::atomic lastReport{0}; std::mutex reportMutex; Counters reported; diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 4a7d20f7a689..d34ddddb13bc 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -43,6 +43,16 @@ struct Counters { std::uint64_t planMisses = 0; std::uint64_t planWrites = 0; std::uint64_t planLoadFailures = 0; + std::uint64_t prebuiltHits = 0; + std::uint64_t prebuiltMisses = 0; + std::uint64_t prebuiltRejected = 0; + std::uint64_t lookupNanoseconds = 0; + std::uint64_t readNanoseconds = 0; + std::uint64_t decodeNanoseconds = 0; + std::uint64_t loadNanoseconds = 0; + std::uint64_t maxLoadNanoseconds = 0; + std::uint64_t prebuiltBytesRead = 0; + std::uint64_t prebuiltLoadNanoseconds = 0; }; [[nodiscard]] std::uint64_t SourceVersion(); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index fbc579d2c00f..d92ef7b1b07f 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -220,6 +220,7 @@ std::uint32_t EmitFPNeg32(SpirvEmitterState& state, std::uint32_t arg0); std::uint32_t EmitFPCmpClass32(SpirvEmitterState& state, std::uint32_t value, std::uint32_t mask); void EmitReference(SpirvValueEmitContext& context); void EmitReferenceU32(SpirvValueEmitContext& context); +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter); void EmitControlNop(SpirvValueEmitContext& context); void EmitWaitcnt(SpirvValueEmitContext& context); void EmitSendmsg(SpirvValueEmitContext& context); @@ -348,6 +349,7 @@ void EmitSharedSrc2U32(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitSharedSrc2U64(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitDataAppend(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitDataConsume(SpirvValueEmitContext& ctx, const IrValue& inst); +std::uint32_t EmitDataOrderedCount(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitImageQueryDimensions(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitImageQueryLod(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitImageRead(SpirvValueEmitContext& ctx, const IrValue& inst); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index 02d48e159bb0..1530c932cd90 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -132,6 +132,7 @@ struct SpirvEmitterState { bool f16DenormPreserveEmitted = false; std::uint32_t dot2F32F16Function = 0; std::array bvhIntersectFunctions {}; + std::map, std::uint32_t> interpolateFunctions; std::array, 2> bdaDwordReadFunctions {}; std::array, 2> bdaSpanReadFunctions {}; std::unordered_map> bdaGroupedLoads; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp index ef3f2ddee93e..1c15a19d015e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp @@ -8,6 +8,7 @@ #include #include #include +#include namespace ShaderRecompiler { namespace { @@ -1094,12 +1095,52 @@ std::uint32_t EmitFPDot2F32F16(SpirvEmitterState& state, std::uint32_t arg0, std }); } +namespace { + +std::uint32_t InterpolateFunction(SpirvEmitterState& state, bool halfResult, std::uint32_t mode) { + const auto key = std::make_pair(halfResult, mode); + if (const auto found = state.interpolateFunctions.find(key); found != state.interpolateFunctions.end()) return found->second; + const auto f32 = TypeF32(state); + const auto id = state.module.AllocateId(); + struct Restore { + SpirvEmitterState& state; + std::uint32_t label; + ~Restore() { + state.module.EndHelperFunction(); + state.currentLabel = label; + } + }; + state.module.BeginHelperFunction(); + const Restore restore{state, state.currentLabel}; + state.module.AddName(id, std::string(halfResult ? "interpolate_f16_mode_" : "interpolate_f32_mode_") + std::to_string(mode)); + state.module.AddFunction(spv::OpFunction, f32, id, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, f32, f32, f32, f32)); + std::array parameters{}; + for (auto& parameter : parameters) { + parameter = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, f32, parameter); + } + EmitLabel(state, state.module.AllocateId()); + state.module.AddFunction(spv::OpReturnValue, Interpolate(state, parameters[0], parameters[1], parameters[2], halfResult, mode)); + state.module.AddFunction(spv::OpFunctionEnd); + state.interpolateFunctions.emplace(key, id); + return id; +} + +std::uint32_t CallInterpolate(SpirvEmitterState& state, std::uint32_t delta, std::uint32_t coordinate, std::uint32_t base, bool halfResult, std::uint32_t mode) { + const auto function = InterpolateFunction(state, halfResult, mode); + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeF32(state), result, function, delta, coordinate, base); + return result; +} + +} + std::uint32_t EmitFPInterpolateF32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode) { - return Interpolate(state, arg0, arg1, arg2, false, mode->ImmediateU32()); + return CallInterpolate(state, arg0, arg1, arg2, false, mode->ImmediateU32()); } std::uint32_t EmitFPInterpolateF16(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2, const IrValue* mode) { - return Interpolate(state, arg0, arg1, arg2, true, mode->ImmediateU32()); + return CallInterpolate(state, arg0, arg1, arg2, true, mode->ImmediateU32()); } std::uint32_t EmitConvertF32F64(SpirvEmitterState& state, std::uint32_t arg0) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp index a749c41ae4fd..2c3e675b6ceb 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp @@ -49,6 +49,7 @@ bool LaneSource(const IrProgram& program, const IrValue& value) { case IrOpcode::DataAppend: case IrOpcode::DataConsume: case IrOpcode::LoadVectorWindow: + case IrOpcode::DataOrderedCount: return true; case IrOpcode::GetBuiltin: { const auto* kind = value.Argument(0)->Resolve(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 11070078b236..c348adb8611c 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -295,6 +295,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::Void: return Invoke(EmitVoid, ctx, inst); case IrOpcode::Reference: return Invoke(EmitReference, ctx, inst); case IrOpcode::ReferenceU32: return Invoke(EmitReferenceU32, ctx, inst); + case IrOpcode::UnlinkedCall: return Invoke(EmitUnlinkedCall, ctx, inst); case IrOpcode::GetUserData: return Invoke(EmitGetUserData, ctx, inst); case IrOpcode::GetShaderBase: return Invoke(EmitGetShaderBase, ctx, inst); case IrOpcode::ShaderClock: return Invoke(EmitShaderClock, ctx, inst); @@ -655,6 +656,7 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::SharedSrc2U64: return Invoke(EmitSharedSrc2U64, ctx, inst); case IrOpcode::DataAppend: return Invoke(EmitDataAppend, ctx, inst); case IrOpcode::DataConsume: return Invoke(EmitDataConsume, ctx, inst); + case IrOpcode::DataOrderedCount: return Invoke(EmitDataOrderedCount, ctx, inst); case IrOpcode::SwizzleU32: return Invoke(EmitSwizzleU32, ctx, inst); case IrOpcode::ImageQueryDimensions: return Invoke(EmitImageQueryDimensions, ctx, inst); case IrOpcode::ImageQueryLod: return Invoke(EmitImageQueryLod, ctx, inst); @@ -938,6 +940,13 @@ void EmitReferenceU32(SpirvValueEmitContext& context) { EmitVoid(context); } +void EmitUnlinkedCall(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t low, std::uint32_t high, std::uint32_t programCounter) { + auto& state = ctx.state; + if (state.faultBufferVariable == 0) ctx.Fail(inst, "calls an unlinked target without a fault buffer"); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 0u), programCounter, BdaAbi::FaultReason::UnlinkedCall); + StopBdaInvocationIf(state, ConstantBool(state, true)); +} + void EmitControlNop(SpirvValueEmitContext& context) { EmitVoid(context); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 1058adbb7534..c2db12f032f9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -1252,7 +1252,7 @@ std::uint32_t AddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) { }); } -std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, bool append) { +std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, bool append, bool ordered = false) { auto& state = ctx.state; if (ctx.half == 1u) { if (ctx.otherHalf == nullptr) { @@ -1263,9 +1263,10 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo const auto& mem = SharedMemory(ctx, inst); const bool wave64 = state.laneCount == 2u; const bool gds = mem.kind == ResourceKind::Gds; + const bool m0Address = gds && !ordered; std::uint32_t rawIndex = ConstantU32(state, mem.offset >> 2u); std::uint32_t m0Bounds = 0; - if (gds) { + if (m0Address) { const auto m0 = ctx.Arg(inst, 0); const auto base = Binary(state, spv::OpShiftRightLogical, TypeU32(state), m0, ConstantU32(state, 16u)); const auto size = Binary(state, spv::OpBitwiseAnd, TypeU32(state), m0, ConstantU32(state, 0xffffu)); @@ -1284,9 +1285,14 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo const auto count = Binary(state, spv::OpIAdd, TypeU32(state), Unary(state, spv::OpBitCount, TypeU32(state), low), Unary(state, spv::OpBitCount, TypeU32(state), high)); const auto first = ctx.FirstLane(ballot); const auto sourceLane = wave64 ? Binary(state, spv::OpBitwiseAnd, TypeU32(state), first, ConstantU32(state, 31u)) : first; + auto amount = count; + if (ordered) { + amount = state.module.AllocateId(); + state.module.AddFunction(spv::OpGroupNonUniformShuffle, TypeU32(state), amount, ConstantU32(state, spv::ScopeSubgroup), ctx.Arg(inst, 4), EmitHostSubgroupLane(state, sourceLane)); + } const auto isFirst = Binary(state, spv::OpIEqual, TypeBool(state), EmitSubgroupLocalInvocationId(state), sourceLane); const auto storageBounds = EmitMemoryElementInBounds(state, access, index); - const auto bounds = gds ? AndCondition(state, storageBounds, m0Bounds) : storageBounds; + const auto bounds = m0Address ? AndCondition(state, storageBounds, m0Bounds) : storageBounds; const auto lanesActive = wave64 ? Binary(state, spv::OpINotEqual, TypeBool(state), count, ConstantU32(state, 0u)) : exec; const auto condition = AndCondition(state, isFirst, AndCondition(state, lanesActive, bounds)); const auto atomic = EmitValueOrZeroIfCondition(state, condition, [&]() { @@ -1294,7 +1300,7 @@ std::uint32_t AppendConsume(SpirvValueEmitContext& ctx, const IrValue& inst, boo return UpdateFunctionElement(state, EmitMemoryElementPointer(state, access, index), [&](std::uint32_t old) { return Binary(state, append ? spv::OpIAdd : spv::OpISub, TypeU32(state), old, count); }); } const auto value = state.module.AllocateId(); - state.module.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value, EmitMemoryElementPointer(state, access, index), ConstantU32(state, gds ? spv::ScopeDevice : spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsMaskNone), count); + state.module.AddFunction(append ? spv::OpAtomicIAdd : spv::OpAtomicISub, TypeU32(state), value, EmitMemoryElementPointer(state, access, index), ConstantU32(state, gds ? spv::ScopeDevice : spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsMaskNone), amount); return value; }); const auto result = state.module.AllocateId(); @@ -1955,6 +1961,10 @@ std::uint32_t EmitDataConsume(SpirvValueEmitContext& ctx, const IrValue& inst) { return AppendConsume(ctx, inst, false); } +std::uint32_t EmitDataOrderedCount(SpirvValueEmitContext& ctx, const IrValue& inst) { + return AppendConsume(ctx, inst, true, true); +} + std::uint32_t EmitSharedAtomicRsub32(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto data = ctx.Arg(inst, 1); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index 08758d2b8566..61ab8168625e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -133,7 +133,7 @@ void DefineModule(SpirvEmitterState& state) { state.module.EmitCapability(spv::CapabilityGroupNonUniformShuffle); } if (state.requirements.computeDerivatives && StageOf(state) == IrShaderStage::Compute) { - state.module.EmitCapability(spv::CapabilityComputeDerivativeGroupQuadsKHR); + state.module.EmitCapability(spv::CapabilityComputeDerivativeGroupLinearKHR); state.module.EmitExtension("SPV_KHR_compute_shader_derivatives"); } const bool fragmentBarycentric = StageOf(state) == IrShaderStage::Pixel && std::any_of(state.inputs.begin(), state.inputs.end(), [](const SpirvInputBinding& input) { @@ -165,10 +165,6 @@ void DefineModule(SpirvEmitterState& state) { localX = ((localX * localY * localZ + 63u) / 64u) * 32u; localY = 1u; localZ = 1u; - if (state.requirements.computeDerivatives) { - localY = localX / 2u; - localX = 2u; - } } state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeLocalSize, localX, localY, localZ); } @@ -205,7 +201,7 @@ void DefineModule(SpirvEmitterState& state) { } } if (state.requirements.computeDerivatives && StageOf(state) == IrShaderStage::Compute) { - state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDerivativeGroupQuadsKHR); + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDerivativeGroupLinearKHR); } state.module.AddName(state.mainFunc, "main"); if (state.requirements.functionLds) { diff --git a/core/shader/recompiler/Translation/include/Translation/RoundingMode.hpp b/core/shader/recompiler/Translation/include/Translation/RoundingMode.hpp new file mode 100644 index 000000000000..5495b99408b9 --- /dev/null +++ b/core/shader/recompiler/Translation/include/Translation/RoundingMode.hpp @@ -0,0 +1,50 @@ +#ifndef CORE_SHADER_RECOMPILIER_TRANSLATION_INCLUDE_TRANSLATION_ROUNDINGMODE_HPP +#define CORE_SHADER_RECOMPILIER_TRANSLATION_INCLUDE_TRANSLATION_ROUNDINGMODE_HPP + +#include "ControlFlow/ControlFlowGraph.hpp" +#include "RdnaDecoder/RdnaOpcode.hpp" +#include "RdnaDecoder/RdnaProgram.hpp" +#include +#include +#include + +namespace ShaderRecompiler { + +inline constexpr std::uint32_t ModeRoundBits = 0xfu; +inline constexpr std::uint32_t ModeRound32Bits = 0x3u; +inline constexpr std::uint32_t ModeRound1664Bits = 0xcu; + +struct RoundingState { + std::uint8_t value = 0; + std::uint8_t unknown = 0; + + bool operator==(const RoundingState&) const = default; +}; + +struct RoundingWrite { + std::uint32_t mask = 0; + std::uint32_t value = 0; + bool known = false; +}; + +enum class RoundingUseKind { + Directed32, + Directed16, + SelfChecked, + Unmodelled +}; + +struct RoundingUse { + std::uint32_t fields = 0; + RoundingUseKind kind = RoundingUseKind::Unmodelled; +}; + +[[nodiscard]] std::optional DecodeRoundingWrite(const RdnaInstruction& instruction); +[[nodiscard]] RoundingState ApplyRoundingWrite(RoundingState state, const RoundingWrite& write); +[[nodiscard]] RoundingState JoinRoundingStates(RoundingState lhs, RoundingState rhs); +[[nodiscard]] RoundingUse RoundingUseOf(RdnaOpcode opcode); +[[nodiscard]] std::unordered_map ComputeEntryRounding(const ControlFlowGraph& cfg, const RdnaProgram& program, std::uint32_t initial); + +} + +#endif diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 290cb9ec34dc..7e7533a3e618 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -2,6 +2,7 @@ #define CORE_SHADER_RECOMPILIER_TRANSLATION_INCLUDE_TRANSLATION_TRANSLATIONCONTEXT_HPP #include "Translation/InstructionTranslator.hpp" +#include "Translation/RoundingMode.hpp" #include #include #include @@ -16,10 +17,17 @@ class TranslationContext { void TranslateEmbeddedFetch(const RdnaInstruction& instruction, std::uint32_t attribute, std::uint32_t components); void TranslateInstruction(const RdnaInstruction& instruction); void SetPixelInput(const ShaderPixelInputInfo* info, bool barycentricEnabled) { pixelInput = info; fragmentShaderBarycentricEnabled = barycentricEnabled; } - void SetFloatMode(const std::optional& mode) { floatMode = mode; ieeeMode = mode.has_value() && mode->ieeeMode; } + void SetFloatMode(const std::optional& mode) { + declaredFloatMode = mode; + floatMode = mode; + ieeeMode = mode.has_value() && mode->ieeeMode; + rounding = RoundingState{static_cast(mode.has_value() ? mode->floatMode & ModeRoundBits : 0u), 0u}; + } + void SetEntryRounding(const RoundingState& state) { setRounding(state); } void SetNativeFloat16(bool enabled) { nativeFloat16 = enabled; } void AddBranchCondition(const BasicBlock& source, BlockInfo& info); void TranslateCodeTableLoad(const RdnaInstruction& instruction, const ControlFlowGraph::CodeTableLoad& table); + void TranslateUnlinkedCall(const RdnaInstruction& instruction); private: struct AddressOperands { @@ -149,6 +157,7 @@ class TranslationContext { bool dsAtomic64(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue); bool dsCondxchg32(const RdnaInstruction& inst); bool dsAppendConsume(const RdnaInstruction& inst, IrOpcode opcode); + bool dsOrderedCount(const RdnaInstruction& inst); bool dsAddtid(const RdnaInstruction& inst, bool write); bool globalAddtid(const RdnaInstruction& inst, bool write); bool dsSwizzleB32(const RdnaInstruction& inst); @@ -347,6 +356,9 @@ class TranslationContext { void vInterpP2F16(const RdnaInstruction& inst); void eXP(const RdnaInstruction& inst); bool emitScalar(const RdnaInstruction& inst); + void setRounding(const RoundingState& state); + void writeRounding(const RdnaInstruction& inst); + void checkRoundingUse(const RdnaInstruction& inst) const; bool emitVector(const RdnaInstruction& inst); std::uint32_t f32DenormalFlushFor(const RdnaInstruction& inst) const; bool emitInterpolation(const RdnaInstruction& inst); @@ -356,6 +368,8 @@ class TranslationContext { const ShaderPixelInputInfo* pixelInput = nullptr; bool fragmentShaderBarycentricEnabled = false; std::optional floatMode; + std::optional declaredFloatMode; + RoundingState rounding; bool ieeeMode = false; bool nativeFloat16 = false; IrBuilder ir; diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 20060cce3070..2f6b88b99738 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -283,6 +283,13 @@ void TranslationContext::sSwappcB64(const RdnaInstruction& inst) { writeU32Pair(inst.destination, extractU64(link)); } +void TranslationContext::TranslateUnlinkedCall(const RdnaInstruction& inst) { + const std::array target = readU32Pair(inst.source0); + (void)ir.Emit(IrOpcode::UnlinkedCall, IrType::Void, {&target[0].Value(), &target[1].Value(), &ir.Constant(inst.programCounter)}); + program.Info().usesFaultBuffer = true; + sSwappcB64(inst); +} + void TranslationContext::sCselectB32(const RdnaInstruction& inst) { const IrU32 trueValue = readU32(sourceAt(inst, 0u)); const IrU32 falseValue = readU32(sourceAt(inst, 1u)); diff --git a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp index e2c9f9f7b582..ce731d97b9d0 100644 --- a/core/shader/recompiler/Translation/src/DispatchInstructions.cpp +++ b/core/shader/recompiler/Translation/src/DispatchInstructions.cpp @@ -24,6 +24,7 @@ void TranslationContext::TranslateInstruction(const RdnaInstruction& decoded) { if (instruction.op == RdnaOpcode::Unsupported) { throw std::runtime_error(instruction.unsupportedReason.empty() ? "unsupported decoded instruction at pc " + std::to_string(instruction.programCounter) : std::string(instruction.unsupportedReason)); } + checkRoundingUse(instruction); f32DenormalFlush = f32DenormalFlushFor(instruction); bool translated = false; switch (instruction.family) { diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index fe36a50ee72b..82c33c1b49c0 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -39,12 +39,22 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc if (high ? operand.negateHi : operand.negate) bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x8000u))); return bits; }; - const auto minMaxLane = [&](bool high) { return minMaxF16Bits(opcode == IrOpcode::FPMax32, laneBits(sourceAt(inst, 0u), high), laneBits(sourceAt(inst, 1u), high)); }; + std::array, 3> sourceBits{}; + std::array, 3> sourceBitsReady{}; + const auto sourceLaneBits = [&](std::uint32_t index, bool high) { + const std::size_t lane = high ? 1u : 0u; + if (!sourceBitsReady[index][lane]) { + const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index); + sourceBits[index][lane] = laneBits(operand, high); + sourceBitsReady[index][lane] = true; + } + return sourceBits[index][lane]; + }; + const auto minMaxLane = [&](bool high) { return minMaxF16Bits(opcode == IrOpcode::FPMax32, sourceLaneBits(0u, high), sourceLaneBits(1u, high)); }; const bool native = nativeFloat16 && inst.destination.omod == 0u && (opcode == IrOpcode::FPAdd32 || opcode == IrOpcode::FPMul32); const auto nativeResult = [&]() { - const auto packed = [&](const RdnaOperand& operand) { return packU16Lanes(laneBits(operand, false), laneBits(operand, true)); }; - const IrU32 lhs = packed(sourceAt(inst, 0u)); - const IrU32 rhs = packed(sourceAt(inst, 1u)); + const IrU32 lhs = packU16Lanes(sourceLaneBits(0u, false), sourceLaneBits(0u, true)); + const IrU32 rhs = packU16Lanes(sourceLaneBits(1u, false), sourceLaneBits(1u, true)); return IrU32(ir.Emit(opcode == IrOpcode::FPAdd32 ? IrOpcode::FPAdd16x2 : IrOpcode::FPMul16x2, IrType::U32, {&lhs.Value(), &rhs.Value()})); }; IrU32 result = quietSnan ? packU16Lanes(minMaxLane(false), minMaxLane(true)) : native ? nativeResult() : packHalf2x16(translateLane(false), translateLane(true)); @@ -67,7 +77,7 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const std::uint32_t count = accumulator ? 3u : inst.sourceCount; std::array bits{}; for (std::uint32_t index = 0u; index < count; ++index) { - bits[index] = laneBits(accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index), high); + bits[index] = sourceLaneBits(index, high); } IrU32 lane(high ? ir.ShiftRightLogical(result.Value(), ir.Constant(16u)) : ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu))); if (native && inst.destination.clamp && dx10Clamp()) return lane; diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index f663697f360f..a3cd6207fb5e 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -89,6 +89,10 @@ const ShaderWorkgroupInputInfo* shaderWorkgroupInput(ShaderStageKind stage, cons } } +bool isUnlinkedCall(const ControlFlowGraph& cfg, std::uint32_t programCounter) { + return std::find(cfg.unlinkedCallProgramCounters.begin(), cfg.unlinkedCallProgramCounters.end(), programCounter) != cfg.unlinkedCallProgramCounters.end(); +} + bool isCodeTableLoad(const ControlFlowGraph& cfg, std::uint32_t programCounter) { return std::find(cfg.codeTableLoadProgramCounters.begin(), cfg.codeTableLoadProgramCounters.end(), programCounter) != cfg.codeTableLoadProgramCounters.end(); } @@ -523,12 +527,14 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con } } + const auto entryRounding = ComputeEntryRounding(cfg, decoded, options.floatMode.has_value() ? options.floatMode->floatMode : 0u); for (const auto& cfgBlock : cfg.blocks) { const auto typedIndex = blockIndices.at(cfgBlock.id); TranslationContext context(program, *blocks[typedIndex], vectorLimit); context.SetPixelInput(options.inputInfo.pixel, options.fragmentShaderBarycentricEnabled); context.SetFloatMode(options.floatMode); context.SetNativeFloat16(options.nativeFloat16); + context.SetEntryRounding(entryRounding.at(cfgBlock.id)); for (std::uint32_t index = cfgBlock.instructionBegin; index < cfgBlock.instructionEnd; index++) { const auto& instruction = decoded.instructions[index]; if (isCodeTableLoad(cfg, instruction.programCounter)) { @@ -537,6 +543,10 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con context.TranslateCodeTableLoad(instruction, *table); continue; } + if (isUnlinkedCall(cfg, instruction.programCounter)) { + context.TranslateUnlinkedCall(instruction); + continue; + } if (options.embeddedFetch != nullptr) { const auto load = std::ranges::find(options.embeddedFetch->loads, instruction.programCounter, &EmbeddedFetchLoad::programCounter); if (load != options.embeddedFetch->loads.end()) { diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp index 1d75e9a28cbd..3f1eb34b4367 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/MemoryInstructionDispatch.cpp @@ -534,7 +534,7 @@ bool TranslationContext::emitMemory(const RdnaInstruction& inst) { case RdnaOpcode::DsGwsBarrier: throw std::runtime_error(std::string(globalWaveSyncName(inst.op)) + " at pc " + std::to_string(inst.programCounter) + ": global wave sync barriers and semaphores shared by waves of different workgroups are not modeled"); case RdnaOpcode::DsOrderedCount: - throw std::runtime_error("ds_ordered_count at pc " + std::to_string(inst.programCounter) + ": GDS counters updated in wave launch order are not modeled"); + return dsOrderedCount(inst); case RdnaOpcode::DsWriteAddtidB32: return dsAddtid(inst, true); case RdnaOpcode::DsReadAddtidB32: diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp index 8f514e7b3abd..099fce6b3804 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp @@ -298,6 +298,24 @@ bool TranslationContext::dsAppendConsume(const RdnaInstruction& inst, IrOpcode o return true; } +bool TranslationContext::dsOrderedCount(const RdnaInstruction& inst) { + const std::uint32_t flags = (inst.memoryOffset >> 8u) & 0xffu; + const auto location = "ds_ordered_count at pc " + std::to_string(inst.programCounter); + if (!inst.gds) throw std::runtime_error(location + " without GDS is not modeled"); + if ((flags & 0x10u) != 0u) throw std::runtime_error(location + ": ordered swap is not modeled"); + if (((flags >> 6u) & 3u) != 0u) throw std::runtime_error(location + ": counting more than one dword is not modeled"); + MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + memory.offset = inst.memoryOffset & 0xffu; + IrValue& address = ir.Constant(0u); + const IrU32 amount = readU32(inst.source0); + IrValue& active = ir.GetExec(); + IrValue& activeLo = ir.GetExecLo(); + IrValue& activeHi = ir.GetExecHi(); + IrValue& result = ir.Emit(IrOpcode::DataOrderedCount, IrOpcodeType(IrOpcode::DataOrderedCount), {&address, &active, &activeLo, &activeHi, &amount.Value()}, addMemoryInfo(memory, inst.programCounter)); + writeOperand(inst.destination, &result); + return true; +} + bool TranslationContext::dsAddtid(const RdnaInstruction& inst, bool write) { const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); IrValue& base = ir.BitwiseAnd(readU32(makeM0Operand()).Value(), ir.Constant(0xffffu)); diff --git a/core/shader/recompiler/Translation/src/RoundingMode.cpp b/core/shader/recompiler/Translation/src/RoundingMode.cpp new file mode 100644 index 000000000000..0edeca52f011 --- /dev/null +++ b/core/shader/recompiler/Translation/src/RoundingMode.cpp @@ -0,0 +1,236 @@ +#include "Translation/RoundingMode.hpp" +#include "Translation/TranslationContext.hpp" +#include +#include +#include + +namespace ShaderRecompiler { + +namespace { + +constexpr std::uint32_t ModeRegisterId = 1u; + +struct ModeField { + std::uint32_t mask = 0; + std::uint32_t offset = 0; + std::uint32_t size = 0; +}; + +std::optional roundingField(std::uint32_t simm16) { + if ((simm16 & 0x3fu) != ModeRegisterId) return std::nullopt; + const std::uint32_t offset = (simm16 >> 6u) & 0x1fu; + const std::uint32_t size = ((simm16 >> 11u) & 0x1fu) + 1u; + if (offset + size > 4u) return std::nullopt; + return ModeField{((1u << size) - 1u) << offset, offset, size}; +} + +RoundingState transfer(RoundingState state, const BasicBlock& block, const RdnaProgram& program) { + for (std::uint32_t index = block.instructionBegin; index < block.instructionEnd; ++index) { + if (const auto write = DecodeRoundingWrite(program.instructions.at(index)); write.has_value()) { + state = ApplyRoundingWrite(state, *write); + } + } + return state; +} + +bool setsDirectedRounding(std::uint32_t value, std::uint32_t initial, std::uint32_t fields) { + for (const std::uint32_t group : {ModeRound32Bits, ModeRound1664Bits}) { + const std::uint32_t used = fields & group; + if (used != 0u && (value & group) != (initial & group) && (value & group) != 0u) return true; + } + return false; +} + +} + +std::optional DecodeRoundingWrite(const RdnaInstruction& instruction) { + switch (instruction.op) { + case RdnaOpcode::SRoundMode: + return RoundingWrite{ModeRoundBits, instruction.source0.value & ModeRoundBits, true}; + case RdnaOpcode::SSetregImm32B32: { + const auto field = roundingField(instruction.source1.value); + if (!field.has_value()) return std::nullopt; + const std::uint32_t written = instruction.source0.value & ((1u << field->size) - 1u); + return RoundingWrite{field->mask, written << field->offset, true}; + } + case RdnaOpcode::SSetregB32: { + const auto field = roundingField(instruction.source1.value); + if (!field.has_value()) return std::nullopt; + return RoundingWrite{field->mask, 0u, false}; + } + default: + return std::nullopt; + } +} + +RoundingState ApplyRoundingWrite(RoundingState state, const RoundingWrite& write) { + const auto mask = static_cast(write.mask & ModeRoundBits); + if (write.known) { + state.value = static_cast((state.value & ~mask) | (write.value & mask)); + state.unknown = static_cast(state.unknown & ~mask); + } else { + state.value = static_cast(state.value & ~mask); + state.unknown = static_cast(state.unknown | mask); + } + return state; +} + +RoundingState JoinRoundingStates(RoundingState lhs, RoundingState rhs) { + RoundingState joined; + joined.unknown = static_cast(lhs.unknown | rhs.unknown | (lhs.value ^ rhs.value)); + joined.value = static_cast(lhs.value & ~joined.unknown & ModeRoundBits); + return joined; +} + +RoundingUse RoundingUseOf(RdnaOpcode opcode) { + constexpr std::uint32_t round32 = ModeRound32Bits; + constexpr std::uint32_t round1664 = ModeRound1664Bits; + switch (opcode) { + case RdnaOpcode::VCvtF32I32: + case RdnaOpcode::VCvtF32U32: + case RdnaOpcode::VLdexpF32: + return {round32, RoundingUseKind::Directed32}; + case RdnaOpcode::VCvtF16F32: + case RdnaOpcode::VCvtF16U16: + case RdnaOpcode::VCvtF16I16: + case RdnaOpcode::VLdexpF16: + return {round1664, RoundingUseKind::Directed16}; + case RdnaOpcode::VMadMixloF16: + case RdnaOpcode::VMadMixhiF16: + case RdnaOpcode::VInterpP1llF16: + case RdnaOpcode::VInterpP1lvF16: + case RdnaOpcode::VInterpP2F16: + return {ModeRoundBits, RoundingUseKind::SelfChecked}; + case RdnaOpcode::VPkAddF16: + case RdnaOpcode::VPkMulF16: + case RdnaOpcode::VPkFmaF16: + case RdnaOpcode::VPkFmacF16: + return {round1664, RoundingUseKind::SelfChecked}; + case RdnaOpcode::VAddF32: + case RdnaOpcode::VSubF32: + case RdnaOpcode::VSubrevF32: + case RdnaOpcode::VMulF32: + case RdnaOpcode::VMulLegacyF32: + case RdnaOpcode::VMadF32: + case RdnaOpcode::VMacF32: + case RdnaOpcode::VMadLegacyF32: + case RdnaOpcode::VMacLegacyF32: + case RdnaOpcode::VMullitF32: + case RdnaOpcode::VFmaF32: + case RdnaOpcode::VMadmkF32: + case RdnaOpcode::VMadakF32: + case RdnaOpcode::VFractF32: + case RdnaOpcode::VCvtPkU8F32: + case RdnaOpcode::VCvtF32F64: + case RdnaOpcode::VDivScaleF32: + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFixupF32: + case RdnaOpcode::VCubemaF32: + case RdnaOpcode::VDot2F32F16: + case RdnaOpcode::VDot2cF32F16: + case RdnaOpcode::VInterpP1F32: + case RdnaOpcode::VInterpP2F32: + return {round32, RoundingUseKind::Unmodelled}; + case RdnaOpcode::VAddF16: + case RdnaOpcode::VSubF16: + case RdnaOpcode::VSubrevF16: + case RdnaOpcode::VMulF16: + case RdnaOpcode::VFmacF16: + case RdnaOpcode::VFmamkF16: + case RdnaOpcode::VFmaakF16: + case RdnaOpcode::VFmaF16: + case RdnaOpcode::VDivFixupF16: + case RdnaOpcode::VFractF16: + case RdnaOpcode::VFmaF64: + case RdnaOpcode::VDivFmasF64: + case RdnaOpcode::VAddF64: + case RdnaOpcode::VMulF64: + case RdnaOpcode::VLdexpF64: + case RdnaOpcode::VFractF64: + return {round1664, RoundingUseKind::Unmodelled}; + default: + return {}; + } +} + +std::unordered_map ComputeEntryRounding(const ControlFlowGraph& cfg, const RdnaProgram& program, std::uint32_t initial) { + std::unordered_map entries; + if (cfg.blocks.empty()) return entries; + std::unordered_map blocks; + for (const auto& block : cfg.blocks) blocks.emplace(block.id, &block); + + RoundingState start; + start.value = static_cast(initial & ModeRoundBits); + entries.emplace(cfg.blocks.front().id, start); + std::deque pending{cfg.blocks.front().id}; + while (!pending.empty()) { + const std::uint32_t id = pending.front(); + pending.pop_front(); + const BasicBlock& block = *blocks.at(id); + const RoundingState exit = transfer(entries.at(id), block, program); + for (const std::uint32_t successor : block.successors) { + const auto target = blocks.find(successor); + if (target == blocks.end()) throw std::runtime_error("control flow graph block " + std::to_string(id) + " has unknown successor " + std::to_string(successor)); + const auto existing = entries.find(successor); + if (existing == entries.end()) { + entries.emplace(successor, exit); + pending.push_back(successor); + continue; + } + const RoundingState joined = JoinRoundingStates(existing->second, exit); + if (joined == existing->second) continue; + existing->second = joined; + pending.push_back(successor); + } + } + for (const auto& block : cfg.blocks) entries.emplace(block.id, start); + return entries; +} + +void TranslationContext::setRounding(const RoundingState& state) { + rounding = state; + if (!declaredFloatMode.has_value()) return; + ShaderFloatMode mode = *declaredFloatMode; + mode.floatMode = (mode.floatMode & ~ModeRoundBits) | (state.value & ModeRoundBits); + floatMode = mode; +} + +void TranslationContext::writeRounding(const RdnaInstruction& inst) { + const auto write = DecodeRoundingWrite(inst); + const char* name = inst.op == RdnaOpcode::SRoundMode ? "s_round_mode" : inst.op == RdnaOpcode::SSetregB32 ? "s_setreg_b32" : "s_setreg_imm32_b32"; + if (!write.has_value()) { + throw std::runtime_error(std::string(name) + " at pc " + std::to_string(inst.programCounter) + " writes a hardware register field other than the MODE rounding fields"); + } + const RoundingState next = ApplyRoundingWrite(rounding, *write); + if (!declaredFloatMode.has_value() && (next.value != 0u || next.unknown != 0u)) { + throw std::runtime_error(std::string(name) + " at pc " + std::to_string(inst.programCounter) + " changes the rounding fields of a shader without a declared FLOAT_MODE: not implemented"); + } + setRounding(next); +} + +void TranslationContext::checkRoundingUse(const RdnaInstruction& inst) const { + const RoundingUse use = RoundingUseOf(inst.op); + if (use.fields == 0u) return; + const std::string where = "instruction (opcode " + std::to_string(inst.opcodeId) + ") at pc " + std::to_string(inst.programCounter); + if ((rounding.unknown & use.fields) != 0u) { + throw std::runtime_error(where + " reads MODE rounding bits that differ between the paths reaching it or were written from an SGPR: not implemented"); + } + const std::uint32_t initial = declaredFloatMode.has_value() ? declaredFloatMode->floatMode : 0u; + const std::uint32_t changed = (rounding.value ^ initial) & use.fields; + if (changed == 0u) return; + switch (use.kind) { + case RoundingUseKind::Directed32: + case RoundingUseKind::SelfChecked: + return; + case RoundingUseKind::Directed16: + if (((declaredFloatMode.has_value() ? declaredFloatMode->floatMode : 0xc0u) & 0xc0u) != 0xc0u) { + throw std::runtime_error(where + " runs with an f16 rounding mode set at run time and f16 denormals not kept: not implemented"); + } + return; + case RoundingUseKind::Unmodelled: + if (!setsDirectedRounding(rounding.value, initial, use.fields)) return; + throw std::runtime_error(where + " runs with MODE rounding " + std::to_string(rounding.value) + " set at run time, and the instruction does not model directed rounding"); + } +} + +} diff --git a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp index 5762f88ccf4f..c1b8ccf0b6bc 100644 --- a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp @@ -1,4 +1,5 @@ #include "Translation/ScalarInstructions.hpp" +#include "Translation/RoundingMode.hpp" #include "Translation/TranslationContext.hpp" #include #include @@ -87,7 +88,7 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { return true; case RdnaOpcode::SSetregB32: if ((inst.source1.value & 0x3fu) == 1u) { - throw std::runtime_error("s_setreg_b32 at pc " + std::to_string(inst.programCounter) + " writes MODE from an SGPR: runtime MODE changes are not implemented"); + writeRounding(inst); } emitControlNop(); return true; @@ -95,17 +96,7 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { emitControlNop(); return true; case RdnaOpcode::SSetregImm32B32: { - const std::uint32_t field = inst.source1.value; - const std::uint32_t offset = (field >> 6u) & 0x1fu; - const std::uint32_t size = ((field >> 11u) & 0x1fu) + 1u; - const std::uint64_t written = static_cast(inst.source0.value) & ((std::uint64_t{1} << size) - 1u); - if ((field & 0x3fu) != 1u || offset + size > 4u) { - throw std::runtime_error("s_setreg_imm32_b32 at pc " + std::to_string(inst.programCounter) + " writes a hardware register field other than the MODE rounding fields"); - } - const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; - if (written != ((initial >> offset) & ((1u << size) - 1u))) { - throw std::runtime_error("s_setreg_imm32_b32 at pc " + std::to_string(inst.programCounter) + " changes the initial MODE rounding fields: runtime MODE changes are not implemented"); - } + writeRounding(inst); emitControlNop(); return true; } @@ -116,8 +107,11 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { if ((field & 0x3fu) != 1u || offset + size > 4u) { throw std::runtime_error("s_getreg_b32 at pc " + std::to_string(inst.programCounter) + " reads hardware register " + std::to_string(field & 0x3fu) + " bits " + std::to_string(offset) + ".." + std::to_string(offset + size - 1u) + ": only the MODE round mode fields are modeled"); } - const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; - writeRawU32(inst.destination, IrU32(ir.Constant((initial >> offset) & ((1u << size) - 1u)))); + const std::uint32_t mask = ((1u << size) - 1u) << offset; + if ((rounding.unknown & mask) != 0u) { + throw std::runtime_error("s_getreg_b32 at pc " + std::to_string(inst.programCounter) + " reads MODE rounding bits that differ between the paths reaching it or were written from an SGPR: not implemented"); + } + writeRawU32(inst.destination, IrU32(ir.Constant((rounding.value >> offset) & (mask >> offset)))); return true; } case RdnaOpcode::SCmovkI32: { @@ -453,14 +447,10 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { case RdnaOpcode::SDecperflevel: emitControlNop(); return true; - case RdnaOpcode::SRoundMode: { - const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; - if ((inst.source0.value & 0xfu) != (initial & 0xfu)) { - throw std::runtime_error("s_round_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + " changes the initial MODE rounding fields: runtime MODE changes are not implemented"); - } + case RdnaOpcode::SRoundMode: + writeRounding(inst); emitControlNop(); return true; - } case RdnaOpcode::SDenormMode: throw std::runtime_error("s_denorm_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + ": the recompiler does not model denormal modes"); case RdnaOpcode::SSethalt: diff --git a/core/shader/recompiler/tests/test_shader_cache_native_paths.py b/core/shader/recompiler/tests/test_shader_cache_native_paths.py new file mode 100644 index 000000000000..b2f1e8c57f36 --- /dev/null +++ b/core/shader/recompiler/tests/test_shader_cache_native_paths.py @@ -0,0 +1,78 @@ +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(sys.argv.pop(1)).resolve() +CMAKE = sys.argv.pop(1) + + +class ShaderCachePathTests(unittest.TestCase): + def fixture(self, root): + source = root / 'recompiler' / 'input.cpp' + source.parent.mkdir(parents=True) + source.write_text('const char* mode = "APS5_TEST_PATH";\n', encoding='utf-8') + listing = root / 'sources.txt' + listing.write_text(f'sources {source.as_posix()}\n', encoding='utf-8') + output = root / 'version.hpp' + return source, listing, output + + def run_script(self, root, listing, output): + return subprocess.run([CMAKE, f'-DROOT={root.as_posix()}', + f'-DRECOMPILER_DIR={(root / "recompiler").as_posix()}', + f'-DLIST_FILE={listing.as_posix()}', f'-DOUTPUT={output.as_posix()}', + '-P', str(SCRIPT)], capture_output=True, encoding='utf-8', errors='replace', timeout=10) + + def check_source_change(self, name): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / name + source, listing, output = self.fixture(root) + result = self.run_script(root, listing, output) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + previous = output.read_bytes() + self.assertIn(b'APS5_TEST_PATH', previous) + source.write_text('const char* mode = "APS5_TEST_CHANGED";\n', encoding='utf-8') + result = self.run_script(root, listing, output) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertNotEqual(previous, output.read_bytes()) + self.assertIn(b'APS5_TEST_CHANGED', output.read_bytes()) + unchanged = output.stat().st_mtime_ns + result = self.run_script(root, listing, output) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(output.stat().st_mtime_ns, unchanged) + + def test_ascii(self): + self.check_source_change('plain path') + + def test_cyrillic(self): + self.check_source_change('\u0441\u0431\u043e\u0440\u043a\u0430 \u0438\u0433\u0440\u044b') + + def test_cjk(self): + self.check_source_change('\u6e2c\u8a66 path') + + def test_unicode_guard_rejects_unhashed_include(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / '\u0441\u0431\u043e\u0440\u043a\u0430' + source, listing, output = self.fixture(root) + (root / 'shared.hpp').write_text('int example();\n', encoding='utf-8') + source.write_text('#include "../shared.hpp"\n', encoding='utf-8') + result = self.run_script(root, listing, output) + self.assertNotEqual(result.returncode, 0) + self.assertIn('does not hash', result.stdout + result.stderr) + self.assertFalse(output.exists()) + + def test_unicode_header_rejects_unhashed_include(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) / 'plain path' + source, listing, output = self.fixture(root) + name = '\u0437\u0430\u0433\u043e\u043b\u043e\u0432\u043e\u043a.hpp' + (root / name).write_text('int example();\n', encoding='utf-8') + source.write_text(f'#include "../{name}"\n', encoding='utf-8') + result = self.run_script(root, listing, output) + self.assertNotEqual(result.returncode, 0) + self.assertIn('does not hash', result.stdout + result.stderr) + self.assertFalse(output.exists()) + + +unittest.main() diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 17d3dbd247c7..259ecc59eb2f 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -1,5 +1,6 @@ # Project technical debt +- [Unlinked calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) - the `s_swappc_b64` a table call's compare chain falls back to, when its target matches no linked copy, records the `UnlinkedCall` fault with the target and the program counter and stops the invocation, or continues after the call in programs whose faults do not stop invocations (see `bdaStopsInvocations`). The driver lists those calls in the request (`linkedCallFallbacks`), and it reports the fault like a BDA fault, naming the target: a dispatch that checks its faults when it completes throws, one whose check runs in a deferred write-back logs it and continues. The hardware would jump to the target, which is not modelled. Any other `s_swappc_b64` whose target is not resolved still throws when the shader is compiled. ### Build - Building on Windows requires a specific version of mingw - MinGW-w64 GCC 15.2.0 (`winlibs-gcc15`, `x86_64-ucrt-posix-seh`) @@ -547,13 +548,13 @@ Throughout the project, every function at every stage either **does exactly what - [8_SRGB and 8_8_SRGB textures](../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. - [Samples and gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. -- [Shader MODE register access](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. +- [Shader MODE register access](../../core/shader/recompiler/Translation/src/RoundingMode.cpp) starts from the rounding fields of `GuestContext::floatMode` and follows immediate rounding writes (`s_round_mode`, `s_setreg_imm32_b32` on MODE bits 0-3) through the control flow graph; `s_getreg_b32` of those bits returns the tracked value. Only `v_cvt_f32_i32`, `v_cvt_f32_u32`, `v_ldexp_f32`, the f16 conversions and `v_ldexp_f16` (with f16 denormals kept) and the instructions that already read `FLOAT_MODE` follow a rounding mode set at run time; any other float instruction that runs while a directed rounding mode differs from the initial one throws at that instruction. Rounding bits written from an SGPR (`s_setreg_b32`) or that differ between the paths reaching an instruction throw where they are read or used. A shader without `FLOAT_MODE` metadata throws on any rounding change. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. - A [flat SRT read](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. - Repeated `s_load`s of one [flat SRT](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. - A value read through a null or unmapped pointer still throws at record time if a [branch condition, uniform fill or table base](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) needs it, or if the pointer comes from user data and is not null, even if the console skips the read. - A [V# or flat SRT word](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp) read through a null pointer taken from user data is zero and counts as a [poisoned SRT read](../../core/shader/recompiler/Recompiler.cpp), so a dispatch that reads it is not cached. The console faults if the program executes the read; PPSA28416 branches around it with `s_cbranch_execz`. The fault is not modeled. - A [V#, T# or S#](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) loaded through a null or unmapped pointer read from memory is zero when the work is recorded, and the capture throws if no flat SRT slot records that read. -- [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; the PS5's gfx1013 is not measured. A plain `image_sample` (implicit LOD) through such a pair takes its LOD from the derivatives of the first coordinate alone, measured on RDNA2 (gfx1036) with the [hardware oracle](../HW_ORACLE.md) on the quads of a compute kernel (wave32 and wave64, IEEE off, DX10 clamp on, denormals preserved, round to nearest even): on a 16-texel, 5-mip 32_UINT T# with point filters it picks the same mip and texel as DIM 1D for first-coordinate steps of 0, 1, 2, 4 and 8 texels per lane along x or along y, whatever the second coordinate does (normalised steps up to 16 per lane), while the same rows on a 16x16 2D T# move the mip with the second coordinate. Outside the pixel stage a plain `image_sample` is emitted with LOD 0 for every image type, so this pair reads mip 0 there, where the kernel measurement shows the quad's derivatives being used. `image_sample_c_lz` and plain `image_sample_c` through such a pair compare against the texel the DIM 1D comparison with the same reference and first coordinate reads, measured on RDNA1 (gfx1010, RX 5700 XT) with the [hardware oracle](../HW_ORACLE.md) template run through the Windows driver's comgr and HIP runtime (wave32 and wave64, IEEE off, DX10 clamp on, denormals preserved, round to nearest even): 8-texel 32_FLOAT and 16_UNORM T#s, also with HEIGHT 3, point and bilinear filters, wrap, clamp and opaque-white border addressing, less and gequal functions, second coordinates 0, 0.5, 1, 1.5, -0.25 and 3; on a 64-texel, 7-mip 32_FLOAT T# `image_sample_c` picks its LOD from the first coordinate's steps alone (1, 4 and 16 texels per lane give mips 0, 2 and 4; second-coordinate steps up to 64 texels leave it unchanged). Only comparisons the runtime does natively (32_FLOAT, 16_UNORM) reach the 1D view; comparisons the runtime emulates for other color formats still throw through such a pair. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips, explicit LOD, bias, offsets, derivatives) still throw. +- [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; `image_sample_l` was measured the same way on gfx1036 against `image_sample_l` with DIM 1D through the same T#, for u inside, on and past the edges, v of 0, 0.5, 1, -0.25 and 3, LOD -1 to 6 on a 5-level R32_FLOAT texture, point, linear and linear-mip filtering, wrap, clamp and border addressing and HEIGHT or DEPTH 3: all 4725 rows matched. The PS5's gfx1013 is not measured. A plain `image_sample` (implicit LOD) through such a pair takes its LOD from the derivatives of the first coordinate alone, measured on RDNA2 (gfx1036) with the [hardware oracle](../HW_ORACLE.md) on the quads of a compute kernel (wave32 and wave64, IEEE off, DX10 clamp on, denormals preserved, round to nearest even): on a 16-texel, 5-mip 32_UINT T# with point filters it picks the same mip and texel as DIM 1D for first-coordinate steps of 0, 1, 2, 4 and 8 texels per lane along x or along y, whatever the second coordinate does (normalised steps up to 16 per lane), while the same rows on a 16x16 2D T# move the mip with the second coordinate. Outside the pixel stage a plain `image_sample` is emitted with LOD 0 for every image type, so this pair reads mip 0 there, where the kernel measurement shows the quad's derivatives being used. `image_sample_c_lz` and plain `image_sample_c` through such a pair compare against the texel the DIM 1D comparison with the same reference and first coordinate reads, measured on RDNA1 (gfx1010, RX 5700 XT) with the [hardware oracle](../HW_ORACLE.md) template run through the Windows driver's comgr and HIP runtime (wave32 and wave64, IEEE off, DX10 clamp on, denormals preserved, round to nearest even): 8-texel 32_FLOAT and 16_UNORM T#s, also with HEIGHT 3, point and bilinear filters, wrap, clamp and opaque-white border addressing, less and gequal functions, second coordinates 0, 0.5, 1, 1.5, -0.25 and 3; on a 64-texel, 7-mip 32_FLOAT T# `image_sample_c` picks its LOD from the first coordinate's steps alone (1, 4 and 16 texels per lane give mips 0, 2 and 4; second-coordinate steps up to 64 texels leave it unchanged). Only comparisons the runtime does natively (32_FLOAT, 16_UNORM) reach the 1D view; comparisons the runtime emulates for other color formats still throw through such a pair. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips other than explicit LOD, explicit LOD with comparison, bias, offsets, derivatives) still throw. - [3D image accesses bound to a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) follow a [hardware oracle](../HW_ORACLE.md) measurement on RDNA1 (RX 5700 XT, gfx1010; wave32 and wave64, IEEE off, DX10 clamp on, denormals preserved, round to nearest even), not the PS5's gfx1013, on 8x1 R8G8B8A8_UNORM linear 1D T#s with HEIGHT and DEPTH 0 or 3 and point filters. `image_sample_lz` reads the texel u selects for v and r in {0, 0.5, 1, 1.5, -0.25, 3}, with wrap, clamp and border addressing, and v and r never produce the border; u wraps, clamps and borders as with DIM 1D. Mip-less `image_load` reads the x texel for any y in {0, 1, 2, 3, -1} when z is 0, returns zero for z in {1, 2, 3, -1}, whatever DEPTH says, and returns zero for x past the width. A plain `image_sample` (implicit LOD) on the quads of a compute kernel takes its texel and LOD from u alone: on a 64-texel, 7-mip 1D T# steps of 1, 4 and 16 texels per lane pick mips 0, 2 and 4, as DIM 1D `image_sample_l` does, and steps in v or r alone stay at mip 0. Outside the pixel stage a plain `image_sample` is emitted with LOD 0, as for 2D accesses through a 1D T#. Every other 3D access of such an image (`image_load` with mips, storage, explicit LOD, bias, derivatives, offsets, comparison, gathers, queries) keeps throwing. SONIC SUPERSTARS (PPSA06888) samples a 1x1 1D T# this way with plain `image_sample` and `image_load`. - [Emulated comparison of a 2D-array access through a 2D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) samples the 2D image and ignores the slice, as the native modes of a 2D-array access through a 2D T# already do. This follows the T# type deciding the addressing; it is not measured on hardware for comparison samples - [Flat aperture bases](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (core/shader/recompiler) - the layout follows the gfx9+ one in the AMD KFD driver's `kfd_flat_memory.c` (`MAKE_LDS_APP_BASE_V9`, `MAKE_LDS_APP_LIMIT`): each aperture is a 4 GB window whose base is the high dword, so a flat address is routed by its whole high dword, as measured on RDNA2 (gfx1035: `src_shared_base` 0x00010000_00000000, `src_shared_limit` 0x00010000_ffffffff). The PS5's base values, high dwords 0x80000000 (shared, `SharedApertureHigh`) and 0x70000000 (private, `PrivateApertureHigh`), are inferred from GTA V's traversal addresses, not measured on a console or read from SH_MEM_BASES. The scalar aperture sources read the same two values @@ -680,6 +681,7 @@ Throughout the project, every function at every stage either **does exactly what - [Raw `buffer_load_dwordx2`](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`. - [Buffer loads without GLC or DLC](../../core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp) (shader recompiler) are not coherent in a program that has GLC or DLC buffer accesses and no `buffer_gl0_inv`, `buffer_gl1_inv`, `s_gl1_inv` or `s_dcache_inv`: only its GLC and DLC loads, stores and atomics use the `Coherent` buffer view, so a plain load can return a value cached before another work-group's write. This follows the [LLVM AMDGPU memory model for GFX10](https://llvm.org/docs/AMDGPUUsage.html#memory-model-gfx10-gfx11), in which a load-acquire is a GLC DLC load followed by `buffer_gl1_inv` and `buffer_gl0_inv`; it is not measured on the console or on RDNA hardware. - [DS src2 operations](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (`ds_*_src2_*`, GFX10.1 only) read `DS[B]`, with `B = A + 4 * sext(offset1[6:0]:offset0)`, and then update `DS[A]` with the matching atomic, so the read of `B` is not atomic with the update of `A`. The 64-bit forms take the same dword offset; a `B` that is not 8-byte aligned reads as 0 and an `A` that is not 8-byte aligned writes nothing, as measured with the alignment mode the Windows driver sets. The f32 and f64 min and max forms follow the [LDS float atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) entry with `DS[B]` as the data operand. Measured on RDNA1 (gfx1010, RX 5700 XT) with the [hardware oracle](HW_ORACLE.md) template run through the Windows driver's comgr and HIP runtime: 78 single-lane cases of the 32-bit integer forms, 624 cases of the 64-bit integer forms with random, equal and adjacent operands, and the f32 and f64 min and max forms against plain `ds_min/max_f32/f64` over 14 x 14 operand sets in IEEE 0 and 1 with denormals flushed and kept, which agreed on every row and follow that entry's rule. The active lanes of a wave run one after another in lane order and a lane's read of `B` sees the writes of lower lanes, as measured on gfx1010 with chains of eight lanes whose `B` is the previous lane's `A` (32-bit add and write, 64-bit add, the same `A` in four lanes, and in wave64 a chain across lanes 31 and 32); the translation runs the active lanes of each host subgroup one at a time with `OpGroupNonUniformElect` and a workgroup memory barrier between lanes, and a wave64 program on a 32-lane host, whose invocations each hold lanes n and n + 32 and run the two halves one after the other, runs lanes 0-31 before 32-63; a wave64 program kept at one lane per invocation on a host whose subgroups are narrower than 64 (`APS5_SINGLE_LANE` on such a host) orders lanes only within each host subgroup, as every other subgroup operation there. `ds_add_src2_f32` throws: on gfx1010 it equals `ds_add_f32`, which flushes both operands to a signed zero when the f32 denormal field flushes, and that is not modelled. The form that takes the offset from the address (`offset1[7]` set) and GDS throw: on gfx1010 that form writes nothing when `A[16]` or `A[17]` is set and otherwise updates `A[15:0]` with a `B` that no documented formula gives. +- [`ds_ordered_count`](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (GDS only) reads its operand fields as LLVM encodes them for `llvm.amdgcn.ds.ordered.add` ([SIISelLowering.cpp](https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/AMDGPU/SIISelLowering.cpp#L8844-L8894)): `offset0` is the counter's byte offset (index times 4) and the whole GDS address, `offset1` bit 0 wave release, bit 1 wave done, bits 2-3 the shader type, bit 4 swap instead of add and bits 6-7 the dword count less one. `M0` holds the ordered wave ID, not a GDS address as for `ds_append` (radeonsi passes `TG_SIZE` bits 6-17 as the intrinsic's pointer in [si_compute_prim_discard.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/mesa-20.3.0/src/gallium/drivers/radeonsi/si_compute_prim_discard.c) and `gs_tg_info` bits 0-11 in [gfx10_shader_ngg.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/mesa-21.3.0/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c), with "The GDS address is always 0 with ordered append"); it only orders the waves, which is not modeled, so it is ignored. Each wave adds the value of its first active lane to the counter with one device-scope atomic and every active lane gets the counter's previous value (the `ORDERED_APPEND_ENBL` dispatch modifier is accepted for it). The hardware also serializes the waves of a dispatch in launch order, which is not modeled: the waves add in any order, so counters a title reads back as ordered offsets give the same totals but may hand out ranges in another order. Which lane's value is added is not documented; the first active lane is assumed and not measured on hardware. Swap, counts of more than one dword and LDS forms throw. Covered by [GdsAppendConsume.cpp](../../core/libs/prx/libSceAgcDriver/tests/execution/GdsAppendConsume.cpp). - [LDS in pixel, vertex-path and fetch programs](../../core/shader/recompiler/SpirvBackend/src/SpirvAnalysis.cpp) (shader recompiler) - is an array private to each host invocation, in `Function` storage, as only compute and mesh programs get workgroup memory; on the console one LDS region is shared by the wave (RDNA 2 ISA sections 3.6.6 and 10.4), so a lane does not see another lane's LDS writes. Sub-dword writes (`ds_write_b8`, `ds_write_b16` and their `_d16_hi` forms), the 32-bit LDS atomics, `ds_append` and `ds_consume` load, update and store the invocation's own dword without an atomic, as Vulkan allows atomics only on `Uniform`, `Workgroup`, `Image`, `StorageBuffer`, `PhysicalStorageBuffer` and `TaskPayloadWorkgroupEXT` pointers (VUID-StandaloneSpirv-None-04686). `ds_append` and `ds_consume` update only the first active lane's copy and return its old value to every lane, so a second append after the first active lane changes, or a read of the counter by another lane, differs from the hardware. Not measured on hardware. - [VALU lane masks written to an SGPR or VCC](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (shader recompiler) - the carry and borrow masks of `v_add_co_u32`, `v_sub_co_u32`, `v_subrev_co_u32` and their `_ci` forms, the carry of `v_mad_u64_u32` and `v_mad_i64_i32`, and the flag of `v_div_scale_f32` and `v_div_scale_f64` have 0 in the bits of lanes that EXEC disables, as `v_cmp` results do. The [RDNA 4 ISA guide](https://gpuopen.com/download/rdna4-instruction-set-architecture.pdf) states this for carry-outs (section 7.2.3, page 72). The RDNA 2 guide does not say it in words, and no guide says it for `v_mad_*64` or `v_div_scale`. Not measured on gfx1013 or another RDNA 2 GPU, in particular whether a disabled lane's bit that was 1 before the instruction is cleared or kept; [ValuMaskInactiveLanes.cpp](../../core/libs/prx/libSceAgcDriver/tests/execution/ValuMaskInactiveLanes.cpp) expects it cleared - [32_AR color exports](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - with `SPI_SHADER_COL_FORMAT` 3 the red value is export channel 0 and the alpha is export channel 1, the GFX10 layout of Mesa's [`emit_ps_color_export`](https://gitlab.freedesktop.org/mesa/mesa/-/blob/3e32b0e1f132231b7a972c1b0b5b06c7e16cf8b9/src/amd/common/nir/ac_nir_lower_ps_late.c#L400-L411), ACO and radeonsi in Mesa 22.0 ([aco_instruction_selection.cpp](https://gitlab.freedesktop.org/mesa/mesa/-/blob/a82920af6efa4a7a66919885c6477fbd8fdc7b9d/src/amd/compiler/aco_instruction_selection.cpp#L11087-L11096), [si_shader_llvm_ps.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/a82920af6efa4a7a66919885c6477fbd8fdc7b9d/src/gallium/drivers/radeonsi/si_shader_llvm_ps.c#L323-L333)) and AMD's [LLPC](https://github.com/GPUOpen-Drivers/llpc/blob/40cb8d95ad8d6f7f1652e3fd47d39667594cce08/lgc/lowering/FragmentColorExport.cpp#L157-L168); before GFX10 the alpha was channel 3. Channels 2 and 3, which none of these compilers enables, are ignored; green and blue are 0 and the alpha of an export without channel 1 is 1.0, the defaults of the other formats, which no source gives. Not measured on hardware, neither on gfx1013 nor on a PC RDNA 2 GPU; pinned in [Pixel32ArExport.cpp](../../core/libs/prx/libSceAgcDriver/tests/execution/Pixel32ArExport.cpp) @@ -700,7 +702,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. - [sceFontGetCharGlyphMetrics, sceFontGetRenderCharGlyphMetrics](../../core/libs/prx/libSceFont/src/Layout.cpp) (libSceFont) report the vertical metrics the way the [FreeType driver](../../core/libs/prx/libSceFontFt/src/Driver.cpp) reports them for a rendered glyph, not confirmed on a title: FreeType's vertical advance, and for a face without vertical metrics a bearing of half the horizontal advance left of the horizontal bearing and of half the advance left over by the glyph height. - [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics`, `sceFontWritingLineGetOrderingSpace` and `sceFontWritingLineClear` follow the PS4 11.00 `libSceFont.sprx` (firmware module, read from its disassembly and compared call for call on random sequences): runs are linked in visual order by embedding level, the space at each spaced join is summed for the line, steps walk the visual order from the left (form `0x12`) or from the right (form `0x11`) and the render metrics grow as steps are consumed. The PS5 library was not measured. Not from that module: form `0x10` is accepted and treated as left to right and a detail with id `0x0FD5` is accepted (the PS4 library rejects both); the memory argument of `sceFontCreateWritingLine` and the rest of the detail are not read (the PS4 library validates its library object there); an unknown writing form, another detail id, a null line pointer on creation, unknown and null handles (except in `sceFontWritingLineRefersRenderStep`, which returns null for a null line) and a null output of `sceFontWritingLineGetRenderMetrics` throw instead of returning the PS4 error codes, which are not confirmed for the PS5; `sceFontDestroyWritingLine` was not compared. Lines are allocated on the host, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. -- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store) throws, apart from the overwrites and shared links that [static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. +- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, hash, store) throws, apart from the overwrites, SOPC compares and shared links that [static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) allow; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. - The scalar aperture sources [src_shared_base, src_shared_limit, src_private_base, src_private_limit](../../core/shader/recompiler/RdnaDecoder/src/RdnaInstructionDecoder.cpp) (core/shader/recompiler, scalar source codes 235 to 238) read as the 64-bit apertures the flat routing uses: a base is the aperture's high dword (`SharedApertureHigh`, `PrivateApertureHigh`) over a low dword of 0, a limit is the same high dword over 0xffffffff. A 64-bit read (`s_mov_b64`, a 64-bit VALU source, a wave64 lane mask) gets the whole value and a 32-bit read gets the low dword. This follows LLVM's `SIRegisterInfo.td` (aperture registers are 64-bit, their low 32 bits are always 0 for a base and -1 for a limit) and `getSegmentAperture` (`AMDGPULegalizerInfo.cpp`, `SIISelLowering.cpp`: a 64-bit read whose high half is the aperture), the KFD's `MAKE_LDS_APP_LIMIT` (limit = base | 0xffffffff) and the gfx1030 and gfx1035 readouts on [#1190](https://github.com/boykopovar/AnyPS5/pull/1190) and [#3803](https://github.com/boykopovar/AnyPS5/issues/3803). Not measured on a console: the PS5's high dwords are the inferred values of the flat aperture bases entry, and the wave is taken to be in 64-bit address mode (PTR32 clear) - [VertexBufferReadSize](../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) and the [vertex staging copy](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - a vertex fetch whose index is at or past the descriptor's `num_records` no longer throws for `OOB_SELECT` 0 and 1. The copy still spans the whole fetch, but the bytes from `num_records * stride` onward read as zeros, so the out-of-range vertex is `(0, 0, 0, 0)`. Measured with `buffer_load_format_xyzw` (`idxen`) through the [hardware oracle](HW_ORACLE.md) on RDNA2 gfx1036 for PPSA01325's word 3 `0x0004dfac` (`OOB_SELECT` 0) in #2241: indices 3, 4 and 5 over 3 records each return `00000000 00000000 00000000 00000000`, W included, unlike Vulkan `robustBufferAccess`, which returns `(0, 0, 0, 1)`. The zeros come from the record check, not from memory: stored values placed past the records are ignored, and a `num_records` 6 control returns them. `RESOURCE_LEVEL` 1, `OOB_SELECT` 1 and FORMAT 74 `32_32_32_FLOAT` give the same zeros, in wave32 and wave64; gfx1013 (the console) is not measured. `OOB_SELECT` 2 returns the stored values past the records and `OOB_SELECT` 3 compares `num_records` against the byte offset, both measured on the same setup in #2241; the two keep throwing past `num_records` here, since each needs its own rule. Zero-stride descriptors (`stride` 0) keep their separate byte-count check and are not covered here. Each out-of-range fetch is copied alone and outside the draw-snapshot cache, which is neither read nor written for it: a reused snapshot would carry another draw's bytes into the zero region, and a stored zero-padded copy would hand zeros to a later in-range draw at the same address. The valid prefix is still flushed for GPU writes before it is read. @@ -715,7 +717,7 @@ Throughout the project, every function at every stage either **does exactly what - [strftime](../../core/libs/prx/libc/src/Time.cpp) (libc) expands FreeBSD's `%k`, `%l`, `%s`, `%v` and `%+` itself, ignoring a leading `E` or `O` as FreeBSD does, and passes every other conversion to the host's `strftime`. On Windows the conversions the CRT lacks (`%P` and the `%-`, `%_` and `%0` padding flags) still produce nothing, and the padding flags are not supported on the five new conversions at all (`%-k`, `%_k` and `%0k` go to the host), an unknown conversion, a doubled `E` or `O` modifier (`%EE`) or a trailing lone `%` returns 0 where FreeBSD prints `%` and the character, and a standalone `%z` or `%Z` uses the host's zone instead of the broken-down time's `tm_zone`. `%s` follows FreeBSD 9 and 11 and is the local-time `mktime` of the broken-down time (current FreeBSD uses `tm_gmtoff`), which is -1 on Windows outside the range above. `%+` is the C locale's `%a %b %e %H:%M:%S %Z %Y` with `%Z` taken from `tm_zone`, or from `tzname[tm_isdst != 0]` when it is null. - [LINE_STIPPLE_TEX](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) reads as 0: line stipple is not emulated. RADV enables the input only as the required pixel input of a shader that reads no interpolants, because it initializes the fewest VGPRs ([Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_shader.c) `radv_compute_spi_ps_input`), and never reads it - [Draw snapshot buffer descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) rewrite each affected array from element zero, preserving unchanged descriptors. MoltenVK 1.4.2 writes auxiliary buffer lengths at index `i` instead of `start + i` for partial array updates ([MVKDescriptorSet.mm, BufferAuxSize](https://github.com/KhronosGroup/MoltenVK/blob/v1.4.2/MoltenVK/MoltenVK/GPUObjects/MVKDescriptorSet.mm#L1051)). Per-element writes therefore let one buffer length replace another even when Vulkan buffer handles and ranges are correct. -- [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair may only be overwritten whole, by `s_mov_b32`, `s_mov_b64`, an SOP2 result or a scalar load, as the caller of a user-data callee does before each call. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. +- [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require an ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and a later `s_setpc_b64` return reading that exact pair. A pair serves one call, or several calls whose regions are disjoint, each closed by the first return reading the pair after its target. Outside every region of its calls the pair is free to use wherever no return address can reach: a dataflow over the decoded instructions follows each dword of the pair from the calls that write it until a scalar load, SALU or VALU result is known to overwrite it (a VALU mask counts as one SGPR unless the request says wave64), and an instruction there that may read a dword still holding a return address throws; SOPC compares are exempt, as they only set SCC. Regions must nest or be disjoint; recursion, other shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. - [Relative vector addressing](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) (`v_movrels_b32`, `v_movreld_b32`, `v_movrelsd_b32`, `v_movrelsd_2_b32`, `v_swaprel_b32`) keeps every vector register from the lowest base register the program addresses relatively up to the last register the program names in a zero-initialised function-scope array per invocation (`LoadVectorWindow`/`StoreVectorWindow`, one element per register), so an indexed access is one load or store instead of a compare-and-select chain over the window; the host compiler decides whether that array stays in registers or spills to scratch, which is not measured. A register in the window the program never wrote reads 0, as an unwritten SSA register does; the console's value is undefined. M0 is unsigned: an index at or past the window reads `v0` and drops its write, and `v_swaprel_b32` writes neither side when either is out of range, as measured on RDNA2 (gfx1035, `agc_driver_relative_indexing`). The window ends at the last register the program names rather than at the shader's VGPR allocation, which the recompiler does not receive, so a relative write past the named registers but inside the allocation is dropped where the hardware keeps it for a later relative read. Killzone: Liberation's PSP vertex program (PPSA11142, 3176 ISA words, 104 relative accesses over 154 registers) went from 649,515 to 392,333 SPIR-V words with this. Pixel programs whose relative span contains the packed ancillary input instead keep that span in SSA and implement indexed operations with compare/select chains. This keeps ancillary definitions visible to field lowering and masked-write proofs; storing the unmodelled whole word in a private array would require composing unknown bits. A fold after masked-select elimination lowers fields whose consumers become visible only after redundant copies are removed. A whole-word use that the masked-select pass keeps gets the composed word described under the packed pixel ancillary word. Both representations include implicit entry inputs in the inferred register limit and retain unsigned M0, inactive-write and DPP update rules; wave32/wave64 native Vulkan readbacks cover relative reads, writes, split-offset swaps, inactive writes, DPP masks and out-of-range accesses, but console comparison and the performance cost of selecting over this span are unverified. - [User-data callees](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped; in the shaders seen they are metadata, and a caller that reads its program counter, which could address them, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. - [DB_RENDER_CONTROL resummarize draws](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - HTILE is not modeled, so a RESUMMARIZE_ENABLE draw without depth or stencil tests and without color writes changes no modeled state and draws nothing. A pure resummarize with pass-through D16/D32 depth, no depth writes or bounds test, no stencil plane, no color writes and no running pixel stage also draws nothing; other resummarize, copy or decompress draws throw. diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index f4d7c9a3baa5..8548f1ff5bb4 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -99,6 +99,7 @@ - [VertexBufferReadSize](../../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) and the [vertex staging copy](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - a vertex fetch whose index is at or past the descriptor's `num_records` no longer throws for `OOB_SELECT` 0 and 1. The copy still spans the whole fetch, but the bytes from `num_records * stride` onward read as zeros, so the out-of-range vertex is `(0, 0, 0, 0)`. Measured with `buffer_load_format_xyzw` (`idxen`) through the [hardware oracle](../HW_ORACLE.md) on RDNA2 gfx1036 for PPSA01325's word 3 `0x0004dfac` (`OOB_SELECT` 0) in #2241: indices 3, 4 and 5 over 3 records each return `00000000 00000000 00000000 00000000`, W included, unlike Vulkan `robustBufferAccess`, which returns `(0, 0, 0, 1)`. The zeros come from the record check, not from memory: stored values placed past the records are ignored, and a `num_records` 6 control returns them. `RESOURCE_LEVEL` 1, `OOB_SELECT` 1 and FORMAT 74 `32_32_32_FLOAT` give the same zeros, in wave32 and wave64; gfx1013 (the console) is not measured. `OOB_SELECT` 2 returns the stored values past the records and `OOB_SELECT` 3 compares `num_records` against the byte offset, both measured on the same setup in #2241; the two keep throwing past `num_records` here, since each needs its own rule. Zero-stride descriptors (`stride` 0) keep their separate byte-count check and are not covered here. Each out-of-range fetch is copied alone and outside the draw-snapshot cache, which is neither read nor written for it: a reused snapshot would carry another draw's bytes into the zero region, and a stored zero-padded copy would hand zeros to a later in-range draw at the same address. The valid prefix is still flushed for GPU writes before it is read. - [Draw snapshot buffer descriptors](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) rewrite each affected array from element zero, preserving unchanged descriptors. MoltenVK 1.4.2 writes auxiliary buffer lengths at index `i` instead of `start + i` for partial array updates ([MVKDescriptorSet.mm, BufferAuxSize](https://github.com/KhronosGroup/MoltenVK/blob/v1.4.2/MoltenVK/MoltenVK/GPUObjects/MVKDescriptorSet.mm#L1051)). Per-element writes therefore let one buffer length replace another even when Vulkan buffer handles and ranges are correct. - [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped; in the shaders seen they are metadata, and a caller that reads its program counter, which could address them, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. +- [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block, calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4` without SOFFSET from a user-data pointer and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB or a copy out of `s_call_b64` range throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. - [DB_RENDER_CONTROL resummarize draws](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - HTILE is not modeled, so a RESUMMARIZE_ENABLE draw without depth or stencil tests and without color writes changes no modeled state and draws nothing. A pure resummarize with pass-through D16/D32 depth, no depth writes or bounds test, no stencil plane, no color writes and no running pixel stage also draws nothing; other resummarize, copy or decompress draws throw. - [Comparison-sampled R32 float and R16 unorm textures](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) whose surface a storage image holds are sampled from a D32_SFLOAT or D16_UNORM image that the GPU refills from the storage image whenever its version moves, because Vulkan compares only depth formats. Only surfaces of at least 1 MiB (`APS5_DEPTH_COPY_MIN_KIB`) that are not 3D take the copy; smaller ones stay CPU snapshots compared against guest memory. With copies of small surfaces, one title hit an intermittent boot-time crash that was not explained. - [Null vertex buffer V#](../../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - a V# whose four dwords are all zero is bound to a zeroed vertex buffer of the attribute's size with stride 0, so every component of its attribute reads 0. This follows the V# fields in the [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) (`NUM_RECORDS` 0, `DST_SEL` 0 selects 0), not measured on hardware.