diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 8e3f18890b81..9bb45a7fa15e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1930,6 +1930,35 @@ std::vector planFixtures() { nestedCode.insert(nestedCode.begin() + 15u, 0x87148114u); add("nested pointer image table", 64u, std::move(nestedCode), srtPointer, {{PlanSrtAddress, nestedSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedImages}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); fixtures.back().groupId = true; + std::vector samplerSrt(20u, 0u); + std::copy(nestedSrt.begin(), nestedSrt.end(), samplerSrt.begin()); + place(samplerSrt, 12u, imageWords(PlanTextureAddress)); + const std::vector pointClamp{0x92u, 0x400000u, 0u, 0u}; + const std::vector pointWrap{0u, 0x400000u, 1u << 24u, 0u}; + std::vector nestedSamplers(32u, 0u); + place(nestedSamplers, 0u, pointWrap); + place(nestedSamplers, 4u, pointClamp); + place(nestedSamplers, 16u, pointClamp); + place(nestedSamplers, 20u, pointWrap); + std::vector samplerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + samplerCode.insert(samplerCode.begin() + 13u, 0x87118311u); + samplerCode.insert(samplerCode.begin() + 15u, 0x87148114u); + add("nested pointer sampler table", 64u, std::move(samplerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedSamplers}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); + fixtures.back().groupId = true; + std::vector pointees(128u, 0u); + place(pointees, 8u, pointWrap); + place(pointees, 72u, pointClamp); + std::vector pointerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + pointerCode.insert(pointerCode.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + add("pointer phi sampler table", 64u, std::move(pointerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, pointees}, {PlanMaterialAddress, {1u, 0u, 0u, 0u, 1u, 0u, 0u, 0u}}}); + fixtures.back().groupId = true; PlanFixture fragment; fragment.name = "fragment parameters"; @@ -2322,6 +2351,7 @@ void verifyMalformedPlans() { rejects("an image table cycle", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->heapSource = 1u; }); rejects("an image table parent that is not a buffer", [](IrResourcePlan& plan) { plan.descriptorSources[2].tableColumn = *plan.descriptorSources[1].tableColumn; }); rejects("an image table selection past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->selection = 4u; }); + rejects("a pointer-keyed table that is not indirect", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->pointerKeyed = true; }); auto foreign = handBuiltPlan(); IrValue stray(IrOpcode::Void, IrType::U32, 99u); foreign.srtReads[0].value = &stray; @@ -2381,6 +2411,12 @@ void verifyPlanRoundTrip() { const auto bufferBytes = encodedPlan(bufferTable); IrResourcePlan bufferDecoded; require(bufferBytes != handBytes && ShaderDiskCache::DecodePlan(bufferBytes, bufferDecoded) && samePlan(bufferTable, bufferDecoded), "the nested buffer column changed in a disk-cache round trip"); + auto pointerTable = handBuiltPlan(); + pointerTable.descriptorSources[1].tableColumn->indirect = true; + pointerTable.descriptorSources[1].tableColumn->pointerKeyed = true; + const auto pointerBytes = encodedPlan(pointerTable); + IrResourcePlan pointerDecoded; + require(pointerBytes != handBytes && ShaderDiskCache::DecodePlan(pointerBytes, pointerDecoded) && samePlan(pointerTable, pointerDecoded) && pointerDecoded.descriptorSources[1].tableColumn->pointerKeyed, "the pointer-keyed column changed in a disk-cache round trip"); PlanFixture handFixture; handFixture.userData = {static_cast(PlanSrtAddress), static_cast(PlanSrtAddress >> 32u), 0u, 0u}; std::vector handSrt(16, 0u); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 90c4b12d916f..8132f9d1ed65 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -3908,12 +3908,19 @@ void verifyRegionMerge(bool strided = false, bool benchmark = false) { } -void verifyNestedSnapshotRetry() { +void verifyNestedSnapshotRetry(bool sampler) { using namespace ShaderRecompiler; constexpr std::uint32_t Parents = ImageTableAbi::MaxParentKeys / 2u + 1u; constexpr std::uint32_t Children = ImageTableAbi::MaxNestedKeys / 2u + 1u; - std::vector> leaves(Children, TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u)); - const auto child = TableV(leaves.data(), 32u, Children); + const std::uint32_t leafDwords = sampler ? 4u : 8u; + const auto texture = TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u); + const std::array pointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; + std::vector leaves; + for (std::uint32_t leaf = 0; leaf < Children; ++leaf) { + if (sampler) leaves.insert(leaves.end(), pointClamp.begin(), pointClamp.end()); + else leaves.insert(leaves.end(), texture.begin(), texture.end()); + } + const auto child = TableV(leaves.data(), leafDwords * 4u, Children); std::vector parents(Parents, reinterpret_cast(child.data())); const auto root = TableV(parents.data(), 8u, Parents); std::vector> values; @@ -3939,17 +3946,25 @@ void verifyNestedSnapshotRetry() { plan.descriptorSources[1].tableColumn = parent; TableColumn nested; nested.heapSource = 1u; - nested.stride = 32u; - nested.dwordCount = 8u; + nested.stride = leafDwords * 4u; + nested.dwordCount = leafDwords; + nested.sampler = sampler; plan.descriptorSources[2].tableColumn = nested; - ImageResource image; - image.source = 2u; - image.table = 0u; - image.resourceClass = ImageResourceClass::Sampled; - image.dimension = RdnaImageDimension::Dim2D; - image.read = true; - plan.info.images.push_back(image); - plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + if (sampler) { + SamplerResource resource; + resource.source = 2u; + resource.table = 0u; + plan.info.samplers.push_back(resource); + } else { + ImageResource image; + image.source = 2u; + image.table = 0u; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2D; + image.read = true; + plan.info.images.push_back(image); + plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + } struct Memory { std::array, 3> ranges; std::uint64_t epoch = 0; @@ -5612,7 +5627,11 @@ bool TableTouched(const std::vector& regions, co void verifyScalarAddressRule() { namespace Abi = ShaderRecompiler::ImageTableAbi; - require(Abi::ScalarAddressDword(0x1000u, 0x23u, 0x41u) == 0x1060u && Abi::ScalarAddressDword(0x1001u, 0x20u, 0u) == 0x1021u, "scalar address rule: the offset and immediate are not dword-aligned on their own"); + require(Abi::ScalarAddressDword(0x1000u, 0x23u, 0x41u) == 0x1060u, "scalar address rule: the offset and immediate are not dword-aligned on their own"); + for (const std::uint64_t low : {1u, 2u, 3u}) { + require(Abi::ScalarAddressDword(0x1000u + low, 0x20u, 0u) == 0x1020u && Abi::ScalarAddressDword(0x1000u + low, 0x23u, 0x41u) == 0x1060u, ("scalar address rule: a base with low bits " + std::to_string(low) + " is not dword-aligned").c_str()); + require(Abi::ScalarAddressDword(0xfffffffffffffff0ull + low, 0x8u, 0x4u) == 0xfffffffffffffffcull && !Abi::ScalarAddressDword(0xfffffffffffffff0ull + low, 0x8u, 0x8u).has_value(), ("scalar address rule: a carry past 2^64 from a base with low bits " + std::to_string(low) + " is wrong").c_str()); + } require(Abi::ScalarAddressDword(0x1000u, 0x20u, 0xfffffff0u) == 0x1010u && !Abi::ScalarAddressDword(0x8u, 0x4u, 0xfffffff0u).has_value(), "scalar address rule: a negative immediate is wrong"); require(!Abi::ScalarAddressDword(0xfffffffffffffff0ull, 0x10u, 0u).has_value() && !Abi::ScalarAddressDword(0xfffffffffffffff0ull, 0x8u, 0x8u).has_value() && Abi::ScalarAddressDword(0xfffffffffffffff0ull, 0x8u, 0x4u) == 0xfffffffffffffffcull, "scalar address rule: a carry past 2^64 is wrong"); for (const std::uint32_t stride : {4u, 32u, 48u, 0x330u}) { @@ -5758,11 +5777,12 @@ void verifyPointerImageTable() { require(loadedTable.capture->snapshot.tables.tables.at(0).codes == snapshot.codes && loadedTable.capture->snapshot.tables.ranges == tables.ranges, "pointer image table: a base loaded from the SRT with a negative immediate snapshots other records"); require(loadedTable.result->imageTableReadRanges == compiled.result->imageTableReadRanges, "pointer image table: a negative immediate changed the consumed ranges"); require(TableCodes(TableMap(*loadedTable.result), 0) == codes, "pointer image table: a loaded base maps other records"); - root[12] = static_cast(pointer) | 2u; - const auto unaligned = CompileTable(loaded); - const auto& unalignedCodes = unaligned.capture->snapshot.tables.tables.at(0).codes; - require(unalignedCodes.size() == Entries && std::all_of(unalignedCodes.begin(), unalignedCodes.end(), [](std::uint32_t code) { return code == Abi::PoisonCode(static_cast(Abi::PoisonReason::Unmapped)); }), "pointer image table: a pointer that is not dword-aligned was read"); - require(unaligned.result->imageTableReadRanges == std::vector>>{{}}, "pointer image table: an unaligned pointer recorded consumed ranges"); + for (const std::uint32_t low : {1u, 2u, 3u}) { + root[12] = static_cast(pointer) | low; + const auto unaligned = CompileTable(loaded); + require(unaligned.capture->snapshot.tables.tables.at(0).codes == snapshot.codes && unaligned.capture->snapshot.tables.ranges == tables.ranges, "pointer image table: a pointer off a dword boundary does not read the records at its dword"); + require(unaligned.result->imageTableReadRanges == compiled.result->imageTableReadRanges, "pointer image table: a pointer off a dword boundary consumed other ranges"); + } root[12] = static_cast(pointer); const std::vector directCode{0xf4080700u, 0xfa000010u, 0xf4080800u, 0xfa000020u, 0xf4000440u, 0xfa000038u, 0x8f128511u, 0xf40c0900u, @@ -11728,10 +11748,15 @@ int main(int argc, char** argv) { return 0; } if (argc == 2 && std::string_view(argv[1]) == "--nested-snapshot-retry") { - verifyNestedSnapshotRetry(); + verifyNestedSnapshotRetry(false); std::cout << "Nested snapshot retry preserves parent and child coverage\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--nested-sampler-snapshot-retry") { + verifyNestedSnapshotRetry(true); + std::cout << "Nested sampler snapshot retry checks the descriptor epoch and preserves parent and child coverage\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake index 2e76d01ccf28..f56fea17b884 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake @@ -4,8 +4,22 @@ if(BUILD_TESTING) add_test(NAME agc_driver_nested_image_table COMMAND agc_driver_image_table_tests --nested) add_test(NAME agc_driver_nested_image_table_large COMMAND agc_driver_image_table_tests --nested-large) add_test(NAME agc_driver_nested_image_table_budget COMMAND agc_driver_image_table_tests --nested-budget) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_nested_sampler_table COMMAND agc_driver_image_table_tests --nested-sampler) + add_test(NAME agc_driver_pointer_phi_sampler_table COMMAND agc_driver_image_table_tests --pointer-phi-sampler) + add_test(NAME agc_driver_pointer_phi_ballot_table COMMAND agc_driver_image_table_tests --pointer-phi-ballot) + add_test(NAME agc_driver_pointer_phi_copies_table COMMAND agc_driver_image_table_tests --pointer-phi-copies) + add_test(NAME agc_driver_pointer_phi_lane_and_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-and) + add_test(NAME agc_driver_pointer_phi_high_table COMMAND agc_driver_image_table_tests --pointer-phi-high) + add_test(NAME agc_driver_pointer_phi_high_second_lane_table COMMAND agc_driver_image_table_tests --pointer-phi-high-second-lane) + add_test(NAME agc_driver_pointer_phi_lane_not_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-not) + add_test(NAME agc_driver_pointer_vsharp_select_table COMMAND agc_driver_image_table_tests --pointer-vsharp-select) + add_test(NAME agc_driver_pointer_hop_table COMMAND agc_driver_image_table_tests --pointer-hop) + add_test(NAME agc_driver_pointer_vsharp_fixed_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed) + add_test(NAME agc_driver_pointer_vsharp_fixed_offset_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed-offset) + add_test(NAME agc_driver_pointer_vsharp_shared_table COMMAND agc_driver_image_table_tests --pointer-vsharp-shared) + add_test(NAME agc_driver_pointer_vsharp_shared_reuse_table COMMAND agc_driver_image_table_tests --pointer-vsharp-shared-reuse) + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table agc_driver_pointer_vsharp_shared_table agc_driver_pointer_vsharp_shared_reuse_table PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table agc_driver_pointer_vsharp_shared_table agc_driver_pointer_vsharp_shared_reuse_table PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake index 1e66e009a013..68f6291c46f9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake @@ -37,8 +37,10 @@ if(BUILD_TESTING) endif() add_test(NAME agc_shader_nested_snapshot_retry COMMAND agc_shader_memory_tests --nested-snapshot-retry) set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES TIMEOUT 20) + add_test(NAME agc_shader_nested_sampler_snapshot_retry COMMAND agc_shader_memory_tests --nested-sampler-snapshot-retry) + set_tests_properties(agc_shader_nested_sampler_snapshot_retry PROPERTIES TIMEOUT 20) if(WIN32) - set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_nested_snapshot_retry agc_shader_nested_sampler_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse) add_test(NAME agc_shader_region_merge COMMAND agc_shader_memory_tests --region-merge) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BdaAnchorReads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BdaAnchorReads.cpp index 9bd219d26731..bba9e87c603b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BdaAnchorReads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BdaAnchorReads.cpp @@ -42,6 +42,10 @@ alignas(256) constexpr std::array Code{ 0xdc708000, 0x00000403, 0xdc708004, 0x00000503, 0xdc708008, 0x00000603, 0xdc70800c, 0x00000703, 0xdc708010, 0x00000803, 0xdc708014, 0x00000903, 0xbf810000}; +alignas(256) constexpr std::array MisalignedBaseCode{ + 0x34060085, 0x4a0606ff, OutputBase, 0xf4000101, 0x0c000000, 0xf4000141, 0x0e000000, 0xbf8cc07f, + 0x7e100204, 0x7e120205, 0xdc708010, 0x00000803, 0xdc708014, 0x00000903, 0xbf810000}; + class GuestBlock { public: GuestBlock() { @@ -93,23 +97,24 @@ std::uint32_t Bytes(const std::vector& memory, std::uint32_t addr return value; } -std::vector Expected() { +std::vector Expected(bool scalarOnly) { const auto initial = Initial(); auto memory = initial; for (std::uint32_t tid = 0; tid < Threads; ++tid) { auto* out = &memory[OutputBase / 4u + tid * OutputDwords]; - out[0] = Bytes(initial, tid * 4u + 0x10u); - out[1] = Bytes(initial, tid * 4u + 0x14u); - out[2] = Bytes(initial, tid * 4u + 0x18u); - out[3] = Bytes(initial, tid * 4u + 0x403u); + if (!scalarOnly) { + out[0] = Bytes(initial, tid * 4u + 0x10u); + out[1] = Bytes(initial, tid * 4u + 0x14u); + out[2] = Bytes(initial, tid * 4u + 0x18u); + out[3] = Bytes(initial, tid * 4u + 0x403u); + } out[4] = Bytes(initial, 0x200u); out[5] = Bytes(initial, 0x2f4u); } return memory; } -ShaderRecompiler::RecompileResult Recompile(std::uint32_t waveSize, std::span userData, std::span guest, const ShaderRecompiler::SpirvTarget& target) { - const std::span code(Code); +ShaderRecompiler::RecompileResult Recompile(std::span code, std::uint32_t waveSize, std::span userData, std::span guest, const ShaderRecompiler::SpirvTarget& target) { const std::array regions{{{reinterpret_cast(code.data()), std::as_bytes(code)}, {reinterpret_cast(guest.data()), guest}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -127,17 +132,20 @@ bool SearchesAnchor(const ShaderRecompiler::SharedSpirv& spirv) { return text.find("bda_anchor_range") != std::string_view::npos; } -void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, const std::string& name) { +void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, const std::string& name, std::uint32_t scalarBaseLow = 0u) { auto memory = Initial(); std::memcpy(guest.Data(), memory.data(), BlockBytes); const auto address = static_cast(reinterpret_cast(guest.Data())); - const std::vector userData{static_cast(address), static_cast(address >> 32u)}; - const auto result = Recompile(waveSize, userData, std::span(reinterpret_cast(guest.Data()), InputDwords * sizeof(std::uint32_t)), target); + const bool scalarOnly = scalarBaseLow != 0u; + const std::span code = scalarOnly ? std::span(MisalignedBaseCode) : std::span(Code); + std::vector userData{static_cast(address), static_cast(address >> 32u)}; + if (scalarOnly) userData.insert(userData.end(), {static_cast(address) + scalarBaseLow, static_cast(address >> 32u), 0u, 0u, 0x200u, 0x2f4u}); + const auto result = Recompile(code, waveSize, userData, std::span(reinterpret_cast(guest.Data()), InputDwords * sizeof(std::uint32_t)), target); Require(SearchesAnchor(result.spirv), "BDA anchor reads: " + name + " reads of one user-data base do not share an anchor"); - device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); std::memcpy(memory.data(), guest.Data(), BlockBytes); - const auto expected = Expected(); + const auto expected = Expected(scalarOnly); for (std::size_t dword = 0; dword < memory.size(); ++dword) { Require(memory[dword] == expected[dword], "BDA anchor reads: " + name + " byte " + std::to_string(dword * 4u) + " is " + Hex(memory[dword]) + ", expected " + Hex(expected[dword])); } @@ -155,6 +163,7 @@ int main() { Run(*device, guest, 32, native, "wave32"); Run(*device, guest, 64, native, "wave64"); if (split.subgroupSize != native.subgroupSize) Run(*device, guest, 64, split, "wave64 split"); + for (const std::uint32_t low : {1u, 2u, 3u}) Run(*device, guest, 64, native, "scalar loads from a base with low bits " + std::to_string(low), low); std::puts("BDA anchor read tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index be41a4c540ed..f2e1eef0b801 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -57,6 +57,9 @@ constexpr std::uint32_t DepthSwizzleMode = 0x18; alignas(256) std::array FloatTexels{}; alignas(256) std::array OtherFloatTexels{}; +constexpr std::uint32_t PlacedTexelStride = 512; +constexpr std::array PlacedTexelOffsets{0x100u, 0x300u}; +alignas(1024) std::array PlacedTexels{}; alignas(256) std::array UintTexels{}; alignas(256) std::array Uint64Texels{}; alignas(256) std::array SintTexels{}; @@ -322,6 +325,174 @@ alignas(256) constexpr std::array NestedRecordCode{ 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, }; +alignas(256) constexpr std::array NestedSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerImageCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf40c0916u, 0xfa000040u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerBallotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820009u, 0xbf068212u, + 0xbf840002u, 0xbeac0334u, 0xbf820004u, 0xbf068312u, 0xbf840001u, 0xbeac0390u, 0xbf800000u, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerCopiesCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf82000eu, 0xbf068212u, 0xbf840007u, 0x3a160613u, + 0x7e2c050bu, 0x8f178416u, 0xf4200b02u, 0x2e000008u, 0xbf8cc07fu, 0xbf820004u, 0xbf068312u, 0xbf840001u, + 0xbeac0390u, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffdeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneAndCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, + 0xbeac03ffu, 0x000000eau, 0xbf820010u, 0xbf068212u, 0xbf840006u, 0xbebc0334u, 0xbf068115u, 0xbf840001u, + 0xbebc0336u, 0x872c343cu, 0xbf820007u, 0xbf068312u, 0xbf840004u, 0xbf068215u, 0xbf840001u, 0xbeac03a0u, + 0x872c342cu, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffd8u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820009u, 0xbf068213u, + 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffdbu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820012u, 0xbf068313u, + 0xbf840007u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbf820009u, + 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, + 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffd2u, + 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondLaneCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820013u, 0xbf068313u, + 0xbf840008u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbeac037eu, + 0xbf820009u, 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, + 0xbf8cc07fu, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, + 0xbf89ffd1u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, + 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneNotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, 0x32000000u, 0xd4c10034u, + 0x00002900u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, 0xbeac0380u, 0xbf82000au, 0xbf068212u, 0xbf840008u, + 0xbebc1334u, 0x8f3d3c81u, 0xbf068115u, 0xbf840002u, 0x8a2c3d34u, 0xbf820002u, 0xbebd073du, 0x872c3d34u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerVsharpSelectCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0xd4c40038u, 0x000200c0u, 0x87ea386au, + 0x7e120215u, 0x4a141210u, 0x7e080211u, 0xbf068116u, 0xbf840001u, 0x7e080217u, 0x02061504u, 0xbeae047eu, + 0xd4c5003eu, 0x000200c0u, 0x87bc3e38u, 0xbeba243cu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, + 0x8f198418u, 0xf4240b02u, 0x32000008u, 0xbf8cc07fu, 0xf4080d16u, 0xfa0000a0u, 0x8f1a8513u, 0xbf8cc07fu, + 0xf42c091au, 0x34000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffedu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerHopCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820003u, 0xbf068212u, + 0xbf840001u, 0xbeac0334u, 0xf4040e16u, 0xfa000000u, 0xbf8cc07fu, 0xf40c091cu, 0xfa000340u, 0xbf8cc07fu, + 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe4u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpOffsetCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerSharedVsharpCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf068012u, 0xbf850002u, 0xbeac03ffu, 0x000000eau, 0xf4080d16u, 0xfa000000u, 0xbf8cc07fu, 0xf42c091au, + 0xfa000000u, 0xf42c0e1au, 0xfa0004c0u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0xf09c0f08u, 0x010e0b01u, + 0xbf8c3f70u, 0x7e0a030bu, 0x8afe3032u, 0xbf89ffe4u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerSharedVsharpReuseCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf068012u, 0xbf850002u, 0xbeac03ffu, 0x000000eau, 0xf4080d16u, 0xfa000000u, 0xbf8cc07fu, 0xf42c091au, + 0xfa000000u, 0xf42c0e1au, 0xfa0004c0u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0xf09c0f08u, 0x010e0b01u, + 0xbf8c3f70u, 0x7e0a030bu, 0xbf068013u, 0xbf850002u, 0xbeac0338u, 0xbead0339u, 0xf4001016u, 0xfa000010u, + 0xbf8cc07fu, 0x7e0c0240u, 0x8afe3032u, 0xbf89ffdcu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + constexpr std::array PointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; constexpr std::array LinearWrap{0u, (4u * 256u) << 12u, (1u << 20u) | (1u << 22u), 0u}; constexpr std::array PointWrap{0u, (4u * 256u) << 12u, 1u << 24u, 0u}; @@ -418,7 +589,7 @@ std::vector> SamplerWords(const std::vector span, const void* root, std::span memory, std::uint32_t groups, std::uint32_t threads, std::uint32_t waveSize) { +Outcome Execute(AgcDriver::VulkanDevice& device, std::span span, const void* root, std::span memory, std::uint32_t groups, std::uint32_t threads, std::uint32_t waveSize, std::span snapshots = {}) { const auto rootAddress = reinterpret_cast(root); const std::array userData{static_cast(rootAddress), static_cast(rootAddress >> 32u)}; std::fill(Output.begin(), Output.end(), 0xdeadbeefu); @@ -445,7 +616,7 @@ Outcome Execute(AgcDriver::VulkanDevice& device, std::span } StderrCapture capture; try { - device.Dispatch(result, groups, 1, 1, {}, reinterpret_cast(span.data())); + device.Dispatch(result, groups, 1, 1, snapshots, reinterpret_cast(span.data())); device.WaitIdle(); } catch (...) { static_cast(capture.Finish()); @@ -541,6 +712,10 @@ bool Faulted(const Outcome& outcome) { return outcome.log.find("image table:") != std::string::npos; } +bool AddressFaulted(const Outcome& outcome) { + return outcome.log.find("BDA access failed") != std::string::npos; +} + bool Holds(const std::vector>& samplers, const std::array& sampler) { return std::find(samplers.begin(), samplers.end(), sampler) != samplers.end(); } @@ -1392,7 +1567,7 @@ void RunNestedRecordTests(AgcDriver::VulkanDevice& device, NestedCase selected) } auto phi = std::vector(NestedRecordCode.begin(), NestedRecordCode.end()); phi.insert(phi.begin() + 17u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); - const auto run = [&](std::span code = NestedRecordCode, std::uint32_t waveSize = 64u) { + const auto run = [&](std::span code = NestedRecordCode, std::uint32_t waveSize = 64u, std::span snapshots = {}) { auto table = Buffer(Table.data(), rootBytes); if (!wideRoots.empty()) { table = Buffer(wideRoots.data(), static_cast(wideRoots.size() / 4u)); @@ -1421,7 +1596,7 @@ void RunNestedRecordTests(AgcDriver::VulkanDevice& device, NestedCase selected) const auto inputPalettes = palettes; const auto inputExtra = extra; const auto inputRoots = wideRoots; - auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize, snapshots); Require(Table == inputTable && Keys == inputKeys && records == inputRecords && palettes == inputPalettes && extra == inputExtra && wideRoots == inputRoots, "nested image table: dispatch modified its input memory"); Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "nested image table: dispatch wrote past its output"); return result; @@ -1508,6 +1683,23 @@ void RunNestedRecordTests(AgcDriver::VulkanDevice& device, NestedCase selected) for (std::uint32_t parent = 0; parent < 2u; ++parent) { Require(covers(records[parent].data() + 40u, 16u) && covers(palettes[parent].data(), 64u), "nested image table: a V# or its palette is absent from the write-overlap ranges"); } + auto consumer = std::vector(NestedRecordCode.begin(), NestedRecordCode.end()); + consumer.insert(consumer.begin() + 30u, 0x7e0e0225u); + const std::array live{{ + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }}; + const auto consumed = run(consumer, 64u, live); + Require(!Faulted(consumed) && !AddressFaulted(consumed), "nested image table: a T# with another consumer faulted:\n" + consumed.log); + Require(consumed.imageTableCount == original.imageTableCount, "nested image table: a T# with another consumer was not bound through the image table"); + for (std::uint32_t group = 0; group < 4u; ++group) { + const auto words = Group(consumed, group); + const auto expected = Group(original, group); + const auto& image = images[keys[group * 2u] + keys[group * 2u + 1u]]; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + Require(std::equal(words.begin() + lane * 4u, words.begin() + lane * 4u + 3u, expected.begin() + lane * 4u) && words[lane * 4u + 3u] == image[1], "nested image table: a T# with another consumer: group " + std::to_string(group) + " lane " + std::to_string(lane) + " alpha is " + Hex(words[lane * 4u + 3u]) + ", expected the T#'s dword 1 " + Hex(image[1])); + } + } auto loop = std::vector(NestedRecordCode.begin(), NestedRecordCode.end() - 1u); const auto secondLow = static_cast(reinterpret_cast(records[1].data())); loop.insert(loop.end(), {0xbf06ff2cu, secondLow, 0xbf850003u, 0xbeac03ffu, secondLow, 0xbf82ffe7u, 0xbf810000u}); @@ -1630,6 +1822,789 @@ void RunPointerChainRecordTests(AgcDriver::VulkanDevice& device) { RequireRed(relinked, 1, Lanes, texels[0], "float texels after record 1's second pointer moved to another T#"); } +void RunNestedSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(512) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + std::vector extra; + const std::array, 2>, 2> samplers{{{PointWrap, PointClamp}, {PointClamp, PointWrap}}}; + Table = {}; + Keys = {}; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(samplers[outer][inner].begin(), samplers[outer][inner].end(), palettes[outer].begin() + inner * 4u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const auto image = Image(FloatTexels.data(), Format32Float); + const auto run = [&](std::span code = NestedSamplerCode, std::uint32_t waveSize = 64u, std::span snapshots = {}) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(image.begin(), image.end(), Srt.begin() + 12u); + std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + if (!extra.empty()) memory.push_back({reinterpret_cast(extra.data()), std::as_bytes(std::span(extra))}); + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + const auto inputPalettes = palettes; + const auto inputExtra = extra; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize, snapshots); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords && palettes == inputPalettes && extra == inputExtra, "nested sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "nested sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [&](std::uint32_t record, std::uint32_t child) { + const bool wrap = samplers[record % 2u][child] == PointWrap; + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}; + std::copy(keys.begin(), keys.end(), Keys.begin()); + const auto original = run(); + Require(!Faulted(original), "nested sampler table: valid independent keys faulted:\n" + original.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(original, group, Lanes, sampled(keys[group * 2u], keys[group * 2u + 1u]), "nested sampler table: the sampler selected through record, pointer and V#"); + const auto wave32 = run(NestedSamplerCode, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "nested sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), palettes[1].begin()); + const auto edited = run(); + Require(!Faulted(edited) && Group(edited, 2) == Group(original, 0) && Group(edited, 3) == Group(original, 3), "nested sampler table: changing only a child S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "nested sampler table: editing a child S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), palettes[1].begin()); + Keys[0] = 3u; + const auto unmapped = run(); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "nested sampler table: an unreadable parent did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "nested sampler table: a faulting dispatch published copied output"); + Keys[0] = 0u; + records[0][43] |= 1u << 30u; + const auto notBuffer = run(); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "nested sampler table: a non-buffer parent was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "nested sampler table: a non-buffer parent published copied output"); + records[0][43] &= ~(1u << 30u); + const auto savedRecords = records; + std::copy_n(records[0].begin() + 40u, 4u, records[1].begin() + 40u); + const auto moved = run(); + Require(!Faulted(moved) && Group(moved, 2) == Group(original, 0) && Group(moved, 3) == Group(original, 1), "nested sampler table: replacing the nested V# retained a stale sampler"); + Require(moved.artifactId == original.artifactId && moved.tableMap.size() < original.tableMap.size(), "nested sampler table: equal V#s did not share one child table"); + records = savedRecords; + auto consumer = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + consumer.insert(consumer.begin() + 31u, 0x7e0e0221u); + const std::array live{{ + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }}; + const auto consumed = run(consumer, 64u, live); + Require(!Faulted(consumed) && !AddressFaulted(consumed), "nested sampler table: an S# with another consumer faulted:\n" + consumed.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + const auto words = Group(consumed, group); + const auto expected = Group(original, group); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto sampler = samplers[keys[group * 2u] % 2u][keys[group * 2u + 1u]]; + Require(std::equal(words.begin() + lane * 4u, words.begin() + lane * 4u + 3u, expected.begin() + lane * 4u) && words[lane * 4u + 3u] == sampler[1], "nested sampler table: an S# with another consumer: group " + std::to_string(group) + " lane " + std::to_string(lane) + " alpha is " + Hex(words[lane * 4u + 3u]) + ", expected the S#'s dword 1 " + Hex(sampler[1])); + } + } + auto phi = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + phi.insert(phi.begin() + 17u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = 1u; + const auto phiValid = run(phi); + Require(!Faulted(phiValid), "nested sampler table: a captured pointer selected through Phi faulted:\n" + phiValid.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(phiValid, group, Lanes, sampled(1u, Keys[group * 2u + 1u]), "nested sampler table: Phi-selected pointer"); + Keys[0] = 0u; + const auto phiFault = run(phi); + Require(Faulted(phiFault) && phiFault.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured pointer did not fault:\n" + phiFault.log); + Require(untouched(phiFault), "nested sampler table: an uncaptured pointer published copied output"); + Require(phiFault.artifactId == phiValid.artifactId, "nested sampler table: the pointer choice was specialized on the CPU"); + std::copy(keys.begin(), keys.end(), Keys.begin()); + extra.assign(320u * 4u, 0u); + for (std::uint32_t entry = 0; entry < 320u; ++entry) { + std::copy(PointWrap.begin(), PointWrap.end(), extra.begin() + entry * 4u); + extra[entry * 4u + 2u] |= entry; + } + const auto many = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(many.begin(), many.end(), records[0].begin() + 40u); + Keys[1] = 319u; + auto unkeyed = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + unkeyed.insert(unkeyed.begin() + 14u, {0x8714ff14u, 0x000001ffu}); + const auto overBudget = run(unkeyed); + Require(Faulted(overBudget) && overBudget.log.find("over budget") != std::string::npos, "nested sampler table: an S# past the sampler budget did not fault:\n" + overBudget.log); + Require(untouched(overBudget), "nested sampler table: an S# past the sampler budget published copied output"); + extra.assign((NestedKeyLimit + 1u) * 4u, 0u); + std::copy(PointClamp.begin(), PointClamp.end(), extra.begin()); + const auto large = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(large.begin(), large.end(), records[0].begin() + 40u); + Keys[1] = 0u; + const auto bounded = run(); + Require(Faulted(bounded) && bounded.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured child did not fault:\n" + bounded.log); + Require(bounded.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: the two levels produced an unbounded map"); + Require(untouched(bounded), "nested sampler table: a dispatch outside the snapshot published copied output"); + std::copy(large.begin(), large.end(), records[1].begin() + 40u); + const auto sharedBudget = run(); + Require(!Faulted(sharedBudget) && sharedBudget.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: equal V#s consumed the snapshot budget twice"); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool clamp = keys[group * 2u + 1u] == 0u; + RequireRed(sharedBudget, group, Lanes, [clamp](std::uint32_t x) { return std::bit_cast(FloatTexels[clamp ? Width - 1u : x]); }, "nested sampler table: aliased parent at the snapshot budget"); + } + std::puts("nested sampler table: S# through record, pointer and V# checked on wave64 and wave32, with sharing and budgets"); +} + +void RunPointerSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + const std::array, 2> samplers{PointWrap, PointClamp}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) { + std::copy(samplers[record].begin(), samplers[record].end(), records[record].begin() + 8u); + std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + auto root = images[0]; + std::uint32_t tableType = 0u; + const auto run = [&](std::span code, std::uint32_t waveSize = 64u) { + auto table = Buffer(Table.data(), 64u); + table[3] |= tableType; + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(root.begin(), root.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords, "pointer sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [](bool wrap) { + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{1u, 2u, 1u, 2u}; + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = keys[group]; + const auto direct = run(PointerSamplerCode); + Require(!Faulted(direct), "pointer sampler table: an S# read through a record's pointer faulted:\n" + direct.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(direct, group, Lanes, sampled(keys[group] % 2u == 0u), "pointer sampler table: the S# read through the record's pointer"); + auto phi = std::vector(PointerSamplerCode.begin(), PointerSamplerCode.end()); + phi.insert(phi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto original = run(phi); + Require(!Faulted(original) && original.words == direct.words, "pointer sampler table: a captured pointer selected through Phi changed the result:\n" + original.log); + const auto wave32 = run(phi, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "pointer sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), records[1].begin() + 8u); + const auto edited = run(phi); + Require(!Faulted(edited) && Group(edited, 0) == Group(original, 1) && Group(edited, 2) == Group(original, 3), "pointer sampler table: changing an S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "pointer sampler table: editing an S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), records[1].begin() + 8u); + Keys[0] = 0u; + const auto uncaptured = run(phi); + Require(Faulted(uncaptured) && uncaptured.log.find("outside the snapshot") != std::string::npos, "pointer sampler table: an uncaptured pointer did not fault:\n" + uncaptured.log); + Require(untouched(uncaptured), "pointer sampler table: an uncaptured pointer published copied output"); + Require(uncaptured.artifactId == original.artifactId, "pointer sampler table: the pointer choice was specialized on the CPU"); + auto alias = phi; + alias[17] = static_cast(reinterpret_cast(records[1].data())); + const auto aliased = run(alias); + Require(!Faulted(aliased) && Group(aliased, 0) == Group(original, 0) && Group(aliased, 1) == Group(original, 1), "pointer sampler table: a Phi constant naming another captured pointer did not read that pointer's S#:\n" + aliased.log); + Keys[0] = 3u; + const auto unmapped = run(phi); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "pointer sampler table: a null pointer did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "pointer sampler table: a null pointer published copied output"); + Keys[0] = 1u; + tableType = 1u << 30u; + const auto notBuffer = run(phi); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "pointer sampler table: a non-buffer record table was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "pointer sampler table: a non-buffer record table published copied output"); + tableType = 0u; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + { + const Registration registered(records.data(), sizeof(records)); + auto consumer = phi; + consumer.insert(consumer.begin() + 30u, 0x7e0e0221u); + const auto shared = run(consumer); + Require(!Faulted(shared), "pointer sampler table: an S# with another consumer faulted:\n" + shared.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + auto expected = Group(original, group); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) expected[lane * 4u + 3u] = samplers[keys[group] % 2u][1]; + Require(Group(shared, group) == expected, "pointer sampler table: an S# with another consumer read the wrong words"); + } + } + root = PointWrap; + auto imagePhi = std::vector(PointerImageCode.begin(), PointerImageCode.end()); + imagePhi.insert(imagePhi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto imageDirect = run(PointerImageCode); + const auto imageMerged = run(imagePhi); + Require(!Faulted(imageDirect) && !Faulted(imageMerged) && imageMerged.words == imageDirect.words, "pointer image table: a captured pointer selected through Phi changed the result:\n" + imageMerged.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool other = keys[group] % 2u != 0u; + RequireRed(imageMerged, group, Lanes, [other](std::uint32_t x) { return std::bit_cast(other ? OtherFloatTexels[x] : FloatTexels[x]); }, "pointer image table: the T# read through a Phi-selected pointer"); + } + Keys[0] = 0u; + const auto imageFault = run(imagePhi); + Require(Faulted(imageFault) && imageFault.log.find("outside the snapshot") != std::string::npos && untouched(imageFault), "pointer image table: an uncaptured pointer did not fault:\n" + imageFault.log); + std::puts("pointer sampler table: S# and T# through a record pointer whose low word passes a Phi checked on wave64 and wave32"); +} + +void RunPointerArmTests(AgcDriver::VulkanDevice& device, bool copies) { + Fill(); + alignas(256) std::array, 2> records{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::span code = copies ? std::span(PointerCopiesCode) : std::span(PointerBallotCode); + const std::string name = copies ? "pointer table with a second record read" : "pointer table with a lane-mask arm"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group, std::uint32_t flip) { + return std::function([&keys, group, flip](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u] ^ flip; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group, 0u), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(3u, "a second constant pointer word"); + if (!copies) { + outside(2u, "a lane-mask pointer word"); + std::puts("pointer table with a lane-mask arm: a T# through a record pointer whose low word passes nested Phis with constants and a Ballot word checked, and the Ballot word faults outside the snapshot"); + return; + } + Keys[2] = 2u; + const auto again = run(); + Require(!Faulted(again) && again.words == taken.words, name + ": a second read of the same record changed the result:\n" + again.log); + Keys[3] = 3u; + const auto flipped = run(); + Keys[2] = 0u; + Keys[3] = 0u; + Require(!Faulted(flipped) && flipped.artifactId == taken.artifactId, name + ": a second read of another record faulted or recompiled:\n" + flipped.log); + RequireRed(flipped, 0u, Lanes, sampled(0u, 3u), name + ": the T# behind the other record's pointer word"); + for (std::uint32_t group = 1; group < 4u; ++group) Require(Group(flipped, group) == Group(taken, group), name + ": a group that kept its record changed its result"); + std::puts("pointer table with a second record read: a T# through a record pointer whose low word passes a second read keyed by another lane checked, by the pointer's value"); +} + +enum class PointerMask { LaneAnd, High, HighSecond, HighSecondLane, LaneNot }; + +void RunPointerMaskTests(AgcDriver::VulkanDevice& device, PointerMask shape) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array second{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + const std::uint32_t pointerWord = shape == PointerMask::LaneNot ? 0u : 2u; + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + pointerWord] = static_cast(pointer); + Table[record * 4u + pointerWord + 1u] = static_cast(pointer >> 32u); + } + const auto secondBuffer = Buffer(second.data() + 8u, 32u); + std::copy(secondBuffer.begin(), secondBuffer.end(), second.begin()); + std::span code = shape == PointerMask::LaneAnd ? std::span(PointerLaneAndCode) : shape == PointerMask::High ? std::span(PointerHighCode) : shape == PointerMask::LaneNot ? std::span(PointerLaneNotCode) : shape == PointerMask::HighSecondLane ? std::span(PointerHighSecondLaneCode) : std::span(PointerHighSecondCode); + const std::string name = shape == PointerMask::LaneAnd ? "pointer table with lane-mask ANDs" : shape == PointerMask::High ? "pointer table with a high-word Phi" : shape == PointerMask::LaneNot ? "pointer table with a lane bit cleared from a lane mask" : shape == PointerMask::HighSecondLane ? "pointer table with a second-buffer high word under a lane-mask low word" : "pointer table with a high word from a second buffer"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + const auto secondAddress = reinterpret_cast(second.data()); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + Srt[16] = static_cast(secondAddress); + Srt[17] = static_cast(secondAddress >> 32u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {secondAddress, std::as_bytes(std::span(second))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group) { + return std::function([&keys, group](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t selector, std::uint32_t arm, std::uint32_t variant, const std::string& what) { + Keys[selector] = arm; + Keys[5] = variant; + const auto faulted = run(); + Keys[selector] = 0u; + Keys[5] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(2u, 1u, 0u, "a constant low word"); + if (shape == PointerMask::LaneAnd) { + outside(2u, 2u, 0u, "a lane-mask Phi ANDed with a lane mask"); + outside(2u, 2u, 1u, "the other lane-mask word ANDed with a lane mask"); + outside(2u, 3u, 0u, "a record word ANDed with a lane mask"); + std::puts("pointer table with lane-mask ANDs: a T# through a record pointer whose low word passes Phis of lane-mask ANDs checked, and every masked word faults outside the snapshot"); + return; + } + if (shape == PointerMask::LaneNot) { + outside(2u, 2u, 0u, "a lane mask whose handled lane's bit s_not and s_and cleared"); + outside(2u, 2u, 1u, "a lane mask whose handled lane's bit s_andn2 cleared"); + std::puts("pointer table with a lane bit cleared from a lane mask: a T# through a record pointer whose low word passes Phis of a zero and of a lane mask without the handled lane's bit checked, and every masked word faults outside the snapshot"); + return; + } + outside(3u, 1u, 0u, "a constant high word"); + if (shape == PointerMask::HighSecondLane) outside(3u, 3u, 0u, "a lane-mask low word with a high word from the second buffer"); + Keys[3] = 2u; + const auto again = run(); + Keys[5] = 3u; + const auto flipped = run(); + Keys[3] = 0u; + Keys[5] = 0u; + Require(!Faulted(again) && again.words == taken.words && again.artifactId == taken.artifactId, name + ": a second read of the same record's high word changed the result:\n" + again.log); + Require(!Faulted(flipped) && flipped.words == taken.words && flipped.artifactId == taken.artifactId, name + ": the high word of the neighbouring record changed the result:\n" + flipped.log); + if (shape == PointerMask::HighSecondLane) { + std::puts("pointer table with a second-buffer high word under a lane-mask low word: a T# through a record pointer whose high word reads a second buffer only on the edge whose low word is a lane mask checked, and that pair faults outside the snapshot"); + return; + } + std::puts("pointer table with a high-word Phi: a T# through a record pointer whose high word passes a Phi of reads under other keys and a constant checked, by the pointer's value"); +} + +void RunPointerVsharpTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float), Image(UintTexels.data(), Format32UInt)}; + const std::array, 3> texels{ + [](std::uint32_t x) { return std::bit_cast(FloatTexels[x]); }, + [](std::uint32_t x) { return std::bit_cast(OtherFloatTexels[x]); }, + [](std::uint32_t x) { return UintTexels[x]; }, + }; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(images[outer + inner].begin(), images[outer + inner].end(), palettes[outer].begin() + inner * 8u); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerVsharpSelectCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, "pointer V# table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer V# table: dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u, 0u}, {2u, 1u, 1u}, {1u, 1u, 1u}, {2u, 2u, 0u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + Keys[group * 8u + 3u] = keys[group][2]; + } + std::array targets{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return texels[targets[record] + keys[group][2]](x); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), "pointer V# table: the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), "pointer V# table: the T# read through the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + targets[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), "pointer V# table: the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, "pointer V# table: moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), "pointer V# table: the T# after record 1's pointer moved to the other V#"); + std::puts("pointer V# table: a T# read through a V# loaded through a record pointer keyed by a first-lane read of a select checked, by the records' values"); +} + +void RunPointerHopTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> links{}; + alignas(256) std::array, 2> targets{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto link = [&](std::uint32_t from, std::uint32_t to) { + const auto pointer = reinterpret_cast(targets[to].data()); + links[from][0] = static_cast(pointer); + links[from][1] = static_cast(pointer >> 32u); + }; + for (std::uint32_t target = 0; target < 2u; ++target) { + std::copy(images[target].begin(), images[target].end(), targets[target].begin() + 0x340u / 4u); + link(target, target); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(links[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::string name = "pointer table with a pointer hop"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerHopCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + {reinterpret_cast(targets.data()), std::as_bytes(std::span(targets))}, + }; + const auto inputTable = Table; + const auto inputLinks = links; + const auto inputTargets = targets; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && links == inputLinks && targets == inputTargets, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array targetOf{0u, 1u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(targetOf[record % 2u] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# behind the record pointer's hop faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the pointer loaded from the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(2u, "a lane-mask pointer word"); + link(1u, 0u); + targetOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved hop faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving the pointer behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after the pointer behind record 1's pointer moved to the other T#"); + std::puts("pointer table with a pointer hop: a T# read through a pointer loaded from a record pointer whose low word passes Phis with a constant and a Ballot word checked, by the records' values"); +} + +void RunPointerFixedVsharpTests(AgcDriver::VulkanDevice& device, std::uint32_t descriptorOffset) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto point = [&](std::uint32_t record, std::uint32_t palette) { + const auto descriptor = Buffer(palettes[palette].data(), static_cast(sizeof(palettes[palette]))); + std::copy(descriptor.begin(), descriptor.end(), records[record].begin() + 40u); + }; + for (std::uint32_t palette = 0; palette < 2u; ++palette) { + std::copy(images[palette].begin(), images[palette].end(), palettes[palette].begin() + descriptorOffset / 4u); + point(palette, palette); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const std::string name = std::string("pointer V# table with a T# at fixed offset ") + Hex(descriptorOffset); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code = descriptorOffset == 0u ? std::span(PointerFixedVsharpCode) : std::span(PointerFixedVsharpOffsetCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array paletteOf{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(paletteOf[record] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read at a fixed offset of the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + paletteOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after record 1's pointer moved to the other V#"); + point(0u, 1u); + paletteOf = {1u, 1u, 1u}; + const auto repointed = run(); + Require(!Faulted(repointed), name + ": the T# behind a moved V# faulted:\n" + repointed.log); + Require(repointed.artifactId == taken.artifactId, name + ": moving the V# behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(repointed, group, Lanes, sampled(group), name + ": the T# after the V# behind records 0 and 1 moved to the other palette"); + std::puts((name + ": a T# read at a fixed offset of a V# loaded through a record pointer checked, by the records' values").c_str()); +} + +void RunPointerSharedVsharpTests(AgcDriver::VulkanDevice& device, bool reuse) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const auto placed = [](std::uint32_t image) { return PlacedTexels.data() + image * PlacedTexelStride + PlacedTexelOffsets[image] / sizeof(float); }; + std::copy(FloatTexels.begin(), FloatTexels.end(), placed(0u)); + std::copy(OtherFloatTexels.begin(), OtherFloatTexels.end(), placed(1u)); + const std::array images{Image(placed(0u), Format32Float), Image(placed(1u), Format32Float)}; + Require((images[0][0] & 3u) == 1u && (images[1][0] & 3u) == 3u, "pointer V# tables: the T#s' first words do not end in the bits 01 and 11"); + const auto point = [&](std::uint32_t record, std::uint32_t palette) { + const auto descriptor = Buffer(palettes[palette].data(), static_cast(sizeof(palettes[palette]))); + std::copy(descriptor.begin(), descriptor.end(), records[record].begin()); + }; + for (std::uint32_t palette = 0; palette < 2u; ++palette) { + std::copy(images[palette].begin(), images[palette].end(), palettes[palette].begin()); + std::copy(images[1u - palette].begin(), images[1u - palette].end(), palettes[palette].begin() + 0x4c0u / 4u); + point(palette, palette); + records[palette][4] = 0x51000000u + palette; + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + std::array, 2> behind{{{0x7b000000u}, {0x7b000001u}}}; + const auto behindAddress = [&](std::uint32_t image) { return ((static_cast(images[image][1]) << 32u | images[image][0]) & ~std::uint64_t{3}) + 0x10u; }; + const std::array live{{ + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + {behindAddress(0u), std::as_bytes(std::span(behind[0]))}, + {behindAddress(1u), std::as_bytes(std::span(behind[1]))}, + }}; + const std::string name = reuse ? "pointer V# table with two T#s whose second T#'s words also reach the record pointer's registers" : "pointer V# table with two T#s read through one V#"; + const std::span shader = reuse ? std::span(PointerSharedVsharpReuseCode) : std::span(PointerSharedVsharpCode); + const auto run = [&](std::span code) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + const auto inputPalettes = palettes; + const auto inputBehind = behind; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u, reuse ? std::span(live) : std::span()); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords && palettes == inputPalettes && behind == inputBehind, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + if (reuse) Keys[group * 4u + 3u] = group % 2u; + } + std::array paletteOf{0u, 1u, 0u}; + std::array targetOf{0u, 1u, 0u}; + const auto check = [&](const Outcome& outcome, const std::string& what) { + for (std::uint32_t group = 0; group < 4u; ++group) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto record = keys[group][lane < 32u ? 0u : 1u]; + const auto palette = paletteOf[record]; + const auto x = lane % Width; + const auto blue = !reuse ? 0u : Keys[group * 4u + 3u] != 0u ? behind[1u - palette][0] : records[targetOf[record]][4]; + const std::array expected{std::bit_cast(palette != 0u ? OtherFloatTexels[x] : FloatTexels[x]), std::bit_cast(palette != 0u ? FloatTexels[x] : OtherFloatTexels[x]), blue, 0x3f800000u}; + const auto* actual = outcome.words.data() + (group * Lanes + lane) * 4u; + Require(std::equal(expected.begin(), expected.end(), actual), name + ": " + what + ": group " + std::to_string(group) + " lane " + std::to_string(lane) + " is " + Hex(actual[0]) + " " + Hex(actual[1]) + " " + Hex(actual[2]) + " " + Hex(actual[3]) + ", expected " + Hex(expected[0]) + " " + Hex(expected[1]) + " " + Hex(expected[2]) + " 0x3f800000"); + } + } + }; + const auto taken = run(shader); + Require(!Faulted(taken) && !AddressFaulted(taken), name + ": the T#s read through the V# behind the record's pointer faulted:\n" + taken.log); + check(taken, "the T#s at offsets 0 and 0x4c0 of the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + paletteOf[1] = 0u; + targetOf[1] = 0u; + const auto moved = run(shader); + Require(!Faulted(moved) && !AddressFaulted(moved), name + ": the T#s behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a record's pointer recompiled the shader"); + check(moved, "the T#s after record 1's pointer moved to the other V#"); + point(0u, 1u); + paletteOf = {1u, 1u, 1u}; + const auto repointed = run(shader); + Require(!Faulted(repointed) && !AddressFaulted(repointed), name + ": the T#s behind a moved V# faulted:\n" + repointed.log); + Require(repointed.artifactId == taken.artifactId, name + ": moving the V# behind a record's pointer recompiled the shader"); + check(repointed, "the T#s after the V# behind records 0 and 1 moved to the other palette"); + if (reuse) { + std::puts((name + ": both T#s sampled at their own offsets, and T#2's dwords 0 and 1 re-read through the guest's V# as the later pointer, checked by the records' values").c_str()); + return; + } + std::puts((name + ": both T#s read at their own offsets of one V# loaded through a record pointer checked, by the records' values").c_str()); +} + void RunSplitRecordTests(AgcDriver::VulkanDevice& device) { Fill(); const auto floatImage = Image(FloatTexels.data(), Format32Float); @@ -1907,6 +2882,62 @@ int main(int argc, char** argv) { RunNestedRecordTests(*device, NestedCase::Budget); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--nested-sampler") == 0) { + RunNestedSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-sampler") == 0) { + RunPointerSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-ballot") == 0) { + RunPointerArmTests(*device, false); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-copies") == 0) { + RunPointerArmTests(*device, true); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-and") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneAnd); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high") == 0) { + RunPointerMaskTests(*device, PointerMask::High); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high-second-lane") == 0) { + RunPointerMaskTests(*device, PointerMask::HighSecondLane); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-not") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneNot); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-select") == 0) { + RunPointerVsharpTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-hop") == 0) { + RunPointerHopTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed") == 0) { + RunPointerFixedVsharpTests(*device, 0u); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed-offset") == 0) { + RunPointerFixedVsharpTests(*device, 0x40u); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-shared") == 0) { + RunPointerSharedVsharpTests(*device, false); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-shared-reuse") == 0) { + RunPointerSharedVsharpTests(*device, true); + return 0; + } RunTests(*device); RunBorderSwizzleTests(*device); RunAnisoOverrideTests(*device); diff --git a/core/shader/recompiler/ImageTableAbi.hpp b/core/shader/recompiler/ImageTableAbi.hpp index b9c9773b5f4b..3ec9612ee12f 100644 --- a/core/shader/recompiler/ImageTableAbi.hpp +++ b/core/shader/recompiler/ImageTableAbi.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler::ImageTableAbi { -inline constexpr std::uint32_t Version = 3; +inline constexpr std::uint32_t Version = 4; inline constexpr std::uint32_t HeaderVersion = 0; inline constexpr std::uint32_t HeaderElementBase = 1; @@ -130,8 +130,9 @@ constexpr std::optional ScalarBufferDword(std::uint32_t offset, s } constexpr std::optional ScalarAddressDword(std::uint64_t base, std::uint32_t offset, std::uint32_t immediate) { - const std::uint64_t first = base + (offset & ~3u); - if (first < base) return std::nullopt; + const std::uint64_t aligned = base & ~std::uint64_t{3}; + const std::uint64_t first = aligned + (offset & ~3u); + if (first < aligned) return std::nullopt; const std::int64_t delta = static_cast(immediate & ~3u); const std::uint64_t magnitude = delta < 0 ? static_cast(-delta) : static_cast(delta); if (delta < 0) return magnitude > first ? std::nullopt : std::optional(first - magnitude); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 646f625d9a38..714c782bf4f3 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -55,9 +55,10 @@ struct TableColumn { std::uint32_t foldedCarry = 0; bool selected = false; std::uint32_t selection = 0; + bool pointerKeyed = false; bool operator==(const TableColumn& other) const { - return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection; + return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection && pointerKeyed == other.pointerKeyed; } }; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index 034d6b7947b5..05979a8ac973 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -132,6 +132,7 @@ struct ImageTableColumnSnapshot { std::uint32_t fault = 0; bool outside = false; bool nested = false; + bool pointerKeyed = false; SharedValues pointers; SharedValues codes; SharedValues words; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 3c562da2bc9d..83e1bf17d99b 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -755,7 +755,10 @@ class TableSnapshotter { for (const auto& sampler : plan.info.samplers) { if (sampler.table != NoTable) { count = std::max(count, sampler.table + 1u); - if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value() && plan.descriptorSources[sampler.source].tableColumn->indirect) hasIndirect = true; + if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value()) { + const auto& column = *plan.descriptorSources[sampler.source].tableColumn; + hasIndirect |= column.indirect || plan.descriptorSources.at(column.heapSource).tableColumn.has_value(); + } } } if (count == 0u) { @@ -1131,7 +1134,7 @@ class TableSnapshotter { const auto end = records.base + records.size; if (column.indirect) { unread(); - if (column.buffer) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); + if (column.buffer || column.pointerKeyed) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); for (std::uint32_t record = 0; record < records.keys; record++) { if (records.states[record] != RecordState::Read) continue; const auto offset = column.addend + record * column.stride; @@ -1142,6 +1145,7 @@ class TableSnapshotter { continue; } std::uint64_t pointer = (static_cast(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord); + if (column.pointerKeyed) records.pointers[record] = pointer; for (const auto& hop : column.pointerHops) { const auto low = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate); const auto high = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate + 4u); @@ -1498,6 +1502,7 @@ class TableSnapshotter { const auto& records = Column(source); auto& out = tables.tables.at(table); out = CopyRecords(records); + out.pointerKeyed = column.pointerKeyed; for (std::uint32_t resource = 0; resource < plan.info.images.size(); ++resource) { const auto& image = plan.info.images[resource]; if (image.table != table || !registeredSampledArrayCandidate(image) || runtime.normalizeImageView == nullptr) continue; @@ -1922,7 +1927,7 @@ bool sameColumns(const std::vector& left, const std::v for (std::size_t index = 0; index < left.size(); index++) { const auto& a = left[index]; const auto& b = right[index]; - if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; + if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || a.pointerKeyed != b.pointerKeyed || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; } return true; } @@ -2092,20 +2097,21 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i out.map[header + Abi::TableFault] = poison(resource, PoisonReason::UnsupportedOperation, nullptr); } outside[table] = outside[table] || column.outside; + const bool byPointer = column.nested || column.pointerKeyed; const auto mapStart = static_cast(out.map.size()); - if (!column.nested) { + if (!byPointer) { growMap(column.keys); out.map[header + Abi::TableMapStart] = mapStart; out.map.resize(out.map.size() + column.keys, 0u); } else if (column.pointers.size() != column.keys) { - throw std::runtime_error("nested image table snapshot has no pointer for each record"); + throw std::runtime_error("pointer-keyed image table snapshot has no pointer for each record"); } std::map pointers; if (column.nested) nestedRecords.Clear(); else records.Clear(); for (std::uint32_t key = 0; key < column.keys; key++) { const auto code = column.codes[key]; - if (column.nested && column.pointers[key] == Abi::UncapturedPointer) continue; + if (byPointer && column.pointers[key] == Abi::UncapturedPointer) continue; std::uint32_t mapped = Abi::NullCode; if ((code & Abi::PoisonFlag) != 0u) { const auto reason = static_cast(code & ~Abi::PoisonFlag); @@ -2133,15 +2139,15 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i records.Set(code, mapped); } } - if (column.nested) { + if (byPointer) { const auto [found, inserted] = pointers.try_emplace(column.pointers[key], mapped); - if (!inserted && found->second != mapped) throw std::runtime_error("nested image table captured conflicting values at one pointer"); + if (!inserted && found->second != mapped) throw std::runtime_error("pointer-keyed image table captured conflicting values at one pointer"); } else { out.map[mapStart + key] = mapped; } } - if (column.nested) { - if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("nested image table has too many pointers"); + if (byPointer) { + if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("pointer-keyed image table has too many pointers"); growMap(pointers.size() * Abi::PointerRecordWords); out.map[header + Abi::TableKeyCount] = static_cast(pointers.size()); out.map[header + Abi::TableMapStart] = static_cast(out.map.size()); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 1d34074f1376..4de155602587 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -155,6 +156,20 @@ class Tracker { memory.sampler = patch.sampler; } } + std::vector> unrewritten; + for (const auto& plan : m_tableColumns) { + for (IrValue* read : plan.reads) { + IrValue* buffer = read->Argument(0)->Resolve(); + if (FindTableColumn(*buffer) == nullptr || std::ranges::find(m_tablePlanningReads, read) != m_tablePlanningReads.end()) continue; + auto copy = std::ranges::find(unrewritten, buffer, &std::pair::first); + if (copy == unrewritten.end()) { + std::vector words; + for (std::uint32_t argument = 0; argument < buffer->ArgumentCount(); argument++) words.push_back(buffer->Argument(argument)); + copy = unrewritten.emplace(unrewritten.end(), buffer, EmitCopy(*buffer, words, *buffer)); + } + read->ReplaceArgument(0, copy->second); + } + } std::vector referenced; for (const auto& plan : m_tableColumns) { const auto& column = *m_sources[plan.source].tableColumn; @@ -202,7 +217,7 @@ class Tracker { IrValue* offset = nullptr; IrValue* parentPointer = nullptr; std::vector memory; - std::vector reads; + std::vector reads; std::vector hopMemory; std::vector hopReads; std::optional keyDomain; @@ -751,6 +766,22 @@ class Tracker { return left.shift == right.shift && EquivalentValue(m_program.Resources(), left.mask, right.mask) && EquivalentValue(m_program.Resources(), left.lane, right.lane); } + static bool DescriptorWordReads(const IrValue& buffer, bool rereadable) { + return std::ranges::all_of(buffer.Uses(), [&](const IrValue* read) { + const auto descriptor = [](const IrValue* user) { return user->Opcode() == IrOpcode::GetImageResource || user->Opcode() == IrOpcode::GetSamplerResource; }; + return read->Opcode() == IrOpcode::ReadConstBuffer && (rereadable ? std::ranges::any_of(read->Uses(), descriptor) : !read->Uses().empty() && std::ranges::all_of(read->Uses(), descriptor)); + }); + } + + bool RereadableVsharp(const IrValue& handle, const TableColumnPlan& parent) const { + if (handle.ArgumentCount() != parent.reads.size()) return false; + for (std::uint32_t dword = 0; dword < handle.ArgumentCount(); dword++) { + const IrValue* read = parent.reads[dword]; + if (handle.Argument(dword)->Resolve() != read || read->Opcode() != IrOpcode::LoadAddressU32 || m_program.Resources().memoryInfo[read->Flags().index].planningOnly) return false; + } + return true; + } + bool TryMakeTableColumn(IrValue& handle, bool sampler, bool r128, TableColumnPlan& plan, bool buffer = false) { const auto expected = buffer ? IrOpcode::GetBufferResource : sampler ? IrOpcode::GetSamplerResource : IrOpcode::GetImageResource; const std::uint32_t width = sampler || buffer ? 4u : 8u; @@ -841,7 +872,13 @@ class Tracker { if (address && immediateU32(offset, descriptorOffset)) { return TryMakeIndirectColumn(handle, *heapHandle, sampler, words, descriptorOffset, first, plan); } - if (!MatchAffineOffset(offset, key, stride, addend)) { + std::uint32_t fixedOffset = 0; + const bool fixed = !address && !buffer && immediateU32(offset, fixedOffset); + if (fixed) { + key = &m_builder.Constant(0u); + stride = width * static_cast(sizeof(std::uint32_t)); + addend = fixedOffset; + } else if (!MatchAffineOffset(offset, key, stride, addend)) { if (!MatchMaskedOffset(offset, key, stride) && (address || !MatchFirstLaneOffset(offset, key, stride))) { return false; } @@ -849,8 +886,10 @@ class Tracker { } DescriptorSource heapSource; std::uint32_t heapIndex = 0; - if (!MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { - if (sampler || address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; + if (MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { + if (fixed) return false; + } else { + if (address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; const auto sameHandle = [&](const IrValue* user) { if (user->Opcode() != handle.Opcode() || user->ArgumentCount() != handle.ArgumentCount()) return false; for (std::uint32_t argument = 0; argument < handle.ArgumentCount(); argument++) { @@ -858,13 +897,17 @@ class Tracker { } return true; }; - if (!std::ranges::all_of(plan.reads, [&](const IrValue* read) { return std::ranges::all_of(read->Uses(), sameHandle); })) return false; + const bool otherUsers = !std::ranges::all_of(plan.reads, [&](const IrValue* read) { return std::ranges::all_of(read->Uses(), sameHandle); }); TableColumnPlan parent; if (!TryMakeTableColumn(*heapHandle, false, false, parent, true)) return false; if (parent.parentPointer == nullptr) return false; + const bool rereadable = RereadableVsharp(*heapHandle, parent); + if (otherUsers && !rereadable) return false; if (!std::ranges::all_of(parent.reads, [&](const IrValue* read) { return std::ranges::all_of(read->Uses(), [&](const IrValue* user) { - return user->Opcode() == IrOpcode::GetBufferResource && std::ranges::all_of(user->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); }); + if (user->Opcode() != IrOpcode::GetBufferResource) return false; + if (std::ranges::all_of(user->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return true; + return SameHandle(*user, *heapHandle) && DescriptorWordReads(*user, rereadable); }); })) return false; heapIndex = parent.source; @@ -914,10 +957,81 @@ class Tracker { return true; } - IrValue* PointerRecordOrigin(IrValue* value) const { + static bool LaneMaskWord(const IrValue& value) { + return value.Opcode() == IrOpcode::CompositeExtractU32x4 && value.ArgumentCount() == 2u && value.Argument(0)->Resolve()->Opcode() == IrOpcode::Ballot && value.Argument(1)->Resolve()->HasImmediate(); + } + + bool SameRecordColumn(IrValue& first, IrValue& second) const { + IrValue* firstKey = nullptr; + IrValue* secondKey = nullptr; + std::uint32_t firstStride = 0; + std::uint32_t firstAddend = 0; + std::uint32_t secondStride = 0; + std::uint32_t secondAddend = 0; + return EquivalentValue(m_program.Resources(), first.Argument(0), second.Argument(0)) && MatchAffineOffset(first.Argument(1), firstKey, firstStride, firstAddend) && MatchAffineOffset(second.Argument(1), secondKey, secondStride, secondAddend) && firstStride == secondStride && firstAddend == secondAddend; + } + + static bool LaneIndex(const IrValue* value) { + std::vector pending{value}; + std::unordered_set seen; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second || (current->HasImmediate() && current->Type() == IrType::U32)) continue; + switch (current->Opcode()) { + case IrOpcode::FindILsb32: + case IrOpcode::FindUMsb32: + case IrOpcode::ReadFirstLane: continue; + case IrOpcode::BitwiseAnd32: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + if (current->ArgumentCount() != 2u) return false; + pending.push_back(current->Argument(0)); + pending.push_back(current->Argument(1)); + continue; + case IrOpcode::SelectU32: + if (current->ArgumentCount() != 3u) return false; + pending.push_back(current->Argument(1)); + pending.push_back(current->Argument(2)); + continue; + default: return false; + } + } + return true; + } + + static bool LaneBit(const IrValue& value) { + if (value.Opcode() != IrOpcode::ShiftLeftLogical32 || value.ArgumentCount() != 2u) return false; + const IrValue* one = value.Argument(0)->Resolve(); + return one->HasImmediate() && one->Type() == IrType::U32 && one->ImmediateU32() == 1u && LaneIndex(value.Argument(1)); + } + + static bool LaneMaskOperand(const IrValue* value) { + const IrValue* root = value->Resolve(); + std::vector pending{root}; + std::unordered_set seen; + bool lane = false; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second) continue; + if (LaneMaskWord(*current) || LaneBit(*current)) { + lane = true; + continue; + } + if (current != root && current->HasImmediate() && current->Type() == IrType::U32) continue; + const auto opcode = current->Opcode(); + const bool combines = (opcode == IrOpcode::BitwiseAnd32 || opcode == IrOpcode::BitwiseOr32) && current->ArgumentCount() == 2u; + const bool inverts = opcode == IrOpcode::BitwiseNot32 && current->ArgumentCount() == 1u; + if (!current->IsPhi() && !combines && !inverts) return false; + for (std::size_t index = 0; index < current->ArgumentCount(); ++index) pending.push_back(current->Argument(index)); + } + return lane; + } + + bool PointerRecordReads(IrValue* value, std::vector& reads) const { std::vector pending{value}; std::unordered_set seen; - IrValue* origin = nullptr; while (!pending.empty()) { auto* current = pending.back()->Resolve(); pending.pop_back(); @@ -928,11 +1042,94 @@ class Tracker { } if (current->HasImmediate() && current->Type() == IrType::U32) continue; std::uint32_t memory = 0; - if (ScalarReadMemory(*current, memory) == nullptr || !MemoryIndexBelongsTo(memory, *current)) return nullptr; - if (origin != nullptr && !EquivalentValue(m_program.Resources(), origin, current)) return nullptr; - origin = current; + const MemoryInfo* info = ScalarReadMemory(*current, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, *current)) return false; + if (!reads.empty() && !EquivalentValue(m_program.Resources(), reads.back(), current)) return false; + reads.push_back(current); + } + return !reads.empty(); + } + + static bool PointerFreeWord(const IrValue& word) { + return (word.HasImmediate() && word.Type() == IrType::U32) || LaneMaskOperand(&word); + } + + static IrValue* MaskedRecordWord(const IrValue& word) { + if (word.Opcode() != IrOpcode::BitwiseAnd32 || word.ArgumentCount() != 2u) return nullptr; + const bool left = LaneMaskOperand(word.Argument(0)); + if (left == LaneMaskOperand(word.Argument(1))) return nullptr; + return word.Argument(left ? 1u : 0u); + } + + bool PointerRecordRead(IrValue& read, std::vector& reads) const { + std::uint32_t memory = 0; + const MemoryInfo* info = ScalarReadMemory(read, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, read)) return false; + if (std::ranges::find(reads, &read) != reads.end()) return true; + if (!reads.empty() && (ScalarReadMemory(*reads.back(), memory)->offset != info->offset || !SameRecordColumn(*reads.back(), read))) return false; + reads.push_back(&read); + return true; + } + + bool PointerRecordPairs(IrValue* low, IrValue* high, std::vector& lowReads, std::vector& highReads) const { + std::vector> pending{{low, high}}; + std::set> seen; + while (!pending.empty()) { + auto* lowWord = pending.back().first->Resolve(); + auto* highWord = pending.back().second->Resolve(); + pending.pop_back(); + if (!seen.emplace(lowWord, highWord).second || PointerFreeWord(*lowWord)) continue; + if (lowWord->IsPhi()) { + const bool paired = highWord->IsPhi() && highWord->Parent() == lowWord->Parent(); + for (std::size_t index = 0; index < lowWord->ArgumentCount(); ++index) { + IrValue* edge = paired ? PhiIncoming(*highWord, lowWord->PhiBlock(index)) : highWord; + if (edge == nullptr) return false; + if (paired && lowWord->Argument(index)->Resolve()->HasImmediate() && edge->Resolve()->HasImmediate()) return false; + pending.emplace_back(lowWord->Argument(index), edge); + } + continue; + } + if (IrValue* word = MaskedRecordWord(*lowWord)) { + pending.emplace_back(word, highWord); + continue; + } + if (!PointerRecordRead(*lowWord, lowReads)) return false; + if (highWord->IsPhi()) { + for (std::size_t index = 0; index < highWord->ArgumentCount(); ++index) pending.emplace_back(lowWord, highWord->Argument(index)); + } else if (IrValue* word = MaskedRecordWord(*highWord)) { + pending.emplace_back(lowWord, word); + } else if (!PointerFreeWord(*highWord) && !PointerRecordRead(*highWord, highReads)) { + return false; + } + } + return !lowReads.empty() && !highReads.empty(); + } + + bool Precedes(const IrValue* value, const IrValue& use) const { + value = value->Resolve(); + const IrBlock* definition = value->Parent(); + const IrBlock* target = use.Parent(); + if (definition == nullptr) return true; + if (target == nullptr) return false; + if (definition == target) { + for (const IrValue* inst : target->Instructions()) { + if (inst == value) return true; + if (inst == &use) return false; + } + return false; + } + std::vector pending{m_program.BlockOrder().front()}; + std::unordered_set seen{pending.front()}; + while (!pending.empty()) { + const IrBlock* block = pending.back(); + pending.pop_back(); + if (block == definition) continue; + if (block == target) return false; + for (const IrBlock* successor : block->Successors()) { + if (seen.insert(successor).second) pending.push_back(successor); + } } - return origin; + return true; } IrValue* UnwrapTableRead(IrValue* value) const { @@ -1033,16 +1230,19 @@ class Tracker { IrValue* bufferHandle = nullptr; IrValue* offset = nullptr; const bool buffer = handle.Opcode() == IrOpcode::GetBufferResource; - if (buffer && !hops.empty()) return false; - const bool merged = buffer && pointer->Argument(0)->Resolve()->IsPhi(); - auto* low = merged ? PointerRecordOrigin(pointer->Argument(0)) : pointer->Argument(0)->Resolve(); + const bool merged = pointer->Argument(0)->Resolve()->IsPhi() && UnwrapTableRead(pointer->Argument(0)) == nullptr; + const bool pointerKeyed = merged && !buffer; + if (!hops.empty() && buffer) return false; + std::vector lowReads; + std::vector highReads; + if (pointerKeyed ? !PointerRecordPairs(pointer->Argument(0), pointer->Argument(1), lowReads, highReads) : merged && !PointerRecordReads(pointer->Argument(0), lowReads)) return false; + auto* low = merged ? lowReads.back() : UnwrapTableRead(pointer->Argument(0)); if (low == nullptr) return false; + auto* high = pointerKeyed ? pointer->Argument(1)->Resolve() : UnwrapTableRead(pointer->Argument(1)); + if (high == nullptr) return false; std::uint32_t first = 0; for (std::uint32_t dword = 0; dword < 2u; dword++) { - IrValue* read = dword == 0u && merged ? low : UnwrapTableRead(pointer->Argument(dword)); - if (read == nullptr) { - return false; - } + IrValue* read = dword == 0u ? low : pointerKeyed ? highReads.back() : high; std::uint32_t memoryIndex = 0; const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex); if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) { @@ -1052,17 +1252,36 @@ class Tracker { first = memory->offset; bufferHandle = read->Argument(0)->Resolve(); offset = read->Argument(1)->Resolve(); - } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1))) { + } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || (pointerKeyed ? !SameRecordColumn(*low, *read) : read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1)))) { return false; } } + IrValue* recordOffset = offset; + if (pointerKeyed) { + IrValue* source = high; + if (highReads.size() != 1u || highReads.front() != high) { + lowReads.insert(lowReads.begin(), highReads.begin(), highReads.end()); + const auto found = std::ranges::find_if(lowReads, [&](const IrValue* read) { + const IrValue* table = read->Argument(0)->Resolve(); + for (std::size_t index = 0; index < table->ArgumentCount(); ++index) { + if (!Precedes(table->Argument(index), handle)) return false; + } + return true; + }); + if (found == lowReads.end()) return false; + source = *found; + } + bufferHandle = source->Argument(0)->Resolve(); + recordOffset = source->Argument(1)->Resolve(); + offset = source == high ? recordOffset : high; + } if (!AvailableAt(offset, handle.Parent())) { return false; } IrValue* key = nullptr; std::uint32_t stride = 0; std::uint32_t addend = 0; - if (!MatchAffineOffset(offset, key, stride, addend)) { + if (!MatchAffineOffset(recordOffset, key, stride, addend)) { return false; } DescriptorSource heapSource; @@ -1082,6 +1301,7 @@ class Tracker { column.buffer = buffer; column.indirectOffset = descriptorOffset; column.indirectImmediate = descriptorImmediate; + column.pointerKeyed = pointerKeyed; column.pointerHops = std::move(hops); const auto domain = merged ? std::optional{} : MatchKeyDomain(key); DescriptorSource columnSource = heapSource; @@ -1098,7 +1318,7 @@ class Tracker { shared.keyDomains.push_back(*domain); } plan.offset = offset; - if (buffer) plan.parentPointer = pointer; + if (buffer || merged) plan.parentPointer = pointer; return true; } @@ -1967,15 +2187,21 @@ class Tracker { } } } - for (const auto& plan : m_tableColumns) { - for (std::size_t index = 0; index < plan.reads.size(); index++) { - const IrValue* read = plan.reads[index]; - const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* user) { - return FindTableColumn(*user) != nullptr; - }); - if (planned && std::ranges::find(m_tablePlanningReads, read) == m_tablePlanningReads.end()) { - m_tablePlanningReads.push_back(read); - m_tablePlanningMemory.push_back(plan.memory[index]); + for (bool marked = true; marked;) { + marked = false; + for (const auto& plan : m_tableColumns) { + for (std::size_t index = 0; index < plan.reads.size(); index++) { + const IrValue* read = plan.reads[index]; + const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* user) { + return FindTableColumn(*user) != nullptr && std::ranges::all_of(user->Uses(), [&](const IrValue* use) { + return BufferAccessOf(use->Opcode()) == BufferAccess::None || std::ranges::find(m_tablePlanningReads, use) != m_tablePlanningReads.end(); + }); + }); + if (planned && std::ranges::find(m_tablePlanningReads, read) == m_tablePlanningReads.end()) { + m_tablePlanningReads.push_back(read); + m_tablePlanningMemory.push_back(plan.memory[index]); + marked = true; + } } } } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index ad91c93b142a..936cffc50022 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1882,6 +1882,7 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(table.fault); mix(table.outside ? 1u : 0u); mix(table.nested ? 1u : 0u); + mix(table.pointerKeyed ? 1u : 0u); mix(table.pointers.Hash()); mix(table.codes.Hash()); mix(table.words.Hash()); diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 8bd1f9f279b0..1af80ab4bc94 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -111,8 +111,8 @@ static_assert(sizeof(IrResourcePlan) == 728, "IrResourcePlan changed: update Enc static_assert(sizeof(IrValue) == 144, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); -static_assert(sizeof(DescriptorSource) == 216, "DescriptorSource changed: update the plan encoder"); -static_assert(sizeof(TableColumn) == 112, "TableColumn changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 224, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(TableColumn) == 120, "TableColumn changed: update the plan encoder"); static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder"); static_assert(sizeof(ResourceBlock) == 104, "ResourceBlock changed: update the plan encoder"); static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); @@ -152,7 +152,7 @@ static_assert(fieldCount() == 25, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 32, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 4, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 19, "TableColumn changed: update the plan encoder"); +static_assert(fieldCount() == 20, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); static_assert(fieldCount() == 5, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); @@ -1187,6 +1187,7 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(column.foldedCarry); out.Value(column.selected); out.Value(column.selection); + out.Value(column.pointerKeyed); out.Values(std::span(source.requiredBlocks)); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { @@ -1320,7 +1321,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { value.SetDetachedOperands(std::move(arguments), std::move(phiBlocks)); } reader.List(plan.memoryInfo, 79, decodeMemoryInfo); - reader.List(plan.descriptorSources, 81, [&](Reader& in, DescriptorSource& source) { + reader.List(plan.descriptorSources, 82, [&](Reader& in, DescriptorSource& source) { for (auto& dword : source.dwords) dword = valueAt(in.Get()); in.Value(source.dwordCount); const bool table = in.Get(); @@ -1352,6 +1353,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(column.foldedCarry); in.Value(column.selected); in.Value(column.selection); + in.Value(column.pointerKeyed); source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt; in.Values(source.requiredBlocks); }); @@ -1393,7 +1395,8 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { const auto& column = *source.tableColumn; if (column.buffer && (column.sampler || column.dwordCount != 4u)) return false; const auto& parent = plan.descriptorSources[column.heapSource].tableColumn; - if (parent.has_value() && (column.buffer || column.sampler || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (parent.has_value() && (column.buffer || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (column.pointerKeyed && (!column.indirect || column.buffer || parent.has_value())) return false; } if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false; if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false; diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 3e43cbb22130..c43427d76d55 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 56; +inline constexpr std::uint32_t FormatVersion = 71; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index b1a94032f7bd..f698931cd306 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -201,6 +201,7 @@ struct SpirvEmitterState { std::uint32_t size = 0; std::uint32_t device = 0; std::uint32_t offset = 0; + std::uint32_t dwordOffset = 0; }; std::vector bdaAnchors; std::uint32_t bdaAnchorFunction = 0; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index 9730b91a5c46..9140f5d3721f 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -482,10 +482,16 @@ std::uint32_t EmitElectedBdaRange(SpirvEmitterState& state, std::uint32_t type, return result; } -std::uint32_t BdaAnchorHolds(SpirvEmitterState& state, const SpirvEmitterState::BdaAnchor& anchor, std::uint32_t low, std::int32_t immediate, std::uint32_t bytes, std::uint32_t& translated) { +bool ScalarAddressRead(const SpirvEmitterState& state, const IrValue& inst) { + const auto index = inst.Flags().index; + const auto& memory = state.program.Resources().memoryInfo; + return index < memory.size() && memory[index].kind == ResourceKind::ScalarAddress; +} + +std::uint32_t BdaAnchorHolds(SpirvEmitterState& state, const SpirvEmitterState::BdaAnchor& anchor, std::uint32_t anchorOffset, std::uint32_t low, std::int32_t immediate, std::uint32_t bytes, std::uint32_t& translated) { const auto u64 = TypeScalarU64(state); const auto boolean = TypeBool(state); - auto offset = Binary(state, spv::OpIAdd, u64, anchor.offset, Unary(state, spv::OpUConvert, u64, low)); + auto offset = Binary(state, spv::OpIAdd, u64, anchorOffset, Unary(state, spv::OpUConvert, u64, low)); if (immediate != 0) offset = Binary(state, spv::OpIAdd, u64, offset, BdaConstant(state, static_cast(static_cast(immediate)))); const auto width = BdaConstant(state, bytes); const auto fits = Binary(state, spv::OpUGreaterThanEqual, boolean, anchor.size, width); @@ -535,6 +541,7 @@ void EmitBdaAnchorRange(SpirvEmitterState& state, SpirvEmitterState::BdaAnchor& const auto end = extract(1u); anchor.device = extract(2u); anchor.offset = Binary(state, spv::OpISub, u64, base, begin); + anchor.dwordOffset = Binary(state, spv::OpISub, u64, Binary(state, spv::OpBitwiseAnd, u64, base, BdaConstant(state, ~std::uint64_t{3})), begin); const auto span = Binary(state, spv::OpISub, u64, end, begin); const auto deviceEnd = Binary(state, spv::OpIAdd, u64, anchor.device, span); auto usable = Binary(state, spv::OpLogicalAnd, boolean, Binary(state, spv::OpINotEqual, boolean, anchor.device, BdaConstant(state, 0u)), Binary(state, spv::OpUGreaterThan, boolean, deviceEnd, anchor.device)); @@ -581,7 +588,8 @@ std::array EmitAnchoredBdaRead(SpirvEmitterState& state, const bytes = (last - first + 1u) * 4u; } std::uint32_t translated = 0; - auto direct = BdaAnchorHolds(state, anchor, low, static_cast(static_cast(immediate) + first * 4u), bytes, translated); + const auto anchorOffset = ScalarAddressRead(state, inst) ? anchor.dwordOffset : anchor.offset; + auto direct = BdaAnchorHolds(state, anchor, anchorOffset, low, static_cast(static_cast(immediate) + first * 4u), bytes, translated); if (bits == 32u) direct = Binary(state, spv::OpLogicalAnd, boolean, direct, Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, translated))); const auto directLabel = state.module.AllocateId(); const auto lookupLabel = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 422dfbd3576a..d232d12793f3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -2198,6 +2198,10 @@ TableWord EmitTableWord(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui const auto offset = ctx.Def(handle->Argument(0)); const auto header = ImageTableAbi::TableHeader(table); const auto& parent = sources.at(column.heapSource).tableColumn; + if (!parent.has_value() && column.pointerKeyed) { + if (!column.indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a pointer-keyed image table without its pointer"); + return LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); + } if (!parent.has_value()) return LookupTableWord(state, column, offset, header); if (!parent->buffer || !parent->indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a nested image table without its buffer pointer"); const auto root = LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 009d1cddde5f..ff82b2a896c9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -190,7 +190,11 @@ std::uint32_t GuestAddressBase(SpirvValueEmitContext& ctx, const IrValue& inst, if (handle == nullptr || handle->Opcode() != IrOpcode::GetAddressResource || handle->ArgumentCount() != 2u) { ctx.Fail(inst, "has no address base pair"); } - const auto base = DeviceAddressFromWords(state, ctx.Arg(*handle, 0), ctx.Arg(*handle, 1)); + auto baseLow = ctx.Arg(*handle, 0); + if (mem.kind == ResourceKind::ScalarAddress) { + baseLow = Binary(state, spv::OpBitwiseAnd, TypeU32(state), baseLow, ConstantU32(state, ~3u)); + } + const auto base = DeviceAddressFromWords(state, baseLow, ctx.Arg(*handle, 1)); return AddBdaAddress(ctx, inst, base, Unary(state, spv::OpUConvert, TypeScalarU64(state), low), false); } diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index fba767a4518f..c75ddd329f43 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -61,8 +61,10 @@ - An [image table](../../../core/shader/recompiler/ImageTableAbi.hpp) entry that cannot be bound exactly reads zeros, and an active lane that selects it raises a BDA fault (reason 9) naming the record's address, words and reason; the dispatch's write-back reports the first one. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) snapshot holds at most 65536 keys per table, 65536 distinct images and 256 S# entries per shader; a key past a longer table's snapshot, or an entry over the budget, faults when selected. A selected column (a T# the SRT walker evaluates) holds its one record wherever it lies. - The [runtime resource snapshot](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) is retried up to three times, with pending descriptor writes forced on the last attempt. Capture throws `runtime resource snapshot did not stabilize` if descriptor-scope changes while snapshotting tables, or mapping changes that the snapshot-validity callback cannot accept, disturb every attempt. A console GPU reads descriptors during execution and has no equivalent host snapshot. Code Violet (PPSA26528) hit this refusal on older main `128b5c49ba`; its logs showed unrelated allocation churn, whose handling was later narrowed. Persistently unstable descriptors or captured views still fail; console-equivalent behavior is unmeasured. -- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. Other origins, shared V# consumers, T# words read by anything but their image handle, nested sampler columns and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. +- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T# or S#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. A V# may feed several buffer handles whose reads are all words of T#s or S#s planned from it, each at its own offset. A T# or S# word that also feeds another instruction is read through the guest's V# again at dispatch, so it can differ from the snapshot. Other origins, data reads through the V# and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. +- [Nested image table children](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp): a T# reached through a nested child is materialized from its raw words, without registered `resourceViews`, so a 1D array past `maxImageArrayLayers` sampled through one keeps its raw descriptor. - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read by `s_load` through a 64-bit pointer hold the keys the key's known bits allow, at most 65536; a key past them, or an entry at a wrapping, unaligned or unmapped address, faults when selected. +- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) T# or S# read through a pointer that a scalar-buffer record holds may have that pointer's low word merged by a Phi with constants, with lane masks (words of a `Ballot`, the waterfall loop's lane bit, and ANDs, ORs and NOTs of them), with the record word ANDed with a lane mask, and with reads of the same record word under another key, and its high word merged the same way from the record's next word. The two words are walked together, edge by edge, and an edge whose low word is a constant or a lane mask is skipped along with its high word. The snapshot then captures the pointer of every record, at most 65536, and the GPU looks up the actual 64-bit pointer; one that was not captured, such as a lane mask, faults when selected. Other origins of the two words remain refused, among them a high word read from a second buffer on an edge whose low word is the record word. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask` with such a key: a key domain would select records by the unmasked key, so that key is not narrowed. A key the walker evaluates selects its one record instead, unless the shader may write the memory the key is read from at run time (the same buffer, or any memory when the read or a write goes through a pointer or a GPU-only descriptor); then the column binds the whole domain too. A write to the key's memory through another V# over the same bytes is not detected. A shader whose selected records would pass 64 images or 256 sampled image/sampler pairs binds, for the heaps with the most such reads, the union of their keys in one table instead. - A shader plans at most 64 [samplers](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (`ShaderInfo::MaxSamplers`, the size of the runtime ABI's `ShaderData` sampler metadata, which [`BindingAllocator`](../../../core/shader/recompiler/Optimization/src/BindingAllocator.cpp) holds every shader to), at most 32 of them direct (each takes two elements of the 64-element sampler heap), and at most 256 image/sampler pairs (`ShaderInfo::MaxSampledPairs`); past them it refuses with `sampler resource limit exceeded`, `shader sampler pairs exceed runtime heap capacity` or `sampled image/sampler pair limit exceeded`. The image-to-sampler masks are 64-bit, one bit per sampler heap element or per sampler, so more samplers need a larger `ShaderData` and wider masks. Ratchet & Clank: Rift Apart (PPSA01474) plans up to 63 samplers and 112 pairs in one shader. - [Image table pointer indexing](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) propagates 32-bit value bounds through `ReadFirstLane` and `SelectU32` so waterfall key masks keep their bit domain. The SRT walker validates runtime select conditions whose active lane mask implies the condition through `LogicalAnd` conjunctions, assuming unselected inactive lanes need not be uniform across the full wave. The same assumption lets the two `s_load`s of one T# record match when one offset is added before `v_readfirstlane` and the other after (`ReadFirstLane(Select(mask, (key << 5) + 16, old), mask)` against `ReadFirstLane(key, mask) << 5`): with no active lane the hardware reads lane 0 of the inactive arm, and the table still binds the whole record.