diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index f77351ffabfc..7a8ccb10bfb2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1899,6 +1899,35 @@ std::vector planFixtures() { nestedCode.insert(nestedCode.begin() + 15u, 0x87148114u); add("nested pointer image table", 64u, std::move(nestedCode), srtPointer, {{PlanSrtAddress, nestedSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedImages}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); fixtures.back().groupId = true; + std::vector samplerSrt(20u, 0u); + std::copy(nestedSrt.begin(), nestedSrt.end(), samplerSrt.begin()); + place(samplerSrt, 12u, imageWords(PlanTextureAddress)); + const std::vector pointClamp{0x92u, 0x400000u, 0u, 0u}; + const std::vector pointWrap{0u, 0x400000u, 1u << 24u, 0u}; + std::vector nestedSamplers(32u, 0u); + place(nestedSamplers, 0u, pointWrap); + place(nestedSamplers, 4u, pointClamp); + place(nestedSamplers, 16u, pointClamp); + place(nestedSamplers, 20u, pointWrap); + std::vector samplerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + samplerCode.insert(samplerCode.begin() + 13u, 0x87118311u); + samplerCode.insert(samplerCode.begin() + 15u, 0x87148114u); + add("nested pointer sampler table", 64u, std::move(samplerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedSamplers}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); + fixtures.back().groupId = true; + std::vector pointees(128u, 0u); + place(pointees, 8u, pointWrap); + place(pointees, 72u, pointClamp); + std::vector pointerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + pointerCode.insert(pointerCode.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + add("pointer phi sampler table", 64u, std::move(pointerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, pointees}, {PlanMaterialAddress, {1u, 0u, 0u, 0u, 1u, 0u, 0u, 0u}}}); + fixtures.back().groupId = true; PlanFixture fragment; fragment.name = "fragment parameters"; @@ -2291,6 +2320,7 @@ void verifyMalformedPlans() { rejects("an image table cycle", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->heapSource = 1u; }); rejects("an image table parent that is not a buffer", [](IrResourcePlan& plan) { plan.descriptorSources[2].tableColumn = *plan.descriptorSources[1].tableColumn; }); rejects("an image table selection past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->selection = 4u; }); + rejects("a pointer-keyed table that is not indirect", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->pointerKeyed = true; }); auto foreign = handBuiltPlan(); IrValue stray(IrOpcode::Void, IrType::U32, 99u); foreign.srtReads[0].value = &stray; @@ -2350,6 +2380,21 @@ void verifyPlanRoundTrip() { const auto bufferBytes = encodedPlan(bufferTable); IrResourcePlan bufferDecoded; require(bufferBytes != handBytes && ShaderDiskCache::DecodePlan(bufferBytes, bufferDecoded) && samePlan(bufferTable, bufferDecoded), "the nested buffer column changed in a disk-cache round trip"); + auto pointerTable = handBuiltPlan(); + pointerTable.descriptorSources[1].tableColumn->indirect = true; + pointerTable.descriptorSources[1].tableColumn->pointerKeyed = true; + const auto pointerBytes = encodedPlan(pointerTable); + IrResourcePlan pointerDecoded; + require(pointerBytes != handBytes && ShaderDiskCache::DecodePlan(pointerBytes, pointerDecoded) && samePlan(pointerTable, pointerDecoded) && pointerDecoded.descriptorSources[1].tableColumn->pointerKeyed, "the pointer-keyed column changed in a disk-cache round trip"); + auto chaseTable = handBuiltPlan(); + auto& chaseColumn = *chaseTable.descriptorSources[1].tableColumn; + chaseColumn.indirect = true; + chaseColumn.pointerKeyed = true; + chaseColumn.pointerChaseDepth = 1u; + chaseColumn.pointerChases = {PointerChase{8u, 16u, 4u, 24u, 40u, {}}}; + const auto chaseBytes = encodedPlan(chaseTable); + IrResourcePlan chaseDecoded; + require(chaseBytes != pointerBytes && ShaderDiskCache::DecodePlan(chaseBytes, chaseDecoded) && samePlan(chaseTable, chaseDecoded) && chaseDecoded.descriptorSources[1].tableColumn->pointerChases == chaseColumn.pointerChases, "a pointer chase whose high word has its own immediate changed in a disk-cache round trip"); PlanFixture handFixture; handFixture.userData = {static_cast(PlanSrtAddress), static_cast(PlanSrtAddress >> 32u), 0u, 0u}; std::vector handSrt(16, 0u); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index effb0b5af54f..f449710032f8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -3828,12 +3828,19 @@ void verifyRegionMerge(bool strided = false, bool benchmark = false) { } -void verifyNestedSnapshotRetry() { +void verifyNestedSnapshotRetry(bool sampler) { using namespace ShaderRecompiler; constexpr std::uint32_t Parents = ImageTableAbi::MaxParentKeys / 2u + 1u; constexpr std::uint32_t Children = ImageTableAbi::MaxNestedKeys / 2u + 1u; - std::vector> leaves(Children, TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u)); - const auto child = TableV(leaves.data(), 32u, Children); + const std::uint32_t leafDwords = sampler ? 4u : 8u; + const auto texture = TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u); + const std::array pointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; + std::vector leaves; + for (std::uint32_t leaf = 0; leaf < Children; ++leaf) { + if (sampler) leaves.insert(leaves.end(), pointClamp.begin(), pointClamp.end()); + else leaves.insert(leaves.end(), texture.begin(), texture.end()); + } + const auto child = TableV(leaves.data(), leafDwords * 4u, Children); std::vector parents(Parents, reinterpret_cast(child.data())); const auto root = TableV(parents.data(), 8u, Parents); std::vector> values; @@ -3859,17 +3866,25 @@ void verifyNestedSnapshotRetry() { plan.descriptorSources[1].tableColumn = parent; TableColumn nested; nested.heapSource = 1u; - nested.stride = 32u; - nested.dwordCount = 8u; + nested.stride = leafDwords * 4u; + nested.dwordCount = leafDwords; + nested.sampler = sampler; plan.descriptorSources[2].tableColumn = nested; - ImageResource image; - image.source = 2u; - image.table = 0u; - image.resourceClass = ImageResourceClass::Sampled; - image.dimension = RdnaImageDimension::Dim2D; - image.read = true; - plan.info.images.push_back(image); - plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + if (sampler) { + SamplerResource resource; + resource.source = 2u; + resource.table = 0u; + plan.info.samplers.push_back(resource); + } else { + ImageResource image; + image.source = 2u; + image.table = 0u; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2D; + image.read = true; + plan.info.images.push_back(image); + plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + } struct Memory { std::array, 3> ranges; std::uint64_t epoch = 0; @@ -11523,10 +11538,15 @@ int main(int argc, char** argv) { return 0; } if (argc == 2 && std::string_view(argv[1]) == "--nested-snapshot-retry") { - verifyNestedSnapshotRetry(); + verifyNestedSnapshotRetry(false); std::cout << "Nested snapshot retry preserves parent and child coverage\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--nested-sampler-snapshot-retry") { + verifyNestedSnapshotRetry(true); + std::cout << "Nested sampler snapshot retry checks the descriptor epoch and preserves parent and child coverage\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake index 2e76d01ccf28..e678a225e907 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake @@ -4,8 +4,24 @@ if(BUILD_TESTING) add_test(NAME agc_driver_nested_image_table COMMAND agc_driver_image_table_tests --nested) add_test(NAME agc_driver_nested_image_table_large COMMAND agc_driver_image_table_tests --nested-large) add_test(NAME agc_driver_nested_image_table_budget COMMAND agc_driver_image_table_tests --nested-budget) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_nested_sampler_table COMMAND agc_driver_image_table_tests --nested-sampler) + add_test(NAME agc_driver_pointer_phi_sampler_table COMMAND agc_driver_image_table_tests --pointer-phi-sampler) + add_test(NAME agc_driver_pointer_phi_ballot_table COMMAND agc_driver_image_table_tests --pointer-phi-ballot) + add_test(NAME agc_driver_pointer_phi_copies_table COMMAND agc_driver_image_table_tests --pointer-phi-copies) + add_test(NAME agc_driver_pointer_phi_lane_and_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-and) + add_test(NAME agc_driver_pointer_phi_high_table COMMAND agc_driver_image_table_tests --pointer-phi-high) + add_test(NAME agc_driver_pointer_phi_high_second_lane_table COMMAND agc_driver_image_table_tests --pointer-phi-high-second-lane) + add_test(NAME agc_driver_pointer_phi_lane_not_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-not) + add_test(NAME agc_driver_pointer_vsharp_select_table COMMAND agc_driver_image_table_tests --pointer-vsharp-select) + add_test(NAME agc_driver_pointer_hop_table COMMAND agc_driver_image_table_tests --pointer-hop) + add_test(NAME agc_driver_pointer_chase_table COMMAND agc_driver_image_table_tests --pointer-chase) + add_test(NAME agc_driver_pointer_chase_split_table COMMAND agc_driver_image_table_tests --pointer-chase-split) + add_test(NAME agc_driver_pointer_chase_words_table COMMAND agc_driver_image_table_tests --pointer-chase-words) + add_test(NAME agc_driver_pointer_chase_edited_table COMMAND agc_driver_image_table_tests --pointer-chase-edited) + add_test(NAME agc_driver_pointer_vsharp_fixed_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed) + add_test(NAME agc_driver_pointer_vsharp_fixed_offset_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed-offset) + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_chase_table agc_driver_pointer_chase_split_table agc_driver_pointer_chase_edited_table agc_driver_pointer_chase_words_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_chase_table agc_driver_pointer_chase_split_table agc_driver_pointer_chase_edited_table agc_driver_pointer_chase_words_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake index 4460a3c25a1e..a5a52685d5ae 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake @@ -32,8 +32,10 @@ if(BUILD_TESTING) endif() add_test(NAME agc_shader_nested_snapshot_retry COMMAND agc_shader_memory_tests --nested-snapshot-retry) set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES TIMEOUT 20) + add_test(NAME agc_shader_nested_sampler_snapshot_retry COMMAND agc_shader_memory_tests --nested-sampler-snapshot-retry) + set_tests_properties(agc_shader_nested_sampler_snapshot_retry PROPERTIES TIMEOUT 20) if(WIN32) - set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_nested_snapshot_retry agc_shader_nested_sampler_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse) add_test(NAME agc_shader_region_merge COMMAND agc_shader_memory_tests --region-merge) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index be41a4c540ed..3959d6a13554 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -322,6 +322,195 @@ alignas(256) constexpr std::array NestedRecordCode{ 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, }; +alignas(256) constexpr std::array NestedSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerImageCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf40c0916u, 0xfa000040u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerBallotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820009u, 0xbf068212u, + 0xbf840002u, 0xbeac0334u, 0xbf820004u, 0xbf068312u, 0xbf840001u, 0xbeac0390u, 0xbf800000u, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerCopiesCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf82000eu, 0xbf068212u, 0xbf840007u, 0x3a160613u, + 0x7e2c050bu, 0x8f178416u, 0xf4200b02u, 0x2e000008u, 0xbf8cc07fu, 0xbf820004u, 0xbf068312u, 0xbf840001u, + 0xbeac0390u, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffdeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneAndCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, + 0xbeac03ffu, 0x000000eau, 0xbf820010u, 0xbf068212u, 0xbf840006u, 0xbebc0334u, 0xbf068115u, 0xbf840001u, + 0xbebc0336u, 0x872c343cu, 0xbf820007u, 0xbf068312u, 0xbf840004u, 0xbf068215u, 0xbf840001u, 0xbeac03a0u, + 0x872c342cu, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffd8u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820009u, 0xbf068213u, + 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffdbu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820012u, 0xbf068313u, + 0xbf840007u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbf820009u, + 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, + 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffd2u, + 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondLaneCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820013u, 0xbf068313u, + 0xbf840008u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbeac037eu, + 0xbf820009u, 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, + 0xbf8cc07fu, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, + 0xbf89ffd1u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, + 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneNotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, 0x32000000u, 0xd4c10034u, + 0x00002900u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, 0xbeac0380u, 0xbf82000au, 0xbf068212u, 0xbf840008u, + 0xbebc1334u, 0x8f3d3c81u, 0xbf068115u, 0xbf840002u, 0x8a2c3d34u, 0xbf820002u, 0xbebd073du, 0x872c3d34u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerVsharpSelectCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0xd4c40038u, 0x000200c0u, 0x87ea386au, + 0x7e120215u, 0x4a141210u, 0x7e080211u, 0xbf068116u, 0xbf840001u, 0x7e080217u, 0x02061504u, 0xbeae047eu, + 0xd4c5003eu, 0x000200c0u, 0x87bc3e38u, 0xbeba243cu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, + 0x8f198418u, 0xf4240b02u, 0x32000008u, 0xbf8cc07fu, 0xf4080d16u, 0xfa0000a0u, 0x8f1a8513u, 0xbf8cc07fu, + 0xf42c091au, 0x34000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffedu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerHopCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820003u, 0xbf068212u, + 0xbf840001u, 0xbeac0334u, 0xf4040e16u, 0xfa000000u, 0xbf8cc07fu, 0xf40c091cu, 0xfa000340u, 0xbf8cc07fu, + 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe4u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerChaseCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf0d8012u, 0xbf840006u, 0xf4080f16u, 0xfa000060u, 0xbf8cc07fu, 0xf4240b1eu, 0xfa000000u, 0xbf8cc07fu, + 0xbf0d8112u, 0xbf840006u, 0xf4081016u, 0xfa000060u, 0xbf8cc07fu, 0xf4240b20u, 0xfa000000u, 0xbf8cc07fu, + 0xf40c0916u, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerChaseSplitCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf0d8012u, 0xbf840006u, 0xf4080f16u, 0xfa000060u, 0xbf8cc07fu, 0xf4240b1eu, 0xfa000000u, 0xbf8cc07fu, + 0xbf0d8112u, 0xbf840006u, 0xf4081016u, 0xfa000060u, 0xbf8cc07fu, 0xf4240b20u, 0xfa000000u, 0xbf8cc07fu, + 0xbf0d8212u, 0xbf840008u, 0xf4081116u, 0xfa000110u, 0xbf8cc07fu, 0xf4301222u, 0xfa000040u, 0xbf8cc07fu, + 0xbeac034du, 0xbead0353u, 0xf40c0916u, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, + 0xbf89ffd7u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, + 0xbf810000u, +}; +alignas(256) constexpr std::array PointerChaseWordsCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080f16u, 0xfa000060u, 0xbf8cc07fu, 0xbf0d8012u, 0xbf840004u, 0xf4240b1eu, 0xfa000000u, 0xbf8cc07fu, + 0xbf820005u, 0xbf0d8112u, 0xbf840003u, 0xf4240b1eu, 0xfa000008u, 0xbf8cc07fu, 0xf40c0916u, 0xfa000000u, + 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe3u, 0xbefe042eu, 0x7e120202u, 0x34101286u, + 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerChaseEditedCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf0d8012u, 0xbf84000bu, 0xf4080f16u, 0xfa000030u, 0xbf8cc07fu, 0x873dff3du, 0x0000ffffu, 0x8f3e843eu, + 0x883fff3fu, 0x30000000u, 0xf4240b1eu, 0xfa000030u, 0xbf8cc07fu, 0xf40c0916u, 0xfa000000u, 0xbf8cc07fu, + 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe4u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpOffsetCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + constexpr std::array PointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; constexpr std::array LinearWrap{0u, (4u * 256u) << 12u, (1u << 20u) | (1u << 22u), 0u}; constexpr std::array PointWrap{0u, (4u * 256u) << 12u, 1u << 24u, 0u}; @@ -409,6 +598,7 @@ struct Outcome { std::vector> samplers; std::vector> samplerTable; std::vector tableMap; + std::vector> imageTableRanges; std::vector>> imageTableReadRanges; }; @@ -436,6 +626,7 @@ Outcome Execute(AgcDriver::VulkanDevice& device, std::span outcome.imageTableFaults = result.imageTableFaults; outcome.poisonedSrtReads = result.poisonedSrtReads; outcome.cacheable = AgcDriver::DriverDetail::CacheableResult(result); + outcome.imageTableRanges = result.imageTableRanges; outcome.imageTableReadRanges = result.imageTableReadRanges; for (const auto& binding : result.bindings) { if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) outcome.imageTableCount = binding.count; @@ -1630,6 +1821,1066 @@ void RunPointerChainRecordTests(AgcDriver::VulkanDevice& device) { RequireRed(relinked, 1, Lanes, texels[0], "float texels after record 1's second pointer moved to another T#"); } +void RunNestedSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(512) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + std::vector extra; + const std::array, 2>, 2> samplers{{{PointWrap, PointClamp}, {PointClamp, PointWrap}}}; + Table = {}; + Keys = {}; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(samplers[outer][inner].begin(), samplers[outer][inner].end(), palettes[outer].begin() + inner * 4u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const auto image = Image(FloatTexels.data(), Format32Float); + const auto run = [&](std::span code = NestedSamplerCode, std::uint32_t waveSize = 64u) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(image.begin(), image.end(), Srt.begin() + 12u); + std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + if (!extra.empty()) memory.push_back({reinterpret_cast(extra.data()), std::as_bytes(std::span(extra))}); + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + const auto inputPalettes = palettes; + const auto inputExtra = extra; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords && palettes == inputPalettes && extra == inputExtra, "nested sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "nested sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [&](std::uint32_t record, std::uint32_t child) { + const bool wrap = samplers[record % 2u][child] == PointWrap; + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}; + std::copy(keys.begin(), keys.end(), Keys.begin()); + const auto original = run(); + Require(!Faulted(original), "nested sampler table: valid independent keys faulted:\n" + original.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(original, group, Lanes, sampled(keys[group * 2u], keys[group * 2u + 1u]), "nested sampler table: the sampler selected through record, pointer and V#"); + const auto wave32 = run(NestedSamplerCode, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "nested sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), palettes[1].begin()); + const auto edited = run(); + Require(!Faulted(edited) && Group(edited, 2) == Group(original, 0) && Group(edited, 3) == Group(original, 3), "nested sampler table: changing only a child S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "nested sampler table: editing a child S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), palettes[1].begin()); + Keys[0] = 3u; + const auto unmapped = run(); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "nested sampler table: an unreadable parent did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "nested sampler table: a faulting dispatch published copied output"); + Keys[0] = 0u; + records[0][43] |= 1u << 30u; + const auto notBuffer = run(); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "nested sampler table: a non-buffer parent was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "nested sampler table: a non-buffer parent published copied output"); + records[0][43] &= ~(1u << 30u); + const auto savedRecords = records; + std::copy_n(records[0].begin() + 40u, 4u, records[1].begin() + 40u); + const auto moved = run(); + Require(!Faulted(moved) && Group(moved, 2) == Group(original, 0) && Group(moved, 3) == Group(original, 1), "nested sampler table: replacing the nested V# retained a stale sampler"); + Require(moved.artifactId == original.artifactId && moved.tableMap.size() < original.tableMap.size(), "nested sampler table: equal V#s did not share one child table"); + records = savedRecords; + auto phi = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + phi.insert(phi.begin() + 17u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = 1u; + const auto phiValid = run(phi); + Require(!Faulted(phiValid), "nested sampler table: a captured pointer selected through Phi faulted:\n" + phiValid.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(phiValid, group, Lanes, sampled(1u, Keys[group * 2u + 1u]), "nested sampler table: Phi-selected pointer"); + Keys[0] = 0u; + const auto phiFault = run(phi); + Require(Faulted(phiFault) && phiFault.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured pointer did not fault:\n" + phiFault.log); + Require(untouched(phiFault), "nested sampler table: an uncaptured pointer published copied output"); + Require(phiFault.artifactId == phiValid.artifactId, "nested sampler table: the pointer choice was specialized on the CPU"); + std::copy(keys.begin(), keys.end(), Keys.begin()); + extra.assign(320u * 4u, 0u); + for (std::uint32_t entry = 0; entry < 320u; ++entry) { + std::copy(PointWrap.begin(), PointWrap.end(), extra.begin() + entry * 4u); + extra[entry * 4u + 2u] |= entry; + } + const auto many = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(many.begin(), many.end(), records[0].begin() + 40u); + Keys[1] = 319u; + auto unkeyed = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + unkeyed.insert(unkeyed.begin() + 14u, {0x8714ff14u, 0x000001ffu}); + const auto overBudget = run(unkeyed); + Require(Faulted(overBudget) && overBudget.log.find("over budget") != std::string::npos, "nested sampler table: an S# past the sampler budget did not fault:\n" + overBudget.log); + Require(untouched(overBudget), "nested sampler table: an S# past the sampler budget published copied output"); + extra.assign((NestedKeyLimit + 1u) * 4u, 0u); + std::copy(PointClamp.begin(), PointClamp.end(), extra.begin()); + const auto large = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(large.begin(), large.end(), records[0].begin() + 40u); + Keys[1] = 0u; + const auto bounded = run(); + Require(Faulted(bounded) && bounded.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured child did not fault:\n" + bounded.log); + Require(bounded.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: the two levels produced an unbounded map"); + Require(untouched(bounded), "nested sampler table: a dispatch outside the snapshot published copied output"); + std::copy(large.begin(), large.end(), records[1].begin() + 40u); + const auto sharedBudget = run(); + Require(!Faulted(sharedBudget) && sharedBudget.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: equal V#s consumed the snapshot budget twice"); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool clamp = keys[group * 2u + 1u] == 0u; + RequireRed(sharedBudget, group, Lanes, [clamp](std::uint32_t x) { return std::bit_cast(FloatTexels[clamp ? Width - 1u : x]); }, "nested sampler table: aliased parent at the snapshot budget"); + } + std::puts("nested sampler table: S# through record, pointer and V# checked on wave64 and wave32, with sharing and budgets"); +} + +void RunPointerSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + const std::array, 2> samplers{PointWrap, PointClamp}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) { + std::copy(samplers[record].begin(), samplers[record].end(), records[record].begin() + 8u); + std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + auto root = images[0]; + std::uint32_t tableType = 0u; + const auto run = [&](std::span code, std::uint32_t waveSize = 64u) { + auto table = Buffer(Table.data(), 64u); + table[3] |= tableType; + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(root.begin(), root.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords, "pointer sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [](bool wrap) { + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{1u, 2u, 1u, 2u}; + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = keys[group]; + const auto direct = run(PointerSamplerCode); + Require(!Faulted(direct), "pointer sampler table: an S# read through a record's pointer faulted:\n" + direct.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(direct, group, Lanes, sampled(keys[group] % 2u == 0u), "pointer sampler table: the S# read through the record's pointer"); + auto phi = std::vector(PointerSamplerCode.begin(), PointerSamplerCode.end()); + phi.insert(phi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto original = run(phi); + Require(!Faulted(original) && original.words == direct.words, "pointer sampler table: a captured pointer selected through Phi changed the result:\n" + original.log); + const auto wave32 = run(phi, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "pointer sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), records[1].begin() + 8u); + const auto edited = run(phi); + Require(!Faulted(edited) && Group(edited, 0) == Group(original, 1) && Group(edited, 2) == Group(original, 3), "pointer sampler table: changing an S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "pointer sampler table: editing an S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), records[1].begin() + 8u); + Keys[0] = 0u; + const auto uncaptured = run(phi); + Require(Faulted(uncaptured) && uncaptured.log.find("outside the snapshot") != std::string::npos, "pointer sampler table: an uncaptured pointer did not fault:\n" + uncaptured.log); + Require(untouched(uncaptured), "pointer sampler table: an uncaptured pointer published copied output"); + Require(uncaptured.artifactId == original.artifactId, "pointer sampler table: the pointer choice was specialized on the CPU"); + auto alias = phi; + alias[17] = static_cast(reinterpret_cast(records[1].data())); + const auto aliased = run(alias); + Require(!Faulted(aliased) && Group(aliased, 0) == Group(original, 0) && Group(aliased, 1) == Group(original, 1), "pointer sampler table: a Phi constant naming another captured pointer did not read that pointer's S#:\n" + aliased.log); + Keys[0] = 3u; + const auto unmapped = run(phi); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "pointer sampler table: a null pointer did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "pointer sampler table: a null pointer published copied output"); + Keys[0] = 1u; + tableType = 1u << 30u; + const auto notBuffer = run(phi); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "pointer sampler table: a non-buffer record table was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "pointer sampler table: a non-buffer record table published copied output"); + tableType = 0u; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + { + const Registration registered(records.data(), sizeof(records)); + auto consumer = phi; + consumer.insert(consumer.begin() + 30u, 0x7e0e0221u); + const auto shared = run(consumer); + Require(!Faulted(shared), "pointer sampler table: an S# with another consumer faulted:\n" + shared.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + auto expected = Group(original, group); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) expected[lane * 4u + 3u] = samplers[keys[group] % 2u][1]; + Require(Group(shared, group) == expected, "pointer sampler table: an S# with another consumer read the wrong words"); + } + } + root = PointWrap; + auto imagePhi = std::vector(PointerImageCode.begin(), PointerImageCode.end()); + imagePhi.insert(imagePhi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto imageDirect = run(PointerImageCode); + const auto imageMerged = run(imagePhi); + Require(!Faulted(imageDirect) && !Faulted(imageMerged) && imageMerged.words == imageDirect.words, "pointer image table: a captured pointer selected through Phi changed the result:\n" + imageMerged.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool other = keys[group] % 2u != 0u; + RequireRed(imageMerged, group, Lanes, [other](std::uint32_t x) { return std::bit_cast(other ? OtherFloatTexels[x] : FloatTexels[x]); }, "pointer image table: the T# read through a Phi-selected pointer"); + } + Keys[0] = 0u; + const auto imageFault = run(imagePhi); + Require(Faulted(imageFault) && imageFault.log.find("outside the snapshot") != std::string::npos && untouched(imageFault), "pointer image table: an uncaptured pointer did not fault:\n" + imageFault.log); + std::puts("pointer sampler table: S# and T# through a record pointer whose low word passes a Phi checked on wave64 and wave32"); +} + +void RunPointerArmTests(AgcDriver::VulkanDevice& device, bool copies) { + Fill(); + alignas(256) std::array, 2> records{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::span code = copies ? std::span(PointerCopiesCode) : std::span(PointerBallotCode); + const std::string name = copies ? "pointer table with a second record read" : "pointer table with a lane-mask arm"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group, std::uint32_t flip) { + return std::function([&keys, group, flip](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u] ^ flip; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group, 0u), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(3u, "a second constant pointer word"); + if (!copies) { + outside(2u, "a lane-mask pointer word"); + std::puts("pointer table with a lane-mask arm: a T# through a record pointer whose low word passes nested Phis with constants and a Ballot word checked, and the Ballot word faults outside the snapshot"); + return; + } + Keys[2] = 2u; + const auto again = run(); + Require(!Faulted(again) && again.words == taken.words, name + ": a second read of the same record changed the result:\n" + again.log); + Keys[3] = 3u; + const auto flipped = run(); + Keys[2] = 0u; + Keys[3] = 0u; + Require(!Faulted(flipped) && flipped.artifactId == taken.artifactId, name + ": a second read of another record faulted or recompiled:\n" + flipped.log); + RequireRed(flipped, 0u, Lanes, sampled(0u, 3u), name + ": the T# behind the other record's pointer word"); + for (std::uint32_t group = 1; group < 4u; ++group) Require(Group(flipped, group) == Group(taken, group), name + ": a group that kept its record changed its result"); + std::puts("pointer table with a second record read: a T# through a record pointer whose low word passes a second read keyed by another lane checked, by the pointer's value"); +} + +enum class PointerMask { LaneAnd, High, HighSecond, HighSecondLane, LaneNot }; + +void RunPointerMaskTests(AgcDriver::VulkanDevice& device, PointerMask shape) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array second{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + const std::uint32_t pointerWord = shape == PointerMask::LaneNot ? 0u : 2u; + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + pointerWord] = static_cast(pointer); + Table[record * 4u + pointerWord + 1u] = static_cast(pointer >> 32u); + } + const auto secondBuffer = Buffer(second.data() + 8u, 32u); + std::copy(secondBuffer.begin(), secondBuffer.end(), second.begin()); + std::span code = shape == PointerMask::LaneAnd ? std::span(PointerLaneAndCode) : shape == PointerMask::High ? std::span(PointerHighCode) : shape == PointerMask::LaneNot ? std::span(PointerLaneNotCode) : shape == PointerMask::HighSecondLane ? std::span(PointerHighSecondLaneCode) : std::span(PointerHighSecondCode); + const std::string name = shape == PointerMask::LaneAnd ? "pointer table with lane-mask ANDs" : shape == PointerMask::High ? "pointer table with a high-word Phi" : shape == PointerMask::LaneNot ? "pointer table with a lane bit cleared from a lane mask" : shape == PointerMask::HighSecondLane ? "pointer table with a second-buffer high word under a lane-mask low word" : "pointer table with a high word from a second buffer"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + const auto secondAddress = reinterpret_cast(second.data()); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + Srt[16] = static_cast(secondAddress); + Srt[17] = static_cast(secondAddress >> 32u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {secondAddress, std::as_bytes(std::span(second))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group) { + return std::function([&keys, group](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t selector, std::uint32_t arm, std::uint32_t variant, const std::string& what) { + Keys[selector] = arm; + Keys[5] = variant; + const auto faulted = run(); + Keys[selector] = 0u; + Keys[5] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(2u, 1u, 0u, "a constant low word"); + if (shape == PointerMask::LaneAnd) { + outside(2u, 2u, 0u, "a lane-mask Phi ANDed with a lane mask"); + outside(2u, 2u, 1u, "the other lane-mask word ANDed with a lane mask"); + outside(2u, 3u, 0u, "a record word ANDed with a lane mask"); + std::puts("pointer table with lane-mask ANDs: a T# through a record pointer whose low word passes Phis of lane-mask ANDs checked, and every masked word faults outside the snapshot"); + return; + } + if (shape == PointerMask::LaneNot) { + outside(2u, 2u, 0u, "a lane mask whose handled lane's bit s_not and s_and cleared"); + outside(2u, 2u, 1u, "a lane mask whose handled lane's bit s_andn2 cleared"); + std::puts("pointer table with a lane bit cleared from a lane mask: a T# through a record pointer whose low word passes Phis of a zero and of a lane mask without the handled lane's bit checked, and every masked word faults outside the snapshot"); + return; + } + outside(3u, 1u, 0u, "a constant high word"); + if (shape == PointerMask::HighSecondLane) outside(3u, 3u, 0u, "a lane-mask low word with a high word from the second buffer"); + Keys[3] = 2u; + const auto again = run(); + Keys[5] = 3u; + const auto flipped = run(); + Keys[3] = 0u; + Keys[5] = 0u; + Require(!Faulted(again) && again.words == taken.words && again.artifactId == taken.artifactId, name + ": a second read of the same record's high word changed the result:\n" + again.log); + Require(!Faulted(flipped) && flipped.words == taken.words && flipped.artifactId == taken.artifactId, name + ": the high word of the neighbouring record changed the result:\n" + flipped.log); + if (shape == PointerMask::HighSecondLane) { + std::puts("pointer table with a second-buffer high word under a lane-mask low word: a T# through a record pointer whose high word reads a second buffer only on the edge whose low word is a lane mask checked, and that pair faults outside the snapshot"); + return; + } + std::puts("pointer table with a high-word Phi: a T# through a record pointer whose high word passes a Phi of reads under other keys and a constant checked, by the pointer's value"); +} + +void RunPointerVsharpTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float), Image(UintTexels.data(), Format32UInt)}; + const std::array, 3> texels{ + [](std::uint32_t x) { return std::bit_cast(FloatTexels[x]); }, + [](std::uint32_t x) { return std::bit_cast(OtherFloatTexels[x]); }, + [](std::uint32_t x) { return UintTexels[x]; }, + }; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(images[outer + inner].begin(), images[outer + inner].end(), palettes[outer].begin() + inner * 8u); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerVsharpSelectCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, "pointer V# table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer V# table: dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u, 0u}, {2u, 1u, 1u}, {1u, 1u, 1u}, {2u, 2u, 0u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + Keys[group * 8u + 3u] = keys[group][2]; + } + std::array targets{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return texels[targets[record] + keys[group][2]](x); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), "pointer V# table: the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), "pointer V# table: the T# read through the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + targets[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), "pointer V# table: the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, "pointer V# table: moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), "pointer V# table: the T# after record 1's pointer moved to the other V#"); + std::puts("pointer V# table: a T# read through a V# loaded through a record pointer keyed by a first-lane read of a select checked, by the records' values"); +} + +void RunPointerHopTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> links{}; + alignas(256) std::array, 2> targets{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto link = [&](std::uint32_t from, std::uint32_t to) { + const auto pointer = reinterpret_cast(targets[to].data()); + links[from][0] = static_cast(pointer); + links[from][1] = static_cast(pointer >> 32u); + }; + for (std::uint32_t target = 0; target < 2u; ++target) { + std::copy(images[target].begin(), images[target].end(), targets[target].begin() + 0x340u / 4u); + link(target, target); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(links[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::string name = "pointer table with a pointer hop"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerHopCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + {reinterpret_cast(targets.data()), std::as_bytes(std::span(targets))}, + }; + const auto inputTable = Table; + const auto inputLinks = links; + const auto inputTargets = targets; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && links == inputLinks && targets == inputTargets, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array targetOf{0u, 1u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(targetOf[record % 2u] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# behind the record pointer's hop faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the pointer loaded from the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(2u, "a lane-mask pointer word"); + link(1u, 0u); + targetOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved hop faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving the pointer behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after the pointer behind record 1's pointer moved to the other T#"); + std::puts("pointer table with a pointer hop: a T# read through a pointer loaded from a record pointer whose low word passes Phis with a constant and a Ballot word checked, by the records' values"); +} + +void RunPointerChaseTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 4> nodes{}; + alignas(256) std::array, 4> links{}; + alignas(256) std::array unsnapshotted{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const std::array imageOf{0u, 1u, 0u, 1u}; + std::array next{}; + const auto address = [](const void* data) { return static_cast(reinterpret_cast(data)); }; + const auto chase = [&](std::uint32_t from, std::uint32_t to, std::uint32_t* link) { + link[0] = static_cast(address(nodes[to].data())); + link[1] = static_cast(address(nodes[to].data()) >> 32u); + const auto descriptor = Buffer(link, 8u); + std::copy(descriptor.begin(), descriptor.end(), nodes[from].begin() + 0x60u / 4u); + next[from] = to; + }; + for (std::uint32_t node = 0; node < 4u; ++node) { + std::copy(images[imageOf[node]].begin(), images[imageOf[node]].end(), nodes[node].begin()); + chase(node, node == 3u ? 3u : (node + 1u) % 3u, links[node].data()); + } + const std::array recordNode{2u, 0u, 1u, 2u}; + for (std::uint32_t record = 0; record < 4u; ++record) { + Table[record * 4u + 2u] = static_cast(address(nodes[recordNode[record]].data())); + Table[record * 4u + 3u] = static_cast(address(nodes[recordNode[record]].data()) >> 32u); + } + const std::string name = "pointer table with a conditional V# chase"; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + const Registration registeredNodes(nodes.data(), sizeof(nodes)); + const Registration registeredLinks(links.data(), sizeof(links)); + const Registration registeredUnsnapshotted(unsnapshotted.data(), sizeof(unsnapshotted)); + const auto run = [&](std::span code) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(nodes.data()), std::as_bytes(std::span(nodes))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + }; + const auto inputTable = Table; + const auto inputNodes = nodes; + const auto inputLinks = links; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && nodes == inputNodes && links == inputLinks, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + Keys[group * 4u + 2u] = group; + } + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + auto node = recordNode[keys[group][x < 32u ? 0u : 1u]]; + for (std::uint32_t step = 0; step < 2u; ++step) { + if ((group >> step) & 1u) node = next[node]; + } + return std::bit_cast(imageOf[node] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(PointerChaseCode); + Require(!Faulted(taken), name + ": the T# at the end of the chase faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# at the pointer reached by the chase steps taken in group " + std::to_string(group)); + chase(0u, 3u, links[0].data()); + const auto moved = run(PointerChaseCode); + Require(!Faulted(moved), name + ": the T# behind a moved chase link faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a chased link recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after node 0's link moved to node 3 in group " + std::to_string(group)); + chase(0u, 1u, links[0].data()); + chase(1u, 3u, unsnapshotted.data()); + const auto outside = run(PointerChaseCode); + Require(Faulted(outside) && outside.log.find("outside the snapshot") != std::string::npos, name + ": a link outside the snapshot did not fault as outside the snapshot:\n" + outside.log); + Require(untouched(outside), name + ": a link outside the snapshot published copied output"); + Require(outside.artifactId == taken.artifactId, name + ": a link outside the snapshot recompiled the shader"); + chase(1u, 2u, links[1].data()); + const std::array, 4> speculativeKeys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {1u, 1u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = speculativeKeys[group][0]; + Keys[group * 4u + 1u] = speculativeKeys[group][1]; + } + nodes[2][0x60u / 4u] += 1u; + const auto unaligned = run(PointerChaseCode); + Require(!Faulted(unaligned), name + ": a V# with an unaligned base on a step no group takes stopped the dispatch:\n" + unaligned.log); + Require(std::ranges::none_of(unaligned.imageTableRanges, [](const auto& range) { return (range.first & 3u) != 0u; }), name + ": the snapshot read guest memory through a V# with an unaligned base on a step no group takes"); + for (std::uint32_t group = 0; group < 4u; ++group) { + const auto expected = std::function([&, group](std::uint32_t x) { + auto node = recordNode[speculativeKeys[group][x < 32u ? 0u : 1u]]; + for (std::uint32_t step = 0; step < 2u; ++step) { + if ((group >> step) & 1u) node = next[node]; + } + return std::bit_cast(imageOf[node] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + RequireRed(unaligned, group, Lanes, expected, name + ": the T# beside a V# with an unaligned base on a step no group takes, in group " + std::to_string(group)); + } + chase(2u, 0u, links[2].data()); + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::puts("pointer table with a conditional V# chase: a T# at a record pointer chased up to twice through a V# at the pointer plus 0x60 checked for every taken and skipped step, a moved link, a link outside the snapshot and a V# with an unaligned base on a step no group takes"); +} + +void RunPointerChaseSplitTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 8> nodes{}; + alignas(256) std::array, 8> nearLinks{}; + alignas(256) std::array, 8> farLinks{}; + alignas(256) std::array unsnapshotted{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const std::array imageOf{0u, 1u, 0u, 1u, 1u, 0u, 1u, 0u}; + std::array nearNext{}; + std::array farNext{}; + const auto address = [](const void* data) { return static_cast(reinterpret_cast(data)); }; + const auto linkNear = [&](std::uint32_t from, std::uint32_t to, std::uint32_t* link) { + link[0] = static_cast(address(nodes[to].data())); + link[1] = static_cast(address(nodes[to].data()) >> 32u); + const auto descriptor = Buffer(link, 8u); + std::copy(descriptor.begin(), descriptor.end(), nodes[from].begin() + 0x60u / 4u); + nearNext[from] = to; + }; + const auto linkFar = [&](std::uint32_t from, std::uint32_t to, std::uint32_t* link) { + link[0x54u / 4u] = static_cast(address(nodes[to].data())); + link[0x6cu / 4u] = static_cast(address(nodes[to].data()) >> 32u); + const auto descriptor = Buffer(link, 0x80u); + std::copy(descriptor.begin(), descriptor.end(), nodes[from].begin() + 0x110u / 4u); + farNext[from] = to; + }; + const std::array nearTo{1u, 2u, 3u, 0u, 0u, 0u, 0u, 7u}; + const std::array farTo{4u, 5u, 6u, 0u, 0u, 0u, 0u, 7u}; + for (std::uint32_t node = 0; node < 8u; ++node) { + std::copy(images[imageOf[node]].begin(), images[imageOf[node]].end(), nodes[node].begin()); + linkNear(node, nearTo[node], nearLinks[node].data()); + linkFar(node, farTo[node], farLinks[node].data()); + } + const std::array recordNode{0u, 0u, 3u, 0u}; + for (std::uint32_t record = 0; record < 4u; ++record) { + Table[record * 4u + 2u] = static_cast(address(nodes[recordNode[record]].data())); + Table[record * 4u + 3u] = static_cast(address(nodes[recordNode[record]].data()) >> 32u); + } + const std::string name = "pointer table with a split-word V# chase"; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + const Registration registeredNodes(nodes.data(), sizeof(nodes)); + const Registration registeredNearLinks(nearLinks.data(), sizeof(nearLinks)); + const Registration registeredFarLinks(farLinks.data(), sizeof(farLinks)); + const Registration registeredUnsnapshotted(unsnapshotted.data(), sizeof(unsnapshotted)); + const auto run = [&](std::span code) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(nodes.data()), std::as_bytes(std::span(nodes))}, + {reinterpret_cast(nearLinks.data()), std::as_bytes(std::span(nearLinks))}, + {reinterpret_cast(farLinks.data()), std::as_bytes(std::span(farLinks))}, + }; + const auto inputTable = Table; + const auto inputNodes = nodes; + const auto inputNearLinks = nearLinks; + const auto inputFarLinks = farLinks; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && nodes == inputNodes && nearLinks == inputNearLinks && farLinks == inputFarLinks, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + const std::array steps{7u, 4u, 3u, 5u}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + Keys[group * 4u + 2u] = steps[group]; + } + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + auto node = recordNode[keys[group][x < 32u ? 0u : 1u]]; + if ((steps[group] & 1u) != 0u) node = nearNext[node]; + if ((steps[group] & 2u) != 0u) node = nearNext[node]; + if ((steps[group] & 4u) != 0u) node = farNext[node]; + return std::bit_cast(imageOf[node] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(PointerChaseSplitCode); + Require(!Faulted(taken), name + ": the T# at the end of the chase faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# at the pointer reached by the chase steps taken in group " + std::to_string(group)); + linkFar(0u, 2u, farLinks[0].data()); + const auto moved = run(PointerChaseSplitCode); + Require(!Faulted(moved), name + ": the T# behind a moved split-word link faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a split-word link recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after node 0's split-word link moved to node 2 in group " + std::to_string(group)); + linkFar(0u, 4u, farLinks[0].data()); + linkFar(1u, 7u, unsnapshotted.data()); + const auto outside = run(PointerChaseSplitCode); + Require(Faulted(outside) && outside.log.find("outside the snapshot") != std::string::npos, name + ": a split-word link outside the snapshot did not fault as outside the snapshot:\n" + outside.log); + Require(untouched(outside), name + ": a split-word link outside the snapshot published copied output"); + Require(outside.artifactId == taken.artifactId, name + ": a split-word link outside the snapshot recompiled the shader"); + std::puts("pointer table with a split-word V# chase: a T# at a record pointer chased twice through a V# at the pointer plus 0x60 and once through a V# at the pointer plus 0x110 whose next pointer is dwords 5 and 11 of one 16-dword load from 0x40, checked for every taken and skipped step with a node only the third chase step reaches, a moved link and a link outside the snapshot"); +} + +void RunPointerChaseWordsTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 6> nodes{}; + alignas(256) std::array, 2> links{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const std::array imageOf{0u, 1u, 0u, 1u, 1u, 0u}; + std::array first{}; + std::array second{}; + const auto address = [](const void* data) { return static_cast(reinterpret_cast(data)); }; + const auto link = [&](std::uint32_t from, std::uint32_t word, std::uint32_t to) { + links[from][word] = static_cast(address(nodes[to].data())); + links[from][word + 1u] = static_cast(address(nodes[to].data()) >> 32u); + (word == 0u ? first : second)[from] = to; + }; + for (std::uint32_t node = 0; node < 6u; ++node) std::copy(images[imageOf[node]].begin(), images[imageOf[node]].end(), nodes[node].begin()); + for (std::uint32_t node = 0; node < 2u; ++node) { + const auto descriptor = Buffer(links[node].data(), 16u); + std::copy(descriptor.begin(), descriptor.end(), nodes[node].begin() + 0x60u / 4u); + link(node, 0u, node + 2u); + link(node, 2u, node + 4u); + } + const std::array recordNode{0u, 0u, 1u, 0u}; + for (std::uint32_t record = 0; record < 4u; ++record) { + Table[record * 4u + 2u] = static_cast(address(nodes[recordNode[record]].data())); + Table[record * 4u + 3u] = static_cast(address(nodes[recordNode[record]].data()) >> 32u); + } + const std::string name = "pointer table with two chase links in one V#"; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + const Registration registeredNodes(nodes.data(), sizeof(nodes)); + const Registration registeredLinks(links.data(), sizeof(links)); + const auto run = [&](std::span code) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(nodes.data()), std::as_bytes(std::span(nodes))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + }; + const auto inputTable = Table; + const auto inputNodes = nodes; + const auto inputLinks = links; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && nodes == inputNodes && links == inputLinks, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + const std::array steps{1u, 2u, 0u, 2u}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + Keys[group * 4u + 2u] = steps[group]; + } + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + auto node = recordNode[keys[group][x < 32u ? 0u : 1u]]; + if ((steps[group] & 1u) != 0u) node = first[node]; + else if ((steps[group] & 2u) != 0u) node = second[node]; + return std::bit_cast(imageOf[node] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto taken = run(PointerChaseWordsCode); + Require(!Faulted(taken), name + ": the T# behind one of the two links faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# at the pointer read from the link taken in group " + std::to_string(group)); + link(0u, 2u, 2u); + const auto moved = run(PointerChaseWordsCode); + Require(!Faulted(moved), name + ": the T# behind a moved second link faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving the second link recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after node 0's second link moved to node 2 in group " + std::to_string(group)); + std::puts("pointer table with two chase links in one V#: a T# at a record pointer chased through words 0 and 1 or words 2 and 3 of one V# at the pointer plus 0x60, each taken in its own groups, checked with a moved second link"); +} + +void RunPointerChaseEditedTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 4> nodes{}; + alignas(256) std::array, 4> links{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const std::array imageOf{0u, 1u, 0u, 1u}; + std::array next{}; + const auto address = [](const void* data) { return static_cast(reinterpret_cast(data)); }; + const auto chase = [&](std::uint32_t from, std::uint32_t to) { + links[from][0x30u / 4u] = static_cast(address(nodes[to].data())); + links[from][0x34u / 4u] = static_cast(address(nodes[to].data()) >> 32u); + auto descriptor = Buffer(links[from].data(), 4u); + descriptor[1] |= 4u << 16u; + descriptor[3] &= ~0x30000000u; + std::copy(descriptor.begin(), descriptor.end(), nodes[from].begin() + 0x30u / 4u); + next[from] = to; + }; + for (std::uint32_t node = 0; node < 4u; ++node) { + std::copy(images[imageOf[node]].begin(), images[imageOf[node]].end(), nodes[node].begin()); + chase(node, node == 3u ? 3u : (node + 1u) % 3u); + } + const std::array recordNode{2u, 0u, 1u, 2u}; + for (std::uint32_t record = 0; record < 4u; ++record) { + Table[record * 4u + 2u] = static_cast(address(nodes[recordNode[record]].data())); + Table[record * 4u + 3u] = static_cast(address(nodes[recordNode[record]].data()) >> 32u); + } + const std::string name = "pointer table with an edited chase V#"; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + const Registration registeredNodes(nodes.data(), sizeof(nodes)); + const Registration registeredLinks(links.data(), sizeof(links)); + const auto run = [&](std::span code) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(nodes.data()), std::as_bytes(std::span(nodes))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + }; + const auto inputTable = Table; + const auto inputNodes = nodes; + const auto inputLinks = links; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && nodes == inputNodes && links == inputLinks, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + Keys[group * 4u + 2u] = group; + } + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + auto node = recordNode[keys[group][x < 32u ? 0u : 1u]]; + if ((group & 1u) != 0u) node = next[node]; + return std::bit_cast(imageOf[node] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto taken = run(PointerChaseEditedCode); + Require(!Faulted(taken), name + ": the T# behind the edited V# faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# at the record pointer or at the pointer read through the edited V# in group " + std::to_string(group)); + chase(0u, 3u); + const auto moved = run(PointerChaseEditedCode); + Require(!Faulted(moved), name + ": the T# behind a moved link faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a link behind the edited V# recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after node 0's link moved to node 3 in group " + std::to_string(group)); + std::puts("pointer table with an edited chase V#: a T# at a record pointer or at the pointer read at 0x30 through a V# from the record pointer plus 0x30 whose words 1, 2 and 3 the shader ANDs with 0xffff, shifts left by 4 and ORs with 0x30000000, a V# that covers the read only after those edits, checked for taken and skipped steps and a moved link"); +} + +void RunPointerFixedVsharpTests(AgcDriver::VulkanDevice& device, std::uint32_t descriptorOffset) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto point = [&](std::uint32_t record, std::uint32_t palette) { + const auto descriptor = Buffer(palettes[palette].data(), static_cast(sizeof(palettes[palette]))); + std::copy(descriptor.begin(), descriptor.end(), records[record].begin() + 40u); + }; + for (std::uint32_t palette = 0; palette < 2u; ++palette) { + std::copy(images[palette].begin(), images[palette].end(), palettes[palette].begin() + descriptorOffset / 4u); + point(palette, palette); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const std::string name = std::string("pointer V# table with a T# at fixed offset ") + Hex(descriptorOffset); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code = descriptorOffset == 0u ? std::span(PointerFixedVsharpCode) : std::span(PointerFixedVsharpOffsetCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array paletteOf{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(paletteOf[record] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read at a fixed offset of the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + paletteOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after record 1's pointer moved to the other V#"); + point(0u, 1u); + paletteOf = {1u, 1u, 1u}; + const auto repointed = run(); + Require(!Faulted(repointed), name + ": the T# behind a moved V# faulted:\n" + repointed.log); + Require(repointed.artifactId == taken.artifactId, name + ": moving the V# behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(repointed, group, Lanes, sampled(group), name + ": the T# after the V# behind records 0 and 1 moved to the other palette"); + std::puts((name + ": a T# read at a fixed offset of a V# loaded through a record pointer checked, by the records' values").c_str()); +} + void RunSplitRecordTests(AgcDriver::VulkanDevice& device) { Fill(); const auto floatImage = Image(FloatTexels.data(), Format32Float); @@ -1907,6 +3158,70 @@ int main(int argc, char** argv) { RunNestedRecordTests(*device, NestedCase::Budget); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--nested-sampler") == 0) { + RunNestedSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-sampler") == 0) { + RunPointerSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-ballot") == 0) { + RunPointerArmTests(*device, false); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-copies") == 0) { + RunPointerArmTests(*device, true); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-and") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneAnd); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high") == 0) { + RunPointerMaskTests(*device, PointerMask::High); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high-second-lane") == 0) { + RunPointerMaskTests(*device, PointerMask::HighSecondLane); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-not") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneNot); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-select") == 0) { + RunPointerVsharpTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-hop") == 0) { + RunPointerHopTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-chase") == 0) { + RunPointerChaseTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-chase-split") == 0) { + RunPointerChaseSplitTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-chase-words") == 0) { + RunPointerChaseWordsTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-chase-edited") == 0) { + RunPointerChaseEditedTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed") == 0) { + RunPointerFixedVsharpTests(*device, 0u); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed-offset") == 0) { + RunPointerFixedVsharpTests(*device, 0x40u); + return 0; + } RunTests(*device); RunBorderSwizzleTests(*device); RunAnisoOverrideTests(*device); diff --git a/core/shader/recompiler/ImageTableAbi.hpp b/core/shader/recompiler/ImageTableAbi.hpp index b9c9773b5f4b..27d982e2e04f 100644 --- a/core/shader/recompiler/ImageTableAbi.hpp +++ b/core/shader/recompiler/ImageTableAbi.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler::ImageTableAbi { -inline constexpr std::uint32_t Version = 3; +inline constexpr std::uint32_t Version = 4; inline constexpr std::uint32_t HeaderVersion = 0; inline constexpr std::uint32_t HeaderElementBase = 1; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 646f625d9a38..0ff7dffe6ff6 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -35,6 +35,28 @@ struct PointerHop { bool operator==(const PointerHop& other) const = default; }; +inline constexpr std::uint32_t MaxPointerChaseDepth = 4; + +struct DescriptorWordEdit { + std::uint32_t shift = 0; + std::uint32_t mask = 0xffffffffu; + std::uint32_t bits = 0; + + [[nodiscard]] std::uint32_t Apply(std::uint32_t word) const { return ((word << shift) & mask) | bits; } + bool operator==(const DescriptorWordEdit& other) const = default; +}; + +struct PointerChase { + std::uint32_t offset = 0; + std::uint32_t immediate = 0; + std::uint32_t readOffset = 0; + std::uint32_t readImmediate = 0; + std::uint32_t readHighImmediate = 0; + std::array edits {}; + + bool operator==(const PointerChase& other) const = default; +}; + struct TableColumn { std::uint32_t heapSource = 0; std::uint32_t stride = 0; @@ -51,13 +73,16 @@ struct TableColumn { std::uint32_t indirectOffset = 0; std::uint32_t indirectImmediate = 0; std::vector pointerHops; + std::vector pointerChases; + std::uint32_t pointerChaseDepth = 0; std::uint32_t foldedAddend = 0; std::uint32_t foldedCarry = 0; bool selected = false; std::uint32_t selection = 0; + bool pointerKeyed = false; bool operator==(const TableColumn& other) const { - return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection; + return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && pointerChases == other.pointerChases && pointerChaseDepth == other.pointerChaseDepth && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection && pointerKeyed == other.pointerKeyed; } }; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index 034d6b7947b5..05979a8ac973 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -132,6 +132,7 @@ struct ImageTableColumnSnapshot { std::uint32_t fault = 0; bool outside = false; bool nested = false; + bool pointerKeyed = false; SharedValues pointers; SharedValues codes; SharedValues words; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 201eaf7c653f..65865502cc5a 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -755,7 +755,10 @@ class TableSnapshotter { for (const auto& sampler : plan.info.samplers) { if (sampler.table != NoTable) { count = std::max(count, sampler.table + 1u); - if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value() && plan.descriptorSources[sampler.source].tableColumn->indirect) hasIndirect = true; + if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value()) { + const auto& column = *plan.descriptorSources[sampler.source].tableColumn; + hasIndirect |= column.indirect || plan.descriptorSources.at(column.heapSource).tableColumn.has_value(); + } } } if (count == 0u) { @@ -1113,6 +1116,57 @@ class TableSnapshotter { if (column.buffer) parentKeysRemaining -= records.keys; } + std::optional Chase(std::uint64_t pointer, const PointerChase& chase) { + DescriptorValue descriptor; + descriptor.dwordCount = 4u; + for (std::uint32_t dword = 0; dword < 4u; dword++) { + const auto address = ImageTableAbi::ScalarAddressDword(pointer, chase.offset, chase.immediate + dword * 4u); + if (!address.has_value() || (*address & 3u) != 0u || !AddressAccessible(*address)) return std::nullopt; + descriptor.dwords[dword] = chase.edits[dword].Apply(Read(*address)); + descriptor.availableDwordMask |= static_cast(1u << dword); + tables.ranges.emplace_back(*address, sizeof(std::uint32_t)); + } + const auto buffer = decodeBufferDescriptor(descriptor); + if (buffer.Type() != 0u) return std::nullopt; + const auto low = ImageTableAbi::ScalarBufferDword(chase.readOffset, chase.readImmediate, buffer.GetSize()); + const auto high = ImageTableAbi::ScalarBufferDword(chase.readOffset, chase.readHighImmediate, buffer.GetSize()); + if (!low.has_value() || !high.has_value()) return std::nullopt; + if (((buffer.Base48() + *low) & 3u) != 0u || ((buffer.Base48() + *high) & 3u) != 0u) return std::nullopt; + if (!AddressAccessible(buffer.Base48() + *low) || !AddressAccessible(buffer.Base48() + *high)) return std::nullopt; + tables.ranges.emplace_back(buffer.Base48() + *low, sizeof(std::uint32_t)); + tables.ranges.emplace_back(buffer.Base48() + *high, sizeof(std::uint32_t)); + return (static_cast(Read(buffer.Base48() + *high)) << 32u) | Read(buffer.Base48() + *low); + } + + template + void ChaseClosure(const TableColumn& column, ColumnRecords& records, std::uint32_t limit, const ReadEntry& readEntry) { + std::unordered_set known; + std::vector frontier; + for (std::uint32_t record = 0; record < records.keys; record++) { + const auto pointer = records.pointers[record]; + if (pointer != ImageTableAbi::UncapturedPointer && known.insert(pointer).second) frontier.push_back(pointer); + } + for (std::uint32_t depth = 0; depth < column.pointerChaseDepth && !frontier.empty(); depth++) { + std::vector reached; + for (const auto pointer : frontier) { + for (const auto& chase : column.pointerChases) { + const auto next = Chase(pointer, chase); + if (!next.has_value() || *next == 0u || !known.insert(*next).second) continue; + if (records.keys >= limit) { + records.outside = true; + continue; + } + records.states.push_back(RecordState::Read); + records.words.push_back(0u); + records.pointers.push_back(*next); + readEntry(records.keys++, *next); + reached.push_back(*next); + } + } + frontier = std::move(reached); + } + } + void ReadColumn(const TableColumn& column, ColumnRecords& records, std::uint32_t limit) { stamp = {}; comparedRuns.clear(); @@ -1131,29 +1185,19 @@ class TableSnapshotter { const auto end = records.base + records.size; if (column.indirect) { unread(); - if (column.buffer) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); - for (std::uint32_t record = 0; record < records.keys; record++) { - if (records.states[record] != RecordState::Read) continue; - const auto offset = column.addend + record * column.stride; - const auto lowWord = ImageTableAbi::ScalarBufferDword(offset, column.offset, records.size); - const auto highWord = ImageTableAbi::ScalarBufferDword(offset, column.offset + 4u, records.size); - if (!lowWord.has_value() || !highWord.has_value() || !PageAccessible(records.base + *lowWord, records.base, end) || !PageAccessible(records.base + *highWord, records.base, end)) { - records.states[record] = RecordState::Unmapped; - continue; - } - std::uint64_t pointer = (static_cast(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord); + if (column.buffer || column.pointerKeyed) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); + const auto readEntry = [&](std::uint32_t entry, std::uint64_t pointer) { for (const auto& hop : column.pointerHops) { const auto low = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate); const auto high = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate + 4u); if (!low.has_value() || !high.has_value() || (*low & 3u) != 0u || *high != *low + sizeof(std::uint32_t) || !AddressAccessible(*low) || !AddressAccessible(*high)) { - records.states[record] = RecordState::Unmapped; - break; + records.states[entry] = RecordState::Unmapped; + return; } pointer = (static_cast(Read(*high)) << 32u) | Read(*low); tables.ranges.emplace_back(*low, 2u * sizeof(std::uint32_t)); } - if (records.states[record] != RecordState::Read) continue; - if (column.buffer) records.pointers[record] = pointer; + if (column.buffer) records.pointers[entry] = pointer; DescriptorValue value; value.dwordCount = column.sampler || column.buffer ? 4u : 8u; std::uint64_t first = std::numeric_limits::max(); @@ -1162,19 +1206,32 @@ class TableSnapshotter { auto address = ImageTableAbi::ScalarAddressDword(pointer, column.indirectOffset, column.indirectImmediate + dword * 4u); if (address.has_value() && (*address & 3u) != 0u) address.reset(); if (!address.has_value() || !AddressAccessible(*address)) { - records.states[record] = RecordState::Unmapped; - break; + records.states[entry] = RecordState::Unmapped; + return; } value.dwords[dword] = Read(*address); value.availableDwordMask |= static_cast(1u << dword); first = std::min(first, *address); past = std::max(past, *address + sizeof(std::uint32_t)); } - if (records.states[record] != RecordState::Read) continue; - records.words[record] = WordsIndex(records, value); + records.words[entry] = WordsIndex(records, value); tables.ranges.emplace_back(first, past - first); if (column.buffer) records.readRanges.emplace_back(first, past - first); + }; + for (std::uint32_t record = 0; record < records.keys; record++) { + if (records.states[record] != RecordState::Read) continue; + const auto offset = column.addend + record * column.stride; + const auto lowWord = ImageTableAbi::ScalarBufferDword(offset, column.offset, records.size); + const auto highWord = ImageTableAbi::ScalarBufferDword(offset, column.offset + 4u, records.size); + if (!lowWord.has_value() || !highWord.has_value() || !PageAccessible(records.base + *lowWord, records.base, end) || !PageAccessible(records.base + *highWord, records.base, end)) { + records.states[record] = RecordState::Unmapped; + continue; + } + const std::uint64_t pointer = (static_cast(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord); + if (column.pointerKeyed) records.pointers[record] = pointer; + readEntry(record, pointer); } + if (column.pointerKeyed && column.pointerChaseDepth != 0u) ChaseClosure(column, records, limit, readEntry); return; } const bool memoized = TableSnapshotReuse() && !records.narrowed && records.keys != 0u; @@ -1494,6 +1551,7 @@ class TableSnapshotter { const auto& records = Column(source); auto& out = tables.tables.at(table); out = CopyRecords(records); + out.pointerKeyed = column.pointerKeyed; for (std::uint32_t resource = 0; resource < plan.info.images.size(); ++resource) { const auto& image = plan.info.images[resource]; if (image.table != table || !registeredSampledArrayCandidate(image) || runtime.normalizeImageView == nullptr) continue; @@ -1918,7 +1976,7 @@ bool sameColumns(const std::vector& left, const std::v for (std::size_t index = 0; index < left.size(); index++) { const auto& a = left[index]; const auto& b = right[index]; - if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; + if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || a.pointerKeyed != b.pointerKeyed || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; } return true; } @@ -2088,20 +2146,21 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i out.map[header + Abi::TableFault] = poison(resource, PoisonReason::UnsupportedOperation, nullptr); } outside[table] = outside[table] || column.outside; + const bool byPointer = column.nested || column.pointerKeyed; const auto mapStart = static_cast(out.map.size()); - if (!column.nested) { + if (!byPointer) { growMap(column.keys); out.map[header + Abi::TableMapStart] = mapStart; out.map.resize(out.map.size() + column.keys, 0u); } else if (column.pointers.size() != column.keys) { - throw std::runtime_error("nested image table snapshot has no pointer for each record"); + throw std::runtime_error("pointer-keyed image table snapshot has no pointer for each record"); } std::map pointers; if (column.nested) nestedRecords.Clear(); else records.Clear(); for (std::uint32_t key = 0; key < column.keys; key++) { const auto code = column.codes[key]; - if (column.nested && column.pointers[key] == Abi::UncapturedPointer) continue; + if (byPointer && column.pointers[key] == Abi::UncapturedPointer) continue; std::uint32_t mapped = Abi::NullCode; if ((code & Abi::PoisonFlag) != 0u) { const auto reason = static_cast(code & ~Abi::PoisonFlag); @@ -2129,15 +2188,15 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i records.Set(code, mapped); } } - if (column.nested) { + if (byPointer) { const auto [found, inserted] = pointers.try_emplace(column.pointers[key], mapped); - if (!inserted && found->second != mapped) throw std::runtime_error("nested image table captured conflicting values at one pointer"); + if (!inserted && found->second != mapped) throw std::runtime_error("pointer-keyed image table captured conflicting values at one pointer"); } else { out.map[mapStart + key] = mapped; } } - if (column.nested) { - if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("nested image table has too many pointers"); + if (byPointer) { + if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("pointer-keyed image table has too many pointers"); growMap(pointers.size() * Abi::PointerRecordWords); out.map[header + Abi::TableKeyCount] = static_cast(pointers.size()); out.map[header + Abi::TableMapStart] = static_cast(out.map.size()); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 3d06941873fe..0bc7fba5d494 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -841,7 +842,13 @@ class Tracker { if (address && immediateU32(offset, descriptorOffset)) { return TryMakeIndirectColumn(handle, *heapHandle, sampler, words, descriptorOffset, first, plan); } - if (!MatchAffineOffset(offset, key, stride, addend)) { + std::uint32_t fixedOffset = 0; + const bool fixed = !address && !buffer && immediateU32(offset, fixedOffset); + if (fixed) { + key = &m_builder.Constant(0u); + stride = width * static_cast(sizeof(std::uint32_t)); + addend = fixedOffset; + } else if (!MatchAffineOffset(offset, key, stride, addend)) { if (!MatchMaskedOffset(offset, key, stride) && (address || !MatchFirstLaneOffset(offset, key, stride))) { return false; } @@ -849,8 +856,10 @@ class Tracker { } DescriptorSource heapSource; std::uint32_t heapIndex = 0; - if (!MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { - if (sampler || address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; + if (MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { + if (fixed) return false; + } else { + if (address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; const auto sameHandle = [&](const IrValue* user) { if (user->Opcode() != handle.Opcode() || user->ArgumentCount() != handle.ArgumentCount()) return false; for (std::uint32_t argument = 0; argument < handle.ArgumentCount(); argument++) { @@ -914,10 +923,81 @@ class Tracker { return true; } - IrValue* PointerRecordOrigin(IrValue* value) const { + static bool LaneMaskWord(const IrValue& value) { + return value.Opcode() == IrOpcode::CompositeExtractU32x4 && value.ArgumentCount() == 2u && value.Argument(0)->Resolve()->Opcode() == IrOpcode::Ballot && value.Argument(1)->Resolve()->HasImmediate(); + } + + bool SameRecordColumn(IrValue& first, IrValue& second) const { + IrValue* firstKey = nullptr; + IrValue* secondKey = nullptr; + std::uint32_t firstStride = 0; + std::uint32_t firstAddend = 0; + std::uint32_t secondStride = 0; + std::uint32_t secondAddend = 0; + return EquivalentValue(m_program.Resources(), first.Argument(0), second.Argument(0)) && MatchAffineOffset(first.Argument(1), firstKey, firstStride, firstAddend) && MatchAffineOffset(second.Argument(1), secondKey, secondStride, secondAddend) && firstStride == secondStride && firstAddend == secondAddend; + } + + static bool LaneIndex(const IrValue* value) { + std::vector pending{value}; + std::unordered_set seen; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second || (current->HasImmediate() && current->Type() == IrType::U32)) continue; + switch (current->Opcode()) { + case IrOpcode::FindILsb32: + case IrOpcode::FindUMsb32: + case IrOpcode::ReadFirstLane: continue; + case IrOpcode::BitwiseAnd32: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + if (current->ArgumentCount() != 2u) return false; + pending.push_back(current->Argument(0)); + pending.push_back(current->Argument(1)); + continue; + case IrOpcode::SelectU32: + if (current->ArgumentCount() != 3u) return false; + pending.push_back(current->Argument(1)); + pending.push_back(current->Argument(2)); + continue; + default: return false; + } + } + return true; + } + + static bool LaneBit(const IrValue& value) { + if (value.Opcode() != IrOpcode::ShiftLeftLogical32 || value.ArgumentCount() != 2u) return false; + const IrValue* one = value.Argument(0)->Resolve(); + return one->HasImmediate() && one->Type() == IrType::U32 && one->ImmediateU32() == 1u && LaneIndex(value.Argument(1)); + } + + static bool LaneMaskOperand(const IrValue* value) { + const IrValue* root = value->Resolve(); + std::vector pending{root}; + std::unordered_set seen; + bool lane = false; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second) continue; + if (LaneMaskWord(*current) || LaneBit(*current)) { + lane = true; + continue; + } + if (current != root && current->HasImmediate() && current->Type() == IrType::U32) continue; + const auto opcode = current->Opcode(); + const bool combines = (opcode == IrOpcode::BitwiseAnd32 || opcode == IrOpcode::BitwiseOr32) && current->ArgumentCount() == 2u; + const bool inverts = opcode == IrOpcode::BitwiseNot32 && current->ArgumentCount() == 1u; + if (!current->IsPhi() && !combines && !inverts) return false; + for (std::size_t index = 0; index < current->ArgumentCount(); ++index) pending.push_back(current->Argument(index)); + } + return lane; + } + + bool PointerRecordReads(IrValue* value, std::vector& reads) const { std::vector pending{value}; std::unordered_set seen; - IrValue* origin = nullptr; while (!pending.empty()) { auto* current = pending.back()->Resolve(); pending.pop_back(); @@ -928,11 +1008,225 @@ class Tracker { } if (current->HasImmediate() && current->Type() == IrType::U32) continue; std::uint32_t memory = 0; - if (ScalarReadMemory(*current, memory) == nullptr || !MemoryIndexBelongsTo(memory, *current)) return nullptr; - if (origin != nullptr && !EquivalentValue(m_program.Resources(), origin, current)) return nullptr; - origin = current; + const MemoryInfo* info = ScalarReadMemory(*current, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, *current)) return false; + if (!reads.empty() && !EquivalentValue(m_program.Resources(), reads.back(), current)) return false; + reads.push_back(current); + } + return !reads.empty(); + } + + static bool PointerFreeWord(const IrValue& word) { + return (word.HasImmediate() && word.Type() == IrType::U32) || LaneMaskOperand(&word); + } + + static IrValue* MaskedRecordWord(const IrValue& word) { + if (word.Opcode() != IrOpcode::BitwiseAnd32 || word.ArgumentCount() != 2u) return nullptr; + const bool left = LaneMaskOperand(word.Argument(0)); + if (left == LaneMaskOperand(word.Argument(1))) return nullptr; + return word.Argument(left ? 1u : 0u); + } + + bool PointerRecordRead(IrValue& read, std::vector& reads) const { + std::uint32_t memory = 0; + const MemoryInfo* info = ScalarReadMemory(read, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, read)) return false; + if (std::ranges::find(reads, &read) != reads.end()) return true; + if (!reads.empty() && (ScalarReadMemory(*reads.back(), memory)->offset != info->offset || !SameRecordColumn(*reads.back(), read))) return false; + reads.push_back(&read); + return true; + } + + struct ChaseSite { + IrValue* handle = nullptr; + IrValue* pointer = nullptr; + PointerChase chase; + std::vector inner; + }; + + bool ChaseRead(IrValue& read, IrValue*& handle, std::uint32_t& readOffset, std::uint32_t& readImmediate) const { + std::uint32_t memory = 0; + const MemoryInfo* info = ScalarReadMemory(read, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, read) || !immediateU32(read.Argument(1), readOffset)) return false; + handle = read.Argument(0)->Resolve(); + readImmediate = info->offset; + return handle->Opcode() == IrOpcode::GetBufferResource && handle->ArgumentCount() == 4u; + } + + static IrValue* EditedWord(IrValue* word, DescriptorWordEdit& edit) { + word = word->Resolve(); + std::uint32_t operand = 0; + if (word->ArgumentCount() != 2u) return word; + if (word->Opcode() == IrOpcode::ShiftLeftLogical32) { + if (!immediateU32(word->Argument(1), operand) || operand > 31u) return word; + edit.shift = operand; + return word->Argument(0)->Resolve(); + } + if (word->Opcode() != IrOpcode::BitwiseAnd32 && word->Opcode() != IrOpcode::BitwiseOr32) return word; + const bool left = immediateU32(word->Argument(0), operand); + if (!left && !immediateU32(word->Argument(1), operand)) return word; + (word->Opcode() == IrOpcode::BitwiseAnd32 ? edit.mask : edit.bits) = operand; + return word->Argument(left ? 1u : 0u)->Resolve(); + } + + bool ChaseVsharp(IrValue& handle, IrValue*& pointer, PointerChase& chase) const { + for (std::uint32_t dword = 0; dword < 4u; dword++) { + IrValue* word = EditedWord(handle.Argument(dword), chase.edits[dword]); + std::uint32_t memoryIndex = 0; + const MemoryInfo* memory = ScalarAddressMemory(*word, memoryIndex); + std::uint32_t offset = 0; + if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *word) || !immediateU32(word->Argument(1), offset)) return false; + IrValue* address = word->Argument(0)->Resolve(); + if (address->Opcode() != IrOpcode::GetAddressResource || address->ArgumentCount() != 2u) return false; + if (dword == 0u) { + pointer = address; + chase.offset = offset; + chase.immediate = memory->offset; + } else if (offset != chase.offset || memory->offset != chase.immediate + dword * static_cast(sizeof(std::uint32_t)) || (address != pointer && !EquivalentValue(m_program.Resources(), pointer, address))) { + return false; + } } - return origin; + return true; + } + + std::optional ChaseHighRead(IrValue& read, const IrValue& handle, std::uint32_t readOffset, std::uint32_t readImmediate) const { + IrValue* highHandle = nullptr; + std::uint32_t highOffset = 0; + std::uint32_t highImmediate = 0; + if (!ChaseRead(read, highHandle, highOffset, highImmediate) || (highHandle != &handle && !EquivalentValue(m_program.Resources(), &handle, highHandle))) return std::nullopt; + if (highOffset != readOffset || highImmediate % sizeof(std::uint32_t) != 0u || highImmediate == readImmediate) return std::nullopt; + return highImmediate; + } + + bool PointerRecordPairs(IrValue* low, IrValue* high, std::vector& lowReads, std::vector& highReads, std::vector& chases, std::uint32_t& chaseDepth) const { + std::vector sites; + std::vector looseHandles; + const auto site = [&](IrValue& handle, IrValue& pointer, const PointerChase& chase) { + const auto found = std::ranges::find_if(sites, [&](const ChaseSite& known) { return known.chase == chase && (known.handle == &handle || EquivalentValue(m_program.Resources(), known.handle, &handle)); }); + if (found != sites.end()) return static_cast(found - sites.begin()); + sites.push_back(ChaseSite{&handle, &pointer, chase, {}}); + return sites.size() - 1u; + }; + const auto walk = [&](IrValue* rootLow, IrValue* rootHigh, std::vector& reached) { + std::vector> pending{{rootLow, rootHigh}}; + std::set> seen; + while (!pending.empty()) { + auto* lowWord = pending.back().first->Resolve(); + auto* highWord = pending.back().second->Resolve(); + pending.pop_back(); + if (!seen.emplace(lowWord, highWord).second || PointerFreeWord(*lowWord)) continue; + if (lowWord->IsPhi()) { + const bool paired = highWord->IsPhi() && highWord->Parent() == lowWord->Parent(); + for (std::size_t index = 0; index < lowWord->ArgumentCount(); ++index) { + IrValue* edge = paired ? PhiIncoming(*highWord, lowWord->PhiBlock(index)) : highWord; + if (edge == nullptr) return false; + if (paired && lowWord->Argument(index)->Resolve()->HasImmediate() && edge->Resolve()->HasImmediate()) return false; + pending.emplace_back(lowWord->Argument(index), edge); + } + continue; + } + if (IrValue* word = MaskedRecordWord(*lowWord)) { + pending.emplace_back(word, highWord); + continue; + } + IrValue* handle = nullptr; + IrValue* pointer = nullptr; + PointerChase chase; + std::uint32_t looseOffset = 0; + std::uint32_t looseImmediate = 0; + if (ChaseRead(*lowWord, handle, chase.readOffset, chase.readImmediate) && ChaseVsharp(*handle, pointer, chase)) { + if (highWord->IsPhi()) { + for (std::size_t index = 0; index < highWord->ArgumentCount(); ++index) pending.emplace_back(lowWord, highWord->Argument(index)); + continue; + } + const auto highImmediate = ChaseHighRead(*highWord, *handle, chase.readOffset, chase.readImmediate); + if (!highImmediate.has_value()) return false; + chase.readHighImmediate = *highImmediate; + const auto index = site(*handle, *pointer, chase); + if (std::ranges::find(reached, index) == reached.end()) reached.push_back(index); + continue; + } + if (!PointerRecordRead(*lowWord, lowReads)) return false; + if (highWord->IsPhi()) { + for (std::size_t index = 0; index < highWord->ArgumentCount(); ++index) pending.emplace_back(lowWord, highWord->Argument(index)); + } else if (IrValue* word = MaskedRecordWord(*highWord)) { + pending.emplace_back(lowWord, word); + } else if (PointerFreeWord(*highWord) || PointerRecordRead(*highWord, highReads)) { + continue; + } else if (IrValue* highHandle = nullptr; ChaseRead(*highWord, highHandle, looseOffset, looseImmediate)) { + looseHandles.push_back(highHandle); + } else { + return false; + } + } + return true; + }; + std::vector roots; + if (!walk(low, high, roots)) return false; + for (std::size_t index = 0; index < sites.size(); ++index) { + std::vector inner; + if (!walk(sites[index].pointer->Argument(0), sites[index].pointer->Argument(1), inner)) return false; + sites[index].inner = std::move(inner); + } + if (!std::ranges::all_of(looseHandles, [&](const IrValue* handle) { + return std::ranges::any_of(sites, [&](const ChaseSite& known) { return known.handle == handle || EquivalentValue(m_program.Resources(), known.handle, handle); }); + })) return false; + std::vector depths(sites.size(), 0u); + std::vector states(sites.size(), 0u); + const auto depthOf = [&](const auto& self, std::size_t index) -> std::optional { + if (states[index] == 2u) return depths[index]; + if (states[index] == 1u) return std::nullopt; + states[index] = 1u; + std::uint32_t deepest = 0; + for (const auto inner : sites[index].inner) { + const auto depth = self(self, inner); + if (!depth.has_value()) return std::nullopt; + deepest = std::max(deepest, *depth); + } + states[index] = 2u; + depths[index] = deepest + 1u; + return depths[index]; + }; + chaseDepth = 0; + for (const auto root : roots) { + const auto depth = depthOf(depthOf, root); + if (!depth.has_value()) return false; + chaseDepth = std::max(chaseDepth, *depth); + } + if (chaseDepth > MaxPointerChaseDepth) return false; + chases.clear(); + for (const auto& known : sites) { + if (std::ranges::find(chases, known.chase) == chases.end()) chases.push_back(known.chase); + } + if (chaseDepth == 0u) chases.clear(); + return !lowReads.empty() && !highReads.empty(); + } + + bool Precedes(const IrValue* value, const IrValue& use) const { + value = value->Resolve(); + const IrBlock* definition = value->Parent(); + const IrBlock* target = use.Parent(); + if (definition == nullptr) return true; + if (target == nullptr) return false; + if (definition == target) { + for (const IrValue* inst : target->Instructions()) { + if (inst == value) return true; + if (inst == &use) return false; + } + return false; + } + std::vector pending{m_program.BlockOrder().front()}; + std::unordered_set seen{pending.front()}; + while (!pending.empty()) { + const IrBlock* block = pending.back(); + pending.pop_back(); + if (block == definition) continue; + if (block == target) return false; + for (const IrBlock* successor : block->Successors()) { + if (seen.insert(successor).second) pending.push_back(successor); + } + } + return true; } IrValue* UnwrapTableRead(IrValue* value) const { @@ -1033,16 +1327,21 @@ class Tracker { IrValue* bufferHandle = nullptr; IrValue* offset = nullptr; const bool buffer = handle.Opcode() == IrOpcode::GetBufferResource; - if (buffer && !hops.empty()) return false; - const bool merged = buffer && pointer->Argument(0)->Resolve()->IsPhi(); - auto* low = merged ? PointerRecordOrigin(pointer->Argument(0)) : pointer->Argument(0)->Resolve(); + const bool merged = pointer->Argument(0)->Resolve()->IsPhi() && UnwrapTableRead(pointer->Argument(0)) == nullptr; + const bool pointerKeyed = merged && !buffer; + if (!hops.empty() && buffer) return false; + std::vector lowReads; + std::vector highReads; + std::vector chases; + std::uint32_t chaseDepth = 0; + if (pointerKeyed ? !PointerRecordPairs(pointer->Argument(0), pointer->Argument(1), lowReads, highReads, chases, chaseDepth) : merged && !PointerRecordReads(pointer->Argument(0), lowReads)) return false; + auto* low = merged ? lowReads.back() : UnwrapTableRead(pointer->Argument(0)); if (low == nullptr) return false; + auto* high = pointerKeyed ? pointer->Argument(1)->Resolve() : UnwrapTableRead(pointer->Argument(1)); + if (high == nullptr) return false; std::uint32_t first = 0; for (std::uint32_t dword = 0; dword < 2u; dword++) { - IrValue* read = dword == 0u && merged ? low : UnwrapTableRead(pointer->Argument(dword)); - if (read == nullptr) { - return false; - } + IrValue* read = dword == 0u ? low : pointerKeyed ? highReads.back() : high; std::uint32_t memoryIndex = 0; const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex); if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) { @@ -1052,17 +1351,36 @@ class Tracker { first = memory->offset; bufferHandle = read->Argument(0)->Resolve(); offset = read->Argument(1)->Resolve(); - } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1))) { + } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || (pointerKeyed ? !SameRecordColumn(*low, *read) : read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1)))) { return false; } } + IrValue* recordOffset = offset; + if (pointerKeyed) { + IrValue* source = high; + if (highReads.size() != 1u || highReads.front() != high) { + lowReads.insert(lowReads.begin(), highReads.begin(), highReads.end()); + const auto found = std::ranges::find_if(lowReads, [&](const IrValue* read) { + const IrValue* table = read->Argument(0)->Resolve(); + for (std::size_t index = 0; index < table->ArgumentCount(); ++index) { + if (!Precedes(table->Argument(index), handle)) return false; + } + return true; + }); + if (found == lowReads.end()) return false; + source = *found; + } + bufferHandle = source->Argument(0)->Resolve(); + recordOffset = source->Argument(1)->Resolve(); + offset = source == high ? recordOffset : high; + } if (!AvailableAt(offset, handle.Parent())) { return false; } IrValue* key = nullptr; std::uint32_t stride = 0; std::uint32_t addend = 0; - if (!MatchAffineOffset(offset, key, stride, addend)) { + if (!MatchAffineOffset(recordOffset, key, stride, addend)) { return false; } DescriptorSource heapSource; @@ -1082,7 +1400,10 @@ class Tracker { column.buffer = buffer; column.indirectOffset = descriptorOffset; column.indirectImmediate = descriptorImmediate; + column.pointerKeyed = pointerKeyed; column.pointerHops = std::move(hops); + column.pointerChases = std::move(chases); + column.pointerChaseDepth = chaseDepth; const auto domain = merged ? std::optional{} : MatchKeyDomain(key); DescriptorSource columnSource = heapSource; columnSource.tableColumn = column; @@ -1098,7 +1419,7 @@ class Tracker { shared.keyDomains.push_back(*domain); } plan.offset = offset; - if (buffer) plan.parentPointer = pointer; + if (buffer || merged) plan.parentPointer = pointer; return true; } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index a00bdfb52a6e..b9edf1847561 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1830,6 +1830,7 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(table.fault); mix(table.outside ? 1u : 0u); mix(table.nested ? 1u : 0u); + mix(table.pointerKeyed ? 1u : 0u); mix(table.pointers.Hash()); mix(table.codes.Hash()); mix(table.words.Hash()); diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 082e55eb8363..44526c3ef422 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -111,8 +111,8 @@ static_assert(sizeof(IrResourcePlan) == 728, "IrResourcePlan changed: update Enc static_assert(sizeof(IrValue) == 144, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); -static_assert(sizeof(DescriptorSource) == 216, "DescriptorSource changed: update the plan encoder"); -static_assert(sizeof(TableColumn) == 112, "TableColumn changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 248, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(TableColumn) == 144, "TableColumn changed: update the plan encoder"); static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder"); static_assert(sizeof(ResourceBlock) == 104, "ResourceBlock changed: update the plan encoder"); static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); @@ -152,7 +152,7 @@ static_assert(fieldCount() == 25, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 32, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 4, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 19, "TableColumn changed: update the plan encoder"); +static_assert(fieldCount() == 22, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); static_assert(fieldCount() == 5, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); @@ -1182,10 +1182,24 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { hops.Value(hop.offset); hops.Value(hop.immediate); }); + out.List(column.pointerChases, [](Writer& chases, const PointerChase& chase) { + chases.Value(chase.offset); + chases.Value(chase.immediate); + chases.Value(chase.readOffset); + chases.Value(chase.readImmediate); + chases.Value(chase.readHighImmediate); + for (const auto& edit : chase.edits) { + chases.Value(edit.shift); + chases.Value(edit.mask); + chases.Value(edit.bits); + } + }); + out.Value(column.pointerChaseDepth); out.Value(column.foldedAddend); out.Value(column.foldedCarry); out.Value(column.selected); out.Value(column.selection); + out.Value(column.pointerKeyed); out.Values(std::span(source.requiredBlocks)); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { @@ -1319,7 +1333,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { value.SetDetachedOperands(std::move(arguments), std::move(phiBlocks)); } reader.List(plan.memoryInfo, 79, decodeMemoryInfo); - reader.List(plan.descriptorSources, 81, [&](Reader& in, DescriptorSource& source) { + reader.List(plan.descriptorSources, 82, [&](Reader& in, DescriptorSource& source) { for (auto& dword : source.dwords) dword = valueAt(in.Get()); in.Value(source.dwordCount); const bool table = in.Get(); @@ -1347,10 +1361,24 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { hops.Value(hop.offset); hops.Value(hop.immediate); }); + in.List(column.pointerChases, 64, [](Reader& chases, PointerChase& chase) { + chases.Value(chase.offset); + chases.Value(chase.immediate); + chases.Value(chase.readOffset); + chases.Value(chase.readImmediate); + chases.Value(chase.readHighImmediate); + for (auto& edit : chase.edits) { + chases.Value(edit.shift); + chases.Value(edit.mask); + chases.Value(edit.bits); + } + }); + in.Value(column.pointerChaseDepth); in.Value(column.foldedAddend); in.Value(column.foldedCarry); in.Value(column.selected); in.Value(column.selection); + in.Value(column.pointerKeyed); source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt; in.Values(source.requiredBlocks); }); @@ -1392,7 +1420,11 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { const auto& column = *source.tableColumn; if (column.buffer && (column.sampler || column.dwordCount != 4u)) return false; const auto& parent = plan.descriptorSources[column.heapSource].tableColumn; - if (parent.has_value() && (column.buffer || column.sampler || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (parent.has_value() && (column.buffer || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (column.pointerKeyed && (!column.indirect || column.buffer || parent.has_value())) return false; + if (column.pointerChases.empty() != (column.pointerChaseDepth == 0u) || column.pointerChaseDepth > MaxPointerChaseDepth || (column.pointerChaseDepth != 0u && !column.pointerKeyed)) return false; + if (std::ranges::any_of(column.pointerChases, [](const PointerChase& chase) { return chase.readHighImmediate % sizeof(std::uint32_t) != 0u || chase.readHighImmediate == chase.readImmediate; })) return false; + if (std::ranges::any_of(column.pointerChases, [](const PointerChase& chase) { return std::ranges::any_of(chase.edits, [](const DescriptorWordEdit& edit) { return edit.shift > 31u; }); })) return false; } if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false; if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false; diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 3e43cbb22130..f928fc60ff54 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 56; +inline constexpr std::uint32_t FormatVersion = 72; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 4e7efa7f940c..e92b69005d36 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -2155,6 +2155,10 @@ TableWord EmitTableWord(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui const auto offset = ctx.Def(handle->Argument(0)); const auto header = ImageTableAbi::TableHeader(table); const auto& parent = sources.at(column.heapSource).tableColumn; + if (!parent.has_value() && column.pointerKeyed) { + if (!column.indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a pointer-keyed image table without its pointer"); + return LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); + } if (!parent.has_value()) return LookupTableWord(state, column, offset, header); if (!parent->buffer || !parent->indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a nested image table without its buffer pointer"); const auto root = LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index 201686b8205b..18880fc4af39 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -58,8 +58,10 @@ - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) (T#s and S#s that `s_buffer_load` reads at `key * stride + addend` from a V#) are snapshotted when the dispatch or draw is recorded; a later CPU write to a record is not seen. - An [image table](../../../core/shader/recompiler/ImageTableAbi.hpp) entry that cannot be bound exactly reads zeros, and an active lane that selects it raises a BDA fault (reason 9) naming the record's address, words and reason; the dispatch's write-back reports the first one. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) snapshot holds at most 65536 keys per table, 65536 distinct images and 256 S# entries per shader; a key past a longer table's snapshot, or an entry over the budget, faults when selected. A selected column (a T# the SRT walker evaluates) holds its one record wherever it lies. -- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. Other origins, shared V# consumers, T# words read by anything but their image handle, nested sampler columns and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. +- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T# or S#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. Other origins, shared V# consumers, T# or S# words read by anything but their descriptor handle, and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. A T# reached through a nested child is materialized from its raw words, without registered `resourceViews`, so a 1D array past `maxImageArrayLayers` keeps its raw descriptor there. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read by `s_load` through a 64-bit pointer hold the keys the key's known bits allow, at most 65536; a key past them, or an entry at a wrapping, unaligned or unmapped address, faults when selected. +- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) T# or S# read through a pointer that a scalar-buffer record holds may have that pointer's low word merged by a Phi with constants, with lane masks (words of a `Ballot`, the waterfall loop's lane bit, and ANDs, ORs and NOTs of them), with the record word ANDed with a lane mask, and with reads of the same record word under another key, and its high word merged the same way from the record's next word. The two words are walked together, edge by edge, and an edge whose low word is a constant or a lane mask is skipped along with its high word. The snapshot then captures the pointer of every record, at most 65536, and the GPU looks up the actual 64-bit pointer; one that was not captured, such as a lane mask, faults when selected. Other origins of the two words remain refused, among them a high word read from a second buffer on an edge whose low word is the record word. +- A pointer-keyed [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) chase stays refused when it loops through a Phi, passes more than 4 chase sites, edits a V# word other than by one constant left shift, AND or OR, or reads the high word at the low word's immediate or through another V# or buffer offset. The [snapshot](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) ends a chase at a V# that is not a buffer, at a read past the V# or outside the snapshot, and at a chased V# base that leaves either word's address unaligned; the GPU faults when it selects the pointer reached there. Whether the console reads the dword-aligned address instead is not measured. A chase arm is two words read by one scalar buffer load offset, each at its own immediate, through a V# that `s_load` reads at a fixed offset of the record pointer or of another arm. The snapshot captures the pointers the chases reach, up to the shader's depth. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask` with such a key: a key domain would select records by the unmasked key, so that key is not narrowed. A key the walker evaluates selects its one record instead, unless the shader may write the memory the key is read from at run time (the same buffer, or any memory when the read or a write goes through a pointer or a GPU-only descriptor); then the column binds the whole domain too. A write to the key's memory through another V# over the same bytes is not detected. A shader whose selected records would pass 64 images or 256 sampled image/sampler pairs binds, for the heaps with the most such reads, the union of their keys in one table instead. - A shader plans at most 64 [samplers](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (`ShaderInfo::MaxSamplers`, the size of the runtime ABI's `ShaderData` sampler metadata, which [`BindingAllocator`](../../../core/shader/recompiler/Optimization/src/BindingAllocator.cpp) holds every shader to), at most 32 of them direct (each takes two elements of the 64-element sampler heap), and at most 256 image/sampler pairs (`ShaderInfo::MaxSampledPairs`); past them it refuses with `sampler resource limit exceeded`, `shader sampler pairs exceed runtime heap capacity` or `sampled image/sampler pair limit exceeded`. The image-to-sampler masks are 64-bit, one bit per sampler heap element or per sampler, so more samplers need a larger `ShaderData` and wider masks. Ratchet & Clank: Rift Apart (PPSA01474) plans up to 63 samplers and 112 pairs in one shader. - [Image table pointer indexing](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) propagates 32-bit value bounds through `ReadFirstLane` and `SelectU32` so waterfall key masks keep their bit domain. The SRT walker validates runtime select conditions whose active lane mask implies the condition through `LogicalAnd` conjunctions, assuming unselected inactive lanes need not be uniform across the full wave. The same assumption lets the two `s_load`s of one T# record match when one offset is added before `v_readfirstlane` and the other after (`ReadFirstLane(Select(mask, (key << 5) + 16, old), mask)` against `ReadFirstLane(key, mask) << 5`): with no active lane the hardware reads lane 0 of the inactive arm, and the table still binds the whole record.