diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 8e3f18890b81..9bb45a7fa15e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -1930,6 +1930,35 @@ std::vector planFixtures() { nestedCode.insert(nestedCode.begin() + 15u, 0x87148114u); add("nested pointer image table", 64u, std::move(nestedCode), srtPointer, {{PlanSrtAddress, nestedSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedImages}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); fixtures.back().groupId = true; + std::vector samplerSrt(20u, 0u); + std::copy(nestedSrt.begin(), nestedSrt.end(), samplerSrt.begin()); + place(samplerSrt, 12u, imageWords(PlanTextureAddress)); + const std::vector pointClamp{0x92u, 0x400000u, 0u, 0u}; + const std::vector pointWrap{0u, 0x400000u, 1u << 24u, 0u}; + std::vector nestedSamplers(32u, 0u); + place(nestedSamplers, 0u, pointWrap); + place(nestedSamplers, 4u, pointClamp); + place(nestedSamplers, 16u, pointClamp); + place(nestedSamplers, 20u, pointWrap); + std::vector samplerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + samplerCode.insert(samplerCode.begin() + 13u, 0x87118311u); + samplerCode.insert(samplerCode.begin() + 15u, 0x87148114u); + add("nested pointer sampler table", 64u, std::move(samplerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, nestedBuffers}, {PlanPayloadAddress, nestedSamplers}, {PlanMaterialAddress, {0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}}}); + fixtures.back().groupId = true; + std::vector pointees(128u, 0u); + place(pointees, 8u, pointWrap); + place(pointees, 72u, pointClamp); + std::vector pointerCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u,}; + pointerCode.insert(pointerCode.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + add("pointer phi sampler table", 64u, std::move(pointerCode), srtPointer, {{PlanSrtAddress, samplerSrt}, {PlanTableAddress, parents}, {PlanPointerAddress, pointees}, {PlanMaterialAddress, {1u, 0u, 0u, 0u, 1u, 0u, 0u, 0u}}}); + fixtures.back().groupId = true; PlanFixture fragment; fragment.name = "fragment parameters"; @@ -2322,6 +2351,7 @@ void verifyMalformedPlans() { rejects("an image table cycle", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->heapSource = 1u; }); rejects("an image table parent that is not a buffer", [](IrResourcePlan& plan) { plan.descriptorSources[2].tableColumn = *plan.descriptorSources[1].tableColumn; }); rejects("an image table selection past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->selection = 4u; }); + rejects("a pointer-keyed table that is not indirect", [](IrResourcePlan& plan) { plan.descriptorSources[1].tableColumn->pointerKeyed = true; }); auto foreign = handBuiltPlan(); IrValue stray(IrOpcode::Void, IrType::U32, 99u); foreign.srtReads[0].value = &stray; @@ -2381,6 +2411,12 @@ void verifyPlanRoundTrip() { const auto bufferBytes = encodedPlan(bufferTable); IrResourcePlan bufferDecoded; require(bufferBytes != handBytes && ShaderDiskCache::DecodePlan(bufferBytes, bufferDecoded) && samePlan(bufferTable, bufferDecoded), "the nested buffer column changed in a disk-cache round trip"); + auto pointerTable = handBuiltPlan(); + pointerTable.descriptorSources[1].tableColumn->indirect = true; + pointerTable.descriptorSources[1].tableColumn->pointerKeyed = true; + const auto pointerBytes = encodedPlan(pointerTable); + IrResourcePlan pointerDecoded; + require(pointerBytes != handBytes && ShaderDiskCache::DecodePlan(pointerBytes, pointerDecoded) && samePlan(pointerTable, pointerDecoded) && pointerDecoded.descriptorSources[1].tableColumn->pointerKeyed, "the pointer-keyed column changed in a disk-cache round trip"); PlanFixture handFixture; handFixture.userData = {static_cast(PlanSrtAddress), static_cast(PlanSrtAddress >> 32u), 0u, 0u}; std::vector handSrt(16, 0u); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 90c4b12d916f..510a3044ceaa 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -3908,12 +3908,19 @@ void verifyRegionMerge(bool strided = false, bool benchmark = false) { } -void verifyNestedSnapshotRetry() { +void verifyNestedSnapshotRetry(bool sampler) { using namespace ShaderRecompiler; constexpr std::uint32_t Parents = ImageTableAbi::MaxParentKeys / 2u + 1u; constexpr std::uint32_t Children = ImageTableAbi::MaxNestedKeys / 2u + 1u; - std::vector> leaves(Children, TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u)); - const auto child = TableV(leaves.data(), 32u, Children); + const std::uint32_t leafDwords = sampler ? 4u : 8u; + const auto texture = TableT(reinterpret_cast(0x100000000ull), TableFormat8888UNorm, TableType2D, 1u); + const std::array pointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; + std::vector leaves; + for (std::uint32_t leaf = 0; leaf < Children; ++leaf) { + if (sampler) leaves.insert(leaves.end(), pointClamp.begin(), pointClamp.end()); + else leaves.insert(leaves.end(), texture.begin(), texture.end()); + } + const auto child = TableV(leaves.data(), leafDwords * 4u, Children); std::vector parents(Parents, reinterpret_cast(child.data())); const auto root = TableV(parents.data(), 8u, Parents); std::vector> values; @@ -3939,17 +3946,25 @@ void verifyNestedSnapshotRetry() { plan.descriptorSources[1].tableColumn = parent; TableColumn nested; nested.heapSource = 1u; - nested.stride = 32u; - nested.dwordCount = 8u; + nested.stride = leafDwords * 4u; + nested.dwordCount = leafDwords; + nested.sampler = sampler; plan.descriptorSources[2].tableColumn = nested; - ImageResource image; - image.source = 2u; - image.table = 0u; - image.resourceClass = ImageResourceClass::Sampled; - image.dimension = RdnaImageDimension::Dim2D; - image.read = true; - plan.info.images.push_back(image); - plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + if (sampler) { + SamplerResource resource; + resource.source = 2u; + resource.table = 0u; + plan.info.samplers.push_back(resource); + } else { + ImageResource image; + image.source = 2u; + image.table = 0u; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2D; + image.read = true; + plan.info.images.push_back(image); + plan.info.runtimeImageModes.push_back(ResourceMaterializer::RuntimeImageModes(image)); + } struct Memory { std::array, 3> ranges; std::uint64_t epoch = 0; @@ -11728,10 +11743,15 @@ int main(int argc, char** argv) { return 0; } if (argc == 2 && std::string_view(argv[1]) == "--nested-snapshot-retry") { - verifyNestedSnapshotRetry(); + verifyNestedSnapshotRetry(false); std::cout << "Nested snapshot retry preserves parent and child coverage\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--nested-sampler-snapshot-retry") { + verifyNestedSnapshotRetry(true); + std::cout << "Nested sampler snapshot retry checks the descriptor epoch and preserves parent and child coverage\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake index 2e76d01ccf28..92ed69cd63aa 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_tests.cmake @@ -4,8 +4,20 @@ if(BUILD_TESTING) add_test(NAME agc_driver_nested_image_table COMMAND agc_driver_image_table_tests --nested) add_test(NAME agc_driver_nested_image_table_large COMMAND agc_driver_image_table_tests --nested-large) add_test(NAME agc_driver_nested_image_table_budget COMMAND agc_driver_image_table_tests --nested-budget) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_nested_sampler_table COMMAND agc_driver_image_table_tests --nested-sampler) + add_test(NAME agc_driver_pointer_phi_sampler_table COMMAND agc_driver_image_table_tests --pointer-phi-sampler) + add_test(NAME agc_driver_pointer_phi_ballot_table COMMAND agc_driver_image_table_tests --pointer-phi-ballot) + add_test(NAME agc_driver_pointer_phi_copies_table COMMAND agc_driver_image_table_tests --pointer-phi-copies) + add_test(NAME agc_driver_pointer_phi_lane_and_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-and) + add_test(NAME agc_driver_pointer_phi_high_table COMMAND agc_driver_image_table_tests --pointer-phi-high) + add_test(NAME agc_driver_pointer_phi_high_second_lane_table COMMAND agc_driver_image_table_tests --pointer-phi-high-second-lane) + add_test(NAME agc_driver_pointer_phi_lane_not_table COMMAND agc_driver_image_table_tests --pointer-phi-lane-not) + add_test(NAME agc_driver_pointer_vsharp_select_table COMMAND agc_driver_image_table_tests --pointer-vsharp-select) + add_test(NAME agc_driver_pointer_hop_table COMMAND agc_driver_image_table_tests --pointer-hop) + add_test(NAME agc_driver_pointer_vsharp_fixed_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed) + add_test(NAME agc_driver_pointer_vsharp_fixed_offset_table COMMAND agc_driver_image_table_tests --pointer-vsharp-fixed-offset) + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PIPELINE_JOURNAL=1" SKIP_RETURN_CODE 77 TIMEOUT 30 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_driver_nested_image_table agc_driver_nested_image_table_large agc_driver_nested_image_table_budget agc_driver_nested_sampler_table agc_driver_pointer_phi_sampler_table agc_driver_pointer_phi_ballot_table agc_driver_pointer_phi_copies_table agc_driver_pointer_phi_lane_and_table agc_driver_pointer_phi_high_table agc_driver_pointer_phi_high_second_lane_table agc_driver_pointer_phi_lane_not_table agc_driver_pointer_vsharp_select_table agc_driver_pointer_hop_table agc_driver_pointer_vsharp_fixed_table agc_driver_pointer_vsharp_fixed_offset_table PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake index 1e66e009a013..68f6291c46f9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake @@ -37,8 +37,10 @@ if(BUILD_TESTING) endif() add_test(NAME agc_shader_nested_snapshot_retry COMMAND agc_shader_memory_tests --nested-snapshot-retry) set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES TIMEOUT 20) + add_test(NAME agc_shader_nested_sampler_snapshot_retry COMMAND agc_shader_memory_tests --nested-sampler-snapshot-retry) + set_tests_properties(agc_shader_nested_sampler_snapshot_retry PROPERTIES TIMEOUT 20) if(WIN32) - set_tests_properties(agc_shader_nested_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_nested_snapshot_retry agc_shader_nested_sampler_snapshot_retry PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_table_snapshot_reuse COMMAND agc_shader_memory_tests --table-snapshot-reuse) add_test(NAME agc_shader_region_merge COMMAND agc_shader_memory_tests --region-merge) diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index be41a4c540ed..6919e0161180 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -322,6 +322,153 @@ alignas(256) constexpr std::array NestedRecordCode{ 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, }; +alignas(256) constexpr std::array NestedSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0xf4200504u, 0x20000004u, 0x8f128411u, 0x8f148414u, + 0xf4240b02u, 0x24000008u, 0xf4080616u, 0xfa0000a0u, 0xf428080cu, 0x28000000u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerSamplerCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf40c0900u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf4080816u, 0xfa000020u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerImageCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108302u, 0xf4200444u, 0x20000000u, 0x8f128411u, 0xf4240b02u, 0x24000008u, 0xf40c0916u, 0xfa000040u, + 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xf09c0f08u, + 0x01090401u, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerBallotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820009u, 0xbf068212u, + 0xbf840002u, 0xbeac0334u, 0xbf820004u, 0xbf068312u, 0xbf840001u, 0xbeac0390u, 0xbf800000u, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerCopiesCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf82000eu, 0xbf068212u, 0xbf840007u, 0x3a160613u, + 0x7e2c050bu, 0x8f178416u, 0xf4200b02u, 0x2e000008u, 0xbf8cc07fu, 0xbf820004u, 0xbf068312u, 0xbf840001u, + 0xbeac0390u, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffdeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneAndCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, + 0xbeac03ffu, 0x000000eau, 0xbf820010u, 0xbf068212u, 0xbf840006u, 0xbebc0334u, 0xbf068115u, 0xbf840001u, + 0xbebc0336u, 0x872c343cu, 0xbf820007u, 0xbf068312u, 0xbf840004u, 0xbf068215u, 0xbf840001u, 0xbeac03a0u, + 0x872c342cu, 0xbf800000u, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffd8u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820009u, 0xbf068213u, + 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, 0xbf800000u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffdbu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820012u, 0xbf068313u, + 0xbf840007u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbf820009u, + 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, 0xbf8cc07fu, + 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffd2u, + 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerHighSecondLaneCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0xf4040300u, 0xfa000040u, 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, + 0x100202ffu, 0x3c800000u, 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, + 0x02061411u, 0xbeae047eu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, + 0x32000008u, 0xd4c10034u, 0x00002900u, 0xd4c40036u, 0x00020014u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, + 0xbeac03ffu, 0x000000eau, 0xbf068113u, 0xbf840003u, 0xbead03ffu, 0x00001234u, 0xbf820013u, 0xbf068313u, + 0xbf840008u, 0xf4080e06u, 0xfa000000u, 0xbf8cc07fu, 0xf4200b5cu, 0xfa000000u, 0xbf8cc07fu, 0xbeac037eu, + 0xbf820009u, 0xbf068213u, 0xbf840006u, 0x3a160615u, 0x7e34050bu, 0x8f1b841au, 0xf4200b42u, 0x3600000cu, + 0xbf8cc07fu, 0xbf800000u, 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, + 0xbf89ffd1u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, + 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerLaneNotCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, 0x8f198418u, 0xf4240b02u, 0x32000000u, 0xd4c10034u, + 0x00002900u, 0xbf8cc07fu, 0xbf068112u, 0xbf840002u, 0xbeac0380u, 0xbf82000au, 0xbf068212u, 0xbf840008u, + 0xbebc1334u, 0x8f3d3c81u, 0xbf068115u, 0xbf840002u, 0x8a2c3d34u, 0xbf820002u, 0xbebd073du, 0x872c3d34u, + 0xf40c0916u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe1u, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; + +alignas(256) constexpr std::array PointerVsharpSelectCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108502u, 0xf42c0404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0xd4c40038u, 0x000200c0u, 0x87ea386au, + 0x7e120215u, 0x4a141210u, 0x7e080211u, 0xbf068116u, 0xbf840001u, 0x7e080217u, 0x02061504u, 0xbeae047eu, + 0xd4c5003eu, 0x000200c0u, 0x87bc3e38u, 0xbeba243cu, 0x7e300503u, 0xd4c20030u, 0x00020618u, 0xbeb22430u, + 0x8f198418u, 0xf4240b02u, 0x32000008u, 0xbf8cc07fu, 0xf4080d16u, 0xfa0000a0u, 0x8f1a8513u, 0xbf8cc07fu, + 0xf42c091au, 0x34000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffedu, 0xbefe042eu, + 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerHopCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xd4c50034u, + 0x00020680u, 0xbf8cc07fu, 0xbf068112u, 0xbf840003u, 0xbeac03ffu, 0x000000eau, 0xbf820003u, 0xbf068212u, + 0xbf840001u, 0xbeac0334u, 0xf4040e16u, 0xfa000000u, 0xbf8cc07fu, 0xf40c091cu, 0xfa000340u, 0xbf8cc07fu, + 0xf09c0f08u, 0x01090401u, 0x8afe3032u, 0xbf89ffe4u, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, + 0x34101084u, 0xe0781000u, 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; +alignas(256) constexpr std::array PointerFixedVsharpOffsetCode{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000020u, 0xf4080800u, 0xfa000030u, + 0x8f108402u, 0xf4280404u, 0x20000000u, 0x360200bfu, 0x7e020d01u, 0x060202f0u, 0x100202ffu, 0x3c800000u, + 0x060202f2u, 0x7e0402ffu, 0x3e000000u, 0xbf8cc07fu, 0x7d8800a0u, 0x7e140210u, 0x02061411u, 0xbeae047eu, + 0x7e280503u, 0xd4c20030u, 0x00020614u, 0xbeb22430u, 0x8f158414u, 0xf4240b02u, 0x2a000008u, 0xbf8cc07fu, + 0xf4080d16u, 0xfa0000a0u, 0xbf8cc07fu, 0xf42c091au, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x01090401u, + 0x8afe3032u, 0xbf89ffeeu, 0xbefe042eu, 0x7e120202u, 0x34101286u, 0x4a101100u, 0x34101084u, 0xe0781000u, + 0x80070408u, 0xbf810000u, +}; + constexpr std::array PointClamp{0x92u, (4u * 256u) << 12u, 0u, 0u}; constexpr std::array LinearWrap{0u, (4u * 256u) << 12u, (1u << 20u) | (1u << 22u), 0u}; constexpr std::array PointWrap{0u, (4u * 256u) << 12u, 1u << 24u, 0u}; @@ -1630,6 +1777,665 @@ void RunPointerChainRecordTests(AgcDriver::VulkanDevice& device) { RequireRed(relinked, 1, Lanes, texels[0], "float texels after record 1's second pointer moved to another T#"); } +void RunNestedSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(512) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + std::vector extra; + const std::array, 2>, 2> samplers{{{PointWrap, PointClamp}, {PointClamp, PointWrap}}}; + Table = {}; + Keys = {}; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(samplers[outer][inner].begin(), samplers[outer][inner].end(), palettes[outer].begin() + inner * 4u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const auto image = Image(FloatTexels.data(), Format32Float); + const auto run = [&](std::span code = NestedSamplerCode, std::uint32_t waveSize = 64u) { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(image.begin(), image.end(), Srt.begin() + 12u); + std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + if (!extra.empty()) memory.push_back({reinterpret_cast(extra.data()), std::as_bytes(std::span(extra))}); + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + const auto inputPalettes = palettes; + const auto inputExtra = extra; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords && palettes == inputPalettes && extra == inputExtra, "nested sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "nested sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [&](std::uint32_t record, std::uint32_t child) { + const bool wrap = samplers[record % 2u][child] == PointWrap; + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{0u, 0u, 0u, 1u, 1u, 0u, 1u, 1u}; + std::copy(keys.begin(), keys.end(), Keys.begin()); + const auto original = run(); + Require(!Faulted(original), "nested sampler table: valid independent keys faulted:\n" + original.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(original, group, Lanes, sampled(keys[group * 2u], keys[group * 2u + 1u]), "nested sampler table: the sampler selected through record, pointer and V#"); + const auto wave32 = run(NestedSamplerCode, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "nested sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), palettes[1].begin()); + const auto edited = run(); + Require(!Faulted(edited) && Group(edited, 2) == Group(original, 0) && Group(edited, 3) == Group(original, 3), "nested sampler table: changing only a child S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "nested sampler table: editing a child S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), palettes[1].begin()); + Keys[0] = 3u; + const auto unmapped = run(); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "nested sampler table: an unreadable parent did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "nested sampler table: a faulting dispatch published copied output"); + Keys[0] = 0u; + records[0][43] |= 1u << 30u; + const auto notBuffer = run(); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "nested sampler table: a non-buffer parent was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "nested sampler table: a non-buffer parent published copied output"); + records[0][43] &= ~(1u << 30u); + const auto savedRecords = records; + std::copy_n(records[0].begin() + 40u, 4u, records[1].begin() + 40u); + const auto moved = run(); + Require(!Faulted(moved) && Group(moved, 2) == Group(original, 0) && Group(moved, 3) == Group(original, 1), "nested sampler table: replacing the nested V# retained a stale sampler"); + Require(moved.artifactId == original.artifactId && moved.tableMap.size() < original.tableMap.size(), "nested sampler table: equal V#s did not share one child table"); + records = savedRecords; + auto phi = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + phi.insert(phi.begin() + 17u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = 1u; + const auto phiValid = run(phi); + Require(!Faulted(phiValid), "nested sampler table: a captured pointer selected through Phi faulted:\n" + phiValid.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(phiValid, group, Lanes, sampled(1u, Keys[group * 2u + 1u]), "nested sampler table: Phi-selected pointer"); + Keys[0] = 0u; + const auto phiFault = run(phi); + Require(Faulted(phiFault) && phiFault.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured pointer did not fault:\n" + phiFault.log); + Require(untouched(phiFault), "nested sampler table: an uncaptured pointer published copied output"); + Require(phiFault.artifactId == phiValid.artifactId, "nested sampler table: the pointer choice was specialized on the CPU"); + std::copy(keys.begin(), keys.end(), Keys.begin()); + extra.assign(320u * 4u, 0u); + for (std::uint32_t entry = 0; entry < 320u; ++entry) { + std::copy(PointWrap.begin(), PointWrap.end(), extra.begin() + entry * 4u); + extra[entry * 4u + 2u] |= entry; + } + const auto many = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(many.begin(), many.end(), records[0].begin() + 40u); + Keys[1] = 319u; + auto unkeyed = std::vector(NestedSamplerCode.begin(), NestedSamplerCode.end()); + unkeyed.insert(unkeyed.begin() + 14u, {0x8714ff14u, 0x000001ffu}); + const auto overBudget = run(unkeyed); + Require(Faulted(overBudget) && overBudget.log.find("over budget") != std::string::npos, "nested sampler table: an S# past the sampler budget did not fault:\n" + overBudget.log); + Require(untouched(overBudget), "nested sampler table: an S# past the sampler budget published copied output"); + extra.assign((NestedKeyLimit + 1u) * 4u, 0u); + std::copy(PointClamp.begin(), PointClamp.end(), extra.begin()); + const auto large = Buffer(extra.data(), static_cast(extra.size() * sizeof(std::uint32_t))); + std::copy(large.begin(), large.end(), records[0].begin() + 40u); + Keys[1] = 0u; + const auto bounded = run(); + Require(Faulted(bounded) && bounded.log.find("outside the snapshot") != std::string::npos, "nested sampler table: an uncaptured child did not fault:\n" + bounded.log); + Require(bounded.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: the two levels produced an unbounded map"); + Require(untouched(bounded), "nested sampler table: a dispatch outside the snapshot published copied output"); + std::copy(large.begin(), large.end(), records[1].begin() + 40u); + const auto sharedBudget = run(); + Require(!Faulted(sharedBudget) && sharedBudget.tableMap.size() < NestedKeyLimit + 1024u, "nested sampler table: equal V#s consumed the snapshot budget twice"); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool clamp = keys[group * 2u + 1u] == 0u; + RequireRed(sharedBudget, group, Lanes, [clamp](std::uint32_t x) { return std::bit_cast(FloatTexels[clamp ? Width - 1u : x]); }, "nested sampler table: aliased parent at the snapshot budget"); + } + std::puts("nested sampler table: S# through record, pointer and V# checked on wave64 and wave32, with sharing and budgets"); +} + +void RunPointerSamplerTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + const std::array, 2> samplers{PointWrap, PointClamp}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) { + std::copy(samplers[record].begin(), samplers[record].end(), records[record].begin() + 8u); + std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + auto root = images[0]; + std::uint32_t tableType = 0u; + const auto run = [&](std::span code, std::uint32_t waveSize = 64u) { + auto table = Buffer(Table.data(), 64u); + table[3] |= tableType; + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(root.begin(), root.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputKeys = Keys; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, waveSize); + Require(Table == inputTable && Keys == inputKeys && records == inputRecords, "pointer sampler table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer sampler table: dispatch wrote past its output"); + return result; + }; + const auto sampled = [](bool wrap) { + return std::function([wrap](std::uint32_t x) { return std::bit_cast(FloatTexels[wrap ? x : Width - 1u]); }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const std::array keys{1u, 2u, 1u, 2u}; + for (std::uint32_t group = 0; group < 4u; ++group) Keys[group * 2u] = keys[group]; + const auto direct = run(PointerSamplerCode); + Require(!Faulted(direct), "pointer sampler table: an S# read through a record's pointer faulted:\n" + direct.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(direct, group, Lanes, sampled(keys[group] % 2u == 0u), "pointer sampler table: the S# read through the record's pointer"); + auto phi = std::vector(PointerSamplerCode.begin(), PointerSamplerCode.end()); + phi.insert(phi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto original = run(phi); + Require(!Faulted(original) && original.words == direct.words, "pointer sampler table: a captured pointer selected through Phi changed the result:\n" + original.log); + const auto wave32 = run(phi, 32u); + Require(!Faulted(wave32) && wave32.words == original.words, "pointer sampler table: wave32 and wave64 disagree"); + std::copy(PointWrap.begin(), PointWrap.end(), records[1].begin() + 8u); + const auto edited = run(phi); + Require(!Faulted(edited) && Group(edited, 0) == Group(original, 1) && Group(edited, 2) == Group(original, 3), "pointer sampler table: changing an S# retained a stale sampler"); + Require(edited.artifactId == original.artifactId, "pointer sampler table: editing an S# recompiled the shader"); + std::copy(PointClamp.begin(), PointClamp.end(), records[1].begin() + 8u); + Keys[0] = 0u; + const auto uncaptured = run(phi); + Require(Faulted(uncaptured) && uncaptured.log.find("outside the snapshot") != std::string::npos, "pointer sampler table: an uncaptured pointer did not fault:\n" + uncaptured.log); + Require(untouched(uncaptured), "pointer sampler table: an uncaptured pointer published copied output"); + Require(uncaptured.artifactId == original.artifactId, "pointer sampler table: the pointer choice was specialized on the CPU"); + auto alias = phi; + alias[17] = static_cast(reinterpret_cast(records[1].data())); + const auto aliased = run(alias); + Require(!Faulted(aliased) && Group(aliased, 0) == Group(original, 0) && Group(aliased, 1) == Group(original, 1), "pointer sampler table: a Phi constant naming another captured pointer did not read that pointer's S#:\n" + aliased.log); + Keys[0] = 3u; + const auto unmapped = run(phi); + Require(Faulted(unmapped) && unmapped.log.find("unmapped") != std::string::npos, "pointer sampler table: a null pointer did not fault:\n" + unmapped.log); + Require(untouched(unmapped), "pointer sampler table: a null pointer published copied output"); + Keys[0] = 1u; + tableType = 1u << 30u; + const auto notBuffer = run(phi); + Require(Faulted(notBuffer) && notBuffer.log.find("not a buffer") != std::string::npos, "pointer sampler table: a non-buffer record table was accepted:\n" + notBuffer.log); + Require(untouched(notBuffer), "pointer sampler table: a non-buffer record table published copied output"); + tableType = 0u; + struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } + ~Registration() { GuestAllocations::Mutation().Remove(data); } + }; + { + const Registration registered(records.data(), sizeof(records)); + auto consumer = phi; + consumer.insert(consumer.begin() + 30u, 0x7e0e0221u); + const auto shared = run(consumer); + Require(!Faulted(shared), "pointer sampler table: an S# with another consumer faulted:\n" + shared.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + auto expected = Group(original, group); + for (std::uint32_t lane = 0; lane < Lanes; ++lane) expected[lane * 4u + 3u] = samplers[keys[group] % 2u][1]; + Require(Group(shared, group) == expected, "pointer sampler table: an S# with another consumer read the wrong words"); + } + } + root = PointWrap; + auto imagePhi = std::vector(PointerImageCode.begin(), PointerImageCode.end()); + imagePhi.insert(imagePhi.begin() + 14u, {0xbf068011u, 0xbf840002u, 0xbeac03ffu, 0x000000eau}); + const auto imageDirect = run(PointerImageCode); + const auto imageMerged = run(imagePhi); + Require(!Faulted(imageDirect) && !Faulted(imageMerged) && imageMerged.words == imageDirect.words, "pointer image table: a captured pointer selected through Phi changed the result:\n" + imageMerged.log); + for (std::uint32_t group = 0; group < 4u; ++group) { + const bool other = keys[group] % 2u != 0u; + RequireRed(imageMerged, group, Lanes, [other](std::uint32_t x) { return std::bit_cast(other ? OtherFloatTexels[x] : FloatTexels[x]); }, "pointer image table: the T# read through a Phi-selected pointer"); + } + Keys[0] = 0u; + const auto imageFault = run(imagePhi); + Require(Faulted(imageFault) && imageFault.log.find("outside the snapshot") != std::string::npos && untouched(imageFault), "pointer image table: an uncaptured pointer did not fault:\n" + imageFault.log); + std::puts("pointer sampler table: S# and T# through a record pointer whose low word passes a Phi checked on wave64 and wave32"); +} + +void RunPointerArmTests(AgcDriver::VulkanDevice& device, bool copies) { + Fill(); + alignas(256) std::array, 2> records{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::span code = copies ? std::span(PointerCopiesCode) : std::span(PointerBallotCode); + const std::string name = copies ? "pointer table with a second record read" : "pointer table with a lane-mask arm"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group, std::uint32_t flip) { + return std::function([&keys, group, flip](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u] ^ flip; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group, 0u), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(3u, "a second constant pointer word"); + if (!copies) { + outside(2u, "a lane-mask pointer word"); + std::puts("pointer table with a lane-mask arm: a T# through a record pointer whose low word passes nested Phis with constants and a Ballot word checked, and the Ballot word faults outside the snapshot"); + return; + } + Keys[2] = 2u; + const auto again = run(); + Require(!Faulted(again) && again.words == taken.words, name + ": a second read of the same record changed the result:\n" + again.log); + Keys[3] = 3u; + const auto flipped = run(); + Keys[2] = 0u; + Keys[3] = 0u; + Require(!Faulted(flipped) && flipped.artifactId == taken.artifactId, name + ": a second read of another record faulted or recompiled:\n" + flipped.log); + RequireRed(flipped, 0u, Lanes, sampled(0u, 3u), name + ": the T# behind the other record's pointer word"); + for (std::uint32_t group = 1; group < 4u; ++group) Require(Group(flipped, group) == Group(taken, group), name + ": a group that kept its record changed its result"); + std::puts("pointer table with a second record read: a T# through a record pointer whose low word passes a second read keyed by another lane checked, by the pointer's value"); +} + +enum class PointerMask { LaneAnd, High, HighSecond, HighSecondLane, LaneNot }; + +void RunPointerMaskTests(AgcDriver::VulkanDevice& device, PointerMask shape) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array second{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + for (std::uint32_t record = 0; record < 2u; ++record) std::copy(images[record].begin(), images[record].end(), records[record].begin() + 16u); + const std::uint32_t pointerWord = shape == PointerMask::LaneNot ? 0u : 2u; + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(records[record % 2u].data()); + Table[record * 4u + pointerWord] = static_cast(pointer); + Table[record * 4u + pointerWord + 1u] = static_cast(pointer >> 32u); + } + const auto secondBuffer = Buffer(second.data() + 8u, 32u); + std::copy(secondBuffer.begin(), secondBuffer.end(), second.begin()); + std::span code = shape == PointerMask::LaneAnd ? std::span(PointerLaneAndCode) : shape == PointerMask::High ? std::span(PointerHighCode) : shape == PointerMask::LaneNot ? std::span(PointerLaneNotCode) : shape == PointerMask::HighSecondLane ? std::span(PointerHighSecondLaneCode) : std::span(PointerHighSecondCode); + const std::string name = shape == PointerMask::LaneAnd ? "pointer table with lane-mask ANDs" : shape == PointerMask::High ? "pointer table with a high-word Phi" : shape == PointerMask::LaneNot ? "pointer table with a lane bit cleared from a lane mask" : shape == PointerMask::HighSecondLane ? "pointer table with a second-buffer high word under a lane-mask low word" : "pointer table with a high word from a second buffer"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + const auto secondAddress = reinterpret_cast(second.data()); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + Srt[16] = static_cast(secondAddress); + Srt[17] = static_cast(secondAddress >> 32u); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {secondAddress, std::as_bytes(std::span(second))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + } + const auto sampled = [&keys](std::uint32_t group) { + return std::function([&keys, group](std::uint32_t x) { + const auto key = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(key % 2u != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t selector, std::uint32_t arm, std::uint32_t variant, const std::string& what) { + Keys[selector] = arm; + Keys[5] = variant; + const auto faulted = run(); + Keys[selector] = 0u; + Keys[5] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(2u, 1u, 0u, "a constant low word"); + if (shape == PointerMask::LaneAnd) { + outside(2u, 2u, 0u, "a lane-mask Phi ANDed with a lane mask"); + outside(2u, 2u, 1u, "the other lane-mask word ANDed with a lane mask"); + outside(2u, 3u, 0u, "a record word ANDed with a lane mask"); + std::puts("pointer table with lane-mask ANDs: a T# through a record pointer whose low word passes Phis of lane-mask ANDs checked, and every masked word faults outside the snapshot"); + return; + } + if (shape == PointerMask::LaneNot) { + outside(2u, 2u, 0u, "a lane mask whose handled lane's bit s_not and s_and cleared"); + outside(2u, 2u, 1u, "a lane mask whose handled lane's bit s_andn2 cleared"); + std::puts("pointer table with a lane bit cleared from a lane mask: a T# through a record pointer whose low word passes Phis of a zero and of a lane mask without the handled lane's bit checked, and every masked word faults outside the snapshot"); + return; + } + outside(3u, 1u, 0u, "a constant high word"); + if (shape == PointerMask::HighSecondLane) outside(3u, 3u, 0u, "a lane-mask low word with a high word from the second buffer"); + Keys[3] = 2u; + const auto again = run(); + Keys[5] = 3u; + const auto flipped = run(); + Keys[3] = 0u; + Keys[5] = 0u; + Require(!Faulted(again) && again.words == taken.words && again.artifactId == taken.artifactId, name + ": a second read of the same record's high word changed the result:\n" + again.log); + Require(!Faulted(flipped) && flipped.words == taken.words && flipped.artifactId == taken.artifactId, name + ": the high word of the neighbouring record changed the result:\n" + flipped.log); + if (shape == PointerMask::HighSecondLane) { + std::puts("pointer table with a second-buffer high word under a lane-mask low word: a T# through a record pointer whose high word reads a second buffer only on the edge whose low word is a lane mask checked, and that pair faults outside the snapshot"); + return; + } + std::puts("pointer table with a high-word Phi: a T# through a record pointer whose high word passes a Phi of reads under other keys and a constant checked, by the pointer's value"); +} + +void RunPointerVsharpTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float), Image(UintTexels.data(), Format32UInt)}; + const std::array, 3> texels{ + [](std::uint32_t x) { return std::bit_cast(FloatTexels[x]); }, + [](std::uint32_t x) { return std::bit_cast(OtherFloatTexels[x]); }, + [](std::uint32_t x) { return UintTexels[x]; }, + }; + for (std::uint32_t outer = 0; outer < 2u; ++outer) { + auto descriptor = Buffer(palettes[outer].data(), 1u); + descriptor[1] |= 64u << 16u; + std::copy(descriptor.begin(), descriptor.end(), records[outer].begin() + 40u); + for (std::uint32_t inner = 0; inner < 2u; ++inner) std::copy(images[outer + inner].begin(), images[outer + inner].end(), palettes[outer].begin() + inner * 8u); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerVsharpSelectCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, "pointer V# table: dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), "pointer V# table: dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u, 0u}, {2u, 1u, 1u}, {1u, 1u, 1u}, {2u, 2u, 0u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 8u] = keys[group][0]; + Keys[group * 8u + 1u] = keys[group][1]; + Keys[group * 8u + 3u] = keys[group][2]; + } + std::array targets{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return texels[targets[record] + keys[group][2]](x); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), "pointer V# table: the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), "pointer V# table: the T# read through the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + targets[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), "pointer V# table: the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, "pointer V# table: moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), "pointer V# table: the T# after record 1's pointer moved to the other V#"); + std::puts("pointer V# table: a T# read through a V# loaded through a record pointer keyed by a first-lane read of a select checked, by the records' values"); +} + +void RunPointerHopTests(AgcDriver::VulkanDevice& device) { + Fill(); + alignas(256) std::array, 2> links{}; + alignas(256) std::array, 2> targets{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto link = [&](std::uint32_t from, std::uint32_t to) { + const auto pointer = reinterpret_cast(targets[to].data()); + links[from][0] = static_cast(pointer); + links[from][1] = static_cast(pointer >> 32u); + }; + for (std::uint32_t target = 0; target < 2u; ++target) { + std::copy(images[target].begin(), images[target].end(), targets[target].begin() + 0x340u / 4u); + link(target, target); + } + for (std::uint32_t record = 0; record < 3u; ++record) { + const auto pointer = reinterpret_cast(links[record % 2u].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + } + const std::string name = "pointer table with a pointer hop"; + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code(PointerHopCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(links.data()), std::as_bytes(std::span(links))}, + {reinterpret_cast(targets.data()), std::as_bytes(std::span(targets))}, + }; + const auto inputTable = Table; + const auto inputLinks = links; + const auto inputTargets = targets; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && links == inputLinks && targets == inputTargets, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array targetOf{0u, 1u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(targetOf[record % 2u] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto untouched = [](const Outcome& outcome) { return std::all_of(outcome.words.begin(), outcome.words.end(), [](auto word) { return word == 0xdeadbeefu; }); }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# behind the record pointer's hop faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read through the pointer loaded from the record's pointer in a waterfall loop"); + const auto outside = [&](std::uint32_t arm, const std::string& what) { + Keys[2] = arm; + const auto faulted = run(); + Keys[2] = 0u; + Require(Faulted(faulted) && faulted.log.find("outside the snapshot") != std::string::npos, name + ": " + what + " did not fault as outside the snapshot:\n" + faulted.log); + Require(untouched(faulted), name + ": " + what + " published copied output"); + Require(faulted.artifactId == taken.artifactId, name + ": " + what + " recompiled the shader"); + }; + outside(1u, "a constant pointer word"); + outside(2u, "a lane-mask pointer word"); + link(1u, 0u); + targetOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved hop faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving the pointer behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after the pointer behind record 1's pointer moved to the other T#"); + std::puts("pointer table with a pointer hop: a T# read through a pointer loaded from a record pointer whose low word passes Phis with a constant and a Ballot word checked, by the records' values"); +} + +void RunPointerFixedVsharpTests(AgcDriver::VulkanDevice& device, std::uint32_t descriptorOffset) { + Fill(); + alignas(256) std::array, 2> records{}; + alignas(256) std::array, 2> palettes{}; + Table = {}; + Keys = {}; + const std::array images{Image(FloatTexels.data(), Format32Float), Image(OtherFloatTexels.data(), Format32Float)}; + const auto point = [&](std::uint32_t record, std::uint32_t palette) { + const auto descriptor = Buffer(palettes[palette].data(), static_cast(sizeof(palettes[palette]))); + std::copy(descriptor.begin(), descriptor.end(), records[record].begin() + 40u); + }; + for (std::uint32_t palette = 0; palette < 2u; ++palette) { + std::copy(images[palette].begin(), images[palette].end(), palettes[palette].begin() + descriptorOffset / 4u); + point(palette, palette); + } + const auto link = [&](std::uint32_t record, std::uint32_t target) { + const auto pointer = reinterpret_cast(records[target].data()); + Table[record * 4u + 2u] = static_cast(pointer); + Table[record * 4u + 3u] = static_cast(pointer >> 32u); + }; + for (std::uint32_t record = 0; record < 3u; ++record) link(record, record % 2u); + const std::string name = std::string("pointer V# table with a T# at fixed offset ") + Hex(descriptorOffset); + const auto run = [&] { + const auto table = Buffer(Table.data(), 64u); + const auto keys = Buffer(Keys.data(), static_cast(sizeof(Keys))); + const auto output = Buffer(Output.data(), static_cast(sizeof(Output))); + Srt = {}; + std::copy(table.begin(), table.end(), Srt.begin()); + std::copy(keys.begin(), keys.end(), Srt.begin() + 4u); + std::copy(output.begin(), output.end(), Srt.begin() + 8u); + std::copy(PointWrap.begin(), PointWrap.end(), Srt.begin() + 12u); + const std::span code = descriptorOffset == 0u ? std::span(PointerFixedVsharpCode) : std::span(PointerFixedVsharpOffsetCode); + const std::vector memory{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(Srt.data()), std::as_bytes(std::span(Srt))}, + {reinterpret_cast(Table.data()), std::as_bytes(std::span(Table))}, + {reinterpret_cast(Keys.data()), std::as_bytes(std::span(Keys))}, + {reinterpret_cast(records.data()), std::as_bytes(std::span(records))}, + {reinterpret_cast(palettes.data()), std::as_bytes(std::span(palettes))}, + }; + const auto inputTable = Table; + const auto inputRecords = records; + const auto inputPalettes = palettes; + auto result = Execute(device, code, Srt.data(), memory, 4u, Lanes, 64u); + Require(Table == inputTable && records == inputRecords && palettes == inputPalettes, name + ": dispatch modified its input memory"); + Require(std::all_of(Output.begin() + 4u * Lanes * 4u, Output.end(), [](auto word) { return word == 0xdeadbeefu; }), name + ": dispatch wrote past its output"); + return result; + }; + const std::array, 4> keys{{{1u, 2u}, {2u, 1u}, {1u, 1u}, {2u, 2u}}}; + for (std::uint32_t group = 0; group < 4u; ++group) { + Keys[group * 4u] = keys[group][0]; + Keys[group * 4u + 1u] = keys[group][1]; + } + std::array paletteOf{0u, 1u, 0u}; + const auto sampled = [&](std::uint32_t group) { + return std::function([&, group](std::uint32_t x) { + const auto record = keys[group][x < 32u ? 0u : 1u]; + return std::bit_cast(paletteOf[record] != 0u ? OtherFloatTexels[x] : FloatTexels[x]); + }); + }; + const auto taken = run(); + Require(!Faulted(taken), name + ": the T# read through the V# behind the record's pointer faulted:\n" + taken.log); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(taken, group, Lanes, sampled(group), name + ": the T# read at a fixed offset of the V# behind the record's pointer in a waterfall loop"); + link(1u, 0u); + paletteOf[1] = 0u; + const auto moved = run(); + Require(!Faulted(moved), name + ": the T# behind a moved record pointer faulted:\n" + moved.log); + Require(moved.artifactId == taken.artifactId, name + ": moving a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(moved, group, Lanes, sampled(group), name + ": the T# after record 1's pointer moved to the other V#"); + point(0u, 1u); + paletteOf = {1u, 1u, 1u}; + const auto repointed = run(); + Require(!Faulted(repointed), name + ": the T# behind a moved V# faulted:\n" + repointed.log); + Require(repointed.artifactId == taken.artifactId, name + ": moving the V# behind a record's pointer recompiled the shader"); + for (std::uint32_t group = 0; group < 4u; ++group) RequireRed(repointed, group, Lanes, sampled(group), name + ": the T# after the V# behind records 0 and 1 moved to the other palette"); + std::puts((name + ": a T# read at a fixed offset of a V# loaded through a record pointer checked, by the records' values").c_str()); +} + void RunSplitRecordTests(AgcDriver::VulkanDevice& device) { Fill(); const auto floatImage = Image(FloatTexels.data(), Format32Float); @@ -1907,6 +2713,54 @@ int main(int argc, char** argv) { RunNestedRecordTests(*device, NestedCase::Budget); return 0; } + if (argc == 2 && std::strcmp(argv[1], "--nested-sampler") == 0) { + RunNestedSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-sampler") == 0) { + RunPointerSamplerTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-ballot") == 0) { + RunPointerArmTests(*device, false); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-copies") == 0) { + RunPointerArmTests(*device, true); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-and") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneAnd); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high") == 0) { + RunPointerMaskTests(*device, PointerMask::High); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-high-second-lane") == 0) { + RunPointerMaskTests(*device, PointerMask::HighSecondLane); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-phi-lane-not") == 0) { + RunPointerMaskTests(*device, PointerMask::LaneNot); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-select") == 0) { + RunPointerVsharpTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-hop") == 0) { + RunPointerHopTests(*device); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed") == 0) { + RunPointerFixedVsharpTests(*device, 0u); + return 0; + } + if (argc == 2 && std::strcmp(argv[1], "--pointer-vsharp-fixed-offset") == 0) { + RunPointerFixedVsharpTests(*device, 0x40u); + return 0; + } RunTests(*device); RunBorderSwizzleTests(*device); RunAnisoOverrideTests(*device); diff --git a/core/shader/recompiler/ImageTableAbi.hpp b/core/shader/recompiler/ImageTableAbi.hpp index b9c9773b5f4b..27d982e2e04f 100644 --- a/core/shader/recompiler/ImageTableAbi.hpp +++ b/core/shader/recompiler/ImageTableAbi.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler::ImageTableAbi { -inline constexpr std::uint32_t Version = 3; +inline constexpr std::uint32_t Version = 4; inline constexpr std::uint32_t HeaderVersion = 0; inline constexpr std::uint32_t HeaderElementBase = 1; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 646f625d9a38..714c782bf4f3 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -55,9 +55,10 @@ struct TableColumn { std::uint32_t foldedCarry = 0; bool selected = false; std::uint32_t selection = 0; + bool pointerKeyed = false; bool operator==(const TableColumn& other) const { - return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection; + return heapSource == other.heapSource && stride == other.stride && addend == other.addend && offset == other.offset && dwordCount == other.dwordCount && sampler == other.sampler && address == other.address && buffer == other.buffer && indirect == other.indirect && indirectOffset == other.indirectOffset && indirectImmediate == other.indirectImmediate && pointerHops == other.pointerHops && foldedAddend == other.foldedAddend && foldedCarry == other.foldedCarry && selected == other.selected && selection == other.selection && pointerKeyed == other.pointerKeyed; } }; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index 034d6b7947b5..05979a8ac973 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -132,6 +132,7 @@ struct ImageTableColumnSnapshot { std::uint32_t fault = 0; bool outside = false; bool nested = false; + bool pointerKeyed = false; SharedValues pointers; SharedValues codes; SharedValues words; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 3c562da2bc9d..83e1bf17d99b 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -755,7 +755,10 @@ class TableSnapshotter { for (const auto& sampler : plan.info.samplers) { if (sampler.table != NoTable) { count = std::max(count, sampler.table + 1u); - if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value() && plan.descriptorSources[sampler.source].tableColumn->indirect) hasIndirect = true; + if (sampler.source < plan.descriptorSources.size() && plan.descriptorSources[sampler.source].tableColumn.has_value()) { + const auto& column = *plan.descriptorSources[sampler.source].tableColumn; + hasIndirect |= column.indirect || plan.descriptorSources.at(column.heapSource).tableColumn.has_value(); + } } } if (count == 0u) { @@ -1131,7 +1134,7 @@ class TableSnapshotter { const auto end = records.base + records.size; if (column.indirect) { unread(); - if (column.buffer) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); + if (column.buffer || column.pointerKeyed) records.pointers.assign(records.keys, ImageTableAbi::UncapturedPointer); for (std::uint32_t record = 0; record < records.keys; record++) { if (records.states[record] != RecordState::Read) continue; const auto offset = column.addend + record * column.stride; @@ -1142,6 +1145,7 @@ class TableSnapshotter { continue; } std::uint64_t pointer = (static_cast(Read(records.base + *highWord)) << 32u) | Read(records.base + *lowWord); + if (column.pointerKeyed) records.pointers[record] = pointer; for (const auto& hop : column.pointerHops) { const auto low = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate); const auto high = ImageTableAbi::ScalarAddressDword(pointer, hop.offset, hop.immediate + 4u); @@ -1498,6 +1502,7 @@ class TableSnapshotter { const auto& records = Column(source); auto& out = tables.tables.at(table); out = CopyRecords(records); + out.pointerKeyed = column.pointerKeyed; for (std::uint32_t resource = 0; resource < plan.info.images.size(); ++resource) { const auto& image = plan.info.images[resource]; if (image.table != table || !registeredSampledArrayCandidate(image) || runtime.normalizeImageView == nullptr) continue; @@ -1922,7 +1927,7 @@ bool sameColumns(const std::vector& left, const std::v for (std::size_t index = 0; index < left.size(); index++) { const auto& a = left[index]; const auto& b = right[index]; - if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; + if (a.base != b.base || a.size != b.size || a.records != b.records || a.keys != b.keys || a.fault != b.fault || a.outside != b.outside || a.nested != b.nested || a.pointerKeyed != b.pointerKeyed || !a.pointers.Shares(b.pointers) || !a.codes.Shares(b.codes) || !a.words.Shares(b.words) || a.resourceViews != b.resourceViews) return false; } return true; } @@ -2092,20 +2097,21 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i out.map[header + Abi::TableFault] = poison(resource, PoisonReason::UnsupportedOperation, nullptr); } outside[table] = outside[table] || column.outside; + const bool byPointer = column.nested || column.pointerKeyed; const auto mapStart = static_cast(out.map.size()); - if (!column.nested) { + if (!byPointer) { growMap(column.keys); out.map[header + Abi::TableMapStart] = mapStart; out.map.resize(out.map.size() + column.keys, 0u); } else if (column.pointers.size() != column.keys) { - throw std::runtime_error("nested image table snapshot has no pointer for each record"); + throw std::runtime_error("pointer-keyed image table snapshot has no pointer for each record"); } std::map pointers; if (column.nested) nestedRecords.Clear(); else records.Clear(); for (std::uint32_t key = 0; key < column.keys; key++) { const auto code = column.codes[key]; - if (column.nested && column.pointers[key] == Abi::UncapturedPointer) continue; + if (byPointer && column.pointers[key] == Abi::UncapturedPointer) continue; std::uint32_t mapped = Abi::NullCode; if ((code & Abi::PoisonFlag) != 0u) { const auto reason = static_cast(code & ~Abi::PoisonFlag); @@ -2133,15 +2139,15 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i records.Set(code, mapped); } } - if (column.nested) { + if (byPointer) { const auto [found, inserted] = pointers.try_emplace(column.pointers[key], mapped); - if (!inserted && found->second != mapped) throw std::runtime_error("nested image table captured conflicting values at one pointer"); + if (!inserted && found->second != mapped) throw std::runtime_error("pointer-keyed image table captured conflicting values at one pointer"); } else { out.map[mapStart + key] = mapped; } } - if (column.nested) { - if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("nested image table has too many pointers"); + if (byPointer) { + if (pointers.size() > Abi::MaxParentKeys) throw std::runtime_error("pointer-keyed image table has too many pointers"); growMap(pointers.size() * Abi::PointerRecordWords); out.map[header + Abi::TableKeyCount] = static_cast(pointers.size()); out.map[header + Abi::TableMapStart] = static_cast(out.map.size()); diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 1d34074f1376..b3d3915bb67c 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -841,7 +842,13 @@ class Tracker { if (address && immediateU32(offset, descriptorOffset)) { return TryMakeIndirectColumn(handle, *heapHandle, sampler, words, descriptorOffset, first, plan); } - if (!MatchAffineOffset(offset, key, stride, addend)) { + std::uint32_t fixedOffset = 0; + const bool fixed = !address && !buffer && immediateU32(offset, fixedOffset); + if (fixed) { + key = &m_builder.Constant(0u); + stride = width * static_cast(sizeof(std::uint32_t)); + addend = fixedOffset; + } else if (!MatchAffineOffset(offset, key, stride, addend)) { if (!MatchMaskedOffset(offset, key, stride) && (address || !MatchFirstLaneOffset(offset, key, stride))) { return false; } @@ -849,8 +856,10 @@ class Tracker { } DescriptorSource heapSource; std::uint32_t heapIndex = 0; - if (!MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { - if (sampler || address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; + if (MakeRuntimeSource(*heapHandle, address ? IrOpcode::GetAddressResource : IrOpcode::GetBufferResource, address ? 2u : 4u, heapIndex, heapSource)) { + if (fixed) return false; + } else { + if (address || buffer || heapHandle->Opcode() != IrOpcode::GetBufferResource || !std::ranges::all_of(heapHandle->Uses(), [&](const IrValue* use) { return std::ranges::find(plan.reads, use) != plan.reads.end(); })) return false; const auto sameHandle = [&](const IrValue* user) { if (user->Opcode() != handle.Opcode() || user->ArgumentCount() != handle.ArgumentCount()) return false; for (std::uint32_t argument = 0; argument < handle.ArgumentCount(); argument++) { @@ -914,10 +923,81 @@ class Tracker { return true; } - IrValue* PointerRecordOrigin(IrValue* value) const { + static bool LaneMaskWord(const IrValue& value) { + return value.Opcode() == IrOpcode::CompositeExtractU32x4 && value.ArgumentCount() == 2u && value.Argument(0)->Resolve()->Opcode() == IrOpcode::Ballot && value.Argument(1)->Resolve()->HasImmediate(); + } + + bool SameRecordColumn(IrValue& first, IrValue& second) const { + IrValue* firstKey = nullptr; + IrValue* secondKey = nullptr; + std::uint32_t firstStride = 0; + std::uint32_t firstAddend = 0; + std::uint32_t secondStride = 0; + std::uint32_t secondAddend = 0; + return EquivalentValue(m_program.Resources(), first.Argument(0), second.Argument(0)) && MatchAffineOffset(first.Argument(1), firstKey, firstStride, firstAddend) && MatchAffineOffset(second.Argument(1), secondKey, secondStride, secondAddend) && firstStride == secondStride && firstAddend == secondAddend; + } + + static bool LaneIndex(const IrValue* value) { + std::vector pending{value}; + std::unordered_set seen; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second || (current->HasImmediate() && current->Type() == IrType::U32)) continue; + switch (current->Opcode()) { + case IrOpcode::FindILsb32: + case IrOpcode::FindUMsb32: + case IrOpcode::ReadFirstLane: continue; + case IrOpcode::BitwiseAnd32: + case IrOpcode::IAdd32: + case IrOpcode::ISub32: + if (current->ArgumentCount() != 2u) return false; + pending.push_back(current->Argument(0)); + pending.push_back(current->Argument(1)); + continue; + case IrOpcode::SelectU32: + if (current->ArgumentCount() != 3u) return false; + pending.push_back(current->Argument(1)); + pending.push_back(current->Argument(2)); + continue; + default: return false; + } + } + return true; + } + + static bool LaneBit(const IrValue& value) { + if (value.Opcode() != IrOpcode::ShiftLeftLogical32 || value.ArgumentCount() != 2u) return false; + const IrValue* one = value.Argument(0)->Resolve(); + return one->HasImmediate() && one->Type() == IrType::U32 && one->ImmediateU32() == 1u && LaneIndex(value.Argument(1)); + } + + static bool LaneMaskOperand(const IrValue* value) { + const IrValue* root = value->Resolve(); + std::vector pending{root}; + std::unordered_set seen; + bool lane = false; + while (!pending.empty()) { + const IrValue* current = pending.back()->Resolve(); + pending.pop_back(); + if (!seen.insert(current).second) continue; + if (LaneMaskWord(*current) || LaneBit(*current)) { + lane = true; + continue; + } + if (current != root && current->HasImmediate() && current->Type() == IrType::U32) continue; + const auto opcode = current->Opcode(); + const bool combines = (opcode == IrOpcode::BitwiseAnd32 || opcode == IrOpcode::BitwiseOr32) && current->ArgumentCount() == 2u; + const bool inverts = opcode == IrOpcode::BitwiseNot32 && current->ArgumentCount() == 1u; + if (!current->IsPhi() && !combines && !inverts) return false; + for (std::size_t index = 0; index < current->ArgumentCount(); ++index) pending.push_back(current->Argument(index)); + } + return lane; + } + + bool PointerRecordReads(IrValue* value, std::vector& reads) const { std::vector pending{value}; std::unordered_set seen; - IrValue* origin = nullptr; while (!pending.empty()) { auto* current = pending.back()->Resolve(); pending.pop_back(); @@ -928,11 +1008,94 @@ class Tracker { } if (current->HasImmediate() && current->Type() == IrType::U32) continue; std::uint32_t memory = 0; - if (ScalarReadMemory(*current, memory) == nullptr || !MemoryIndexBelongsTo(memory, *current)) return nullptr; - if (origin != nullptr && !EquivalentValue(m_program.Resources(), origin, current)) return nullptr; - origin = current; + const MemoryInfo* info = ScalarReadMemory(*current, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, *current)) return false; + if (!reads.empty() && !EquivalentValue(m_program.Resources(), reads.back(), current)) return false; + reads.push_back(current); } - return origin; + return !reads.empty(); + } + + static bool PointerFreeWord(const IrValue& word) { + return (word.HasImmediate() && word.Type() == IrType::U32) || LaneMaskOperand(&word); + } + + static IrValue* MaskedRecordWord(const IrValue& word) { + if (word.Opcode() != IrOpcode::BitwiseAnd32 || word.ArgumentCount() != 2u) return nullptr; + const bool left = LaneMaskOperand(word.Argument(0)); + if (left == LaneMaskOperand(word.Argument(1))) return nullptr; + return word.Argument(left ? 1u : 0u); + } + + bool PointerRecordRead(IrValue& read, std::vector& reads) const { + std::uint32_t memory = 0; + const MemoryInfo* info = ScalarReadMemory(read, memory); + if (info == nullptr || !MemoryIndexBelongsTo(memory, read)) return false; + if (std::ranges::find(reads, &read) != reads.end()) return true; + if (!reads.empty() && (ScalarReadMemory(*reads.back(), memory)->offset != info->offset || !SameRecordColumn(*reads.back(), read))) return false; + reads.push_back(&read); + return true; + } + + bool PointerRecordPairs(IrValue* low, IrValue* high, std::vector& lowReads, std::vector& highReads) const { + std::vector> pending{{low, high}}; + std::set> seen; + while (!pending.empty()) { + auto* lowWord = pending.back().first->Resolve(); + auto* highWord = pending.back().second->Resolve(); + pending.pop_back(); + if (!seen.emplace(lowWord, highWord).second || PointerFreeWord(*lowWord)) continue; + if (lowWord->IsPhi()) { + const bool paired = highWord->IsPhi() && highWord->Parent() == lowWord->Parent(); + for (std::size_t index = 0; index < lowWord->ArgumentCount(); ++index) { + IrValue* edge = paired ? PhiIncoming(*highWord, lowWord->PhiBlock(index)) : highWord; + if (edge == nullptr) return false; + if (paired && lowWord->Argument(index)->Resolve()->HasImmediate() && edge->Resolve()->HasImmediate()) return false; + pending.emplace_back(lowWord->Argument(index), edge); + } + continue; + } + if (IrValue* word = MaskedRecordWord(*lowWord)) { + pending.emplace_back(word, highWord); + continue; + } + if (!PointerRecordRead(*lowWord, lowReads)) return false; + if (highWord->IsPhi()) { + for (std::size_t index = 0; index < highWord->ArgumentCount(); ++index) pending.emplace_back(lowWord, highWord->Argument(index)); + } else if (IrValue* word = MaskedRecordWord(*highWord)) { + pending.emplace_back(lowWord, word); + } else if (!PointerFreeWord(*highWord) && !PointerRecordRead(*highWord, highReads)) { + return false; + } + } + return !lowReads.empty() && !highReads.empty(); + } + + bool Precedes(const IrValue* value, const IrValue& use) const { + value = value->Resolve(); + const IrBlock* definition = value->Parent(); + const IrBlock* target = use.Parent(); + if (definition == nullptr) return true; + if (target == nullptr) return false; + if (definition == target) { + for (const IrValue* inst : target->Instructions()) { + if (inst == value) return true; + if (inst == &use) return false; + } + return false; + } + std::vector pending{m_program.BlockOrder().front()}; + std::unordered_set seen{pending.front()}; + while (!pending.empty()) { + const IrBlock* block = pending.back(); + pending.pop_back(); + if (block == definition) continue; + if (block == target) return false; + for (const IrBlock* successor : block->Successors()) { + if (seen.insert(successor).second) pending.push_back(successor); + } + } + return true; } IrValue* UnwrapTableRead(IrValue* value) const { @@ -1033,16 +1196,19 @@ class Tracker { IrValue* bufferHandle = nullptr; IrValue* offset = nullptr; const bool buffer = handle.Opcode() == IrOpcode::GetBufferResource; - if (buffer && !hops.empty()) return false; - const bool merged = buffer && pointer->Argument(0)->Resolve()->IsPhi(); - auto* low = merged ? PointerRecordOrigin(pointer->Argument(0)) : pointer->Argument(0)->Resolve(); + const bool merged = pointer->Argument(0)->Resolve()->IsPhi() && UnwrapTableRead(pointer->Argument(0)) == nullptr; + const bool pointerKeyed = merged && !buffer; + if (!hops.empty() && buffer) return false; + std::vector lowReads; + std::vector highReads; + if (pointerKeyed ? !PointerRecordPairs(pointer->Argument(0), pointer->Argument(1), lowReads, highReads) : merged && !PointerRecordReads(pointer->Argument(0), lowReads)) return false; + auto* low = merged ? lowReads.back() : UnwrapTableRead(pointer->Argument(0)); if (low == nullptr) return false; + auto* high = pointerKeyed ? pointer->Argument(1)->Resolve() : UnwrapTableRead(pointer->Argument(1)); + if (high == nullptr) return false; std::uint32_t first = 0; for (std::uint32_t dword = 0; dword < 2u; dword++) { - IrValue* read = dword == 0u && merged ? low : UnwrapTableRead(pointer->Argument(dword)); - if (read == nullptr) { - return false; - } + IrValue* read = dword == 0u ? low : pointerKeyed ? highReads.back() : high; std::uint32_t memoryIndex = 0; const MemoryInfo* memory = ScalarReadMemory(*read, memoryIndex); if (memory == nullptr || !MemoryIndexBelongsTo(memoryIndex, *read)) { @@ -1052,17 +1218,36 @@ class Tracker { first = memory->offset; bufferHandle = read->Argument(0)->Resolve(); offset = read->Argument(1)->Resolve(); - } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1))) { + } else if (memory->offset != first + static_cast(sizeof(std::uint32_t)) || (pointerKeyed ? !SameRecordColumn(*low, *read) : read->Argument(0)->Resolve() != bufferHandle || !EquivalentValue(m_program.Resources(), offset, read->Argument(1)))) { return false; } } + IrValue* recordOffset = offset; + if (pointerKeyed) { + IrValue* source = high; + if (highReads.size() != 1u || highReads.front() != high) { + lowReads.insert(lowReads.begin(), highReads.begin(), highReads.end()); + const auto found = std::ranges::find_if(lowReads, [&](const IrValue* read) { + const IrValue* table = read->Argument(0)->Resolve(); + for (std::size_t index = 0; index < table->ArgumentCount(); ++index) { + if (!Precedes(table->Argument(index), handle)) return false; + } + return true; + }); + if (found == lowReads.end()) return false; + source = *found; + } + bufferHandle = source->Argument(0)->Resolve(); + recordOffset = source->Argument(1)->Resolve(); + offset = source == high ? recordOffset : high; + } if (!AvailableAt(offset, handle.Parent())) { return false; } IrValue* key = nullptr; std::uint32_t stride = 0; std::uint32_t addend = 0; - if (!MatchAffineOffset(offset, key, stride, addend)) { + if (!MatchAffineOffset(recordOffset, key, stride, addend)) { return false; } DescriptorSource heapSource; @@ -1082,6 +1267,7 @@ class Tracker { column.buffer = buffer; column.indirectOffset = descriptorOffset; column.indirectImmediate = descriptorImmediate; + column.pointerKeyed = pointerKeyed; column.pointerHops = std::move(hops); const auto domain = merged ? std::optional{} : MatchKeyDomain(key); DescriptorSource columnSource = heapSource; @@ -1098,7 +1284,7 @@ class Tracker { shared.keyDomains.push_back(*domain); } plan.offset = offset; - if (buffer) plan.parentPointer = pointer; + if (buffer || merged) plan.parentPointer = pointer; return true; } diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index ad91c93b142a..936cffc50022 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1882,6 +1882,7 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(table.fault); mix(table.outside ? 1u : 0u); mix(table.nested ? 1u : 0u); + mix(table.pointerKeyed ? 1u : 0u); mix(table.pointers.Hash()); mix(table.codes.Hash()); mix(table.words.Hash()); diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 8bd1f9f279b0..1af80ab4bc94 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -111,8 +111,8 @@ static_assert(sizeof(IrResourcePlan) == 728, "IrResourcePlan changed: update Enc static_assert(sizeof(IrValue) == 144, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); -static_assert(sizeof(DescriptorSource) == 216, "DescriptorSource changed: update the plan encoder"); -static_assert(sizeof(TableColumn) == 112, "TableColumn changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 224, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(TableColumn) == 120, "TableColumn changed: update the plan encoder"); static_assert(sizeof(KeyDomain) == 16, "KeyDomain changed: update the plan encoder"); static_assert(sizeof(ResourceBlock) == 104, "ResourceBlock changed: update the plan encoder"); static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); @@ -152,7 +152,7 @@ static_assert(fieldCount() == 25, "IrResourcePlan changed: updat static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); static_assert(fieldCount() == 32, "MemoryInfo changed: update the plan encoder"); static_assert(fieldCount() == 4, "DescriptorSource changed: update the plan encoder"); -static_assert(fieldCount() == 19, "TableColumn changed: update the plan encoder"); +static_assert(fieldCount() == 20, "TableColumn changed: update the plan encoder"); static_assert(fieldCount() == 4, "KeyDomain changed: update the plan encoder"); static_assert(fieldCount() == 5, "ResourceBlock changed: update the plan encoder"); static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); @@ -1187,6 +1187,7 @@ bool encodePlan(Writer& writer, const IrResourcePlan& plan) { out.Value(column.foldedCarry); out.Value(column.selected); out.Value(column.selection); + out.Value(column.pointerKeyed); out.Values(std::span(source.requiredBlocks)); }); writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { @@ -1320,7 +1321,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { value.SetDetachedOperands(std::move(arguments), std::move(phiBlocks)); } reader.List(plan.memoryInfo, 79, decodeMemoryInfo); - reader.List(plan.descriptorSources, 81, [&](Reader& in, DescriptorSource& source) { + reader.List(plan.descriptorSources, 82, [&](Reader& in, DescriptorSource& source) { for (auto& dword : source.dwords) dword = valueAt(in.Get()); in.Value(source.dwordCount); const bool table = in.Get(); @@ -1352,6 +1353,7 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { in.Value(column.foldedCarry); in.Value(column.selected); in.Value(column.selection); + in.Value(column.pointerKeyed); source.tableColumn = table ? std::optional(std::move(column)) : std::nullopt; in.Values(source.requiredBlocks); }); @@ -1393,7 +1395,8 @@ bool decodePlan(Reader& reader, IrResourcePlan& plan) { const auto& column = *source.tableColumn; if (column.buffer && (column.sampler || column.dwordCount != 4u)) return false; const auto& parent = plan.descriptorSources[column.heapSource].tableColumn; - if (parent.has_value() && (column.buffer || column.sampler || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (parent.has_value() && (column.buffer || column.address || column.indirect || !parent->buffer || !parent->indirect || plan.descriptorSources[parent->heapSource].tableColumn.has_value())) return false; + if (column.pointerKeyed && (!column.indirect || column.buffer || parent.has_value())) return false; } if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false; if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false; diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 3e43cbb22130..0f8a38b5a9f2 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 56; +inline constexpr std::uint32_t FormatVersion = 67; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 422dfbd3576a..d232d12793f3 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -2198,6 +2198,10 @@ TableWord EmitTableWord(SpirvValueEmitContext& ctx, const IrValue& inst, std::ui const auto offset = ctx.Def(handle->Argument(0)); const auto header = ImageTableAbi::TableHeader(table); const auto& parent = sources.at(column.heapSource).tableColumn; + if (!parent.has_value() && column.pointerKeyed) { + if (!column.indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a pointer-keyed image table without its pointer"); + return LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); + } if (!parent.has_value()) return LookupTableWord(state, column, offset, header); if (!parent->buffer || !parent->indirect || handle->ArgumentCount() < 3u) ctx.Fail(inst, "has a nested image table without its buffer pointer"); const auto root = LookupTablePointer(state, ctx.Def(handle->Argument(1)), ctx.Def(handle->Argument(2)), header); diff --git a/docs/dev/debt/shader.md b/docs/dev/debt/shader.md index fba767a4518f..4b7ceab4a208 100644 --- a/docs/dev/debt/shader.md +++ b/docs/dev/debt/shader.md @@ -61,8 +61,9 @@ - An [image table](../../../core/shader/recompiler/ImageTableAbi.hpp) entry that cannot be bound exactly reads zeros, and an active lane that selects it raises a BDA fault (reason 9) naming the record's address, words and reason; the dispatch's write-back reports the first one. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) snapshot holds at most 65536 keys per table, 65536 distinct images and 256 S# entries per shader; a key past a longer table's snapshot, or an entry over the budget, faults when selected. A selected column (a T# the SRT walker evaluates) holds its one record wherever it lies. - The [runtime resource snapshot](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) is retried up to three times, with pending descriptor writes forced on the last attempt. Capture throws `runtime resource snapshot did not stabilize` if descriptor-scope changes while snapshotting tables, or mapping changes that the snapshot-validity callback cannot accept, disturb every attempt. A console GPU reads descriptors during execution and has no equivalent host snapshot. Code Violet (PPSA26528) hit this refusal on older main `128b5c49ba`; its logs showed unrelated allocation churn, whose handling was later narrowed. Persistently unstable descriptors or captured views still fail; console-equivalent behavior is unmeasured. -- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. Other origins, shared V# consumers, T# words read by anything but their image handle, nested sampler columns and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. +- [Nested image tables](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) support a V# loaded through a pointer from one scalar-buffer record origin, followed by an independently indexed T# or S#. A low-pointer Phi may merge that origin with constants; the GPU looks up the actual 64-bit pointer and faults if it was not captured. Other origins, shared V# consumers, T# or S# words read by anything but their descriptor handle, and deeper buffer nesting remain refused. A snapshot scans at most 524288 parent records and holds 1048576 inner key slots across its nested columns; equal V#s share a child snapshot within a column. The final image-table map is limited to 32 MiB. A T# reached through a nested child is materialized from its raw words, without registered `resourceViews`, so a 1D array past `maxImageArrayLayers` keeps its raw descriptor there. Synthetic readback covers the domains reported in #6191; complete title behavior remains unverified. - [Image tables](../../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read by `s_load` through a 64-bit pointer hold the keys the key's known bits allow, at most 65536; a key past them, or an entry at a wrapping, unaligned or unmapped address, faults when selected. +- An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) T# or S# read through a pointer that a scalar-buffer record holds may have that pointer's low word merged by a Phi with constants, with lane masks (words of a `Ballot`, the waterfall loop's lane bit, and ANDs, ORs and NOTs of them), with the record word ANDed with a lane mask, and with reads of the same record word under another key, and its high word merged the same way from the record's next word. The two words are walked together, edge by edge, and an edge whose low word is a constant or a lane mask is skipped along with its high word. The snapshot then captures the pointer of every record, at most 65536, and the GPU looks up the actual 64-bit pointer; one that was not captured, such as a lane mask, faults when selected. Other origins of the two words remain refused, among them a high word read from a second buffer on an edge whose low word is the record word. - An [image table](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) whose key is neither a V#-relative scalar read nor a value the SRT walker evaluates (for example a shifted or masked runtime-offset V# read, a per-lane value made uniform by `v_readfirstlane`, or a loop-carried phi) binds every record of the snapshot domain instead of the selected ones. So does a table read at `(key << n) & mask` with such a key: a key domain would select records by the unmasked key, so that key is not narrowed. A key the walker evaluates selects its one record instead, unless the shader may write the memory the key is read from at run time (the same buffer, or any memory when the read or a write goes through a pointer or a GPU-only descriptor); then the column binds the whole domain too. A write to the key's memory through another V# over the same bytes is not detected. A shader whose selected records would pass 64 images or 256 sampled image/sampler pairs binds, for the heaps with the most such reads, the union of their keys in one table instead. - A shader plans at most 64 [samplers](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (`ShaderInfo::MaxSamplers`, the size of the runtime ABI's `ShaderData` sampler metadata, which [`BindingAllocator`](../../../core/shader/recompiler/Optimization/src/BindingAllocator.cpp) holds every shader to), at most 32 of them direct (each takes two elements of the 64-element sampler heap), and at most 256 image/sampler pairs (`ShaderInfo::MaxSampledPairs`); past them it refuses with `sampler resource limit exceeded`, `shader sampler pairs exceed runtime heap capacity` or `sampled image/sampler pair limit exceeded`. The image-to-sampler masks are 64-bit, one bit per sampler heap element or per sampler, so more samplers need a larger `ShaderData` and wider masks. Ratchet & Clank: Rift Apart (PPSA01474) plans up to 63 samplers and 112 pairs in one shader. - [Image table pointer indexing](../../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) propagates 32-bit value bounds through `ReadFirstLane` and `SelectU32` so waterfall key masks keep their bit domain. The SRT walker validates runtime select conditions whose active lane mask implies the condition through `LogicalAnd` conjunctions, assuming unselected inactive lanes need not be uniform across the full wave. The same assumption lets the two `s_load`s of one T# record match when one offset is added before `v_readfirstlane` and the other after (`ReadFirstLane(Select(mask, (key << 5) + 16, old), mask)` against `ReadFirstLane(key, mask) << 5`): with no active lane the hardware reads lane 0 of the inactive arm, and the table still binds the whole record.