From c4e90e71d287944155734aaa533e42da7133aca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Tue, 27 Oct 2026 17:40:08 +0100 Subject: [PATCH 1/5] perf(shader): replay an image table column's decoded records when its content recurs An image table column whose content (codes and words) and key count match a column any shader resolved before, through an image resource description that agrees in every field the decode reads, replays that column's decoded records and elements instead of decoding them again. The memo holds the decoded records per (content, usage) under a word budget (APS5_TABLE_COLUMN_MEMO_WORDS, default 1<<25), shares one content's codes and words across compiled variants and across shaders, and evicts by least recent use when the budget or the entry count is reached. APS5_NO_TABLE_COLUMN_MEMO resolves every column from its records as before. APS5_VERIFY_TABLE_COLUMN_MEMO also resolves every replayed snapshot from its records and counts the resolutions that differ in the column memo report of APS5_PROFILE_DRAW. APS5_TABLE_COLUMN_MEMO_DIAG extends that report with the distinct contents and usages of the window and the records that changed between a column's contents. The four switches change memo behaviour only, never the recompiled output, so they are listed as neutral to the shader disk cache key. Tests: agc_shader_table_column_memo, with the verify, off and diagnosed variants. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 8 + .../libSceAgcDriver/tests/ShaderMemory.cpp | 128 ++++ .../Optimization/ResourceMaterializer.hpp | 16 +- .../Optimization/src/ResourceMaterializer.cpp | 569 +++++++++++++++++- core/shader/recompiler/ShaderDiskCache.cpp | 4 + docs/dev/ENVIRONMENT.md | 4 + 6 files changed, 702 insertions(+), 27 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 401e0a2b9d66..00554b21679c 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2625,6 +2625,14 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_snapshot_reuse_unmasked COMMAND agc_shader_memory_tests --table-snapshot-reuse) set_tests_properties(agc_shader_table_snapshot_reuse_unmasked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_verified COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo_verified PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_TABLE_COLUMN_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_off COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_TABLE_COLUMN_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_diagnosed COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo_diagnosed PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 3d4fd33ab0c9..42bf48d14001 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2941,6 +2941,130 @@ void verifyImageTableScratchReuse() { require(warm.allocations < Entries / 4u, "image table scratch: snapshotting and resolving a table allocated per record"); } +void verifyTableColumnMemo() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + constexpr std::uint32_t Entries = 3000; + static TableTexture textures[4]; + alignas(4096) static TableGuest guests[2]; + alignas(4096) static std::array, Entries> heaps[2]; + for (std::uint32_t entry = 0; entry < Entries; entry++) { + heaps[0][entry] = entry % 97u == 0u ? std::array{} : TableT(textures[entry % 4u].bytes.data(), TableFormat8888UNorm, TableType2D, 1u + entry % 5u); + } + heaps[0][11][3] = (heaps[0][11][3] & 0x0fffffffu) | (1u << 28u); + heaps[1] = heaps[0]; + const bool enabled = std::getenv("APS5_NO_TABLE_COLUMN_MEMO") == nullptr; + const bool verified = enabled && std::getenv("APS5_VERIFY_TABLE_COLUMN_MEMO") != nullptr; + const bool diagnosed = enabled && std::getenv("APS5_TABLE_COLUMN_MEMO_DIAG") != nullptr; + const auto same = [](const ResolvedImageTables& left, const ResolvedImageTables& right) { + return left.map == right.map && left.elements == right.elements && left.samplers == right.samplers && left.poison == right.poison && left.faults == right.faults; + }; + std::array, 2> userData{}; + std::vector requests; + for (std::size_t side = 0; side < 2u; side++) { + FillTableSrt(guests[side], heaps[side].data(), 32u, Entries); + userData[side] = TableUserData(guests[side]); + requests.push_back(TableRequest(TableWholeCode, userData[side])); + } + auto staticRequest = requests[0]; + const std::array absentResources{}; + staticRequest.context.userData = absentResources; + staticRequest.context.memory = {}; + auto staticProgram = PrepareResourceProgram(staticRequest); + ResourceMaterializer{}.ApplyStaticInterface(staticProgram); + const auto& info = staticProgram.Resources().info; + const auto capture = [&](const RecompileRequest& request) { + CountingMemory memory; + SrtRuntime runtime; + runtime.userData = request.context.userData; + runtime.shaderBase = request.shader.codeAddress; + runtime.userContext = &memory; + runtime.readMemory = &CountingRead; + runtime.readSpecializationMemory = &CountingRead; + runtime.accessible = &CountingAccessible; + runtime.readWords = &CountingReadWords; + return CaptureResources(request, runtime); + }; + constexpr std::uint64_t MemoKey = 0x7ab1ec01u; + constexpr std::uint64_t OtherKey = 0x7ab1ec02u; + constexpr std::uint64_t ModesKey = 0x7ab1ec03u; + constexpr std::uint64_t ImageKey = 0x7ab1ec04u; + constexpr std::uint64_t SourceKey = 0x7ab1ec05u; + constexpr std::uint64_t DecodeKey = 0x7ab1ec06u; + const auto resolveWith = [&](const ShaderInfo& used, std::uint64_t memoKey, const RecompileRequest& request, const std::string& what) { + const auto captured = capture(request); + require(captured->snapshot.tables.tables.size() == 1u && captured->snapshot.tables.tables.at(0).keys == Entries, ("image table column memo: " + what + ": the snapshot does not hold the whole table").c_str()); + auto memoized = ResourceMaterializer::ResolveImageTables(used, captured->snapshot, memoKey); + const auto plain = ResourceMaterializer::ResolveImageTables(used, captured->snapshot, memoKey, false); + require(same(memoized, plain), ("image table column memo: " + what + ": the memoized resolution differs from the plain one").c_str()); + return memoized; + }; + const auto resolveBoth = [&](const RecompileRequest& request, const std::string& what) { return resolveWith(info, MemoKey, request, what); }; + const auto before = ResourceMaterializer::TableReuse(); + const auto first = resolveBoth(requests[0], "the first table"); + const auto afterFirst = ResourceMaterializer::TableReuse(); + require(afterFirst.columnHits == before.columnHits && afterFirst.columnMisses - before.columnMisses == (enabled ? 1u : 0u), "image table column memo: the first table did not miss exactly once"); + require(first.map[Abi::HeaderElementCount] == 20u && first.poison.size() == 3u && first.poison.at(2).reason == Abi::PoisonReason::NotImage, "image table column memo: the first table did not index its twenty distinct records and poison its invalid one"); + const auto moved = resolveBoth(requests[1], "a moved table with the same records"); + const auto afterMoved = ResourceMaterializer::TableReuse(); + require(afterMoved.columnHits - afterFirst.columnHits == (enabled ? 1u : 0u) && afterMoved.columnMisses == afterFirst.columnMisses, "image table column memo: a moved table with the same records was not replayed"); + const auto header = Abi::TableHeader(0); + require(first.map.at(header + Abi::TableBaseLow) == static_cast(reinterpret_cast(heaps[0].data())) && moved.map.at(header + Abi::TableBaseLow) == static_cast(reinterpret_cast(heaps[1].data())), "image table column memo: a resolution does not carry its own table base"); + require(moved.elements == first.elements && moved.map.size() == first.map.size() && moved.poison == first.poison, "image table column memo: a moved table resolved to other elements"); + heaps[1][5] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D, 2u); + const auto changed = resolveBoth(requests[1], "a table with a changed record"); + const auto afterChanged = ResourceMaterializer::TableReuse(); + require(afterChanged.columnHits == afterMoved.columnHits && afterChanged.columnMisses - afterMoved.columnMisses == (enabled ? 1u : 0u), "image table column memo: a table with a changed record was replayed"); + require(changed.elements.size() == first.elements.size() + 1u, "image table column memo: the changed record did not add one element"); + static_cast(resolveBoth(requests[1], "the changed table again")); + const auto afterAgain = ResourceMaterializer::TableReuse(); + require(afterAgain.columnHits - afterChanged.columnHits == (enabled ? 1u : 0u) && afterAgain.columnMisses == afterChanged.columnMisses, "image table column memo: the changed table was not replayed the second time"); + require(afterAgain.columnVerified - before.columnVerified == (verified ? 2u : 0u), "image table column memo: the replayed snapshots were not verified exactly when asked"); + require(afterAgain.columnMismatches == before.columnMismatches, "image table column memo: the verification found a resolution that differs"); + const auto shared = resolveWith(info, OtherKey, requests[1], "the changed table under a second variant"); + const auto afterShared = ResourceMaterializer::TableReuse(); + require(afterShared.columnHits - afterAgain.columnHits == (enabled ? 1u : 0u) && afterShared.columnMisses == afterAgain.columnMisses, "image table column memo: a second variant did not replay the column the first variant resolved"); + require(shared.map == changed.map && shared.elements == changed.elements && shared.poison == changed.poison, "image table column memo: the second variant's resolution differs from the first variant's"); + const auto codes = TableCodes(changed.map, 0); + const auto mode = changed.map.at(codes.at(1) + Abi::ImageRecordFlags) >> Abi::ModeShift; + auto alteredModes = info; + auto& altered = alteredModes.runtimeImageModes.at(0).at(mode); + altered.depthCompare = !altered.depthCompare; + const auto otherModes = resolveWith(alteredModes, ModesKey, requests[1], "the changed table with a changed mode list"); + const auto afterModes = ResourceMaterializer::TableReuse(); + require(afterModes.columnHits == afterShared.columnHits && afterModes.columnMisses - afterShared.columnMisses == (enabled ? 1u : 0u), "image table column memo: a changed mode list was replayed from another mode list's records"); + require(otherModes.map == changed.map && otherModes.elements.size() == changed.elements.size() && otherModes.elements != changed.elements, "image table column memo: the changed mode did not reach the elements"); + static_cast(resolveWith(alteredModes, ModesKey, requests[1], "the changed mode list again")); + const auto afterModesAgain = ResourceMaterializer::TableReuse(); + require(afterModesAgain.columnHits - afterModes.columnHits == (enabled ? 1u : 0u) && afterModesAgain.columnMisses == afterModes.columnMisses, "image table column memo: the changed mode list was not replayed the second time"); + auto alteredImage = info; + alteredImage.images.at(0).firstUsePc += 4u; + alteredImage.runtimeImageModes.at(0) = ResourceMaterializer::RuntimeImageModes(alteredImage.images.at(0)); + const auto otherImage = resolveWith(alteredImage, ImageKey, requests[1], "the changed table through a resource with another first use"); + const auto afterImage = ResourceMaterializer::TableReuse(); + require(afterImage.columnHits - afterModesAgain.columnHits == (enabled ? 1u : 0u) && afterImage.columnMisses == afterModesAgain.columnMisses, "image table column memo: a resource with another first use did not replay the shared records"); + require(otherImage.map == changed.map && otherImage.elements == changed.elements && otherImage.poison == changed.poison, "image table column memo: another first use resolved to other records"); + auto alteredSource = info; + alteredSource.images.at(0).source += 1u; + alteredSource.runtimeImageModes.at(0) = ResourceMaterializer::RuntimeImageModes(alteredSource.images.at(0)); + const auto otherSource = resolveWith(alteredSource, SourceKey, requests[1], "the changed table through a resource with another descriptor source"); + const auto afterSource = ResourceMaterializer::TableReuse(); + require(afterSource.columnHits - afterImage.columnHits == (enabled ? 1u : 0u) && afterSource.columnMisses == afterImage.columnMisses, "image table column memo: a resource with another descriptor source did not replay the shared records"); + require(otherSource.map == changed.map && otherSource.elements == changed.elements && otherSource.poison == changed.poison, "image table column memo: another descriptor source resolved to other records"); + auto alteredDecode = info; + alteredDecode.images.at(0).r128 = !alteredDecode.images.at(0).r128; + alteredDecode.runtimeImageModes.at(0) = ResourceMaterializer::RuntimeImageModes(alteredDecode.images.at(0)); + const auto otherDecode = resolveWith(alteredDecode, DecodeKey, requests[1], "the changed table through a resource with another decode input"); + const auto afterDecode = ResourceMaterializer::TableReuse(); + require(afterDecode.columnHits == afterSource.columnHits && afterDecode.columnMisses - afterSource.columnMisses == (enabled ? 1u : 0u), "image table column memo: a changed decode input was replayed from another description's records"); + require(otherDecode.map == changed.map && otherDecode.elements == changed.elements && otherDecode.poison == changed.poison, "image table column memo: the changed decode input resolved to other records"); + require(afterDecode.columnVerified - before.columnVerified == (verified ? 6u : 0u), "image table column memo: the shared replays were not verified exactly when asked"); + require(afterDecode.columnMismatches == before.columnMismatches, "image table column memo: the verification found a shared resolution that differs"); + require(afterDecode.columnMissesNeverSeen - before.columnMissesNeverSeen == (enabled ? 3u : 0u) && afterDecode.columnMissesChangedContent - before.columnMissesChangedContent == (enabled ? 1u : 0u), "image table column memo: the miss diagnostics did not count three first sights and one content change"); + require(afterDecode.columnDiffedMisses - before.columnDiffedMisses == (diagnosed ? 1u : 0u) && afterDecode.columnChangedKeys - before.columnChangedKeys == (diagnosed ? 1u : 0u), "image table column memo: the diagnostics did not diff the one changed record exactly when asked"); + std::cout << "image table column memo passed\n"; +} + void verifyImageTableBulkKeyScan() { using namespace ShaderRecompiler; namespace Abi = ImageTableAbi; @@ -8737,6 +8861,10 @@ int main(int argc, char** argv) { std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--table-column-memo") { + verifyTableColumnMemo(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index ad1a0f01b5ce..3cf3d9b7592d 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -22,12 +22,16 @@ struct ImageTableElement { std::uint32_t samplers = 0; bool atomic = false; bool atomic64 = false; + + bool operator==(const ImageTableElement& other) const = default; }; struct ImageTableSamplerElement { std::array words{}; bool compare = false; bool point = false; + + bool operator==(const ImageTableSamplerElement& other) const = default; }; struct ImageTablePoisonRecord { @@ -35,6 +39,8 @@ struct ImageTablePoisonRecord { std::uint32_t dwordCount = 0; std::uint32_t resource = 0; ImageTableAbi::PoisonReason reason = ImageTableAbi::PoisonReason::OutsideSnapshot; + + bool operator==(const ImageTablePoisonRecord& other) const = default; }; struct ResolvedImageTables { @@ -53,6 +59,14 @@ struct ImageTableReuseCounts { std::uint64_t columns = 0; std::uint64_t resolutions = 0; std::uint64_t resolutionsShared = 0; + std::uint64_t columnHits = 0; + std::uint64_t columnMisses = 0; + std::uint64_t columnVerified = 0; + std::uint64_t columnMismatches = 0; + std::uint64_t columnMissesNeverSeen = 0; + std::uint64_t columnMissesChangedContent = 0; + std::uint64_t columnDiffedMisses = 0; + std::uint64_t columnChangedKeys = 0; }; struct RuntimeImageModeMatch { @@ -77,7 +91,7 @@ class ResourceMaterializer { static const char* UnnormalizedSampleMismatch(const ImageResource& image); static std::shared_ptr ResolveImageTablesShared(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey); static bool NullImageDescriptor(const ImageResource& image, const DescriptorValue& descriptor); - static ResolvedImageTables ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey = 0); + static ResolvedImageTables ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey = 0, bool columnMemo = true); [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; static void CompactPlan(IrResourcePlan& plan); void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index de9855c810f3..68de4394f67d 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -369,6 +370,34 @@ bool TableTraced() { return traced; } +bool TableColumnMemo() { + static const bool enabled = std::getenv("APS5_NO_TABLE_COLUMN_MEMO") == nullptr; + return enabled; +} + +bool VerifyTableColumnMemo() { + static const bool verify = std::getenv("APS5_VERIFY_TABLE_COLUMN_MEMO") != nullptr; + return verify; +} + +bool TableColumnMemoDiagnosed() { + static const bool diagnosed = std::getenv("APS5_TABLE_COLUMN_MEMO_DIAG") != nullptr; + return diagnosed; +} + +std::size_t TableColumnMemoWords() { + static const std::size_t words = [] { + const char* text = std::getenv("APS5_TABLE_COLUMN_MEMO_WORDS"); + if (text == nullptr) return std::size_t{1} << 25u; + char* end = nullptr; + errno = 0; + const auto value = std::strtoull(text, &end, 0); + if (std::strchr(text, '-') != nullptr || end == text || *end != '\0' || errno == ERANGE || value == 0u) throw std::runtime_error("APS5_TABLE_COLUMN_MEMO_WORDS is not a positive word count"); + return static_cast(value); + }(); + return words; +} + constexpr std::size_t PoisonReasonCount = static_cast(PoisonReason::Count); enum class NarrowSkip : std::size_t { Value, NotBuffer, KeyScan, Pending, Unmapped, Count }; @@ -387,6 +416,13 @@ struct TableCounters { std::atomic reusedUnwritten{0}; std::atomic reusedCompared{0}; std::atomic nanoseconds{0}; + std::atomic columnHits{0}; + std::atomic columnMisses{0}; + std::atomic columnEvictions{0}; + std::atomic columnVerified{0}; + std::atomic columnMismatches{0}; + std::atomic columnMissesNeverSeen{0}; + std::atomic columnMissesChangedContent{0}; std::atomic lastReport{0}; }; @@ -403,6 +439,14 @@ struct TableReuseTotals { std::atomic columns{0}; std::atomic resolutions{0}; std::atomic resolutionsShared{0}; + std::atomic columnHits{0}; + std::atomic columnMisses{0}; + std::atomic columnVerified{0}; + std::atomic columnMismatches{0}; + std::atomic columnMissesNeverSeen{0}; + std::atomic columnMissesChangedContent{0}; + std::atomic columnDiffedMisses{0}; + std::atomic columnChangedKeys{0}; }; TableReuseTotals& tableReuseTotals() { @@ -410,6 +454,325 @@ TableReuseTotals& tableReuseTotals() { return totals; } +struct ColumnResolutionKey { + std::uint64_t codesHash = 0; + std::uint64_t wordsHash = 0; + std::uint64_t usageHash = 0; + std::uint32_t keys = 0; + bool outside = false; + bool reducing = false; + + bool operator==(const ColumnResolutionKey& other) const = default; +}; + +struct ColumnResolutionKeyHash { + std::size_t operator()(const ColumnResolutionKey& key) const { + std::uint64_t hash = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { hash = (hash ^ value) * 0x100000001b3ull; }; + mix(key.codesHash); + mix(key.wordsHash); + mix(key.usageHash); + mix(key.keys); + mix(key.outside ? 1u : 0u); + mix(key.reducing ? 1u : 0u); + return static_cast(hash); + } +}; + +ImageResource columnUsage(const ImageResource& image) { + ImageResource usage = image; + usage.source = 0; + usage.firstUsePc = 0; + if (usage.table != NoTable) usage.table = 0; + return usage; +} + +void mixImageResource(std::uint64_t& hash, const ImageResource& image) { + const auto mix = [&](std::uint64_t value) { hash = (hash ^ value) * 0x100000001b3ull; }; + mix(static_cast(image.resourceClass)); + mix(static_cast(image.numericClass)); + mix(static_cast(image.dimension)); + mix(static_cast(image.mipMode)); + mix(image.mipCount); + mix(static_cast(image.conversionFormat)); + mix(image.shaderSwizzle); + std::uint64_t flags = 0; + for (const bool flag : {image.read, image.written, image.atomic, image.atomic64, image.depthCompare, image.cube, image.r128, image.srgbDecode, image.srgbDecodeCompatible, image.depthBits, image.depthUnorm16, image.packed, image.fmaskCompatible, image.depthBitsCompatible, image.constantSwizzle, image.constantSwizzleCompatible, image.cubeFaceCompatible, image.flatVolumeCompatible, image.singleMipPlaneBiasCompatible, image.volumeArraySampleCompatible, image.flatLineCompatible, image.flatVolumeLineCompatible, image.dimensionQueryOnly, image.foldTexelOffsetsOnly, image.convertedSamples, image.remappedDepth1D, image.targetMerge}) flags = (flags << 1u) | (flag ? 1u : 0u); + mix(flags); + mix(image.srgbDecodeFormats); + mix(image.byElements); + mix(image.byComponents); + mix(static_cast(image.packedFormat)); + mix(image.emulatedCompare); + mix(image.table != NoTable ? 1u : 0u); + mix(static_cast(image.tableOperation)); +} + +std::uint64_t imageUsageHash(const ImageResource& image, std::span modes) { + std::uint64_t hash = 0xcbf29ce484222325ull; + mixImageResource(hash, image); + hash = (hash ^ modes.size()) * 0x100000001b3ull; + for (const auto& mode : modes) mixImageResource(hash, mode); + return hash; +} + +std::uint64_t strictUsageHash(std::uint64_t usage, const ImageResource& image) { + const auto mix = [&](std::uint64_t value) { usage = (usage ^ value) * 0x100000001b3ull; }; + mix(image.source); + mix(image.firstUsePc); + mix(image.table); + return usage; +} + +struct ColumnUsageKey { + std::uint64_t memoKey = 0; + std::uint32_t resource = 0; + std::uint32_t table = 0; + + bool operator==(const ColumnUsageKey& other) const = default; +}; + +struct ColumnUsageKeyHash { + std::size_t operator()(const ColumnUsageKey& key) const { + std::uint64_t hash = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { hash = (hash ^ value) * 0x100000001b3ull; }; + mix(key.memoKey); + mix(key.resource); + mix(key.table); + return static_cast(hash); + } +}; + +struct ColumnSlot { + std::uint64_t content = 0; + SharedValues codes; + SharedValues words; +}; + +struct ColumnUsageSets { + std::unordered_set strict; + std::unordered_set relaxed; +}; + +struct ColumnMissDiagnostics { + std::mutex mutex; + std::unordered_map slots; + std::unordered_set missedContents; + std::unordered_set allContents; + std::unordered_map usages; + std::vector changedKeys; + std::vector touchedChunks; + std::uint64_t resized = 0; + std::size_t pinnedWords = 0; +}; + +ColumnMissDiagnostics& columnMissDiagnostics() { + static ColumnMissDiagnostics diagnostics; + return diagnostics; +} + +constexpr std::size_t ColumnMissDiagnosticsEntries = std::size_t{1} << 17u; + +constexpr std::uint32_t ColumnDiffChunkKeys = 1024; +constexpr std::size_t ColumnDiagnosticsPinnedWords = std::size_t{1} << 26u; + +std::size_t columnWords(const SharedValues& codes, const SharedValues& words) { + return codes.size() + words.size() * (sizeof(DescriptorValue) / sizeof(std::uint32_t)); +} + +std::pair diffColumns(const ColumnSlot& previous, const SharedValues& codes, const SharedValues& words) { + std::uint32_t changed = 0; + std::uint32_t chunks = 0; + bool touched = false; + for (std::uint32_t key = 0; key < codes.size(); key++) { + if (key % ColumnDiffChunkKeys == 0u) { + chunks += touched ? 1u : 0u; + touched = false; + } + const auto before = previous.codes[key]; + const auto after = codes[key]; + const bool poisoned = ((before | after) & ImageTableAbi::PoisonFlag) != 0u; + const bool differs = poisoned ? before != after : !(previous.words.at(before) == words.at(after)); + if (differs) { + changed++; + touched = true; + } + } + chunks += touched ? 1u : 0u; + return {changed, chunks}; +} + +void noteColumnLookup(const ColumnUsageKey& usage, std::uint64_t content, bool hit, const SharedValues& codes, const SharedValues& words, std::uint64_t relaxedUsage, std::uint64_t strictUsage) { + auto& diagnostics = columnMissDiagnostics(); + auto& counters = tableCounters(); + auto& totals = tableReuseTotals(); + const bool diagnosed = TableColumnMemoDiagnosed(); + std::optional previous; + { + std::lock_guard lock(diagnostics.mutex); + if (diagnostics.slots.size() >= ColumnMissDiagnosticsEntries) { + diagnostics.slots.clear(); + diagnostics.pinnedWords = 0; + } + if (diagnostics.missedContents.size() >= ColumnMissDiagnosticsEntries) diagnostics.missedContents.clear(); + const auto [slot, inserted] = diagnostics.slots.try_emplace(usage); + const auto heldWords = columnWords(slot->second.codes, slot->second.words); + if (!hit) { + if (inserted) { + counters.columnMissesNeverSeen.fetch_add(1, std::memory_order_relaxed); + totals.columnMissesNeverSeen.fetch_add(1, std::memory_order_relaxed); + } else if (slot->second.content != content) { + counters.columnMissesChangedContent.fetch_add(1, std::memory_order_relaxed); + totals.columnMissesChangedContent.fetch_add(1, std::memory_order_relaxed); + if (diagnosed) previous = std::move(slot->second); + } + diagnostics.missedContents.insert(content); + } + slot->second.content = content; + if (diagnosed) { + diagnostics.pinnedWords -= heldWords; + const auto pinWords = columnWords(codes, words); + if (diagnostics.pinnedWords + pinWords <= ColumnDiagnosticsPinnedWords) { + slot->second.codes = codes; + slot->second.words = words; + diagnostics.pinnedWords += pinWords; + } else { + slot->second.codes = {}; + slot->second.words = {}; + } + if (diagnostics.allContents.size() < ColumnMissDiagnosticsEntries) diagnostics.allContents.insert(content); + if (const auto found = diagnostics.usages.find(content); found != diagnostics.usages.end()) { + found->second.strict.insert(strictUsage); + found->second.relaxed.insert(relaxedUsage); + } else if (diagnostics.usages.size() < ColumnMissDiagnosticsEntries) { + auto& sets = diagnostics.usages[content]; + sets.strict.insert(strictUsage); + sets.relaxed.insert(relaxedUsage); + } + } + } + if (!previous.has_value() || previous->codes.empty()) return; + if (previous->codes.size() != codes.size()) { + std::lock_guard lock(diagnostics.mutex); + diagnostics.resized++; + return; + } + const auto [changed, chunks] = diffColumns(*previous, codes, words); + totals.columnDiffedMisses.fetch_add(1, std::memory_order_relaxed); + totals.columnChangedKeys.fetch_add(changed, std::memory_order_relaxed); + std::lock_guard lock(diagnostics.mutex); + if (diagnostics.changedKeys.size() >= ColumnMissDiagnosticsEntries) return; + diagnostics.changedKeys.push_back(changed); + diagnostics.touchedChunks.push_back(chunks); +} + +enum class ColumnDecodeKind : std::uint8_t { Unknown, Null, Mode, Poison }; + +struct ColumnDecode { + ColumnDecodeKind kind = ColumnDecodeKind::Unknown; + std::uint32_t value = 0; + std::uint32_t element = 0; +}; + +struct ColumnResolutionElement { + ImageTableElement element; + std::uint64_t hash = 0; +}; + +struct ColumnResolution { + ColumnResolutionKey key; + ImageResource image; + std::vector modes; + SharedValues codes; + SharedValues words; + std::vector decodes; + std::vector elements; + std::size_t footprint = 0; + std::uint64_t used = 0; +}; + +bool sameColumnUsage(const ColumnResolution& known, const ImageResource& image, std::span modes) { + if (known.modes.size() != modes.size() || known.image != columnUsage(image)) return false; + for (std::size_t mode = 0; mode < modes.size(); mode++) { + if (known.modes[mode] != columnUsage(modes[mode])) return false; + } + return true; +} + +constexpr std::size_t ColumnResolutionMemoEntries = 4096; +constexpr std::uint32_t ColumnResolutionMinimumKeys = 64; +constexpr std::uint32_t NoElement = std::numeric_limits::max(); + +std::size_t columnResolutionFootprint(const ColumnResolution& entry) { + constexpr std::size_t Overhead = sizeof(ColumnResolution) + 160u; + const auto bytes = Overhead + entry.modes.size() * sizeof(ImageResource) + entry.codes.size() * sizeof(std::uint32_t) + entry.words.size() * sizeof(DescriptorValue) + entry.decodes.size() * sizeof(ColumnDecode) + entry.elements.size() * sizeof(ColumnResolutionElement); + return (bytes + sizeof(std::uint32_t) - 1u) / sizeof(std::uint32_t); +} + +struct ColumnResolutionMemo { + std::mutex mutex; + std::unordered_map, ColumnResolutionKeyHash> entries; + std::size_t words = 0; + std::size_t records = 0; + std::size_t elements = 0; + std::uint64_t clock = 0; +}; + +ColumnResolutionMemo& columnResolutionMemo() { + static ColumnResolutionMemo memo; + return memo; +} + +std::shared_ptr findColumnResolution(const ColumnResolutionKey& key) { + auto& memo = columnResolutionMemo(); + std::lock_guard lock(memo.mutex); + const auto found = memo.entries.find(key); + if (found == memo.entries.end()) return nullptr; + found->second->used = ++memo.clock; + return found->second; +} + +void rememberColumnResolution(std::shared_ptr entry) { + entry->elements.shrink_to_fit(); + entry->footprint = columnResolutionFootprint(*entry); + const auto budget = TableColumnMemoWords(); + if (entry->footprint > budget) return; + auto& memo = columnResolutionMemo(); + auto& counters = tableCounters(); + std::lock_guard lock(memo.mutex); + if (const auto held = memo.entries.find(entry->key); held != memo.entries.end()) { + const auto& known = *held->second; + if (known.codes == entry->codes && known.words == entry->words && known.image == entry->image && known.modes == entry->modes) return; + memo.words -= known.footprint; + memo.records -= known.words.size(); + memo.elements -= known.elements.size(); + memo.entries.erase(held); + counters.columnEvictions.fetch_add(1, std::memory_order_relaxed); + } + if (memo.words + entry->footprint > budget || memo.entries.size() >= ColumnResolutionMemoEntries) { + std::vector> order; + order.reserve(memo.entries.size()); + for (const auto& [key, held] : memo.entries) order.emplace_back(held->used, &key); + std::sort(order.begin(), order.end(), [](const auto& left, const auto& right) { return left.first < right.first; }); + std::size_t evicted = 0; + for (const auto& [used, key] : order) { + if (memo.words + entry->footprint <= budget && memo.entries.size() < ColumnResolutionMemoEntries) break; + const auto victim = memo.entries.find(*key); + memo.words -= victim->second->footprint; + memo.records -= victim->second->words.size(); + memo.elements -= victim->second->elements.size(); + memo.entries.erase(victim); + evicted++; + } + counters.columnEvictions.fetch_add(evicted, std::memory_order_relaxed); + } + entry->used = ++memo.clock; + memo.words += entry->footprint; + memo.records += entry->words.size(); + memo.elements += entry->elements.size(); + memo.entries.emplace(entry->key, std::move(entry)); +} + void reportTables() { if (!MaterializeProfiled()) return; auto& counters = tableCounters(); @@ -438,6 +801,52 @@ void reportTables() { std::fprintf(stderr, "[image-table] (10 s): %llu snapshots, %llu tables, %llu keys, %llu selected records, structural poison:%s, narrowing %llu applied / %llu skipped%s%s, reused %llu unwritten / %llu compared, %.1f ms\n", static_cast(snapshots), static_cast(counters.tables.exchange(0, std::memory_order_relaxed)), static_cast(counters.keys.exchange(0, std::memory_order_relaxed)), static_cast(counters.selected.exchange(0, std::memory_order_relaxed)), poison.empty() ? " none" : poison.c_str(), static_cast(counters.narrowed.exchange(0, std::memory_order_relaxed)), static_cast(counters.narrowSkipped.exchange(0, std::memory_order_relaxed)), skips.empty() ? "" : ":", skips.c_str(), static_cast(counters.reusedUnwritten.exchange(0, std::memory_order_relaxed)), static_cast(counters.reusedCompared.exchange(0, std::memory_order_relaxed)), static_cast(counters.nanoseconds.exchange(0, std::memory_order_relaxed)) / 1e6); + if (!TableColumnMemo()) return; + auto& memo = columnResolutionMemo(); + std::size_t entries = 0; + std::size_t words = 0; + std::size_t records = 0; + std::size_t elements = 0; + { + std::lock_guard lock(memo.mutex); + entries = memo.entries.size(); + words = memo.words; + records = memo.records; + elements = memo.elements; + } + std::size_t missedContents = 0; + std::string diagnosed; + { + auto& diagnostics = columnMissDiagnostics(); + std::lock_guard lock(diagnostics.mutex); + missedContents = diagnostics.missedContents.size(); + diagnostics.missedContents.clear(); + if (TableColumnMemoDiagnosed()) { + std::size_t strict = 0; + std::size_t relaxed = 0; + for (const auto& [content, sets] : diagnostics.usages) { + strict += sets.strict.size(); + relaxed += sets.relaxed.size(); + } + const auto quantiles = [](std::vector& values) { + if (values.empty()) return std::array{}; + std::sort(values.begin(), values.end()); + return std::array{values.front(), values[values.size() / 2u], values.back()}; + }; + const auto keys = quantiles(diagnostics.changedKeys); + const auto chunks = quantiles(diagnostics.touchedChunks); + diagnosed = ", " + std::to_string(diagnostics.allContents.size()) + " distinct contents so far, " + std::to_string(diagnostics.usages.size()) + " contents in the window, " + std::to_string(strict) + " strict usages, " + std::to_string(relaxed) + " relaxed usages, " + std::to_string(diagnostics.changedKeys.size()) + " diffed misses, " + + std::to_string(keys[0]) + " changed keys min, " + std::to_string(keys[1]) + " changed keys median, " + std::to_string(keys[2]) + " changed keys max, " + std::to_string(chunks[0]) + " touched chunks min, " + std::to_string(chunks[1]) + " touched chunks median, " + std::to_string(chunks[2]) + " touched chunks max, " + std::to_string(diagnostics.resized) + " resized misses"; + diagnostics.usages.clear(); + diagnostics.changedKeys.clear(); + diagnostics.touchedChunks.clear(); + diagnostics.resized = 0; + } + } + std::fprintf(stderr, "[image-table] column memo (10 s): %llu hits, %llu misses, %llu evictions, %llu entries holding %llu words, %llu verified, %llu verify mismatches, %llu misses never seen, %llu misses with changed content, %llu distinct contents among the misses, %llu records held, %llu elements held%s\n", + static_cast(counters.columnHits.exchange(0, std::memory_order_relaxed)), static_cast(counters.columnMisses.exchange(0, std::memory_order_relaxed)), static_cast(counters.columnEvictions.exchange(0, std::memory_order_relaxed)), + static_cast(entries), static_cast(words), static_cast(counters.columnVerified.exchange(0, std::memory_order_relaxed)), static_cast(counters.columnMismatches.exchange(0, std::memory_order_relaxed)), + static_cast(counters.columnMissesNeverSeen.exchange(0, std::memory_order_relaxed)), static_cast(counters.columnMissesChangedContent.exchange(0, std::memory_order_relaxed)), static_cast(missedContents), static_cast(records), static_cast(elements), diagnosed.c_str()); } enum class RecordState : std::uint8_t { OutsideDomain, Read, Unmapped, Carried }; @@ -1662,9 +2071,11 @@ std::size_t resolutionFootprint(const ResolvedImageTables& resolved, const std:: } -ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey) { +ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey, bool columnMemo) { namespace Abi = ImageTableAbi; ResolvedImageTables out; + const bool memoized = columnMemo && memoKey != 0u && TableColumnMemo(); + std::uint32_t served = 0; std::uint32_t count = 0; for (const auto& image : info.images) { if (image.table != NoTable) count = std::max(count, image.table + 1u); @@ -1709,51 +2120,98 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i if (info.samplers.at(pair.sampler).table == NoTable) directSamplers.at(pair.image) |= 1u << pair.sampler; } const auto reducingSampler = reducingSamplers(info, snapshot); - const auto imageRecord = [&](std::uint32_t resource, const DescriptorValue& words) { + const auto placeElement = [&](const ImageTableElement& candidate, std::uint64_t hash) -> std::optional { + auto index = elementIndex.Find(hash, [&](std::uint32_t known) { + const auto& entry = out.elements[known]; + return entry.words == candidate.words && entry.dimension == candidate.dimension && entry.depthCompare == candidate.depthCompare && entry.kind == candidate.kind && entry.atomic == candidate.atomic && entry.atomic64 == candidate.atomic64; + }); + if (!index.has_value()) { + if (out.elements.size() >= Abi::ImageElementBudget) return std::nullopt; + index = static_cast(out.elements.size()); + out.elements.push_back(candidate); + elementIndex.Insert(hash, *index); + } + return index; + }; + struct ColumnRecorder { + ColumnResolution* resolution = nullptr; + std::unordered_map localByHash; + }; + const auto localElement = [](ColumnRecorder& recorder, const ImageTableElement& candidate, std::uint64_t hash) { + auto& elements = recorder.resolution->elements; + const auto found = recorder.localByHash.find(hash); + if (found != recorder.localByHash.end() && elements[found->second].element == candidate) return found->second; + const auto index = static_cast(elements.size()); + elements.push_back({candidate, hash}); + recorder.localByHash.try_emplace(hash, index); + return index; + }; + const auto appendRecord = [&](const std::array& record) { + const auto offset = static_cast(out.map.size()); + out.map.insert(out.map.end(), record.begin(), record.end()); + return offset; + }; + const auto imageRecord = [&](std::uint32_t resource, const DescriptorValue& words, ColumnRecorder* recorder, ColumnDecode* decode) { const auto& image = info.images[resource]; const auto& modes = info.runtimeImageModes[resource]; std::array record{0u, words.dwords[3], 0u}; + const auto poisoned = [&](PoisonReason reason) { + if (decode != nullptr) *decode = {ColumnDecodeKind::Poison, static_cast(reason), 0u}; + record[Abi::ImageRecordFlags] = poison(resource, reason, &words); + }; if ((words.availableDwordMask & (1u << 3u)) == 0u) { - record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::OutsideSnapshot, &words); - const auto offset = static_cast(out.map.size()); - out.map.insert(out.map.end(), record.begin(), record.end()); - return offset; + poisoned(PoisonReason::OutsideSnapshot); + return appendRecord(record); } const auto decoded = decodeTableEntry(image, modes, words, memoKey, resource); if ((decoded & DecodeFailed) != 0u) { - record[Abi::ImageRecordFlags] = poison(resource, static_cast(decoded & ~DecodeFailed), &words); + poisoned(static_cast(decoded & ~DecodeFailed)); } else { const auto& mode = modes.at(decoded); if (mode.dimension == RdnaImageDimension::Dim2DMsaa || mode.dimension == RdnaImageDimension::Dim2DMsaaArray) { - record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::Multisampled, &words); + poisoned(PoisonReason::Multisampled); } else if (requiresPointSampler(mode) && reducingSampler[resource]) { - record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::Reduction, &words); + poisoned(PoisonReason::Reduction); } else { auto element = words.dwords; if (mode.conversionFormat != IrBufferFormat::Invalid || mode.depthBits) element[3] = (element[3] & ~0xfffu) | ShaderImageIdentitySwizzle; const auto kind = DescriptorBindingForImage(mode); + const ImageTableElement candidate{element, mode.dimension, mode.depthCompare, kind, 0u, image.atomic, image.atomic64}; const auto hash = hashWords(element, (static_cast(image.atomic64) << 25u) ^ (static_cast(image.atomic) << 24u) ^ (static_cast(mode.dimension) << 16u) ^ (static_cast(mode.depthCompare) << 8u) ^ static_cast(kind)); - auto index = elementIndex.Find(hash, [&](std::uint32_t known) { - const auto& entry = out.elements[known]; - return entry.words == element && entry.dimension == mode.dimension && entry.depthCompare == mode.depthCompare && entry.kind == kind && entry.atomic == image.atomic && entry.atomic64 == image.atomic64; - }); - if (!index.has_value() && out.elements.size() >= Abi::ImageElementBudget) { + if (decode != nullptr) *decode = {ColumnDecodeKind::Mode, decoded, localElement(*recorder, candidate, hash)}; + const auto index = placeElement(candidate, hash); + if (!index.has_value()) { record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::Budget, &words); } else { - if (!index.has_value()) { - index = static_cast(out.elements.size()); - out.elements.push_back({element, mode.dimension, mode.depthCompare, kind, 0u, image.atomic, image.atomic64}); - elementIndex.Insert(hash, *index); - } out.elements[*index].samplers |= directSamplers[resource]; record[Abi::ImageRecordFlags] = decoded << Abi::ModeShift; record[Abi::ImageRecordElement] = *index; } } } - const auto offset = static_cast(out.map.size()); - out.map.insert(out.map.end(), record.begin(), record.end()); - return offset; + return appendRecord(record); + }; + const auto replayRecord = [&](std::uint32_t resource, const DescriptorValue& words, const ColumnDecode& decode, const ColumnResolution& known, std::vector& placed) { + if (decode.kind == ColumnDecodeKind::Unknown) throw std::runtime_error("image table column memo replays a record it never decoded"); + if (decode.kind == ColumnDecodeKind::Null) return Abi::NullCode; + std::array record{0u, words.dwords[3], 0u}; + if (decode.kind == ColumnDecodeKind::Poison) { + record[Abi::ImageRecordFlags] = poison(resource, static_cast(decode.value), &words); + return appendRecord(record); + } + const auto& cached = known.elements.at(decode.element); + auto& at = placed.at(decode.element); + if (at == NoElement) { + if (const auto index = placeElement(cached.element, cached.hash)) at = *index; + } + if (at == NoElement) { + record[Abi::ImageRecordFlags] = poison(resource, PoisonReason::Budget, &words); + } else { + out.elements[at].samplers |= directSamplers[resource]; + record[Abi::ImageRecordFlags] = decode.value << Abi::ModeShift; + record[Abi::ImageRecordElement] = at; + } + return appendRecord(record); }; const auto samplerRecord = [&](std::uint32_t resource, const DescriptorValue& words) { std::array record{0u, 0u}; @@ -1813,6 +2271,36 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i out.map[header + Abi::TableMapStart] = mapStart; out.map.resize(out.map.size() + column.keys, 0u); records.Clear(); + std::shared_ptr known; + std::shared_ptr fresh; + ColumnRecorder recorder; + std::vector placed; + std::uint64_t content = 0; + std::uint64_t relaxedUsage = 0; + std::uint64_t strictUsage = 0; + if (memoized && !sampler && out.map[header + Abi::TableFault] == 0u && column.keys >= ColumnResolutionMinimumKeys) { + const auto& image = info.images[resource]; + const auto& modes = info.runtimeImageModes[resource]; + const ColumnResolutionKey columnKey{column.codes.Hash(), column.words.Hash(), imageUsageHash(image, modes), column.keys, column.outside, reducingSampler[resource]}; + known = findColumnResolution(columnKey); + if (known != nullptr && !(known->codes == column.codes && known->words == column.words && sameColumnUsage(*known, image, modes))) known = nullptr; + content = (columnKey.codesHash * 0x9e3779b97f4a7c15ull) ^ columnKey.wordsHash; + relaxedUsage = columnKey.usageHash; + if (TableColumnMemoDiagnosed()) strictUsage = strictUsageHash(relaxedUsage, image); + if (known != nullptr) { + placed.assign(known->elements.size(), NoElement); + } else { + fresh = std::make_shared(); + fresh->key = columnKey; + fresh->image = columnUsage(image); + fresh->modes.reserve(modes.size()); + for (const auto& mode : modes) fresh->modes.push_back(columnUsage(mode)); + fresh->codes = column.codes; + fresh->words = column.words; + fresh->decodes.assign(column.words.size(), ColumnDecode{}); + recorder.resolution = fresh.get(); + } + } for (std::uint32_t key = 0; key < column.keys; key++) { const auto code = column.codes[key]; std::uint32_t mapped = Abi::NullCode; @@ -1820,16 +2308,34 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i const auto reason = static_cast(code & ~Abi::PoisonFlag); mapped = reason == PoisonReason::OutsideDomain ? Abi::PoisonCode(Abi::OutsideDomainPoison) : poison(resource, reason, nullptr); } else if (out.map[header + Abi::TableFault] == 0u) { - if (const auto known = records.Find(code)) { - mapped = *known; + if (const auto found = records.Find(code)) { + mapped = *found; } else { const auto& words = column.words.at(code); - if (sampler || (words.availableDwordMask & (1u << 3u)) == 0u || !nullImageDescriptor(info.images[resource], words)) mapped = sampler ? samplerRecord(resource, words) : imageRecord(resource, words); + if (known != nullptr) { + mapped = replayRecord(resource, words, known->decodes.at(code), *known, placed); + } else if (sampler || (words.availableDwordMask & (1u << 3u)) == 0u || !nullImageDescriptor(info.images[resource], words)) { + mapped = sampler ? samplerRecord(resource, words) : imageRecord(resource, words, fresh != nullptr ? &recorder : nullptr, fresh != nullptr ? &fresh->decodes.at(code) : nullptr); + } else if (fresh != nullptr) { + fresh->decodes.at(code) = {ColumnDecodeKind::Null, 0u, 0u}; + } records.Set(code, mapped); } } out.map[mapStart + key] = mapped; } + if (known != nullptr) { + served++; + tableCounters().columnHits.fetch_add(1, std::memory_order_relaxed); + tableReuseTotals().columnHits.fetch_add(1, std::memory_order_relaxed); + noteColumnLookup({memoKey, resource, table}, content, true, column.codes, column.words, relaxedUsage, strictUsage); + } else if (fresh != nullptr) { + tableCounters().columnMisses.fetch_add(1, std::memory_order_relaxed); + tableReuseTotals().columnMisses.fetch_add(1, std::memory_order_relaxed); + noteColumnLookup({memoKey, resource, table}, content, false, column.codes, column.words, relaxedUsage, strictUsage); + const bool complete = std::none_of(fresh->decodes.begin(), fresh->decodes.end(), [](const ColumnDecode& decode) { return decode.kind == ColumnDecodeKind::Unknown; }); + if (complete) rememberColumnResolution(std::move(fresh)); + } if (sampler && out.map[header + Abi::TableFault] == 0u) { DescriptorValue zero; zero.dwordCount = 4u; @@ -1851,6 +2357,17 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i out.map[Abi::HeaderSamplerFaults] = static_cast(out.map.size()); out.map.resize(out.map.size() + out.samplers.size() / Abi::SamplerEntryElements, 0u); out.faults = static_cast(expectedCount + static_cast(std::count(outside.begin(), outside.end(), true))); + if (served != 0u && VerifyTableColumnMemo()) { + tableCounters().columnVerified.fetch_add(1, std::memory_order_relaxed); + tableReuseTotals().columnVerified.fetch_add(1, std::memory_order_relaxed); + const auto reference = ResolveImageTables(info, snapshot, memoKey, false); + const char* differs = out.map != reference.map ? "map" : out.elements != reference.elements ? "elements" : out.samplers != reference.samplers ? "samplers" : out.poison != reference.poison ? "poison" : out.faults != reference.faults ? "faults" : nullptr; + if (differs != nullptr) { + tableCounters().columnMismatches.fetch_add(1, std::memory_order_relaxed); + const auto mismatch = tableReuseTotals().columnMismatches.fetch_add(1, std::memory_order_relaxed) + 1u; + if (mismatch <= 20u) std::fprintf(stderr, "[image-table] column memo mismatch %llu: shader 0x%llx variant %016llx differs in its %s\n", static_cast(mismatch), static_cast(tables.shader), static_cast(memoKey), differs); + } + } return out; } @@ -2532,7 +3049,7 @@ std::uint64_t ResourceMaterializer::SpecializationNanoseconds() { ImageTableReuseCounts ResourceMaterializer::TableReuse() { auto& totals = tableReuseTotals(); - return {totals.unwritten.load(std::memory_order_relaxed), totals.compared.load(std::memory_order_relaxed), totals.wordsRead.load(std::memory_order_relaxed), totals.shared.load(std::memory_order_relaxed), totals.columns.load(std::memory_order_relaxed), totals.resolutions.load(std::memory_order_relaxed), totals.resolutionsShared.load(std::memory_order_relaxed)}; + return {totals.unwritten.load(std::memory_order_relaxed), totals.compared.load(std::memory_order_relaxed), totals.wordsRead.load(std::memory_order_relaxed), totals.shared.load(std::memory_order_relaxed), totals.columns.load(std::memory_order_relaxed), totals.resolutions.load(std::memory_order_relaxed), totals.resolutionsShared.load(std::memory_order_relaxed), totals.columnHits.load(std::memory_order_relaxed), totals.columnMisses.load(std::memory_order_relaxed), totals.columnVerified.load(std::memory_order_relaxed), totals.columnMismatches.load(std::memory_order_relaxed), totals.columnMissesNeverSeen.load(std::memory_order_relaxed), totals.columnMissesChangedContent.load(std::memory_order_relaxed), totals.columnDiffedMisses.load(std::memory_order_relaxed), totals.columnChangedKeys.load(std::memory_order_relaxed)}; } std::uint64_t ResourceMaterializer::EvaluateNanoseconds() { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 7622edf1b901..9772ffff9970 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -1384,9 +1384,13 @@ constexpr std::string_view NeutralSwitches[] = { "APS5_NO_PERF_ITERATION_SELECTION", "APS5_NO_PERF_LAYOUT_PAIR", "APS5_NO_RESULT_MEMO", + "APS5_NO_TABLE_COLUMN_MEMO", "APS5_NO_TABLE_RESOLUTION_REUSE", "APS5_NO_TABLE_SNAPSHOT_REUSE", "APS5_RESULT_MEMO_MB", + "APS5_TABLE_COLUMN_MEMO_DIAG", + "APS5_TABLE_COLUMN_MEMO_WORDS", + "APS5_VERIFY_TABLE_COLUMN_MEMO", }; const std::vector& switchKey() { diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 13dd3116109b..5e48d8b058cd 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -47,6 +47,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them | | `APS5_NO_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes a fresh result for every recompile instead of reusing the result memo | | `APS5_NO_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | runs the runtime-source walk of a plan in the IR evaluator instead of the compiled tape | +| `APS5_NO_TABLE_COLUMN_MEMO` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | resolves every image table column from its records instead of replaying the decoded records of a column of at least 64 keys with the same content, key count and records-outside-the-snapshot state that any shader resolved before through an image resource description, runtime mode list and reducing-sampler state that agree in every field the decode reads (all but the descriptor source, the first use and the table index); the memo serves the runtime resolution only | | `APS5_NO_TABLE_RESOLUTION_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | stops a result memo miss from reusing an unchanged image table's resolution, so the table is resolved again | | `APS5_NO_TABLE_SNAPSHOT_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | turns off the image table snapshot reuse, so every capture reads and compares each table column | | `APS5_NO_WINDOW_NARROWING` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp` | | indexes every slot of the shader's vector window instead of only the slots the program uses | @@ -57,12 +58,15 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_SINGLE_LANE` | string | `core/shader/recompiler/Recompiler.cpp` | | hex code addresses (comma separated, or all) of wave64 programs kept at one guest lane per invocation | | `APS5_SRT_DEBUG` | presence | `core/shader/recompiler/Optimization/src/RequestMemoryView.cpp` | | prints extra SRT descriptor-walk debug output, such as guest reads outside the request memory | | `APS5_STRICT_DESCRIPTOR_WEBS` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | makes a T# or S# read in a descriptor web throw when a path's descriptor dwords depend on the program counter, EXEC or a lane mask, instead of selecting the first descriptor and recording an `InvalidDescriptor` fault when a lane takes that path; 0 leaves it off | +| `APS5_TABLE_COLUMN_MEMO_DIAG` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | extends the column memo report of `APS5_PROFILE_DRAW` with the distinct column contents seen so far, the distinct contents, strict and relaxed usages of the window (the two content counts stop at 131,072), and, over the misses whose column changed content while its previous column was still pinned, the least, median and most keys whose descriptor differs from the previous content, the 1,024-key chunks holding them, and the count whose key count changed; prints with `APS5_PROFILE_DRAW`; keeps one previous column per (variant, resource, table) within 64 Mi words | +| `APS5_TABLE_COLUMN_MEMO_WORDS` | integer | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | 33554432 | 4-byte words of estimated host memory (each entry's decodes, elements, codes, words, modes and overhead) the image table column memo may hold; zero, negative or malformed values throw | | `APS5_TRACE_BDA` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | prints the first few accesses that make a program take the address-based path | | `APS5_TRACE_BUFFER_WRITTEN` | presence | `core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp` | | prints how many guest buffer elements a program may store to and how many it only loads | | `APS5_TRACE_IMAGE_TABLE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | traces image-table resolution | | `APS5_VERIFY_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | also materializes every patched result memo hit, prints the first twenty whose patched result differs and counts them in the result memo report of `APS5_PROFILE_DRAW` | | `APS5_VERIFY_SRT_COMPACT` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp` | | walks every SRT descriptor a second time through the IR evaluator and throws when the two walks differ | | `APS5_VERIFY_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | walks a plan's runtime sources a second time through the evaluator, adopts its answers where the two differ and prints the first twenty differences | +| `APS5_VERIFY_TABLE_COLUMN_MEMO` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | also resolves every image table snapshot that replayed a column's decoded records from its records, prints the first twenty whose resolution differs and counts them in the column memo report of `APS5_PROFILE_DRAW` | | `APS5_WAVE_LDS_SUBGROUP` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp` | | restores subgroup-scope LDS barriers for single-wave workgroups instead of workgroup scope | ### libSceAgc From f5cf0db16c28ee4a7575c7cd1dda342fd01a9fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Tue, 27 Oct 2026 19:03:33 +0100 Subject: [PATCH 2/5] perf(shader): keep the column memo's lookup bookkeeping to the draw profile noteColumnLookup fed the never-seen and changed-content counters of the column memo report on every lookup, taking the diagnostics mutex and inserting into its slot and content maps whether or not APS5_PROFILE_DRAW, the report's only reader, was set. It now returns first unless the draw profile is on, and the strict usage hash of APS5_TABLE_COLUMN_MEMO_DIAG is computed under the same condition. The column memo test gains a reported variant with APS5_PROFILE_DRAW=1 that expects the three first sights and the one content change, and a diagnosed variant without the profile switch; without it every variant expects those counters to stay at zero, and the diagnosed variant sets the profile switch its report needs. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 6 +++++- core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp | 5 +++-- .../recompiler/Optimization/src/ResourceMaterializer.cpp | 3 ++- docs/dev/ENVIRONMENT.md | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 00554b21679c..0282d35e17d3 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2631,8 +2631,12 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_table_column_memo_verified PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_VERIFY_TABLE_COLUMN_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_column_memo_off COMMAND agc_shader_memory_tests --table-column-memo) set_tests_properties(agc_shader_table_column_memo_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_TABLE_COLUMN_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_reported COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo_reported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PROFILE_DRAW=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_column_memo_diagnosed COMMAND agc_shader_memory_tests --table-column-memo) - set_tests_properties(agc_shader_table_column_memo_diagnosed PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_shader_table_column_memo_diagnosed PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PROFILE_DRAW=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_diagnosed_unreported COMMAND agc_shader_memory_tests --table-column-memo) + set_tests_properties(agc_shader_table_column_memo_diagnosed_unreported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 42bf48d14001..c39cf01727d3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -2955,7 +2955,8 @@ void verifyTableColumnMemo() { heaps[1] = heaps[0]; const bool enabled = std::getenv("APS5_NO_TABLE_COLUMN_MEMO") == nullptr; const bool verified = enabled && std::getenv("APS5_VERIFY_TABLE_COLUMN_MEMO") != nullptr; - const bool diagnosed = enabled && std::getenv("APS5_TABLE_COLUMN_MEMO_DIAG") != nullptr; + const bool reported = enabled && std::getenv("APS5_PROFILE_DRAW") != nullptr; + const bool diagnosed = reported && std::getenv("APS5_TABLE_COLUMN_MEMO_DIAG") != nullptr; const auto same = [](const ResolvedImageTables& left, const ResolvedImageTables& right) { return left.map == right.map && left.elements == right.elements && left.samplers == right.samplers && left.poison == right.poison && left.faults == right.faults; }; @@ -3060,7 +3061,7 @@ void verifyTableColumnMemo() { require(otherDecode.map == changed.map && otherDecode.elements == changed.elements && otherDecode.poison == changed.poison, "image table column memo: the changed decode input resolved to other records"); require(afterDecode.columnVerified - before.columnVerified == (verified ? 6u : 0u), "image table column memo: the shared replays were not verified exactly when asked"); require(afterDecode.columnMismatches == before.columnMismatches, "image table column memo: the verification found a shared resolution that differs"); - require(afterDecode.columnMissesNeverSeen - before.columnMissesNeverSeen == (enabled ? 3u : 0u) && afterDecode.columnMissesChangedContent - before.columnMissesChangedContent == (enabled ? 1u : 0u), "image table column memo: the miss diagnostics did not count three first sights and one content change"); + require(afterDecode.columnMissesNeverSeen - before.columnMissesNeverSeen == (reported ? 3u : 0u) && afterDecode.columnMissesChangedContent - before.columnMissesChangedContent == (reported ? 1u : 0u), "image table column memo: the miss diagnostics did not count three first sights and one content change exactly when the report is on"); require(afterDecode.columnDiffedMisses - before.columnDiffedMisses == (diagnosed ? 1u : 0u) && afterDecode.columnChangedKeys - before.columnChangedKeys == (diagnosed ? 1u : 0u), "image table column memo: the diagnostics did not diff the one changed record exactly when asked"); std::cout << "image table column memo passed\n"; } diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 68de4394f67d..31804b844c4c 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -603,6 +603,7 @@ std::pair diffColumns(const ColumnSlot& previous, } void noteColumnLookup(const ColumnUsageKey& usage, std::uint64_t content, bool hit, const SharedValues& codes, const SharedValues& words, std::uint64_t relaxedUsage, std::uint64_t strictUsage) { + if (!MaterializeProfiled()) return; auto& diagnostics = columnMissDiagnostics(); auto& counters = tableCounters(); auto& totals = tableReuseTotals(); @@ -2286,7 +2287,7 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i if (known != nullptr && !(known->codes == column.codes && known->words == column.words && sameColumnUsage(*known, image, modes))) known = nullptr; content = (columnKey.codesHash * 0x9e3779b97f4a7c15ull) ^ columnKey.wordsHash; relaxedUsage = columnKey.usageHash; - if (TableColumnMemoDiagnosed()) strictUsage = strictUsageHash(relaxedUsage, image); + if (MaterializeProfiled() && TableColumnMemoDiagnosed()) strictUsage = strictUsageHash(relaxedUsage, image); if (known != nullptr) { placed.assign(known->elements.size(), NoElement); } else { diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 5e48d8b058cd..340e1ccf1210 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -58,7 +58,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_SINGLE_LANE` | string | `core/shader/recompiler/Recompiler.cpp` | | hex code addresses (comma separated, or all) of wave64 programs kept at one guest lane per invocation | | `APS5_SRT_DEBUG` | presence | `core/shader/recompiler/Optimization/src/RequestMemoryView.cpp` | | prints extra SRT descriptor-walk debug output, such as guest reads outside the request memory | | `APS5_STRICT_DESCRIPTOR_WEBS` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | makes a T# or S# read in a descriptor web throw when a path's descriptor dwords depend on the program counter, EXEC or a lane mask, instead of selecting the first descriptor and recording an `InvalidDescriptor` fault when a lane takes that path; 0 leaves it off | -| `APS5_TABLE_COLUMN_MEMO_DIAG` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | extends the column memo report of `APS5_PROFILE_DRAW` with the distinct column contents seen so far, the distinct contents, strict and relaxed usages of the window (the two content counts stop at 131,072), and, over the misses whose column changed content while its previous column was still pinned, the least, median and most keys whose descriptor differs from the previous content, the 1,024-key chunks holding them, and the count whose key count changed; prints with `APS5_PROFILE_DRAW`; keeps one previous column per (variant, resource, table) within 64 Mi words | +| `APS5_TABLE_COLUMN_MEMO_DIAG` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | extends the column memo report of `APS5_PROFILE_DRAW` with the distinct column contents seen so far, the distinct contents, strict and relaxed usages of the window (the two content counts stop at 131,072), and, over the misses whose column changed content while its previous column was still pinned, the least, median and most keys whose descriptor differs from the previous content, the 1,024-key chunks holding them, and the count whose key count changed; takes effect only with `APS5_PROFILE_DRAW`; keeps one previous column per (variant, resource, table) within 64 Mi words | | `APS5_TABLE_COLUMN_MEMO_WORDS` | integer | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | 33554432 | 4-byte words of estimated host memory (each entry's decodes, elements, codes, words, modes and overhead) the image table column memo may hold; zero, negative or malformed values throw | | `APS5_TRACE_BDA` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | prints the first few accesses that make a program take the address-based path | | `APS5_TRACE_BUFFER_WRITTEN` | presence | `core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp` | | prints how many guest buffer elements a program may store to and how many it only loads | From dc1e7e5695ab32e8f3186c58eae3c44574f9588c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Wed, 28 Oct 2026 11:57:33 +0100 Subject: [PATCH 3/5] test(shader): cover the image table column memo's placement, eviction and budget parsing agc_shader_table_column_memo_placement and _placement_before resolve thirty selected single-record tables and one merged column, whose narrowed domain binds its sixty-five keyed records and record 0, in one resolution, with the merged column resolved after or before the selected tables. Thirty of its records equal the selected tables' records: the replayed column must land them on the elements the selected tables placed, or the selected tables on the elements the replayed column placed, 66 elements in all, equal to the plain resolution. _placement_off runs the first order with the memo off. agc_shader_table_column_memo_evicted_by_count fills the memo to its 4,096 entries and reads through hits and misses that the least recently used entry leaves first; _evicted_by_budget does the same under APS5_TABLE_COLUMN_MEMO_WORDS=65536 with the first content replayed after every insertion, and _oversize_entry checks that an entry over a budget of 0x1 words is never kept. agc_shader_table_column_memo_words*: accepted values (unset, 1048576, 0x100000) resolve the same column twice and check that the memo keeps it; refused values (0, -1, empty, abc, 12x, a 20-digit overflow) check that the first resolution fails with the variable's message. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 30 +++ .../libSceAgcDriver/tests/ShaderMemory.cpp | 199 ++++++++++++++++++ 2 files changed, 229 insertions(+) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 0282d35e17d3..8e32e9046bb8 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2637,6 +2637,36 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_table_column_memo_diagnosed PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_PROFILE_DRAW=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_column_memo_diagnosed_unreported COMMAND agc_shader_memory_tests --table-column-memo) set_tests_properties(agc_shader_table_column_memo_diagnosed_unreported PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_DIAG=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_placement COMMAND agc_shader_memory_tests --table-column-memo-placement after) + set_tests_properties(agc_shader_table_column_memo_placement PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_placement_before COMMAND agc_shader_memory_tests --table-column-memo-placement before) + set_tests_properties(agc_shader_table_column_memo_placement_before PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_placement_off COMMAND agc_shader_memory_tests --table-column-memo-placement after) + set_tests_properties(agc_shader_table_column_memo_placement_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_TABLE_COLUMN_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_evicted_by_count COMMAND agc_shader_memory_tests --table-column-memo-eviction count) + set_tests_properties(agc_shader_table_column_memo_evicted_by_count PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_evicted_by_budget COMMAND agc_shader_memory_tests --table-column-memo-eviction budget) + set_tests_properties(agc_shader_table_column_memo_evicted_by_budget PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=65536" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_oversize_entry COMMAND agc_shader_memory_tests --table-column-memo-eviction oversize) + set_tests_properties(agc_shader_table_column_memo_oversize_entry PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=0x1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words COMMAND agc_shader_memory_tests --table-column-memo-words accepted) + set_tests_properties(agc_shader_table_column_memo_words PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_set COMMAND agc_shader_memory_tests --table-column-memo-words accepted) + set_tests_properties(agc_shader_table_column_memo_words_set PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=1048576" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_hex COMMAND agc_shader_memory_tests --table-column-memo-words accepted) + set_tests_properties(agc_shader_table_column_memo_words_hex PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=0x100000" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_zero COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_zero PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=0" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_negative COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_negative PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=-1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_empty COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_empty PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_text COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_text PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=abc" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_suffix COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_suffix PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=12x" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_words_overflow COMMAND agc_shader_memory_tests --table-column-memo-words refused) + set_tests_properties(agc_shader_table_column_memo_words_overflow PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=99999999999999999999" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index c39cf01727d3..7c7e21a25d0a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -3515,6 +3516,192 @@ void verifySelectedRecordsWithinLimits(bool pairs, bool direct) { require(twoHeaps->info.images.size() == 31u && selectedTables(*twoHeaps) == 30u && unselected != twoHeaps->info.images.end() && columnOf(*twoHeaps, *unselected).keyDomains.size() == 40u, (name + ": the larger heap was not merged first, or the smaller one lost its split").c_str()); } +void verifyTableColumnMemoPlacement(std::string_view order) { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + constexpr std::uint32_t Selected = 30; + constexpr std::uint32_t Replayed = 65; + require(order == "after" || order == "before", "image table column memo placement: the order is after or before"); + const bool after = order == "after"; + const std::uint32_t selectedBase = after ? 2u : 2u + Replayed; + const std::uint32_t mergedBase = after ? 2u + Selected : 2u; + const std::string name = "image table column memo placement (replayed " + std::string(order) + " the selected tables)"; + const auto step = [&](const char* what, auto&& action) { + try { + return action(); + } catch (const std::exception& error) { + throw std::runtime_error(name + ": " + what + ": " + error.what()); + } + }; + static TableTexture textures[4]; + alignas(4096) static TableManyGuest guest; + auto& heap = guest.heap; + for (std::uint32_t record = 0; record < heap.size(); record++) heap[record] = TableT(textures[record % 4u].bytes.data(), TableFormat8888UNorm, TableType2D, record + 1u); + for (std::uint32_t shared = 0; shared < Selected; shared++) heap[mergedBase + shared] = heap[selectedBase + shared]; + const auto heapV = TableV(heap.data(), 32u, static_cast(heap.size())); + const auto outputV = TableV(guest.output.data(), 0u, 16u); + guest.srt = {}; + std::copy(heapV.begin(), heapV.end(), guest.srt.begin()); + guest.srt[8] = 0x92u; + std::copy(outputV.begin(), outputV.end(), guest.srt.begin() + 12); + std::copy(heapV.begin(), heapV.end(), guest.srt.begin() + 16); + for (std::uint32_t read = 0; read + 20u < guest.srt.size(); read++) guest.srt[20u + read] = read + 2u; + const auto address = static_cast(reinterpret_cast(guest.srt.data())); + const std::array userData{static_cast(address), static_cast(address >> 32u)}; + const auto code = after ? TableManyKeyCode(Selected, Replayed, false) : TableManyKeyCode(Replayed, Selected, false); + const auto request = TableRequest(code, userData); + const auto plan = step("plan", [&] { return GetResourcePlan(request); }); + const auto& info = plan->info; + require(info.images.size() == Selected + 1u, (name + ": the reads were not planned as thirty selected tables and one merged table").c_str()); + const auto capture = [&] { + CountingMemory memory; + SrtRuntime runtime; + runtime.userData = request.context.userData; + runtime.shaderBase = request.shader.codeAddress; + runtime.userContext = &memory; + runtime.readMemory = &CountingRead; + runtime.readSpecializationMemory = &CountingRead; + runtime.accessible = &CountingAccessible; + runtime.readWords = &CountingReadWords; + return step("capture", [&] { return CaptureResources(request, runtime); }); + }; + const auto same = [](const ResolvedImageTables& left, const ResolvedImageTables& right) { + return left.map == right.map && left.elements == right.elements && left.samplers == right.samplers && left.poison == right.poison && left.faults == right.faults; + }; + constexpr std::uint64_t MemoKey = 0x7ab1ec11u; + const bool enabled = std::getenv("APS5_NO_TABLE_COLUMN_MEMO") == nullptr; + const auto first = capture(); + const auto& tables = first->snapshot.tables.tables; + require(tables.size() == Selected + 1u, (name + ": the snapshot does not hold thirty-one tables").c_str()); + auto merged = std::numeric_limits::max(); + for (std::uint32_t table = 0; table < tables.size(); table++) { + if (tables[table].keys == 1u) continue; + require(merged == std::numeric_limits::max() && tables[table].keys >= 64u, (name + ": more than one table holds the merged reads, or the merged table has fewer than 64 keys").c_str()); + merged = table; + } + require(merged != std::numeric_limits::max(), (name + ": no table holds the merged reads").c_str()); + const auto mergedImage = std::find_if(info.images.begin(), info.images.end(), [&](const ImageResource& image) { return image.table == merged; }); + require(mergedImage != info.images.end() && static_cast(mergedImage - info.images.begin()) == (after ? Selected : 0u), (name + ": the merged column is not resolved at the expected position among the resources").c_str()); + const auto before = ResourceMaterializer::TableReuse(); + const auto plain = step("plain resolution", [&] { return ResourceMaterializer::ResolveImageTables(info, first->snapshot, MemoKey, false); }); + const auto missed = step("recording resolution", [&] { return ResourceMaterializer::ResolveImageTables(info, first->snapshot, MemoKey); }); + const auto afterMiss = ResourceMaterializer::TableReuse(); + require(afterMiss.columnMisses - before.columnMisses == (enabled ? 1u : 0u) && afterMiss.columnHits == before.columnHits, (name + ": the merged column did not miss exactly once").c_str()); + require(same(missed, plain), (name + ": the recording resolution differs from the plain one").c_str()); + const auto second = capture(); + const auto replayed = step("replaying resolution", [&] { return ResourceMaterializer::ResolveImageTables(info, second->snapshot, MemoKey); }); + const auto afterHit = ResourceMaterializer::TableReuse(); + require(afterHit.columnHits - afterMiss.columnHits == (enabled ? 1u : 0u) && afterHit.columnMisses == afterMiss.columnMisses, (name + ": the merged column was not replayed").c_str()); + require(same(replayed, plain) && same(replayed, missed), (name + ": the replayed resolution differs from the plain one").c_str()); + require(replayed.map[Abi::HeaderElementCount] == Replayed + 1u && replayed.elements.size() == Replayed + 1u, (name + ": the merged column's 65 keyed records and its record 0 with the selected tables' thirty shared records did not resolve to 66 elements").c_str()); + std::map, std::uint32_t> placedBySelected; + for (std::uint32_t table = 0; table < tables.size(); table++) { + if (table == merged) continue; + const auto selectedCodes = TableCodes(replayed.map, table); + require(selectedCodes.size() == 1u && (selectedCodes[0] & Abi::PoisonFlag) == 0u, (name + ": a selected table's map does not hold one record").c_str()); + const auto element = replayed.map.at(selectedCodes[0] + Abi::ImageRecordElement); + placedBySelected.emplace(replayed.elements.at(element).words, element); + } + require(placedBySelected.size() == Selected, (name + ": the selected tables did not place thirty distinct elements").c_str()); + const auto codes = TableCodes(replayed.map, merged); + require(codes.size() >= mergedBase + Replayed, (name + ": the merged table's map is shorter than its key domain").c_str()); + for (std::uint32_t shared = 0; shared < Selected; shared++) { + const auto mapped = codes.at(mergedBase + shared); + require((mapped & Abi::PoisonFlag) == 0u, (name + ": a shared key of the merged table is poisoned").c_str()); + const auto element = replayed.map.at(mapped + Abi::ImageRecordElement); + const auto found = placedBySelected.find(replayed.elements.at(element).words); + require(found != placedBySelected.end() && found->second == element, (name + ": a shared record of the merged column and its selected table do not share one element").c_str()); + require(replayed.elements.at(element).words[2] == heap[mergedBase + shared][2], (name + ": a shared element does not hold its record").c_str()); + } + const auto zero = codes.at(0); + require((zero & Abi::PoisonFlag) == 0u && placedBySelected.find(replayed.elements.at(replayed.map.at(zero + Abi::ImageRecordElement)).words) == placedBySelected.end(), (name + ": record 0 of the narrowed merged column was not bound to an element of its own").c_str()); + std::cout << name << ": " << Selected << " selected tables and one merged column of " << Replayed << " keyed records plus record 0 share " << Selected << " elements, " << replayed.elements.size() << " elements in all\n"; +} + +ShaderRecompiler::ResolvedImageTables ResolveTableColumnMemoProbe(std::uint64_t content) { + using namespace ShaderRecompiler; + constexpr std::uint32_t Keys = 64; + static TableTexture textures[4]; + alignas(4096) static TableGuest guest; + alignas(4096) static std::array, Keys> heap; + static const auto userData = [&] { + for (std::uint32_t record = 0; record < Keys; record++) heap[record] = TableT(textures[record % 4u].bytes.data(), TableFormat8888UNorm, TableType2D, 1u + record); + FillTableSrt(guest, heap.data(), 32u, Keys); + return TableUserData(guest); + }(); + static const auto request = TableRequest(TableWholeCode, userData); + static const auto plan = GetResourcePlan(request); + heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D, Keys + 1u + static_cast(content)); + CountingMemory memory; + SrtRuntime runtime; + runtime.userData = request.context.userData; + runtime.shaderBase = request.shader.codeAddress; + runtime.userContext = &memory; + runtime.readMemory = &CountingRead; + runtime.readSpecializationMemory = &CountingRead; + runtime.accessible = &CountingAccessible; + runtime.readWords = &CountingReadWords; + const auto captured = CaptureResources(request, runtime); + require(captured->snapshot.tables.tables.size() == 1u && captured->snapshot.tables.tables.at(0).keys == Keys, "image table column memo probe: the snapshot does not hold the whole table"); + return ResourceMaterializer::ResolveImageTables(plan->info, captured->snapshot, 0x7ab1ec21u); +} + +void verifyTableColumnMemoEviction(std::string_view mode) { + using namespace ShaderRecompiler; + constexpr std::uint64_t Entries = 4096; + constexpr std::uint64_t Budgeted = 200; + require(mode == "count" || mode == "budget" || mode == "oversize", "image table column memo eviction: the mode is count, budget or oversize"); + const auto expect = [&](std::uint64_t content, bool hit, const char* what) { + const auto previous = ResourceMaterializer::TableReuse(); + static_cast(ResolveTableColumnMemoProbe(content)); + const auto now = ResourceMaterializer::TableReuse(); + require(now.columnHits - previous.columnHits == (hit ? 1u : 0u) && now.columnMisses - previous.columnMisses == (hit ? 0u : 1u), what); + }; + expect(0u, false, "image table column memo eviction: the first content did not miss"); + if (mode == "oversize") { + expect(0u, false, "image table column memo eviction: a content over the budget was kept"); + expect(0u, false, "image table column memo eviction: a content over the budget was kept on the third resolution"); + std::cout << "image table column memo eviction: an entry over the budget is never kept\n"; + return; + } + expect(0u, true, "image table column memo eviction: the first content was not replayed"); + if (mode == "budget") { + for (std::uint64_t content = 1; content < Budgeted; content++) { + expect(content, false, "image table column memo eviction: a new content did not miss"); + expect(0u, true, "image table column memo eviction: the first content, replayed after every insertion, was evicted under the budget"); + } + expect(Budgeted - 1u, true, "image table column memo eviction: the newest entry was not kept under the budget"); + expect(1u, false, "image table column memo eviction: the least recently used entry survived the budget"); + expect(0u, true, "image table column memo eviction: re-adding the least recently used content evicted the entry replayed most"); + std::cout << "image table column memo eviction: " << Budgeted << " contents under the budget; the least recently used entry left first\n"; + return; + } + for (std::uint64_t content = 1; content < Entries; content++) expect(content, false, "image table column memo eviction: a new content did not miss while the memo had room"); + expect(0u, true, "image table column memo eviction: the oldest entry was not replayed before the cap was crossed"); + expect(Entries, false, "image table column memo eviction: a content past the entry cap did not miss"); + expect(0u, true, "image table column memo eviction: the entry replayed just before the cap was evicted"); + expect(1u, false, "image table column memo eviction: the least recently used entry survived the cap"); + expect(3u, true, "image table column memo eviction: an entry that was not the oldest was evicted"); + expect(2u, false, "image table column memo eviction: the next oldest entry survived"); + std::cout << "image table column memo eviction: " << Entries << " entries, then the least recently used entry left first\n"; +} + +void verifyTableColumnMemoWords(std::string_view expectation) { + using namespace ShaderRecompiler; + if (expectation == "refused") { + expectFailure([] { static_cast(ResolveTableColumnMemoProbe(0u)); }, "APS5_TABLE_COLUMN_MEMO_WORDS is not a positive word count", "image table column memo words: a malformed word count was accepted"); + std::cout << "image table column memo words: the variable was refused\n"; + return; + } + require(expectation == "accepted", "image table column memo words: the expectation is accepted or refused"); + const auto before = ResourceMaterializer::TableReuse(); + static_cast(ResolveTableColumnMemoProbe(0u)); + static_cast(ResolveTableColumnMemoProbe(0u)); + const auto after = ResourceMaterializer::TableReuse(); + require(after.columnMisses - before.columnMisses == 1u && after.columnHits - before.columnHits == 1u, "image table column memo words: the memo did not keep a column under the accepted budget"); + std::cout << "image table column memo words: the variable was accepted\n"; +} + void verifyImageTableSrtKey() { using namespace ShaderRecompiler; namespace Abi = ImageTableAbi; @@ -8866,6 +9053,18 @@ int main(int argc, char** argv) { verifyTableColumnMemo(); return 0; } + if (argc == 3 && std::string_view(argv[1]) == "--table-column-memo-placement") { + verifyTableColumnMemoPlacement(argv[2]); + return 0; + } + if (argc == 3 && std::string_view(argv[1]) == "--table-column-memo-eviction") { + verifyTableColumnMemoEviction(argv[2]); + return 0; + } + if (argc == 3 && std::string_view(argv[1]) == "--table-column-memo-words") { + verifyTableColumnMemoWords(argv[2]); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); From 9360de223ac4f82b0554af634692d14685d9261b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Wed, 28 Oct 2026 16:57:52 +0100 Subject: [PATCH 4/5] perf(shader): hash every compared field into the column usage A column memo hit compares the stored usage's whole image resource and mode list (ImageResource's defaulted operator==), while the usage hash lists fields by name. Two fields were compared and not hashed: `sampledOnly`, which the image descriptor decode reads since #11507 (a cube descriptor with fewer than six slices is poisoned unless the image is only sampled), and `optionalDescriptor`. Two usages that differed only in one of them shared a key; the hit check rejected the entry and the next miss replaced it, so the two missed in turn instead of replaying. The hash now mixes both, and the docs/dev/ENVIRONMENT.md row of APS5_NO_TABLE_COLUMN_MEMO says what a hit compares: every field but the descriptor source, the first use and the table index. Test: `agc_shader_table_column_memo_usage` resolves one column under a sampled-only usage, under the same usage with `sampledOnly` cleared, and under the same usage with `optionalDescriptor` set, each under its own variant key: every new usage misses once, and every usage then hits. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 2 + .../libSceAgcDriver/tests/ShaderMemory.cpp | 40 ++++++++++++++++++- .../Optimization/src/ResourceMaterializer.cpp | 2 +- docs/dev/ENVIRONMENT.md | 2 +- 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index dfde5b46fc46..b05becea5ddc 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2720,6 +2720,8 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_table_column_memo_words_suffix PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=12x" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_column_memo_words_overflow COMMAND agc_shader_memory_tests --table-column-memo-words refused) set_tests_properties(agc_shader_table_column_memo_words_overflow PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_TABLE_COLUMN_MEMO_WORDS=99999999999999999999" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_column_memo_usage COMMAND agc_shader_memory_tests --table-column-memo-usage) + set_tests_properties(agc_shader_table_column_memo_usage PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 93f97f95e4a3..93f75a2c06ce 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -42,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -4088,7 +4089,7 @@ void verifyTableColumnMemoPlacement(std::string_view order) { std::cout << name << ": " << Selected << " selected tables and one merged column of " << Replayed << " keyed records plus record 0 share " << Selected << " elements, " << replayed.elements.size() << " elements in all\n"; } -ShaderRecompiler::ResolvedImageTables ResolveTableColumnMemoProbe(std::uint64_t content) { +ShaderRecompiler::ResolvedImageTables ResolveTableColumnMemoProbe(std::uint64_t content, std::optional sampledOnly = std::nullopt, std::optional optionalDescriptor = std::nullopt) { using namespace ShaderRecompiler; constexpr std::uint32_t Keys = 64; static TableTexture textures[4]; @@ -4113,7 +4114,20 @@ ShaderRecompiler::ResolvedImageTables ResolveTableColumnMemoProbe(std::uint64_t runtime.readWords = &CountingReadWords; const auto captured = CaptureResources(request, runtime); require(captured->snapshot.tables.tables.size() == 1u && captured->snapshot.tables.tables.at(0).keys == Keys, "image table column memo probe: the snapshot does not hold the whole table"); - return ResourceMaterializer::ResolveImageTables(plan->info, captured->snapshot, 0x7ab1ec21u); + std::uint64_t key = 0x7ab1ec21u; + if (sampledOnly == false) key ^= 0x100u; + if (optionalDescriptor == true) key ^= 0x200u; + if (!sampledOnly.has_value() && !optionalDescriptor.has_value()) return ResourceMaterializer::ResolveImageTables(plan->info, captured->snapshot, key); + ShaderInfo usage = plan->info; + const auto apply = [&](ImageResource& image) { + if (sampledOnly.has_value()) image.sampledOnly = *sampledOnly; + if (optionalDescriptor.has_value()) image.optionalDescriptor = *optionalDescriptor; + }; + for (auto& image : usage.images) apply(image); + for (auto& modes : usage.runtimeImageModes) { + for (auto& mode : modes) apply(mode); + } + return ResourceMaterializer::ResolveImageTables(usage, captured->snapshot, key); } void verifyTableColumnMemoEviction(std::string_view mode) { @@ -4172,6 +4186,24 @@ void verifyTableColumnMemoWords(std::string_view expectation) { std::cout << "image table column memo words: the variable was accepted\n"; } +void verifyTableColumnMemoUsage() { + using namespace ShaderRecompiler; + const auto expect = [](bool sampledOnly, bool optionalDescriptor, bool hit, const char* what) { + const auto before = ResourceMaterializer::TableReuse(); + static_cast(ResolveTableColumnMemoProbe(0u, sampledOnly, optionalDescriptor)); + const auto after = ResourceMaterializer::TableReuse(); + require(after.columnHits - before.columnHits == (hit ? 1u : 0u) && after.columnMisses - before.columnMisses == (hit ? 0u : 1u), what); + }; + expect(true, false, false, "image table column memo usage: the first sampled-only resolution did not miss"); + expect(false, false, false, "image table column memo usage: a usage that differs only in sampledOnly did not miss"); + expect(true, false, true, "image table column memo usage: the sampled-only entry was lost to the other usage"); + expect(false, false, true, "image table column memo usage: the entry with sampledOnly cleared was lost to the sampled-only one"); + expect(true, true, false, "image table column memo usage: a usage that differs only in optionalDescriptor did not miss"); + expect(true, true, true, "image table column memo usage: the optional-descriptor entry was not kept"); + expect(true, false, true, "image table column memo usage: the sampled-only entry was lost to the optional-descriptor usage"); + std::cout << "image table column memo usage: usages that differ only in sampledOnly or optionalDescriptor keep separate entries\n"; +} + void verifyImageTableSrtKey() { using namespace ShaderRecompiler; namespace Abi = ImageTableAbi; @@ -9715,6 +9747,10 @@ int main(int argc, char** argv) { verifyTableColumnMemoWords(argv[2]); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--table-column-memo-usage") { + verifyTableColumnMemoUsage(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--table-snapshot-reuse") { verifyTableSnapshotReuse(); verifyTableStampRace(); diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 5592b3bdfd65..3dc1e2bb8b6c 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -506,7 +506,7 @@ void mixImageResource(std::uint64_t& hash, const ImageResource& image) { mix(static_cast(image.conversionFormat)); mix(image.shaderSwizzle); std::uint64_t flags = 0; - for (const bool flag : {image.read, image.written, image.atomic, image.atomic64, image.depthCompare, image.cube, image.r128, image.srgbDecode, image.srgbDecodeCompatible, image.depthBits, image.depthUnorm16, image.packed, image.fmaskCompatible, image.depthBitsCompatible, image.constantSwizzle, image.constantSwizzleCompatible, image.cubeFaceCompatible, image.flatVolumeCompatible, image.singleMipPlaneBiasCompatible, image.volumeArraySampleCompatible, image.flatLineCompatible, image.flatVolumeLineCompatible, image.dimensionQueryOnly, image.foldTexelOffsetsOnly, image.convertedSamples, image.remappedDepth1D, image.targetMerge}) flags = (flags << 1u) | (flag ? 1u : 0u); + for (const bool flag : {image.read, image.written, image.atomic, image.atomic64, image.depthCompare, image.cube, image.r128, image.srgbDecode, image.srgbDecodeCompatible, image.depthBits, image.depthUnorm16, image.packed, image.fmaskCompatible, image.depthBitsCompatible, image.constantSwizzle, image.constantSwizzleCompatible, image.cubeFaceCompatible, image.flatVolumeCompatible, image.singleMipPlaneBiasCompatible, image.volumeArraySampleCompatible, image.flatLineCompatible, image.flatVolumeLineCompatible, image.dimensionQueryOnly, image.sampledOnly, image.foldTexelOffsetsOnly, image.convertedSamples, image.remappedDepth1D, image.optionalDescriptor, image.targetMerge}) flags = (flags << 1u) | (flag ? 1u : 0u); mix(flags); mix(image.srgbDecodeFormats); mix(image.byElements); diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 86ed941ee028..9238d6d99e1d 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -49,7 +49,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_PERF_ITERATION_SELECTION` | presence | `core/shader/recompiler/ControlFlow/src/Structurizer.cpp` | | recomputes the post-dominator sets when finding a loop's iteration-selection merge instead of reusing them | | `APS5_NO_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes a fresh result for every recompile instead of reusing the result memo | | `APS5_NO_SRT_TAPE` | presence | `core/shader/recompiler/Optimization/src/SrtWalker/SrtTape.cpp` | | runs the runtime-source walk of a plan in the IR evaluator instead of the compiled tape | -| `APS5_NO_TABLE_COLUMN_MEMO` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | resolves every image table column from its records instead of replaying the decoded records of a column of at least 64 keys with the same content, key count and records-outside-the-snapshot state that any shader resolved before through an image resource description, runtime mode list and reducing-sampler state that agree in every field the decode reads (all but the descriptor source, the first use and the table index); the memo serves the runtime resolution only | +| `APS5_NO_TABLE_COLUMN_MEMO` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | resolves every image table column from its records instead of replaying the decoded records of a column of at least 64 keys with the same content, key count and records-outside-the-snapshot state that any shader resolved before through an image resource description, runtime mode list and reducing-sampler state that agree in every field but the descriptor source, the first use and the table index; the memo serves the runtime resolution only | | `APS5_NO_TABLE_RESOLUTION_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | stops a result memo miss from reusing an unchanged image table's resolution, so the table is resolved again | | `APS5_NO_TABLE_SNAPSHOT_REUSE` | presence | `core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp` | | turns off the image table snapshot reuse, so every capture reads and compares each table column | | `APS5_NO_WINDOW_NARROWING` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp` | | indexes every slot of the shader's vector window instead of only the slots the program uses | From c5dcde982eb63a59661c78481d3c357d69da750b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20=C5=A0l=C3=A9gl?= Date: Wed, 28 Oct 2026 17:33:28 +0100 Subject: [PATCH 5/5] test(shader): put libc on the PATH of the column memo tests on Windows Every other agc_shader_memory_tests entry prepends libc's directory to PATH on Windows; the 22 column memo tests did not. They need libc's directory on PATH there, like the other entries. Co-Authored-By: Claude Fable 5.1 --- core/libs/prx/libSceAgcDriver/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index b05becea5ddc..7ebb5fce499e 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -2728,6 +2728,7 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) if(WIN32) set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_table_column_memo agc_shader_table_column_memo_verified agc_shader_table_column_memo_off agc_shader_table_column_memo_reported agc_shader_table_column_memo_diagnosed agc_shader_table_column_memo_diagnosed_unreported agc_shader_table_column_memo_placement agc_shader_table_column_memo_placement_before agc_shader_table_column_memo_placement_off agc_shader_table_column_memo_evicted_by_count agc_shader_table_column_memo_evicted_by_budget agc_shader_table_column_memo_oversize_entry agc_shader_table_column_memo_words agc_shader_table_column_memo_words_set agc_shader_table_column_memo_words_hex agc_shader_table_column_memo_words_zero agc_shader_table_column_memo_words_negative agc_shader_table_column_memo_words_empty agc_shader_table_column_memo_words_text agc_shader_table_column_memo_words_suffix agc_shader_table_column_memo_words_overflow agc_shader_table_column_memo_usage PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization) set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")