feat(agc): look up table images at run time through a device image table on the GPU heap (APS5_GPU_PATH)

CMakeLists.txt

@@ -155,6 +155,14 @@ if(BUILD_TESTING)
         ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include)
     add_test(NAME request_serializer COMMAND request_serializer_tests)
     set_tests_properties(request_serializer PROPERTIES TIMEOUT 20)
+    add_test_executable(heap_template_budget_tests
+        ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/HeapTemplateBudgetTests.cpp)
+    target_include_directories(heap_template_budget_tests PRIVATE
+        ${CMAKE_CURRENT_SOURCE_DIR}/core/libs
+        ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler
+        ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include)
+    add_test(NAME heap_template_budget COMMAND heap_template_budget_tests)
+    set_tests_properties(heap_template_budget PROPERTIES TIMEOUT 20)
     find_package(Python3 QUIET COMPONENTS Interpreter)
     add_test_executable(shader_ir_use_rewrite_tests
         core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -195,6 +195,18 @@ add_custom_command(
         VERBATIM
 )
 
+set(agcTextureDetileAddressedShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileAddressed.spv)
+set(agcTextureDetileAddressedShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileAddressed_spv.h)
+add_custom_command(
+        OUTPUT "${agcTextureDetileAddressedShaderHeader}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+        COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DADDRESSED=1 -o "${agcTextureDetileAddressedShaderSpv}" "${agcTextureDetileShaderSource}"
+        COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileAddressedShaderSpv} -DOUTPUT=${agcTextureDetileAddressedShaderHeader}
+        -DSYMBOL=TEXTURE_DETILE_ADDRESSED_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+        DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
+        VERBATIM
+)
+
 set(agcTextureRepackShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/TextureRepack.comp)
 set(agcTextureRepackShaderSpv ${agcTextureDetileGeneratedDir}/TextureRepack.spv)
 set(agcTextureRepackShaderHeader ${agcTextureDetileGeneratedDir}/TextureRepack_spv.h)
@@ -207,6 +219,17 @@ add_custom_command(
         DEPENDS "${agcTextureRepackShaderSource}" glslang-standalone
         VERBATIM
 )
+set(agcTextureRepackAddressedShaderSpv ${agcTextureDetileGeneratedDir}/TextureRepackAddressed.spv)
+set(agcTextureRepackAddressedShaderHeader ${agcTextureDetileGeneratedDir}/TextureRepackAddressed_spv.h)
+add_custom_command(
+        OUTPUT "${agcTextureRepackAddressedShaderHeader}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+        COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DADDRESSED=1 -o "${agcTextureRepackAddressedShaderSpv}" "${agcTextureRepackShaderSource}"
+        COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureRepackAddressedShaderSpv} -DOUTPUT=${agcTextureRepackAddressedShaderHeader}
+        -DSYMBOL=TEXTURE_REPACK_ADDRESSED_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+        DEPENDS "${agcTextureRepackShaderSource}" glslang-standalone
+        VERBATIM
+)
 
 set(agcDispatchArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/DispatchArguments.comp)
 set(agcDispatchArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/DispatchArguments.spv)
@@ -357,6 +380,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCache.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCacheStatistics.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCapture.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/HeapStages.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawDecode.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawLookup.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp
@@ -435,6 +459,7 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeferredDestruction.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/MemoryAccounting.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DescriptorHeapObjects.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp
@@ -465,6 +490,9 @@ set(agcDriverSources
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp
         ${agcSampleOrderShaderHeaders}
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BindlessHeap.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeviceImageKeys.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeviceImageTable.cpp
         ${agcTextureDetileShaderHeader}
         ${agcReadbackHeaders}
         ${agcMultisampleUploadHeader}
@@ -474,6 +502,8 @@ set(agcDriverSources
         ${agcStagingCopyBackShaderHeader}
         ${agcTextureDetileImageShaderHeader}
         ${agcTextureRepackShaderHeader}
+        ${agcTextureDetileAddressedShaderHeader}
+        ${agcTextureRepackAddressedShaderHeader}
         ${agcDispatchArgumentsShaderHeader}
         ${agcTextureDetileWordsShaderHeader}
         ${agcTextureDetileImageWordsShaderHeader}
@@ -711,14 +741,31 @@ function(agc_add_default_test agcTest)
     endif()
 endfunction()
 
-function(agc_add_draw_pipeline_test agcTestName)
-    add_test(NAME ${agcTestName}_draw_pipeline COMMAND ${agcTestName}_tests)
-    set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_PIPELINE=1;APS5_PIPELINED_DRAWS=32;APS5_PIPELINE_DISPATCH=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+function(agc_add_env_test agcTestName agcTestTarget)
+    cmake_parse_arguments(PARSE_ARGV 2 agcEnvTest "NO_WORKING_DIRECTORY" "TIMEOUT" "ARGS;ENV")
+    add_test(NAME ${agcTestName} COMMAND ${agcTestTarget} ${agcEnvTest_ARGS})
+    set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT "${agcEnvTest_ENV}" SKIP_RETURN_CODE 77)
+    if(NOT agcEnvTest_NO_WORKING_DIRECTORY)
+        set_tests_properties(${agcTestName} PROPERTIES WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    endif()
+    if(agcEnvTest_TIMEOUT)
+        set_tests_properties(${agcTestName} PROPERTIES TIMEOUT ${agcEnvTest_TIMEOUT})
+    endif()
     if(WIN32)
-        set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+        set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
     endif()
 endfunction()
 
+function(agc_add_draw_pipeline_test agcTestName)
+    agc_add_env_test(${agcTestName}_draw_pipeline ${agcTestName}_tests
+            ENV ANYPS5_NO_SHADER_CACHE=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1)
+endfunction()
+
+function(agc_add_gpu_heap_test agcTestName)
+    agc_add_env_test(${agcTestName}_gpu_heap ${agcTestName}_tests
+            ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1)
+endfunction()
+
 function(agc_add_spirv_mode_tests agcTest)
     if(NOT TARGET ${agcTest})
         return()
@@ -1059,7 +1106,7 @@ endif()
 
 if(BUILD_TESTING)
     get_directory_property(agcRegisteredTests TESTS)
-    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$")
+    list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_driver_pipeline_journal_gpu_heap|agc_pipeline_store_replay_growth)$")
     set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1")
 endif()
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp

@@ -72,6 +72,7 @@ struct DispatchVariant {
     std::vector<std::pair<std::uint32_t, std::uint32_t>> shaderDataPatches;
     std::uint64_t userDataMask = ~std::uint64_t{0};
     std::shared_ptr<const ShaderRecompiler::ShaderVertexStageInfo> vertexInfo;
+    std::shared_ptr<const ShaderRecompiler::HeapStageWords> ringTemplate;
 
     std::shared_ptr<ShaderMemory> memory;
     std::vector<ShaderRecompiler::MemoryRegion> captured;
@@ -88,7 +89,7 @@ struct DispatchEntry {
 };
 
 struct EntryCounters {
-    std::uint64_t lookups = 0, absent = 0, equal = 0, differing = 0, inaccessible = 0, queuedLabel = 0, flushingImage = 0, publishMoved = 0, pendingMoved = 0, forgetMoved = 0, imagesFlushed = 0, runsSynced = 0, forgetSinceInsert = 0, replaced = 0, inserts = 0, unstable = 0, touches = 0;
+    std::uint64_t lookups = 0, absent = 0, equal = 0, differing = 0, inaccessible = 0, queuedLabel = 0, flushingImage = 0, publishMoved = 0, pendingMoved = 0, forgetMoved = 0, deviceImageFallbacks = 0, imagesFlushed = 0, runsSynced = 0, forgetSinceInsert = 0, replaced = 0, inserts = 0, unstable = 0, touches = 0;
 
     std::uint64_t runsValidated = 0, runsInserted = 0, retriesEqual = 0, retriesMoved = 0;
     double validateUs = 0;
@@ -112,7 +113,7 @@ struct DataMask {
     std::vector<std::pair<std::uint32_t, std::uint32_t>>* live;
 };
 
-enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved };
+enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved, DeviceImageFallback };
 
 bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled);
 

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp

@@ -58,6 +58,7 @@ struct PreparedGraphicsStage {
     ShaderRecompiler::ShaderStage stage = ShaderRecompiler::ShaderStage::Vertex;
     std::uint32_t pushOffset = 0;
     std::shared_ptr<const ShaderRecompiler::RecompileResult> registration;
+    bool heapVariant = false;
 };
 
 std::vector<PreparedGraphicsStage> PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false);
@@ -131,6 +132,12 @@ struct StageCapture {
     std::uint32_t pushOffset = 0;
     ShaderRecompiler::SrtUserDataUse userDataUse;
     std::uint64_t userDataMask = ~std::uint64_t{0};
+    bool direct = false;
+    bool ring = false;
+    std::shared_ptr<ShaderRecompiler::HeapStageWords> heapWords;
+    std::shared_ptr<const std::vector<std::uint32_t>> heapReadWords;
+
+    bool Cacheable() const { return !direct || ring; }
 };
 
 struct RelocatableUserWord {

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp

@@ -0,0 +1,91 @@
+#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPSTAGES_HPP
+#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPSTAGES_HPP
+
+#include "Optimization/ResourceProgram.hpp"
+#include "UserDataUse.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp"
+#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp"
+#include <cstdint>
+#include <memory>
+#include <optional>
+#include <span>
+#include <utility>
+#include <vector>
+
+namespace AgcDriver {
+class VulkanDevice;
+}
+
+namespace AgcDriver::DriverDetail {
+
+bool HeapFragmentsEnabled();
+bool HeapRefused();
+bool HeapStageExcluded(ShaderRecompiler::ShaderStage stage);
+bool HeapPreRasterStage(ShaderRecompiler::ShaderStage stage);
+Graphics::HeapStageReason HeapStageIneligibility(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan,
+                                                 bool heapVertex);
+bool HeapStageEligible(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan, bool heapVertex);
+enum class HeapKeyClass { Draw, Compute, HeapVertex };
+
+constexpr std::size_t HeapFragmentKey(std::size_t codeOffset, HeapKeyClass keyClass) {
+    switch (keyClass) {
+    case HeapKeyClass::Compute: return codeOffset | std::size_t{1} << 61u;
+    case HeapKeyClass::HeapVertex: return codeOffset | std::size_t{1} << 62u;
+    case HeapKeyClass::Draw: break;
+    }
+    return codeOffset;
+}
+
+std::optional<bool> HeapFragmentKnown(const void* snapshot, std::size_t codeOffset);
+void NoteHeapFragment(const void* snapshot, std::size_t codeOffset, bool eligible);
+bool HeapStageMappable(const ShaderRecompiler::RecompileResult& result, bool workgroupAddress = false);
+void SetHeapStageUnmapped(std::optional<ShaderRecompiler::ShaderStage> stage);
+bool HeapDrawStageMappable(const ShaderRecompiler::RecompileResult& result, ShaderRecompiler::ShaderStage stage);
+bool HeapPairRefused(std::uint64_t pair);
+bool ExpectDeviceImages(const VulkanDevice& device, std::span<const std::uint32_t> userData, std::uint64_t codeAddress, const ShaderRecompiler::RecompileResult& result, bool dispatch);
+void RefuseHeapPair(std::uint64_t pair);
+
+class HeapDrawOff {
+public:
+    HeapDrawOff();
+    ~HeapDrawOff();
+    HeapDrawOff(const HeapDrawOff&) = delete;
+    HeapDrawOff& operator=(const HeapDrawOff&) = delete;
+
+private:
+    bool previous;
+};
+
+bool SampleHeapStageVerify();
+
+struct HeapStageReads {
+    std::vector<std::pair<std::uint64_t, std::uint64_t>> ranges;
+    std::vector<std::uint32_t> values;
+    ShaderRecompiler::SrtUserDataUse userDataUse;
+    std::shared_ptr<const ShaderRecompiler::ResourceCapture> capture;
+    bool complete = false;
+};
+
+struct HeapStageMaterialized {
+    std::shared_ptr<const ShaderRecompiler::RecompileResult> result;
+    std::shared_ptr<const ShaderRecompiler::HeapStageWords> words;
+    HeapStageReads reads;
+};
+
+HeapStageMaterialized MaterializeHeapStage(const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span<const std::uint32_t> userData,
+                                           std::uint64_t shaderBase);
+void NoteHeapDirect();
+void NoteHeapDirectVerify(const ShaderRecompiler::RecompileResult& direct, const ShaderRecompiler::RecompileResult& expected,
+                          const ShaderRecompiler::ResourceCapture* expectedCapture = nullptr);
+std::uint64_t HeapStagesTaken();
+std::uint64_t HeapRingHitsVerified();
+std::uint64_t HeapRingHitsDiffering();
+std::shared_ptr<const ShaderRecompiler::RecompileResult> PatchedHeapResult(const ShaderRecompiler::RecompileResult& shared,
+                                                                           const ShaderRecompiler::HeapStageWords& words);
+void VerifyHeapAccessor(const Graphics::CompiledShader& stage);
+void VerifyRingWords(const ShaderRecompiler::RecompileResult& kept, const ShaderRecompiler::HeapStageWords& ring,
+                     const ShaderRecompiler::RecompileResult& fresh, const ShaderRecompiler::HeapStageWords* freshWords);
+
+}
+
+#endif