diff --git a/3rdparty/Vulkan-Headers b/3rdparty/Vulkan-Headers index 2fa203425eb4..5d94bb4dcc96 160000 --- a/3rdparty/Vulkan-Headers +++ b/3rdparty/Vulkan-Headers @@ -1 +1 @@ -Subproject commit 2fa203425eb4af9dfc6b03f97ef72b0b5bcb8350 +Subproject commit 5d94bb4dcc968cccce1f601324fcaf3eda92a52b diff --git a/CMakeLists.txt b/CMakeLists.txt index c96a38e33627..372e8966dba5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -155,6 +155,14 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include) add_test(NAME request_serializer COMMAND request_serializer_tests) set_tests_properties(request_serializer PROPERTIES TIMEOUT 20) + add_test_executable(heap_template_budget_tests + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/tests/HeapTemplateBudgetTests.cpp) + target_include_directories(heap_template_budget_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/core/libs + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_CURRENT_SOURCE_DIR}/core/shader/recompiler/ControlFlow/include) + add_test(NAME heap_template_budget COMMAND heap_template_budget_tests) + set_tests_properties(heap_template_budget PROPERTIES TIMEOUT 20) find_package(Python3 QUIET COMPONENTS Interpreter) add_test_executable(shader_ir_use_rewrite_tests core/shader/recompiler/IntermediateRepresentation/tests/UseRewrite.cpp diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 66bf094bca08..961c83180695 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -195,6 +195,18 @@ add_custom_command( VERBATIM ) +set(agcTextureDetileAddressedShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileAddressed.spv) +set(agcTextureDetileAddressedShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileAddressed_spv.h) +add_custom_command( + OUTPUT "${agcTextureDetileAddressedShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -DADDRESSED=1 -o "${agcTextureDetileAddressedShaderSpv}" "${agcTextureDetileShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileAddressedShaderSpv} -DOUTPUT=${agcTextureDetileAddressedShaderHeader} + -DSYMBOL=TEXTURE_DETILE_ADDRESSED_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone + VERBATIM +) + set(agcTextureRepackShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/TextureRepack.comp) set(agcTextureRepackShaderSpv ${agcTextureDetileGeneratedDir}/TextureRepack.spv) set(agcTextureRepackShaderHeader ${agcTextureDetileGeneratedDir}/TextureRepack_spv.h) @@ -207,6 +219,17 @@ add_custom_command( DEPENDS "${agcTextureRepackShaderSource}" glslang-standalone VERBATIM ) +set(agcTextureRepackAddressedShaderSpv ${agcTextureDetileGeneratedDir}/TextureRepackAddressed.spv) +set(agcTextureRepackAddressedShaderHeader ${agcTextureDetileGeneratedDir}/TextureRepackAddressed_spv.h) +add_custom_command( + OUTPUT "${agcTextureRepackAddressedShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -DADDRESSED=1 -o "${agcTextureRepackAddressedShaderSpv}" "${agcTextureRepackShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureRepackAddressedShaderSpv} -DOUTPUT=${agcTextureRepackAddressedShaderHeader} + -DSYMBOL=TEXTURE_REPACK_ADDRESSED_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTextureRepackShaderSource}" glslang-standalone + VERBATIM +) set(agcDispatchArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/DispatchArguments.comp) set(agcDispatchArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/DispatchArguments.spv) @@ -357,6 +380,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCacheStatistics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawCapture.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/HeapStages.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawDecode.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawLookup.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -435,6 +459,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeferredDestruction.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/MemoryAccounting.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PlaceholderImages.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DescriptorHeapObjects.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp @@ -465,6 +490,9 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcSampleOrderShaderHeaders} + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/BindlessHeap.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeviceImageKeys.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/DeviceImageTable.cpp ${agcTextureDetileShaderHeader} ${agcReadbackHeaders} ${agcMultisampleUploadHeader} @@ -474,6 +502,8 @@ set(agcDriverSources ${agcStagingCopyBackShaderHeader} ${agcTextureDetileImageShaderHeader} ${agcTextureRepackShaderHeader} + ${agcTextureDetileAddressedShaderHeader} + ${agcTextureRepackAddressedShaderHeader} ${agcDispatchArgumentsShaderHeader} ${agcTextureDetileWordsShaderHeader} ${agcTextureDetileImageWordsShaderHeader} @@ -711,14 +741,31 @@ function(agc_add_default_test agcTest) endif() endfunction() -function(agc_add_draw_pipeline_test agcTestName) - add_test(NAME ${agcTestName}_draw_pipeline COMMAND ${agcTestName}_tests) - set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_DRAW_PIPELINE=1;APS5_PIPELINED_DRAWS=32;APS5_PIPELINE_DISPATCH=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") +function(agc_add_env_test agcTestName agcTestTarget) + cmake_parse_arguments(PARSE_ARGV 2 agcEnvTest "NO_WORKING_DIRECTORY" "TIMEOUT" "ARGS;ENV") + add_test(NAME ${agcTestName} COMMAND ${agcTestTarget} ${agcEnvTest_ARGS}) + set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT "${agcEnvTest_ENV}" SKIP_RETURN_CODE 77) + if(NOT agcEnvTest_NO_WORKING_DIRECTORY) + set_tests_properties(${agcTestName} PROPERTIES WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + endif() + if(agcEnvTest_TIMEOUT) + set_tests_properties(${agcTestName} PROPERTIES TIMEOUT ${agcEnvTest_TIMEOUT}) + endif() if(WIN32) - set_tests_properties(${agcTestName}_draw_pipeline PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(${agcTestName} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endfunction() +function(agc_add_draw_pipeline_test agcTestName) + agc_add_env_test(${agcTestName}_draw_pipeline ${agcTestName}_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1) +endfunction() + +function(agc_add_gpu_heap_test agcTestName) + agc_add_env_test(${agcTestName}_gpu_heap ${agcTestName}_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1) +endfunction() + function(agc_add_spirv_mode_tests agcTest) if(NOT TARGET ${agcTest}) return() @@ -1059,7 +1106,7 @@ endif() if(BUILD_TESTING) get_directory_property(agcRegisteredTests TESTS) - list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_pipeline_store_replay_growth)$") + list(FILTER agcRegisteredTests EXCLUDE REGEX "^(agc_pipeline_journal(_long_paths|_extended_paths)?|agc_driver_pipeline_journal|agc_driver_pipeline_journal_gpu_heap|agc_pipeline_store_replay_growth)$") set_property(TEST ${agcRegisteredTests} APPEND PROPERTY ENVIRONMENT "APS5_NO_PIPELINE_JOURNAL=1") endif() diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index 47863c827643..3aa6e8a6b59b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -72,6 +72,7 @@ struct DispatchVariant { std::vector> shaderDataPatches; std::uint64_t userDataMask = ~std::uint64_t{0}; std::shared_ptr vertexInfo; + std::shared_ptr ringTemplate; std::shared_ptr memory; std::vector captured; @@ -88,7 +89,7 @@ struct DispatchEntry { }; struct EntryCounters { - std::uint64_t lookups = 0, absent = 0, equal = 0, differing = 0, inaccessible = 0, queuedLabel = 0, flushingImage = 0, publishMoved = 0, pendingMoved = 0, forgetMoved = 0, imagesFlushed = 0, runsSynced = 0, forgetSinceInsert = 0, replaced = 0, inserts = 0, unstable = 0, touches = 0; + std::uint64_t lookups = 0, absent = 0, equal = 0, differing = 0, inaccessible = 0, queuedLabel = 0, flushingImage = 0, publishMoved = 0, pendingMoved = 0, forgetMoved = 0, deviceImageFallbacks = 0, imagesFlushed = 0, runsSynced = 0, forgetSinceInsert = 0, replaced = 0, inserts = 0, unstable = 0, touches = 0; std::uint64_t runsValidated = 0, runsInserted = 0, retriesEqual = 0, retriesMoved = 0; double validateUs = 0; @@ -112,7 +113,7 @@ struct DataMask { std::vector>* live; }; -enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved }; +enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved, DeviceImageFallback }; bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp index e1357753b899..95d465faf642 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp @@ -58,6 +58,7 @@ struct PreparedGraphicsStage { ShaderRecompiler::ShaderStage stage = ShaderRecompiler::ShaderStage::Vertex; std::uint32_t pushOffset = 0; std::shared_ptr registration; + bool heapVariant = false; }; std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared = false); @@ -131,6 +132,12 @@ struct StageCapture { std::uint32_t pushOffset = 0; ShaderRecompiler::SrtUserDataUse userDataUse; std::uint64_t userDataMask = ~std::uint64_t{0}; + bool direct = false; + bool ring = false; + std::shared_ptr heapWords; + std::shared_ptr> heapReadWords; + + bool Cacheable() const { return !direct || ring; } }; struct RelocatableUserWord { diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp new file mode 100644 index 000000000000..2bd374619548 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp @@ -0,0 +1,91 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPSTAGES_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPSTAGES_HPP + +#include "Optimization/ResourceProgram.hpp" +#include "UserDataUse.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { +class VulkanDevice; +} + +namespace AgcDriver::DriverDetail { + +bool HeapFragmentsEnabled(); +bool HeapRefused(); +bool HeapStageExcluded(ShaderRecompiler::ShaderStage stage); +bool HeapPreRasterStage(ShaderRecompiler::ShaderStage stage); +Graphics::HeapStageReason HeapStageIneligibility(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan, + bool heapVertex); +bool HeapStageEligible(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan, bool heapVertex); +enum class HeapKeyClass { Draw, Compute, HeapVertex }; + +constexpr std::size_t HeapFragmentKey(std::size_t codeOffset, HeapKeyClass keyClass) { + switch (keyClass) { + case HeapKeyClass::Compute: return codeOffset | std::size_t{1} << 61u; + case HeapKeyClass::HeapVertex: return codeOffset | std::size_t{1} << 62u; + case HeapKeyClass::Draw: break; + } + return codeOffset; +} + +std::optional HeapFragmentKnown(const void* snapshot, std::size_t codeOffset); +void NoteHeapFragment(const void* snapshot, std::size_t codeOffset, bool eligible); +bool HeapStageMappable(const ShaderRecompiler::RecompileResult& result, bool workgroupAddress = false); +void SetHeapStageUnmapped(std::optional stage); +bool HeapDrawStageMappable(const ShaderRecompiler::RecompileResult& result, ShaderRecompiler::ShaderStage stage); +bool HeapPairRefused(std::uint64_t pair); +bool ExpectDeviceImages(const VulkanDevice& device, std::span userData, std::uint64_t codeAddress, const ShaderRecompiler::RecompileResult& result, bool dispatch); +void RefuseHeapPair(std::uint64_t pair); + +class HeapDrawOff { +public: + HeapDrawOff(); + ~HeapDrawOff(); + HeapDrawOff(const HeapDrawOff&) = delete; + HeapDrawOff& operator=(const HeapDrawOff&) = delete; + +private: + bool previous; +}; + +bool SampleHeapStageVerify(); + +struct HeapStageReads { + std::vector> ranges; + std::vector values; + ShaderRecompiler::SrtUserDataUse userDataUse; + std::shared_ptr capture; + bool complete = false; +}; + +struct HeapStageMaterialized { + std::shared_ptr result; + std::shared_ptr words; + HeapStageReads reads; +}; + +HeapStageMaterialized MaterializeHeapStage(const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span userData, + std::uint64_t shaderBase); +void NoteHeapDirect(); +void NoteHeapDirectVerify(const ShaderRecompiler::RecompileResult& direct, const ShaderRecompiler::RecompileResult& expected, + const ShaderRecompiler::ResourceCapture* expectedCapture = nullptr); +std::uint64_t HeapStagesTaken(); +std::uint64_t HeapRingHitsVerified(); +std::uint64_t HeapRingHitsDiffering(); +std::shared_ptr PatchedHeapResult(const ShaderRecompiler::RecompileResult& shared, + const ShaderRecompiler::HeapStageWords& words); +void VerifyHeapAccessor(const Graphics::CompiledShader& stage); +void VerifyRingWords(const ShaderRecompiler::RecompileResult& kept, const ShaderRecompiler::HeapStageWords& ring, + const ShaderRecompiler::RecompileResult& fresh, const ShaderRecompiler::HeapStageWords* freshWords); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapWalkMemo.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapWalkMemo.hpp new file mode 100644 index 000000000000..da1ea6b54f6a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapWalkMemo.hpp @@ -0,0 +1,154 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPWALKMEMO_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_DRAW_HEAPWALKMEMO_HPP + +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +struct HeapWalkEntry { + std::uint64_t variant = 0; + std::uint64_t shaderBase = 0; + std::uint32_t pushOffset = 0; + std::vector userData; + std::vector> ranges; + std::vector values; + std::vector> accessible; + ShaderRecompiler::SrtUserDataUse userDataUse; + std::shared_ptr capture; + std::shared_ptr result; + std::shared_ptr words; + std::uint64_t bytes = 0; + std::uint64_t stored = 0; +}; + +constexpr std::size_t HeapWalkWays = 4; + +constexpr std::size_t HeapWalkMemoKeys = 16384; + +constexpr std::size_t HeapWalkDrain = 2; + +constexpr std::size_t HeapWalkSeenKeys = 1u << 16; + +std::uint64_t HeapWalkEntryBytes(const HeapWalkEntry& entry); + +std::uint64_t HeapWalkEntryOwnBytes(const HeapWalkEntry& entry); + +std::uint64_t HeapWalkCaptureBytes(const ShaderRecompiler::ResourceCapture& capture); + +std::uint64_t HeapWalkResultBytes(const ShaderRecompiler::RecompileResult& result); + +std::uint64_t HeapWalkWordsBytes(const ShaderRecompiler::HeapStageWords& words); + +std::uint64_t HeapWalkMemoBudget(); + +struct HeapWalkMemo { + std::unordered_map> current; + std::unordered_map> previous; + std::vector seen = std::vector(HeapWalkSeenKeys, 0); + std::unordered_map> pins; + std::uint64_t heldBytes = 0; + std::uint64_t stores = 0; + std::uint64_t budget = HeapWalkMemoBudget(); + + bool admit(std::uint64_t key) { + auto& last = seen[(key ^ (key >> 29)) & (HeapWalkSeenKeys - 1u)]; + if (last == key) return true; + last = key; + return false; + } + + std::vector* find(std::uint64_t key) { + if (const auto found = current.find(key); found != current.end()) return &found->second; + const auto old = previous.find(key); + if (old == previous.end()) return nullptr; + auto moved = std::move(old->second); + previous.erase(old); + auto& ways = slot(key); + ways = std::move(moved); + return &ways; + } + + void remember(std::uint64_t key, HeapWalkEntry entry) { + auto& ways = slot(key); + if (ways.size() >= HeapWalkWays) { + drop(ways.back()); + ways.pop_back(); + } + charge(entry); + touch(entry); + ways.insert(ways.begin(), std::move(entry)); + if (budget != 0 && heldBytes > budget) trim(); + } + + void touch(HeapWalkEntry& entry) { + entry.stored = ++stores; + } + +private: + std::vector& slot(std::uint64_t key) { + for (std::size_t drained = 0; drained < HeapWalkDrain && !previous.empty(); ++drained) { + release(previous.begin()->second); + previous.erase(previous.begin()); + } + if (current.size() >= HeapWalkMemoKeys && !current.contains(key)) { + retirePrevious(); + previous = std::move(current); + current.clear(); + } + return current[key]; + } + + template + void pin(const std::shared_ptr& object, TSize size) { + if (object == nullptr) return; + auto& held = pins[object.get()]; + if (held.first++ == 0) { + held.second = size(*object); + heldBytes += held.second; + } + } + + void unpin(const void* object) { + if (object == nullptr) return; + const auto held = pins.find(object); + if (--held->second.first != 0) return; + heldBytes -= held->second.second; + pins.erase(held); + } + + void charge(HeapWalkEntry& entry) { + entry.bytes = HeapWalkEntryOwnBytes(entry); + heldBytes += entry.bytes; + pin(entry.capture, HeapWalkCaptureBytes); + pin(entry.result, HeapWalkResultBytes); + pin(entry.words, HeapWalkWordsBytes); + } + + void drop(const HeapWalkEntry& entry) { + heldBytes -= entry.bytes; + unpin(entry.capture.get()); + unpin(entry.result.get()); + unpin(entry.words.get()); + } + + void release(const std::vector& ways) { + for (const auto& entry : ways) drop(entry); + } + + void retirePrevious() { + for (const auto& [key, ways] : previous) release(ways); + previous.clear(); + } + + void trim(); +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index db5745139e2a..d06a52deb400 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -57,6 +57,33 @@ inline std::uint32_t ThreadDimensionGroups(std::uint32_t threads, std::uint32_t struct DrawTail; +struct DrawStageContext { + const QueueState& queue; + const Submission& submission; + std::size_t packetOffset; + std::uint64_t traceToken; + const std::vector& programs; + const Graphics::State& graphics; + const ShaderRecompiler::ShaderPixelStageInfo& pixel; + const std::vector>& vertexInfos; + std::vector& memory; + const std::vector& linked; + const std::shared_ptr& localDevice; + ShaderMemory& shaderMemory; + std::vector& stageCaptures; + std::vector& recompiled; + const bool& drawHit; + const std::vector>& matched; + const std::vector>& matchedRegions; + bool profile; + std::uint64_t dumpTarget; + std::uint64_t dumpSlot1; + std::uint64_t& captures; + DrawPhaseTiming& phaseTiming; + std::array& phaseMs; + const std::optional& meshBarycentrics; +}; + class Driver { public: static Driver& Get(); @@ -102,9 +129,9 @@ class Driver { [[nodiscard]] std::vector> PreparedEntryHashes(const Shader* shader); [[nodiscard]] std::array PrebuiltLibraries(); static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters); - static std::optional ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); + static std::optional ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect, const ShaderRecompiler::HeapStageWords* heap = nullptr); static void tagVariantRuns(DispatchVariant& variant, std::span regions, const ShaderRecompiler::SrtUserDataUse& use, bool eligible, std::uint32_t hiddenWords, std::span> supplied = {}); - static void tagUserValueWords(DispatchVariant& variant, const DrawProgram& program); + static void tagUserValueWords(DispatchVariant& variant, const DrawProgram& program, const ShaderRecompiler::HeapStageWords* heap = nullptr); static std::uint64_t relocatedContentKey(const RelocatableUserWord& word, std::uint32_t value, bool& readable); static void applyShaderDataPatches(DispatchVariant& variant); static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request, std::string_view suffix = {}); @@ -218,7 +245,10 @@ class Driver { std::shared_ptr decodeDraw(const QueueState& queue, const Submission& submission, const Graphics::Context& context); void resolveDrawDecode(const QueueState& queue, const Submission& submission, const Graphics::Context& context, std::shared_ptr& decode, bool registerKey, std::uint64_t drawKey, bool profile); void lookupDraw(const Submission& submission, const std::shared_ptr& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector& programs, const std::vector& roles, const std::vector>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr& entry, std::vector>& matched, std::vector>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array& phaseMs, const std::shared_ptr& dataEntry, DrawDataCandidates& dataCandidates, std::vector& stageData); - std::shared_ptr materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, std::size_t packetOffset, std::uint64_t traceToken, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected, const std::optional& meshBarycentrics); + enum class HeapDirect { Declined, Unmapped, Taken }; + std::shared_ptr materializeDrawStage(const DrawStageContext& context, std::size_t i, std::uint32_t pushOffset, const Pm4::DrawParameters& drawParameters, bool heapPartners); + HeapDirect materializeHeapDirect(const DrawStageContext& context, std::size_t i, const ShaderRecompiler::PreparedShaderInvocation& invocation, const ShaderRecompiler::RecompileRequest& request, PerformanceTimer& timing); + static std::runtime_error stageFailure(const DrawProgram& program, const ShaderRecompiler::RecompileRequest& request, const std::exception& error, std::span regions); static void recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector& decodeReads); void cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawParameters& drawParameters, const std::optional& indirectCpu, const std::vector& programs, const std::vector& stageCaptures, const std::vector>& vertexInfos, const std::vector>& decodeReads, bool verifyHit, const std::vector>& matched, const std::vector& stageData, std::vector>& fresh, std::uint64_t drawKey, bool registerKey, const std::shared_ptr& decode, DrawPhaseTiming& phaseTiming, const std::vector& reused, const std::shared_ptr& entry, std::uint64_t shapeKey, bool shapeNeedsEntry = false); static bool drawPrecheck(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 253d285864c0..f16e54280259 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -349,6 +349,7 @@ ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, con std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); std::shared_ptr CaptureInvocation(ShaderMemory& memory, const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::PreparedShaderInvocation& invocation); +bool PreparedHeapStage(const ShaderSnapshot& snapshot, std::size_t codeOffset); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index 32fb2f68804e..80839a0993ba 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -122,6 +122,7 @@ std::uint64_t CollectEpoch(); std::uint64_t CollectEpochBumps(); std::uint64_t ThreadCollectEpoch(); std::uint64_t UnwatchSerial(); +std::uint64_t OwnWriteSerial(); std::uint64_t CollectWritesUncached(std::uint64_t address, std::size_t bytes); // The tracker's current generation (every collect and MarkWritten bumps it): a stamp taken after // a set of driver stores, for UnchangedSinceCollected to compare against later. Read under the diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp index 319dc9c15e55..954234c6ace3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/PresentationPass.hpp @@ -3,6 +3,7 @@ #define VK_NO_PROTOTYPES #include +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include #include @@ -43,8 +44,10 @@ class PresentationPass { struct Target { VkRenderPass renderPass = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline heapPipeline = VK_NULL_HANDLE; }; - struct Slot { + struct Slot : Graphics::HeapSlotOwner { + using Graphics::HeapSlotOwner::ReleaseHeapSlots; VkDescriptorSet set = VK_NULL_HANDLE; VkImageView source = VK_NULL_HANDLE; std::uint32_t width = 0; @@ -71,11 +74,13 @@ class PresentationPass { void releaseDumpTarget(Slot& slot) noexcept; void ensureUploadImage(Slot& slot, VkExtent2D extent); void releaseUploadImage(Slot& slot) noexcept; - void draw(VkCommandBuffer commands, const Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (¶meters)[9]) const; + void draw(VkCommandBuffer commands, Slot& slot, const Target& target, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (¶meters)[9]) const; void release() noexcept; Graphics::Context context; + VkSamplerCreateInfo samplerInfo{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; VkSampler sampler = VK_NULL_HANDLE; + bool heap = false; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; VkDescriptorPool descriptorPool = VK_NULL_HANDLE; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 2e398841cb1a..9b96a2b10f22 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -4,6 +4,7 @@ #define VK_NO_PROTOTYPES #include #include "Recompiler.hpp" +#include "DeviceImageTableAbi.hpp" #include "prx/libSceAgcDriver/Execution/include/Presentation.hpp" #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Execution/include/Recipe.hpp" @@ -75,6 +76,7 @@ class VulkanDevice { void SavePipelineCache(); // Compute pipelines this device built, by dispatches and by the journal replay (tests). std::uint64_t ComputePipelinesCreated() const; + std::uint64_t DescriptorHeapComputePipelines() const; ~VulkanDevice(); VulkanDevice(const VulkanDevice&) = delete; VulkanDevice& operator=(const VulkanDevice&) = delete; @@ -91,6 +93,11 @@ class VulkanDevice { const std::uint32_t* WorkGroupCountLimit() const; std::uint32_t VendorId() const; ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const; + bool DeviceImagePath() const; + bool DeviceImagesServe(std::uint64_t codeAddress) const; + void NoteDeviceImageProgram(const ShaderRecompiler::RecompileResult& result, std::uint64_t codeAddress) const; + std::optional ExpectDeviceImages(std::span images) const; + void FallBackDeviceImages(const ShaderRecompiler::RecompileResult& result, std::uint64_t codeAddress, const std::string& reason) const; bool ComputeFullSubgroups() const; // Distinguishes this device from every earlier one in the process (a memo keyed by the device // cannot rely on the pointer, which a replacement may reuse). @@ -374,6 +381,7 @@ class VulkanDevice { IndirectDrawSupport DrawIndirectSupport() const; void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}); bool PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span shaders); + const Graphics::Context& contextRef(std::optional& scratch) const; public: // The device's Graphics::Context: a copy of the one built at setup (its instance functions diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index f6eaa8838ab0..29dc18ddfd1d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -2,6 +2,8 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferCopy.hpp" @@ -192,6 +194,9 @@ void Driver::dispatch(QueueState& queue, std::span packet, localDevice->ComputeTarget((packet[4] & 0x8000u) != 0 ? 32u : 64u), {0, 0, 0, threadGroup != std::array{} ? 0u : 128u} }; + if (request.target.deviceImageTables) request.target.deviceImageTables = localDevice->DeviceImagesServe(address); + const std::size_t computeHeapKey = HeapFragmentKey(codeOffset, HeapKeyClass::Compute); + if (request.target.gpuResolvedImages && (HeapRefused() || HeapStageExcluded(ShaderRecompiler::ShaderStage::Compute) || HeapFragmentKnown(&snapshot, computeHeapKey) == std::optional(false))) request.target.gpuResolvedImages = false; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; static std::atomic captureMs{0}, keyMs{0}, recompileMs{0}, deviceMs{0}; static std::atomic cacheHits{0}; @@ -301,6 +306,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, } mix(reinterpret_cast(registeredShader.get())); + mix(request.target.gpuResolvedImages); if (fromAhead && key != ahead.key) { fromAhead = false; ++groupStats.keyMismatches; @@ -400,6 +406,16 @@ void Driver::dispatch(QueueState& queue, std::span packet, timing.Mark("capture_resources"); compiledResult = invocation.Materialize(*capture); + if (compiledResult->gpuResolvedImages && !HeapStageMappable(*compiledResult, compiledResult->workgroupMemoryDwords != 0)) { + Graphics::NoteHeapFallback(Graphics::HeapFallback::DispatchUnmapped); + NoteHeapFragment(&snapshot, computeHeapKey, false); + request.target.gpuResolvedImages = false; + invocation = InvocationFor(snapshot, codeOffset, request); + capture = shaderMemory->Capture(invocation); + captured = shaderMemory->Regions(); + request.context.memory = captured; + compiledResult = invocation.Materialize(*capture); + } timing.Mark("materialize"); if (profile && compiledResult->cacheHit) cacheHits.fetch_add(1, std::memory_order_relaxed); const auto elapsed = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); @@ -469,6 +485,10 @@ void Driver::dispatch(QueueState& queue, std::span packet, } phaseTiming.Phase(PhaseQueuedLabels); const auto& compiled = *compiledResult; + if (!ExpectDeviceImages(*localDevice, userData, address, compiled, true)) { + dispatch(queue, packet, submission, indirectArguments); + return; + } std::vector snapshots; for (const auto& region : captured) snapshots.push_back({region.guestAddress, region.bytes}); std::array groups{packet[1], packet[2], packet[3]}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp index 5495a8c0aac3..ace3e7df7390 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp @@ -58,7 +58,7 @@ void Driver::classifyDiffering(std::uint64_t program, std::uint64_t key, const D void Driver::reportDispatchCache(EntryCounters& counters) { const auto count = [](std::uint64_t value) { return static_cast(value); }; const auto validated = counters.lookups - counters.absent; - AgcDriver::ProfilePrint_nid_no_patch("[dispatch-cache] %llu lookups (10 s): %llu no entry, %llu validated by value in %.1f us each: %llu equal, %llu differing, %llu page not mapped, %llu queued label, %llu image being stored, %llu publish generation moved, %llu pending serial moved, %llu forget serial moved (%llu pending images stored first, %llu pending runs synced first, %llu equal entries inserted before a forget; gate retried %llu: %llu equal, %llu moved again); runs per entry %.1f validated / %.1f inserted; %llu misses found the entry replaced meanwhile; %llu inserts (%llu captures not kept: unstable), %llu evictions in total, %zu entries, %llu LRU moves\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast(validated) : 0.0, count(counters.equal), count(counters.differing), count(counters.inaccessible), count(counters.queuedLabel), count(counters.flushingImage), count(counters.publishMoved), count(counters.pendingMoved), count(counters.forgetMoved), count(counters.imagesFlushed), count(counters.runsSynced), count(counters.forgetSinceInsert), count(counters.retriesEqual + counters.retriesMoved), count(counters.retriesEqual), count(counters.retriesMoved), validated != 0 ? static_cast(counters.runsValidated) / static_cast(validated) : 0.0, counters.inserts != 0 ? static_cast(counters.runsInserted) / static_cast(counters.inserts) : 0.0, count(counters.replaced), count(counters.inserts), count(counters.unstable), count(dispatchCacheEvictions), dispatchCache.size(), count(counters.touches)); + AgcDriver::ProfilePrint_nid_no_patch("[dispatch-cache] %llu lookups (10 s): %llu no entry, %llu validated by value in %.1f us each: %llu equal, %llu differing, %llu page not mapped, %llu queued label, %llu image being stored, %llu publish generation moved, %llu pending serial moved, %llu forget serial moved, %llu fell back from the device image table (%llu pending images stored first, %llu pending runs synced first, %llu equal entries inserted before a forget; gate retried %llu: %llu equal, %llu moved again); runs per entry %.1f validated / %.1f inserted; %llu misses found the entry replaced meanwhile; %llu inserts (%llu captures not kept: unstable), %llu evictions in total, %zu entries, %llu LRU moves\n", count(counters.lookups), count(counters.absent), count(validated), validated != 0 ? counters.validateUs / static_cast(validated) : 0.0, count(counters.equal), count(counters.differing), count(counters.inaccessible), count(counters.queuedLabel), count(counters.flushingImage), count(counters.publishMoved), count(counters.pendingMoved), count(counters.forgetMoved), count(counters.deviceImageFallbacks), count(counters.imagesFlushed), count(counters.runsSynced), count(counters.forgetSinceInsert), count(counters.retriesEqual + counters.retriesMoved), count(counters.retriesEqual), count(counters.retriesMoved), validated != 0 ? static_cast(counters.runsValidated) / static_cast(validated) : 0.0, counters.inserts != 0 ? static_cast(counters.runsInserted) / static_cast(counters.inserts) : 0.0, count(counters.replaced), count(counters.inserts), count(counters.unstable), count(dispatchCacheEvictions), dispatchCache.size(), count(counters.touches)); std::vector> programs(counters.differingByProgram.begin(), counters.differingByProgram.end()); std::sort(programs.begin(), programs.end(), [](const auto& a, const auto& b) { return a.second > b.second; }); std::string top; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchLookup.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchLookup.cpp index 6b9bdf10f29c..8123f35f01e0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchLookup.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchLookup.cpp @@ -119,6 +119,7 @@ void Driver::lookupDispatch(std::uint64_t address, const Submission& submission, case EntryOutcome::PublishMoved: ++counters.publishMoved; break; case EntryOutcome::PendingMoved: ++counters.pendingMoved; break; case EntryOutcome::ForgetMoved: ++counters.forgetMoved; break; + case EntryOutcome::DeviceImageFallback: ++counters.deviceImageFallbacks; break; } if (current) { ++counters.variantHitsByRank[rank]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp index 3eb679308ecb..c1b5a6ff0c11 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/VariantValidation.cpp @@ -38,6 +38,7 @@ void Driver::appendEntryRegions(const DispatchVariant& variant, std::vector Driver::patchDataWords(const DispatchVariant& variant, const std::vector& words) { + if (variant.ringTemplate != nullptr && !variant.ringTemplate->flattenedSrt.empty()) return variant.compiled; auto patched = std::make_shared(*variant.compiled); auto& descriptor = patched->bindings[variant.flatBinding].guestDescriptor; for (std::size_t k = 0; k < variant.dataPositions.size(); ++k) { @@ -108,6 +109,10 @@ bool Driver::vouchedValidation() { EntryOutcome Driver::validateVariant(std::uint64_t program, std::uint32_t queue, const DispatchVariant& variant, std::span regions, std::uint64_t& imagesFlushed, std::uint64_t& runsSynced, std::optional& sampling, std::vector>* live) { if (live != nullptr) live->clear(); + if (variant.compiled->deviceImageTables) { + const auto localDevice = device.Load(); + if (localDevice == nullptr || !localDevice->DeviceImagesServe(program)) return EntryOutcome::DeviceImageFallback; + } if (DrawPipeline::Active()) { auto& pipeline = DrawPipeline::Queue0(); if (pipeline.Busy() && pipeline.OverlapsAny(variant.runs)) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index 03bdda6fdf55..45bcb7a30a72 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -7,6 +7,8 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawProgramTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" @@ -30,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -201,6 +204,42 @@ void noteFinisherHandOff(bool handed) { namespace { +bool MixesHeapAndGuestStages(std::span stages, std::span programResults) { + const auto guest = [&](const Graphics::CompiledShader& stage) { return std::ranges::find(programResults, stage.program) != programResults.end(); }; + return std::ranges::any_of(stages, [](const Graphics::CompiledShader& stage) { return stage.program->gpuResolvedImages; }) && std::ranges::any_of(stages, [&](const Graphics::CompiledShader& stage) { return guest(stage) && !stage.program->gpuResolvedImages; }); +} + +class DrawStageCheckpoint { +public: + DrawStageCheckpoint(std::vector& stages, std::vector>& compiledStages, std::uint32_t& pushCursorBytes, Pm4::DrawParameters& drawParameters, ShaderRecompiler::ShaderPixelStageInfo& pixel, bool keepCopies) + : stages(stages), compiledStages(compiledStages), pushCursorBytes(pushCursorBytes), drawParameters(drawParameters), pixel(pixel), stagesBefore(stages.size()), compiledBefore(compiledStages.size()), pushBefore(pushCursorBytes), + parametersBefore(keepCopies ? std::optional(drawParameters) : std::nullopt), pixelBefore(keepCopies ? std::optional(pixel) : std::nullopt) {} + + void Restore() { + stages.resize(stagesBefore); + compiledStages.resize(compiledBefore); + pushCursorBytes = pushBefore; + drawParameters = *parametersBefore; + pixel = *pixelBefore; + } + +private: + std::vector& stages; + std::vector>& compiledStages; + std::uint32_t& pushCursorBytes; + Pm4::DrawParameters& drawParameters; + ShaderRecompiler::ShaderPixelStageInfo& pixel; + std::size_t stagesBefore; + std::size_t compiledBefore; + std::uint32_t pushBefore; + std::optional parametersBefore; + std::optional pixelBefore; +}; + +} + +namespace { + enum class HeldReason : std::size_t { Known, NoDecode, Size, NullVertex, NullFragment, NullOther, Bda, Buffer, Storage, Count }; std::array, static_cast(HeldReason::Count)> heldReasons{}; @@ -224,7 +263,7 @@ void noteHeld(HeldReason reason) { std::optional writesBeyondTargets(const ShaderRecompiler::RecompileResult& compiled) { using Role = ShaderRecompiler::DescriptorRole; for (const auto& binding : compiled.bindings) { - if (binding.role == Role::BdaPagetable && Graphics::ShaderStoresThroughAddresses(compiled.spirv)) return HeldReason::Bda; + if (binding.role == Role::BdaPagetable && compiled.bdaStores) return HeldReason::Bda; if (binding.role == Role::GuestBuffers) { for (std::uint32_t element = 0; element < binding.count; ++element) { if (element >= binding.bufferWritten.size() || binding.bufferWritten[element]) return HeldReason::Buffer; @@ -879,11 +918,53 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) { std::memcpy(words.data() + offset, region.bytes.data(), count * sizeof(std::uint32_t)); } }; + const auto buildRingWords = [&](std::size_t i, const DispatchVariant& variant) { + auto words = std::make_shared(); + const auto& user = programs[i].userData; + for (const auto index : variant.compiled->heapUserWords) { + require(index < user.size(), "a ring vertex template reads a user word the draw does not have"); + words->userData.push_back(user[index]); + } + const auto& ringTemplate = *variant.ringTemplate; + words->vertexAttributes = ringTemplate.vertexAttributes; + if (!ringTemplate.flattenedSrt.empty()) { + words->flattenedSrt = ringTemplate.flattenedSrt; + if (&variant == matched[i].get() && stageData[i].compiled != nullptr) { + for (std::size_t k = 0; k < variant.dataPositions.size(); ++k) { + if (variant.dataSlots[k] < words->flattenedSrt.size()) words->flattenedSrt[variant.dataSlots[k]] = stageData[i].words[variant.dataPositions[k]]; + } + } + } + if (!ringTemplate.bindings.empty()) { + words->bindings = ringTemplate.bindings; + require(words->bindings.size() == variant.compiled->bindings.size(), "a ring vertex hit's bindings differ in number from its template"); + for (std::size_t index = 0; index < words->bindings.size(); ++index) { + const auto role = variant.compiled->bindings[index].role; + auto& descriptor = words->bindings[index]; + if (role == ShaderRecompiler::DescriptorRole::FlattenedSrt) descriptor.clear(); + if (role != ShaderRecompiler::DescriptorRole::ShaderData || descriptor.empty()) continue; + std::copy_n(words->userData.begin(), std::min(words->userData.size(), descriptor.size()), descriptor.begin()); + if (descriptor == variant.compiled->bindings[index].guestDescriptor) descriptor.clear(); + } + } + ShaderRecompiler::CompleteHeapDataWords(*variant.compiled, *words); + if (words->vertexAttributes.empty()) return words; + decodeVertexInfo(i); + require(vertexInfos[i].has_value(), "a ring vertex hit has no vertex stage info"); + const auto& vertex = *vertexInfos[i]; + for (auto& attribute : words->vertexAttributes) { + require(attribute.location < vertex.resourcesNum && attribute.location < vertex.resources.size(), "a ring vertex attribute lies outside the draw's vertex resources"); + attribute.resource = vertex.resources[attribute.location]; + if (vertex.fetchEmbedded) attribute.fetchIndex = vertex.resourcesDst[attribute.location].fetchIndex; + } + return words; + }; const auto verifyDataStage = [&](std::size_t i, const DispatchVariant& kept) { std::vector> runs; std::vector words; freshRuns(i, runs, words); const auto& compiled = *stageCaptures[i].compiled; + if (kept.ringTemplate != nullptr) VerifyRingWords(*kept.compiled, *buildRingWords(i, kept), *stageCaptures[i].compiled, stageCaptures[i].heapWords.get()); const char* what = runs != kept.runs ? "runs" : words != kept.words ? "words" : vertexInfos[i].has_value() != (kept.vertexInfo != nullptr) || (vertexInfos[i] && !sameVertexInfo(*vertexInfos[i], *kept.vertexInfo)) ? "vertex stage info" : compiled.variantId != kept.compiled->variantId ? "variant" : compiled.pushConstants != kept.compiled->pushConstants ? "push constants" : nullptr; ++dataVerified; if (what == nullptr) return; @@ -907,63 +988,118 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) { std::snprintf(message, sizeof(message), "APS5_VERIFY_DRAW_DATA_HITS: stage %zu (program 0x%llx) of a stage data hit differs from its full build in its %s (%zu data positions refreshed)", i, static_cast(programs[i].binary.codeAddress), what, kept.dataPositions.size()); throw std::runtime_error(message); }; + const auto ringWords = [&](std::size_t i) { + if (matched[i]->ringTemplate == nullptr) return; + stageCaptures[i].heapWords = buildRingWords(i, *matched[i]); + stageCaptures[i].direct = true; + stageCaptures[i].ring = true; + }; std::optional indirectCpu; std::size_t completedSubdraws = 0; std::vector pushOffsets(programs.size(), 0); std::optional meshBarycentrics; std::vector resultIndex(programs.size(), 0); - for (std::size_t i = 0; i < programs.size(); ++i) { - if (roles[i] == Role::GeometryBack) continue; - const auto& program = programs[i]; - pushCursorBytes = Graphics::StagePushOffset(pushCursorBytes, program.binary.stage, localDevice->GraphicsPipelineLibraries()); - pushOffsets[i] = pushCursorBytes; - if (drawHit) { + const bool heapCheck = !drawHit && Graphics::DescriptorHeapEnabled(); + std::uint64_t heapPair = 0xcbf29ce484222325ull; + for (const auto& program : programs) heapPair = (heapPair ^ reinterpret_cast(program.snapshot.get()) ^ (static_cast(program.codeOffset) << 1u)) * 0x100000001b3ull; + const bool heapPairRefused = heapCheck && HeapPairRefused(heapPair); + bool heapRetry = false; + const DrawStageContext stageContext{queue, submission, packetOffset, trace.token, programs, graphics, pixel, vertexInfos, memory, linked, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, meshBarycentrics}; + const auto candidateForStage = [&](std::size_t i, bool afterHeap) { + return std::find_if(dataCandidates[i].begin(), dataCandidates[i].end(), [&](const auto& kept) { return kept.first->pushOffset == pushCursorBytes && (!afterHeap || kept.first->compiled->gpuResolvedImages); }); + }; + const auto runStages = [&]() -> std::optional { + for (std::size_t i = 0; i < programs.size(); ++i) { + if (roles[i] == Role::GeometryBack) continue; + const auto& program = programs[i]; + pushCursorBytes = Graphics::StagePushOffset(pushCursorBytes, program.binary.stage, localDevice->GraphicsPipelineLibraries()); + pushOffsets[i] = pushCursorBytes; + if (drawHit) { - programResults[i] = stageData[i].compiled != nullptr ? stageData[i].compiled.get() : matched[i]->compiled.get(); - memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); - } else { - const auto candidate = std::find_if(dataCandidates[i].begin(), dataCandidates[i].end(), [&](const auto& kept) { return kept.first->pushOffset == pushCursorBytes; }); - if (candidate != dataCandidates[i].end() && !verifyDrawDataHits()) { - matched[i] = candidate->first; - stageCaptures[i].registeredViews = matched[i]->registeredViews; - matchedRegions[i] = std::move(candidate->second); - reused[i] = true; programResults[i] = stageData[i].compiled != nullptr ? stageData[i].compiled.get() : matched[i]->compiled.get(); memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); + ringWords(i); } else { - if (!dataCandidates[i].empty() && !verifyDrawDataHits()) decodeVertexInfo(i); - if (program.binary.stage == ShaderRecompiler::ShaderStage::Fragment) { - for (auto previous = i; previous-- > 0;) { - if (programResults[previous] == nullptr) continue; - pixel = Graphics::DefaultUnexportedInputs(decode->pixel, programResults[previous]->parameterExports); - break; + const bool afterHeap = heapRetry || std::any_of(programResults.begin(), programResults.begin() + static_cast(i), [](const ShaderRecompiler::RecompileResult* earlier) { return earlier != nullptr && earlier->gpuResolvedImages; }); + if (reused[i] && afterHeap && !matched[i]->compiled->gpuResolvedImages) { + reused[i] = false; + matched[i] = nullptr; + matchedRegions[i].clear(); + } + const auto candidate = reused[i] ? dataCandidates[i].end() : candidateForStage(i, afterHeap); + if (reused[i]) { + programResults[i] = stageData[i].compiled != nullptr ? stageData[i].compiled.get() : matched[i]->compiled.get(); + memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); + ringWords(i); + } else if (candidate != dataCandidates[i].end() && !verifyDrawDataHits()) { + matched[i] = candidate->first; + stageCaptures[i].registeredViews = matched[i]->registeredViews; + matchedRegions[i] = std::move(candidate->second); + reused[i] = true; + programResults[i] = stageData[i].compiled != nullptr ? stageData[i].compiled.get() : matched[i]->compiled.get(); + memory.insert(memory.end(), matchedRegions[i].begin(), matchedRegions[i].end()); + ringWords(i); + } else { + if (!dataCandidates[i].empty() && !verifyDrawDataHits()) decodeVertexInfo(i); + if (program.binary.stage == ShaderRecompiler::ShaderStage::Fragment) { + for (auto previous = i; previous-- > 0;) { + if (programResults[previous] == nullptr) continue; + pixel = Graphics::DefaultUnexportedInputs(decode->pixel, programResults[previous]->parameterExports); + break; + } } + compiledStages.push_back(materializeDrawStage(stageContext, i, pushCursorBytes, drawParameters, !heapPairRefused)); + if (!rejected.empty()) return DrawVerdict::Rejected; + programResults[i] = compiledStages.back().get(); + if (verifyHit && matched[i] != nullptr && matched[i]->ringTemplate != nullptr) VerifyRingWords(*matched[i]->compiled, *buildRingWords(i, *matched[i]), *programResults[i], stageCaptures[i].heapWords.get()); + if (candidate != dataCandidates[i].end()) verifyDataStage(i, *candidate->first); + if (stageData[i].compiled != nullptr && verifyDrawDataHits()) verifyStageData(i); } - compiledStages.push_back(materializeDrawStage(i, pushCursorBytes, queue, submission, packetOffset, trace.token, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected, meshBarycentrics)); - if (!rejected.empty()) return DrawVerdict::Rejected; - programResults[i] = compiledStages.back().get(); - if (candidate != dataCandidates[i].end()) verifyDataStage(i, *candidate->first); - if (stageData[i].compiled != nullptr && verifyDrawDataHits()) verifyStageData(i); + if (dataEntry != nullptr && reused[i]) ++dataStagesReused; + else if (dataEntry != nullptr) ++dataStagesCompiled; } - if (dataEntry != nullptr && reused[i]) ++dataStagesReused; - else if (dataEntry != nullptr) ++dataStagesCompiled; - } - const auto& result = *programResults[i]; - if (i == 0 && drawParameters.indirect) { - indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect); - if (EmptyIndirectSkip()) CpuIndirectPrograms()[programs.front().binary.codeAddress] = indirectCpu.has_value(); - } else if (i == 0) { - fold(result, drawParameters); + const auto& result = *programResults[i]; + if (i == 0 && drawParameters.indirect) { + indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect, stageCaptures[0].heapWords.get()); + if (EmptyIndirectSkip()) CpuIndirectPrograms()[programs.front().binary.codeAddress] = indirectCpu.has_value(); + } else if (i == 0) { + fold(result, drawParameters); + } + try { + const auto slotEnd = (pushCursorBytes / Graphics::PipelinePushSlotBytes + 1u) * Graphics::PipelinePushSlotBytes; + require(result.pushConstants.size() <= slotEnd - pushCursorBytes, "stage push constants exceed the pipeline push constant block"); + stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes, stageCaptures[i].registeredViews.get(), stageCaptures[i].heapWords.get()}); + pushCursorBytes += static_cast(result.pushConstants.size()); + } catch (...) { + TraceMaterializationFailure(trace.token, submission, packetOffset, "push-constant-validation", completedSubdraws, std::current_exception()); + throw; + } + if (stages.back().heap != nullptr && SampleHeapStageVerify()) VerifyHeapAccessor(stages.back()); } - try { - const auto slotEnd = (pushCursorBytes / Graphics::PipelinePushSlotBytes + 1u) * Graphics::PipelinePushSlotBytes; - require(result.pushConstants.size() <= slotEnd - pushCursorBytes, "stage push constants exceed the pipeline push constant block"); - stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes, stageCaptures[i].registeredViews.get()}); - pushCursorBytes += static_cast(result.pushConstants.size()); - } catch (...) { - TraceMaterializationFailure(trace.token, submission, packetOffset, "push-constant-validation", completedSubdraws, std::current_exception()); - throw; + return std::nullopt; + }; + DrawStageCheckpoint checkpoint(stages, compiledStages, pushCursorBytes, drawParameters, pixel, heapCheck); + if (const auto verdict = runStages()) return *verdict; + for (std::uint32_t retry = 0; heapCheck && MixesHeapAndGuestStages(stages, programResults); ++retry) { + require(retry < 2u, "a draw mixes GPU-mode and per-draw stages after taking the per-draw path"); + Graphics::NoteHeapFallback(retry == 0u ? Graphics::HeapFallback::DrawMixedRetry : Graphics::HeapFallback::DrawRefusedPair); + if (retry == 1u) RefuseHeapPair(heapPair); + const auto reusedGuest = [&](std::size_t i) { return reused[i] && programResults[i] != nullptr && !programResults[i]->gpuResolvedImages; }; + if (std::ranges::any_of(std::views::iota(std::size_t{0}, programs.size()), reusedGuest)) Graphics::NoteHeapFallback(Graphics::HeapFallback::DrawMixedReused); + heapRetry = retry == 0u; + checkpoint.Restore(); + std::fill(programResults.begin(), programResults.end(), nullptr); + for (std::size_t i = 0; i < programs.size(); ++i) { + if (!reused[i]) stageCaptures[i] = {}; } + dataStagesReused = dataStagesCompiled = dataVerified = stageDataVerified = 0; + indirectCpu.reset(); + std::optional heapOff; + if (retry == 1u) heapOff.emplace(); + if (const auto verdict = runStages()) return *verdict; + } + for (std::size_t i = 0; i < programs.size(); ++i) { + if (programResults[i] != nullptr) static_cast(ExpectDeviceImages(*localDevice, programs[i].userData, programs[i].binary.codeAddress, *programResults[i], false)); } if (stageDataVerified != 0) { @@ -985,7 +1121,7 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) { }; for (std::size_t i = 0; i < programs.size(); ++i) { const auto& capture = stageCaptures[i]; - if (reused[i] || capture.compiled == nullptr || !CacheableResult(*capture.compiled)) continue; + if (reused[i] || capture.compiled == nullptr || !capture.Cacheable() || !CacheableResult(*capture.compiled)) continue; stored = true; const auto& words = programs[i].userData; for (std::size_t w = 0; w < words.size(); ++w) { @@ -1057,11 +1193,12 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) { if (programResults[0]->meshBarycentrics != meshBarycentrics) { const auto pushBytes = programResults[0]->pushConstants.size(); const auto exports = programResults[0]->parameterExports; - compiledStages.push_back(materializeDrawStage(0, pushOffsets[0], queue, submission, packetOffset, traceToken, programs, graphics, pixel, vertexInfos, memory, linked, drawParameters, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected, meshBarycentrics)); + compiledStages.push_back(materializeDrawStage(stageContext, 0, pushOffsets[0], drawParameters, !heapPairRefused)); if (!rejected.empty()) return DrawVerdict::Rejected; programResults[0] = compiledStages.back().get(); stages[0].program = programResults[0]; stages[0].registeredImageViews = stageCaptures[0].registeredViews.get(); + stages[0].heap = stageCaptures[0].heapWords.get(); require(programResults[0]->pushConstants.size() == pushBytes && programResults[0]->parameterExports == exports, "pairing the mesh program with its pixel shader changed the push constants or the parameter exports the pixel shader was built for"); } require(programResults[0]->meshBarycentrics == meshBarycentrics, "the mesh program was compiled for a different barycentric layout than its pixel shader"); @@ -1178,10 +1315,13 @@ DrawVerdict Driver::finishDraw(DrawTail& tail, PerformanceTimer& timing) { auto& result = results[resultIndex[programIndex]]; const auto pushBytes = result.pushConstants.size(); decodeVertexInfo(programIndex); - result = *materializeDrawStage(programIndex, pushOffsets[programIndex], queue, submission, packetOffset, trace.token, programs, graphics, pixel, vertexInfos, memory, linked, programs[programIndex].binary.stage == Stage::TessellationControl || programs[programIndex].binary.stage == Stage::TessellationEvaluation ? direct : drawParameters, localDevice, *shaderMemory, stageCaptures, recompiled, drawHit, matched, matchedRegions, profile, dumpTarget, dumpSlot1, captures, phaseTiming, phaseMs, rejected, meshBarycentrics); + result = *materializeDrawStage(stageContext, programIndex, pushOffsets[programIndex], programs[programIndex].binary.stage == Stage::TessellationControl || programs[programIndex].binary.stage == Stage::TessellationEvaluation ? direct : drawParameters, !heapPairRefused); for (auto& stage : stages) if (stage.program == &result) stage.registeredImageViews = stageCaptures[programIndex].registeredViews.get(); if (!rejected.empty()) return DrawVerdict::Rejected; require(result.pushConstants.size() == pushBytes, "patched program changed its push constant layout"); + for (auto& stage : stages) { + if (stage.program == &result) stage.heap = stageCaptures[programIndex].heapWords.get(); + } } fold(*programResults[0], direct); if (graphics.rectList && patched.contains(0)) buildRectList(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 735319763bd3..7448f80f1cb3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -1,3 +1,14 @@ +#include +#include +#include +#include +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include +#include +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" @@ -37,6 +48,19 @@ void TraceMaterializationFailure(std::uint64_t token, const Submission& submissi } } +std::uint64_t UserDataMask(const DrawProgram& program, const std::optional& vertexInfo, std::uint64_t walked, const ShaderRecompiler::RecompileResult& compiled) { + std::uint64_t mask = walked | compiled.userDataRegisterReads; + const auto& tail = program.snapshot->userDataTailRegister; + if (tail && *tail >= program.firstUserSgpr) { + const auto index = *tail - program.firstUserSgpr; + mask |= index + 1u < 64u ? std::uint64_t{3} << index : ~std::uint64_t{0}; + } + if (vertexInfo) { + for (const auto reg : {static_cast(vertexInfo->fetchAttribReg), static_cast(vertexInfo->fetchBufferReg)}) mask |= reg + 1u < 64u ? std::uint64_t{3} << reg : ~std::uint64_t{0}; + } + return mask; +} + void TraceShaderRequestFailure(std::uint64_t token, const Submission& submission, std::size_t packetOffset, const DrawProgram& program, const std::exception_ptr& failure) noexcept { if (token == 0) return; try { @@ -60,131 +84,240 @@ void TraceShaderRequestFailure(std::uint64_t token, const Submission& submission } -std::shared_ptr Driver::materializeDrawStage(std::size_t i, std::uint32_t pushOffset, const QueueState& queue, const Submission& submission, std::size_t packetOffset, std::uint64_t traceToken, const std::vector& programs, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector>& vertexInfos, std::vector& memory, const std::vector& linked, const Pm4::DrawParameters& drawParameters, const std::shared_ptr& localDevice, ShaderMemory& shaderMemory, std::vector& stageCaptures, std::vector& recompiled, bool drawHit, const std::vector>& matched, const std::vector>& matchedRegions, bool profile, std::uint64_t dumpTarget, std::uint64_t dumpSlot1, std::uint64_t& captures, DrawPhaseTiming& phaseTiming, std::array& phaseMs, std::string& rejected, const std::optional& meshBarycentrics) try { +Driver::HeapDirect Driver::materializeHeapDirect(const DrawStageContext& context, std::size_t i, const ShaderRecompiler::PreparedShaderInvocation& invocation, const ShaderRecompiler::RecompileRequest& request, PerformanceTimer& timing) { + const auto& program = context.programs[i]; + auto& stageCapture = context.stageCaptures[i]; + HeapStageMaterialized materialized; + Graphics::HeapCostScope callCost(Graphics::HeapCost::HeapStageCall); + try { + materialized = MaterializeHeapStage(invocation, program.userData, program.binary.codeAddress); + } catch (const std::exception& error) { + throw stageFailure(program, request, error, request.context.memory); + } + callCost.Stop(); + Graphics::HeapCostScope finishCost(Graphics::HeapCost::StageFinish); + if (materialized.result == nullptr) return HeapDirect::Declined; + if (!HeapDrawStageMappable(*materialized.result, program.binary.stage)) return HeapDirect::Unmapped; + NoteHeapDirect(); + stageCapture.compiled = std::move(materialized.result); + stageCapture.heapWords = stageCapture.compiled->heapTemplateId != 0 ? std::const_pointer_cast(std::move(materialized.words)) : nullptr; + if (SampleHeapStageVerify()) { + const auto fullCapture = context.shaderMemory.Capture(invocation); + const auto expected = invocation.Materialize(*fullCapture); + static_cast(context.shaderMemory.TakeRecentRegions()); + NoteHeapDirectVerify(stageCapture.heapWords != nullptr ? *PatchedHeapResult(*stageCapture.compiled, *stageCapture.heapWords) : *stageCapture.compiled, *expected, fullCapture.get()); + } + stageCapture.regions.clear(); + stageCapture.direct = true; + auto& reads = materialized.reads; + if (reads.complete) { + auto owned = std::make_shared>(std::move(reads.values)); + std::size_t cursor = 0; + for (const auto& [begin, end] : reads.ranges) { + const auto count = static_cast((end - begin) / sizeof(std::uint32_t)); + stageCapture.regions.push_back({begin, std::as_bytes(std::span(owned->data() + cursor, count))}); + cursor += count; + } + stageCapture.heapReadWords = std::move(owned); + stageCapture.userDataUse = reads.userDataUse; + stageCapture.capture = std::move(reads.capture); + stageCapture.userDataMask = UserDataMask(program, context.vertexInfos[i], reads.userDataUse.asValue | reads.userDataUse.asBase, *stageCapture.compiled); + stageCapture.ring = true; + } + context.recompiled[i] = true; + timing.Mark("heap_direct"); + context.phaseTiming.Phase(DrawRowRecompile); + return HeapDirect::Taken; +} + +std::runtime_error Driver::stageFailure(const DrawProgram& program, const ShaderRecompiler::RecompileRequest& request, const std::exception& error, std::span regions) { + static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; + std::string dump; + if (dumpShaders) { + try { + auto failed = request; + failed.context.memory = regions; + dump = dumpRequest(program.binary.codeAddress, failed, "_failed"); + } catch (...) {} + } + std::ostringstream where; + where << "shader 0x" << std::hex << program.binary.codeAddress << " stage " << std::dec << static_cast(program.binary.stage); + if (!dump.empty()) where << " (" << dump << ")"; + where << ": "; + return std::runtime_error(where.str() + error.what()); +} + +std::shared_ptr Driver::materializeDrawStage(const DrawStageContext& context, std::size_t i, std::uint32_t pushOffset, const Pm4::DrawParameters& drawParameters, bool heapPartners) try { PerformanceTimer timing("Shader.DrawStage"); using Stage = ShaderRecompiler::ShaderStage; - phaseTiming.Phase(DrawRowVectors); - const auto& program = programs[i]; - const auto waveSize = Graphics::StageWaveSize(graphics.stages, program.binary.stage); + context.phaseTiming.Phase(DrawRowVectors); + const auto& program = context.programs[i]; + Graphics::HeapCostScope prepareCost(Graphics::HeapCost::StagePrepare); + const auto waveSize = Graphics::StageWaveSize(context.graphics.stages, program.binary.stage); ShaderRecompiler::RecompileRequest request{ program.binary, - {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)}, - localDevice->Target(), - {0, 0, pushOffset, Graphics::StagePushLimit(graphics.stages, pushOffset, graphics.depthRemap.mode != 0u) - pushOffset % Graphics::PipelinePushSlotBytes}, - ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, Graphics::StageMeshConfiguration(graphics.stages, program.binary.stage), graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}, program.binary.stage == Stage::Mesh ? meshBarycentrics : std::nullopt} + {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(context.pixel) : std::nullopt, context.vertexInfos[i], context.memory, RegisteredFloatMode(*program.snapshot)}, + context.localDevice->Target(), + {0, 0, pushOffset, Graphics::StagePushLimit(context.graphics.stages, pushOffset, context.graphics.depthRemap.mode != 0u) - pushOffset % Graphics::PipelinePushSlotBytes}, + ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, context.linked, Graphics::StageMeshConfiguration(context.graphics.stages, program.binary.stage), context.graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}, program.binary.stage == Stage::Mesh ? context.meshBarycentrics : std::nullopt} }; - if (graphics.depthRemap.mode != 0u && pushOffset < Graphics::PipelinePushSlotBytes) { + if (context.graphics.depthRemap.mode != 0u && pushOffset < Graphics::PipelinePushSlotBytes) { request.layout.pushConstantSizeBytes = std::min(request.layout.pushConstantSizeBytes, ShaderRecompiler::DepthRemapPushOffsetBytes - pushOffset); } - const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; - auto invocation = [&] { + if (request.target.deviceImageTables) request.target.deviceImageTables = context.localDevice->DeviceImagesServe(program.binary.codeAddress); + const auto waitedBefore = traceCapSync() || context.profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; + const bool heapCandidate = heapPartners && (program.binary.stage == Stage::Fragment || HeapPreRasterStage(program.binary.stage)) && !HeapStageExcluded(program.binary.stage) && HeapFragmentsEnabled(); + const auto fragmentOnHeap = [&] { + const auto fragment = std::find_if(context.programs.begin(), context.programs.end(), [](const DrawProgram& candidate) { return candidate.binary.stage == Stage::Fragment; }); + if (fragment == context.programs.end()) return true; + const auto known = HeapFragmentKnown(fragment->snapshot.get(), fragment->codeOffset); + return known.has_value() ? *known : PreparedHeapStage(*fragment->snapshot, fragment->codeOffset); + }; + const bool heapVertex = request.target.descriptorMemory && HeapPreRasterStage(program.binary.stage) && fragmentOnHeap(); + const auto heapKey = HeapFragmentKey(program.codeOffset, heapVertex ? HeapKeyClass::HeapVertex : HeapKeyClass::Draw); + const auto heapKnown = heapCandidate ? HeapFragmentKnown(program.snapshot.get(), heapKey) : std::nullopt; + if (heapKnown.value_or(false)) request.target.gpuResolvedImages = true; + const auto invocationFor = [&] { try { return InvocationFor(*program.snapshot, program.codeOffset, request); } catch (...) { - TraceShaderRequestFailure(traceToken, submission, packetOffset, program, std::current_exception()); + TraceShaderRequestFailure(context.traceToken, context.submission, context.packetOffset, program, std::current_exception()); throw; } - }(); + }; + auto invocation = invocationFor(); + const auto heapEligible = [&](const std::shared_ptr& plan) { return HeapStageEligible(program.binary.stage, plan.get(), heapVertex); }; + const auto plan = heapCandidate && invocation.Plan() == nullptr ? invocation.LoadPlan() : invocation.Plan(); + using StageReason = Graphics::HeapStageReason; + StageReason stageReason = StageReason::Heap; + if (!heapCandidate) stageReason = !HeapFragmentsEnabled() ? StageReason::Disabled : !heapPartners ? StageReason::PairRefused : StageReason::StageKind; + else if (heapKnown == std::optional(false)) stageReason = StageReason::KnownIneligible; + if (heapKnown.value_or(false) && !heapEligible(plan)) { + stageReason = HeapStageIneligibility(program.binary.stage, plan.get(), heapVertex); + NoteHeapFragment(program.snapshot.get(), heapKey, false); + request.target.gpuResolvedImages = false; + invocation = invocationFor(); + } else if (heapCandidate && !heapKnown.has_value()) { + stageReason = HeapStageIneligibility(program.binary.stage, plan.get(), heapVertex); + const bool eligible = stageReason == StageReason::Heap; + NoteHeapFragment(program.snapshot.get(), heapKey, eligible); + if (eligible) { + request.target.gpuResolvedImages = true; + invocation = invocationFor(); + } + } + Graphics::NoteHeapStageReason(stageReason); + prepareCost.Stop(); timing.Mark("prepared_invocation"); - auto& stageCapture = stageCaptures[i]; + auto& stageCapture = context.stageCaptures[i]; + stageCapture.direct = false; + stageCapture.ring = false; + stageCapture.heapReadWords.reset(); + stageCapture.capture.reset(); stageCapture.forgetSerial = GuestMemory::ForgetSerial(); stageCapture.pushOffset = pushOffset; - const auto named = [&](const std::exception& error, std::span regions) { - static const bool dumpShaders = std::getenv("APS5_DUMP_SHADERS") != nullptr; - std::string dump; - if (dumpShaders) { - try { - auto failed = request; - failed.context.memory = regions; - dump = dumpRequest(program.binary.codeAddress, failed, "_failed"); - } catch (...) {} - } - std::ostringstream where; - where << "shader 0x" << std::hex << program.binary.codeAddress << " stage " << std::dec << static_cast(program.binary.stage); - if (!dump.empty()) where << " (" << dump << ")"; - where << ": "; - return std::runtime_error(where.str() + error.what()); + const auto unmapped = [&] { + Graphics::NoteHeapFallback(Graphics::HeapFallback::DrawUnmapped); + NoteHeapFragment(program.snapshot.get(), heapKey, false); + return materializeDrawStage(context, i, pushOffset, drawParameters, heapPartners); }; + if (request.target.gpuResolvedImages) { + switch (materializeHeapDirect(context, i, invocation, request, timing)) { + case HeapDirect::Taken: return stageCapture.compiled; + case HeapDirect::Unmapped: return unmapped(); + case HeapDirect::Declined: break; + } + } const auto capture = [&] { const SampledReadScope sampling(evidenceReads); try { - return CaptureInvocation(shaderMemory, *program.snapshot, program.codeOffset, request, invocation); + return CaptureInvocation(context.shaderMemory, *program.snapshot, program.codeOffset, request, invocation); } catch (const std::exception& error) { - throw named(error, shaderMemory.Regions()); + throw stageFailure(program, request, error, context.shaderMemory.Regions()); } }(); - stageCapture.regions = shaderMemory.TakeRecentRegions(); - stageCapture.supplied = shaderMemory.SuppliedBlocks(); + stageCapture.regions = context.shaderMemory.TakeRecentRegions(); + stageCapture.supplied = context.shaderMemory.SuppliedBlocks(); stageCapture.userDataUse = capture->userDataUse; stageCapture.registeredViews = std::make_shared>(capture->snapshot.registeredImages); - recompiled[i] = true; - memory = shaderMemory.Regions(); + context.recompiled[i] = true; + context.memory = context.shaderMemory.Regions(); - for (std::size_t j = 0; j < programs.size(); ++j) { - if (matched[j] != nullptr && !recompiled[j] && (drawHit || j < i)) memory.insert(memory.end(), matchedRegions[j].begin(), matchedRegions[j].end()); + for (std::size_t j = 0; j < context.programs.size(); ++j) { + if (context.matched[j] != nullptr && !context.recompiled[j] && (context.drawHit || j < i)) context.memory.insert(context.memory.end(), context.matchedRegions[j].begin(), context.matchedRegions[j].end()); } - request.context.memory = memory; - if (traceCapSync()) traceCapture("draw-capture", program.binary.codeAddress, submission.queue, memory, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); - if (profile) { - ++captures; - phaseTiming.Phase(DrawRowCapture); + request.context.memory = context.memory; + if (traceCapSync()) traceCapture("draw-capture", program.binary.codeAddress, context.submission.queue, context.memory, Graphics::Recorder::ThreadWaitedMs() - waitedBefore); + if (context.profile) { + ++context.captures; + context.phaseTiming.Phase(DrawRowCapture); - const auto waited = std::min(Graphics::Recorder::ThreadWaitedMs() - waitedBefore, phaseMs[DrawRowCapture]); - phaseMs[DrawRowCapture] -= waited; - phaseMs[DrawRowCaptureHookWaits] += waited; + const auto waited = std::min(Graphics::Recorder::ThreadWaitedMs() - waitedBefore, context.phaseMs[DrawRowCapture]); + context.phaseMs[DrawRowCapture] -= waited; + context.phaseMs[DrawRowCaptureHookWaits] += waited; } - if (dumpTarget != 0) { + if (context.dumpTarget != 0) { - const auto slot0 = (static_cast(readRegister(queue.context, 0x390)) << 40u) | (static_cast(readRegister(queue.context, 0x318)) << 8u); - if ((graphics.hasColorTarget && graphics.color.address == dumpTarget) || slot0 == dumpTarget || program.binary.codeAddress == dumpTarget) static_cast(dumpRequest(program.binary.codeAddress, request)); + const auto slot0 = (static_cast(readRegister(context.queue.context, 0x390)) << 40u) | (static_cast(readRegister(context.queue.context, 0x318)) << 8u); + if ((context.graphics.hasColorTarget && context.graphics.color.address == context.dumpTarget) || slot0 == context.dumpTarget || program.binary.codeAddress == context.dumpTarget) static_cast(dumpRequest(program.binary.codeAddress, request)); } - if (dumpSlot1 != 0) { - const auto value = [&](std::uint32_t offset) -> std::uint64_t { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; + if (context.dumpSlot1 != 0) { + const auto value = [&](std::uint32_t offset) -> std::uint64_t { const auto it = context.queue.context.find(offset); return it == context.queue.context.end() ? 0u : it->second; }; const auto slot1 = (value(0x391) << 40u) | (value(0x327) << 8u); - if (slot1 == dumpSlot1) { + if (slot1 == context.dumpSlot1) { static_cast(dumpRequest(program.binary.codeAddress, request)); if (std::FILE* file = std::fopen("draw_slot1.regs", "w")) { - for (const auto& [offset, value] : queue.context) std::fprintf(file, "context %x %08x\n", offset, value); - for (const auto& [offset, value] : queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); - for (const auto& [offset, value] : queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); + for (const auto& [offset, value] : context.queue.context) std::fprintf(file, "context %x %08x\n", offset, value); + for (const auto& [offset, value] : context.queue.userConfig) std::fprintf(file, "uconfig %x %08x\n", offset, value); + for (const auto& [offset, value] : context.queue.shader) std::fprintf(file, "shader %x %08x\n", offset, value); std::fclose(file); } } } - phaseTiming.Phase(DrawRowCapture); + context.phaseTiming.Phase(DrawRowCapture); timing.Mark("capture_resources"); - try { - stageCapture.compiled = invocation.Materialize(*capture); - stageCapture.capture = capture; - } catch (const std::exception& error) { - TraceMaterializationFailure(traceToken, submission, packetOffset, i, "capture-or-materialize", std::current_exception()); - throw named(error, request.context.memory); + stageCapture.heapWords.reset(); + if (request.target.gpuResolvedImages) { + auto words = std::make_shared(); + if (auto shared = invocation.MaterializeHeap(*capture, *words)) { + stageCapture.compiled = std::move(shared); + stageCapture.heapWords = std::move(words); + stageCapture.direct = true; + stageCapture.ring = true; + if (SampleHeapStageVerify()) NoteHeapDirectVerify(*PatchedHeapResult(*stageCapture.compiled, *stageCapture.heapWords), *invocation.Materialize(*capture)); + } } - timing.Mark("materialize"); - const auto walked = capture->userDataUse.asValue | capture->userDataUse.asBase; - auto mask = walked == 0 ? ~std::uint64_t{0} : walked | stageCapture.compiled->userDataRegisterReads; - if (vertexInfos[i]) { - for (const auto reg : {static_cast(vertexInfos[i]->fetchAttribReg), static_cast(vertexInfos[i]->fetchBufferReg)}) { - mask |= reg + 1u < 64u ? std::uint64_t{3} << reg : ~std::uint64_t{0}; + if (stageCapture.heapWords == nullptr) { + try { + stageCapture.compiled = invocation.Materialize(*capture); + } catch (const std::exception& error) { + TraceMaterializationFailure(context.traceToken, context.submission, context.packetOffset, i, "capture-or-materialize", std::current_exception()); + throw stageFailure(program, request, error, request.context.memory); } } - stageCapture.userDataMask = mask; - phaseTiming.Phase(DrawRowRecompile); + if (stageCapture.compiled->gpuResolvedImages && !HeapDrawStageMappable(*stageCapture.compiled, program.binary.stage)) return unmapped(); + stageCapture.capture = capture; + timing.Mark("materialize"); + const auto walked = capture->userDataUse.asValue | capture->userDataUse.asBase; + stageCapture.userDataMask = walked == 0 ? ~std::uint64_t{0} : UserDataMask(program, context.vertexInfos[i], walked, *stageCapture.compiled); + context.phaseTiming.Phase(DrawRowRecompile); return stageCapture.compiled; } catch (const std::exception& error) { if (!SkipRefusedDraws()) throw; - throw RefusedDraw(programs[i].binary.codeAddress, error.what()); + throw RefusedDraw(context.programs[i].binary.codeAddress, error.what()); } + + void Driver::recordStageDataPositions(DispatchVariant& variant, const ShaderRecompiler::ResourceCapture& capture, const std::vector& decodeReads) { const auto& compiled = *variant.compiled; for (std::size_t b = 0; b < compiled.bindings.size(); ++b) { if (compiled.bindings[b].role != ShaderRecompiler::DescriptorRole::FlattenedSrt) continue; variant.flatBinding = static_cast(b); - static_cast(DataWordPositions(variant.runs, capture.readTrace.leaves, capture.readTrace.otherReads, variant.words, compiled.bindings[b].guestDescriptor, variant.dataPositions, variant.dataSlots)); + static_cast(DataWordPositions(variant.runs, capture.readTrace.leaves, capture.readTrace.otherReads, variant.words, variant.ringTemplate != nullptr && !variant.ringTemplate->flattenedSrt.empty() ? variant.ringTemplate->flattenedSrt : compiled.bindings[b].guestDescriptor, variant.dataPositions, variant.dataSlots)); break; } if (variant.dataPositions.empty() || decodeReads.empty()) return; @@ -223,7 +356,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP fresh[i] = matched[i]; continue; } - if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue; + if (stageCapture.compiled == nullptr || !stageCapture.Cacheable() || !CacheableResult(*stageCapture.compiled)) continue; auto variant = std::make_shared(); variant->compiled = stageCapture.compiled; variant->registeredViews = stageCapture.registeredViews; @@ -234,6 +367,13 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP variant->forgetSerial = stageCapture.forgetSerial; variant->pushOffset = stageCapture.pushOffset; variant->userDataMask = stageCapture.userDataMask; + if (stageCapture.ring && stageCapture.heapWords != nullptr) { + auto ringTemplate = std::make_shared(); + ringTemplate->vertexAttributes = stageCapture.heapWords->vertexAttributes; + ringTemplate->flattenedSrt = stageCapture.heapWords->flattenedSrt; + ringTemplate->bindings = stageCapture.heapWords->bindings; + variant->ringTemplate = std::move(ringTemplate); + } if (vertexInfos[i]) variant->vertexInfo = std::make_shared(*vertexInfos[i]); std::vector regions(stageCapture.regions.begin(), stageCapture.regions.end()); @@ -241,7 +381,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP std::stable_sort(regions.begin(), regions.end(), [](const ShaderRecompiler::MemoryRegion& a, const ShaderRecompiler::MemoryRegion& b) { return a.guestAddress < b.guestAddress; }); const bool relocatable = decodeReads[i].empty() && (!vertexInfos[i] || vertexInfos[i]->resourcesNum == 0); tagVariantRuns(*variant, regions, stageCapture.userDataUse, relocatable, programs[i].hiddenUserWords, stageCapture.supplied); - if (relocatable) tagUserValueWords(*variant, programs[i]); + if (relocatable && (stageCapture.heapWords == nullptr || variant->ringTemplate != nullptr)) tagUserValueWords(*variant, programs[i], stageCapture.heapWords.get()); if (drawStageDataHits() && stageCapture.capture != nullptr && !stageCapture.capture->readTrace.leaves.empty()) recordStageDataPositions(*variant, *stageCapture.capture, decodeReads[i]); if (traceDrawCache() && entry != nullptr && matched[i] == nullptr && i < entry->stages.size()) { const auto& variants = entry->stages[i]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp index c61e3112c581..b852725b4a35 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "Optimization/ResourceProgram.hpp" #include @@ -9,6 +10,7 @@ namespace AgcDriver::DriverDetail { void Driver::lookupDraw(const Submission& submission, const std::shared_ptr& localDevice, const Graphics::State& graphics, const ShaderRecompiler::ShaderPixelStageInfo& pixel, const std::vector& programs, const std::vector& roles, const std::vector>& vertexInfos, bool useDrawEntries, bool registerKey, bool profile, std::uint64_t& drawKey, std::shared_ptr& entry, std::vector>& matched, std::vector>& matchedRegions, bool& drawHit, bool& verifyHit, DrawPhaseTiming& phaseTiming, std::array& phaseMs, const std::shared_ptr& dataEntry, DrawDataCandidates& dataCandidates, std::vector& stageData) { using Role = ShaderRecompiler::ProgramRole; + const bool heapRefused = HeapRefused(); if (useDrawEntries) { if (!registerKey) { drawKey = 0xcbf29ce484222325ull; @@ -129,6 +131,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptrstages.size() != programs.size()) miss = DrawMiss::Stages; std::uint32_t cursor = 0; + std::optional heapMode; { const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::DrawCache); @@ -143,7 +146,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptrregisteredViews != nullptr && !CaptureRegisteredViewsCurrent(*variant->registeredViews)) continue; - if (variant->pushOffset != cursor) continue; + if (variant->pushOffset != cursor || (heapRefused && variant->compiled->gpuResolvedImages) || (heapMode && variant->compiled->gpuResolvedImages != *heapMode)) continue; if (programs[i].binary.stage == ShaderRecompiler::ShaderStage::Mesh && !std::equal(variant->meshIndexDescriptor.begin(), variant->meshIndexDescriptor.end(), programs[i].userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord)) { if (!anyLayout) outcome = EntryOutcome::Differing; anyLayout = true; @@ -170,6 +173,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptrcompiled->gpuResolvedImages; ++stageEqual; cursor += static_cast(variant->compiled->pushConstants.size()); break; @@ -252,6 +256,7 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptrstages[i]) { if (variant->registeredViews != nullptr && !CaptureRegisteredViewsCurrent(*variant->registeredViews)) continue; + if (heapRefused && variant->compiled->gpuResolvedImages) continue; bool usedEqual = true; for (std::size_t w = 0; w < now.size() && usedEqual; ++w) { if (now[w] != was[w] && (w >= 64u || ((variant->userDataMask >> w) & 1u) != 0)) usedEqual = false; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp index 731c5505aa4b..c6deb7beedaf 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/WorkerSampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include #include diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp index 52a8c3c29243..6c2e100c4f99 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRelocate.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include #include #include @@ -61,14 +62,18 @@ std::uint64_t Driver::relocatedContentKey(const RelocatableUserWord& word, std:: return hash; } -void Driver::tagUserValueWords(DispatchVariant& variant, const DrawProgram& program) { +void Driver::tagUserValueWords(DispatchVariant& variant, const DrawProgram& program, const ShaderRecompiler::HeapStageWords* heap) { if (!drawRelocationEnabled() || variant.compiled == nullptr) return; const auto& compiled = *variant.compiled; const auto& forwarded = compiled.shaderDataUserWords; const auto users = std::min(program.userData.size(), 64); const auto bound = [&](std::uint32_t value) { std::size_t count = 0; - for (const auto& binding : compiled.bindings) count += static_cast(std::count(binding.guestDescriptor.begin(), binding.guestDescriptor.end(), value)); + for (const auto& binding : compiled.bindings) { + if (heap != nullptr && binding.role == ShaderRecompiler::DescriptorRole::ShaderData) continue; + const auto& words = Graphics::DrawGuestWords(compiled, heap, binding); + count += static_cast(std::count(words.begin(), words.end(), value)); + } return count; }; const auto elsewhere = [&](std::uint32_t index, std::uint32_t value) { @@ -115,16 +120,18 @@ void Driver::tagUserValueWords(DispatchVariant& variant, const DrawProgram& prog if (base == 0 || taken(index) || bound(base) != 1 || elsewhere(index, base)) continue; std::uint32_t binding = 0, word = 0; for (std::size_t b = 0; b < compiled.bindings.size(); ++b) { - const auto& descriptor = compiled.bindings[b].guestDescriptor; + if (heap != nullptr && compiled.bindings[b].role == ShaderRecompiler::DescriptorRole::ShaderData) continue; + const auto& descriptor = Graphics::DrawGuestWords(compiled, heap, compiled.bindings[b]); const auto found = std::find(descriptor.begin(), descriptor.end(), base); if (found == descriptor.end()) continue; binding = static_cast(b); word = static_cast(found - descriptor.begin()); } const auto& target = compiled.bindings[binding]; + const auto& targetWords = Graphics::DrawGuestWords(compiled, heap, target); const auto element = word / 4u; - const bool readOnly = target.role == ShaderRecompiler::DescriptorRole::GuestBuffers && target.guestDescriptor.size() == static_cast(target.count) * 4u && word % 4u == 0 && element < target.bufferWritten.size() && !target.bufferWritten[element] && (element >= target.bufferAtomic.size() || !target.bufferAtomic[element]); - if (!readOnly || target.guestDescriptor[word + 1] != program.userData[index + 1] || target.guestDescriptor[word + 2] != program.userData[index + 2] || readThrough(target.guestDescriptor.data() + word)) continue; + const bool readOnly = target.role == ShaderRecompiler::DescriptorRole::GuestBuffers && targetWords.size() == static_cast(target.count) * 4u && word % 4u == 0 && element < target.bufferWritten.size() && !target.bufferWritten[element] && (element >= target.bufferAtomic.size() || !target.bufferAtomic[element]); + if (!readOnly || targetWords[word + 1] != program.userData[index + 1] || targetWords[word + 2] != program.userData[index + 2] || readThrough(targetWords.data() + word)) continue; variant.valueWords |= std::uint64_t {1} << index; variant.valueSlots.push_back({index, binding, word}); const auto records = program.userData[index + 2]; @@ -324,6 +331,7 @@ void Driver::relocateDrawEntry(const QueueState& queue, std::uint64_t alias, std copy->flatBinding = variant->flatBinding; copy->pushOffset = variant->pushOffset; copy->vertexInfo = variant->vertexInfo; + copy->ringTemplate = variant->ringTemplate; for (const auto& move : moves) { if (move.program != i) continue; const auto delta = AlignedWord(move.to) - AlignedWord(move.from); @@ -354,7 +362,7 @@ void Driver::applyShaderDataPatches(DispatchVariant& variant) { auto rebased = std::make_shared(*variant.compiled); for (const auto& [slot, value] : variant.shaderDataPatches) { const auto at = static_cast(slot) * sizeof(std::uint32_t); - if (at + sizeof(std::uint32_t) <= rebased->pushConstants.size()) std::memcpy(rebased->pushConstants.data() + at, &value, sizeof(value)); + if (!rebased->dataAddressPush && at + sizeof(std::uint32_t) <= rebased->pushConstants.size()) std::memcpy(rebased->pushConstants.data() + at, &value, sizeof(value)); for (auto& binding : rebased->bindings) { if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData && slot < binding.guestDescriptor.size()) binding.guestDescriptor[slot] = value; } @@ -362,6 +370,17 @@ void Driver::applyShaderDataPatches(DispatchVariant& variant) { for (const auto& [binding, word, value] : variant.descriptorPatches) { if (binding < rebased->bindings.size() && word < rebased->bindings[binding].guestDescriptor.size()) rebased->bindings[binding].guestDescriptor[word] = value; } + if (variant.ringTemplate != nullptr && !variant.descriptorPatches.empty()) { + auto ringTemplate = std::make_shared(*variant.ringTemplate); + if (ringTemplate->bindings.empty()) ringTemplate->bindings.resize(variant.compiled->bindings.size()); + for (const auto& [binding, word, value] : variant.descriptorPatches) { + if (binding >= ringTemplate->bindings.size()) continue; + auto& words = ringTemplate->bindings[binding]; + if (words.empty()) words = variant.compiled->bindings[binding].guestDescriptor; + if (word < words.size()) words[word] = value; + } + variant.ringTemplate = std::move(ringTemplate); + } variant.compiled = std::move(rebased); variant.shaderDataPatches.clear(); variant.descriptorPatches.clear(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp new file mode 100644 index 000000000000..bedfbbf2602f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp @@ -0,0 +1,909 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapWalkMemo.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libc/include/HostThreadLocal.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +namespace { + +constexpr std::size_t ThreadCacheLimit = 65536u; +constexpr std::uint64_t WalkPageBytes = 0x1000u; +constexpr std::uint64_t WordBytes = sizeof(std::uint32_t); +constexpr std::uint64_t DefaultWalkMemoMiB = 256u; + +std::atomic& heapStages() { + static std::atomic count{0}; + return count; +} + +const char* resultDifference(const ShaderRecompiler::RecompileResult& left, const ShaderRecompiler::RecompileResult& right, + std::size_t& index) { + index = 0; + if (left.PipelineVariantId() != right.PipelineVariantId()) return "variant"; + if (left.pushConstants != right.pushConstants) return "push constants"; + if (left.bindings.size() != right.bindings.size()) return "binding count"; + if (left.specialization.size() != right.specialization.size()) return "specialization count"; + for (index = 0; index < left.bindings.size(); ++index) { + const auto& a = left.bindings[index]; + const auto& b = right.bindings[index]; + if (a.role != b.role || a.kind != b.kind || a.binding != b.binding || a.count != b.count) return "binding layout"; + if (a.guestDescriptor != b.guestDescriptor) return "binding words"; + } + for (index = 0; index < left.specialization.size(); ++index) { + if (left.specialization[index].id != right.specialization[index].id || + left.specialization[index].value != right.specialization[index].value) + return "specialization"; + } + index = 0; + if (left.vertexAttributes.size() != right.vertexAttributes.size()) return "attribute count"; + for (index = 0; index < left.vertexAttributes.size(); ++index) { + const auto& a = left.vertexAttributes[index]; + const auto& b = right.vertexAttributes[index]; + const bool sameAttribute = a.location == b.location && a.components == b.components && a.resource.fields == b.resource.fields && + a.fetchIndex == b.fetchIndex && a.formatComponents == b.formatComponents; + if (!sameAttribute) return "attribute"; + } + return nullptr; +} + +void reportHeapMismatch(const char* check, const char* what, const ShaderRecompiler::RecompileResult& result, std::size_t index) { + static std::atomic printed{0}; + if (printed.fetch_add(1, std::memory_order_relaxed) >= 64u) return; + const int role = index < result.bindings.size() ? static_cast(result.bindings[index].role) : -1; + const int kind = index < result.bindings.size() ? static_cast(result.bindings[index].kind) : -1; + std::fprintf(stderr, "[heap-verify] %s: %s differ at %zu (role %d kind %d, template %llu, variant 0x%llx)\n", check, what, index, role, + kind, static_cast(result.heapTemplateId), static_cast(result.PipelineVariantId())); + std::fflush(stderr); +} + +struct VerifyCounts { + std::atomic verified{0}; + std::atomic differ{0}; + std::atomic notComparable{0}; + std::atomic accessor{0}; + std::atomic accessorDiffer{0}; + std::atomic ring{0}; + std::atomic ringDiffer{0}; +}; + +unsigned long long loaded(const std::atomic& counter) { + return static_cast(counter.load()); +} + +void reportVerifyCounts(const VerifyCounts& c) { + if (c.verified.load() != 0u || c.notComparable.load() != 0u) { + std::fprintf( + stderr, + "[heap-verify] %llu heap stages compared with a full capture, %llu differed, %llu not comparable (the full capture could not read every descriptor)\n", + loaded(c.verified), loaded(c.differ), loaded(c.notComparable)); + } + if (c.accessor.load() != 0u || c.ring.load() != 0u) { + std::fprintf(stderr, "[heap-verify] %llu accessor checks, %llu differed; %llu ring hits checked, %llu differed\n", + loaded(c.accessor), loaded(c.accessorDiffer), loaded(c.ring), loaded(c.ringDiffer)); + } +} + +VerifyCounts& verifyCounts() { + static VerifyCounts counts; + static std::once_flag registered; + std::call_once(registered, [] { std::atexit([] { reportVerifyCounts(verifyCounts()); }); }); + return counts; +} + +} + +bool HeapRefused() { + return Graphics::BindlessHeap::NearlyFull(); +} + +namespace { + +bool& heapDrawOff() { + struct DrawOffStorage {}; + return HostThreadLocal(); +} + +} + +HeapDrawOff::HeapDrawOff() : previous(heapDrawOff()) { + heapDrawOff() = true; +} + +HeapDrawOff::~HeapDrawOff() { + heapDrawOff() = previous; +} + +bool HeapFragmentsEnabled() { + return Graphics::DescriptorHeapEnabled() && !HeapRefused() && !heapDrawOff(); +} + +bool HeapPreRasterStage(ShaderRecompiler::ShaderStage stage) { + using Stage = ShaderRecompiler::ShaderStage; + return stage == Stage::Vertex || stage == Stage::Local || stage == Stage::TessellationControl || + stage == Stage::TessellationEvaluation || stage == Stage::Mesh; +} + +Graphics::HeapStageReason HeapStageIneligibility(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan, + bool heapVertex) { + using Reason = Graphics::HeapStageReason; + if (stage != ShaderRecompiler::ShaderStage::Fragment && !HeapPreRasterStage(stage)) return Reason::StageKind; + if (plan == nullptr) return Reason::NoPlan; + if (stage != ShaderRecompiler::ShaderStage::Fragment && !heapVertex) return Reason::FragmentOffHeap; + if (plan->stage == ShaderRecompiler::IrShaderStage::Unknown || plan->stage == ShaderRecompiler::IrShaderStage::Fetch) + return Reason::PlanStage; + if (!ShaderRecompiler::HeapFragmentEligible(*plan)) return Reason::UnresolvedImageSource; + return Reason::Heap; +} + +bool HeapStageEligible(ShaderRecompiler::ShaderStage stage, const ShaderRecompiler::IrResourcePlan* plan, bool heapVertex) { + return HeapStageIneligibility(stage, plan, heapVertex) == Graphics::HeapStageReason::Heap; +} + +namespace { + +std::unordered_set& refusedPairs() { + struct PairsStorage {}; + return HostThreadLocal, PairsStorage>(); +} + +} + +bool HeapPairRefused(std::uint64_t pair) { + return refusedPairs().contains(pair); +} + +void RefuseHeapPair(std::uint64_t pair) { + auto& refused = refusedPairs(); + if (refused.size() >= ThreadCacheLimit) refused.clear(); + refused.insert(pair); +} + +namespace { + +struct HeapExclusions { + std::uint32_t stages = 0; + bool tables = false; + bool storage = false; + bool sampled = false; +}; + +using ExcludableStage = std::pair; + +constexpr ExcludableStage ExcludableStages[] = { + {"compute", ShaderRecompiler::ShaderStage::Compute}, + {"vertex", ShaderRecompiler::ShaderStage::Vertex}, + {"hull", ShaderRecompiler::ShaderStage::TessellationControl}, + {"domain", ShaderRecompiler::ShaderStage::TessellationEvaluation}, + {"geometry", ShaderRecompiler::ShaderStage::Geometry}, + {"pixel", ShaderRecompiler::ShaderStage::Fragment}, + {"local", ShaderRecompiler::ShaderStage::Local}, + {"mesh", ShaderRecompiler::ShaderStage::Mesh}}; + +bool applyExclusion(HeapExclusions& exclusions, std::string_view item) { + if (item == "tables") { + exclusions.tables = true; + return true; + } + if (item == "storage") { + exclusions.storage = true; + return true; + } + if (item == "sampled") { + exclusions.sampled = true; + return true; + } + const auto stage = std::find_if(std::begin(ExcludableStages), std::end(ExcludableStages), [&](const auto& entry) { return entry.first == item; }); + if (stage == std::end(ExcludableStages)) return false; + exclusions.stages |= 1u << static_cast(stage->second); + return true; +} + +const HeapExclusions& heapExclusions() { + static const HeapExclusions exclusions = [] { + HeapExclusions result; + const char* value = std::getenv("APS5_GPU_HEAP_EXCLUDE"); + std::string_view list = value != nullptr ? value : ""; + while (!list.empty()) { + const auto comma = list.find(','); + const auto item = list.substr(0, comma); + if (!applyExclusion(result, item)) throw std::runtime_error("APS5_GPU_HEAP_EXCLUDE: unknown class '" + std::string(item) + "'; use compute, vertex, hull, domain, geometry, pixel, local, mesh, tables, storage or sampled"); + if (comma == std::string_view::npos) break; + list.remove_prefix(comma + 1u); + } + return result; + }(); + return exclusions; +} + +bool heapClassesAllowed(const ShaderRecompiler::RecompileResult& result) { + using Role = ShaderRecompiler::DescriptorRole; + using Kind = ShaderRecompiler::DescriptorKind; + const auto& exclusions = heapExclusions(); + std::uint64_t demand = 0; + for (const auto& binding : result.bindings) { + const bool table = binding.role == Role::ImageTable || binding.role == Role::ImageStorageTable; + const bool image = binding.role == Role::GuestImages && (binding.kind == Kind::SampledImage || binding.kind == Kind::StorageImage); + if (table || image) demand += binding.count; + if (table && exclusions.tables) return false; + if (image && binding.kind == Kind::StorageImage && exclusions.storage) return false; + if (image && binding.kind == Kind::SampledImage && exclusions.sampled) return false; + } + return demand <= Graphics::BindlessHeap::ImageSlotCapacity(); +} + +} + +bool HeapStageExcluded(ShaderRecompiler::ShaderStage stage) { + return (heapExclusions().stages >> static_cast(stage) & 1u) != 0u; +} + +bool HeapStageMappable(const ShaderRecompiler::RecompileResult& result, bool workgroupAddress) { + struct MappableStorage {}; + auto& known = HostThreadLocal, MappableStorage>(); + const auto key = result.variantId * 2u + (workgroupAddress ? 1u : 0u); + if (const auto found = known.find(key); found != known.end()) return found->second; + if (known.size() >= ThreadCacheLimit) known.clear(); + const bool mappable = heapClassesAllowed(result) && + Graphics::DescriptorHeapMappable(std::span(result.spirv.data(), result.spirv.size()), workgroupAddress, result.dataAddressPush, result.functionLdsBuffer, result.meshWindowBuffer, result.scratchPoolSlotDwords != 0u); + known.emplace(key, mappable); + return mappable; +} + +namespace { + +struct HeapFragmentKey { + const void* snapshot; + std::size_t codeOffset; + bool operator==(const HeapFragmentKey&) const = default; +}; + +struct HeapFragmentKeyHash { + std::size_t operator()(const HeapFragmentKey& key) const { + return std::hash{}(key.snapshot) ^ (key.codeOffset * 0x9e3779b97f4a7c15ull); + } +}; + +std::unordered_map& heapFragments() { + struct FragmentsStorage {}; + auto& value = HostThreadLocal, FragmentsStorage>(); + return value; +} + +} + +std::optional HeapFragmentKnown(const void* snapshot, std::size_t codeOffset) { + const auto& known = heapFragments(); + const auto found = known.find({snapshot, codeOffset}); + return found == known.end() ? std::nullopt : std::optional(found->second); +} + +void NoteHeapFragment(const void* snapshot, std::size_t codeOffset, bool eligible) { + heapFragments()[{snapshot, codeOffset}] = eligible; +} + +bool SampleHeapStageVerify() { + static const std::uint64_t every = [] { + const char* text = std::getenv("APS5_VERIFY_GPU_HEAP"); + if (text == nullptr) return std::uint64_t{0}; + const auto value = std::strtoull(text, nullptr, 10); + return value == 0 ? std::uint64_t{1} : static_cast(value); + }(); + if (every == 0) return false; + thread_local std::uint64_t calls = 0; + return calls++ % every == 0; +} + +namespace { + +template +std::uint64_t vectorBytes(const std::vector& values) { + return static_cast(values.capacity()) * sizeof(TValue); +} + +std::uint64_t vectorBytes(const std::vector& values) { + return static_cast(values.capacity() + 7u) / 8u; +} + +template +std::uint64_t sharedBytes(const ShaderRecompiler::SharedValues& values) { + return static_cast(values.size()) * sizeof(TValue); +} + +std::uint64_t descriptorBindingBytes(const ShaderRecompiler::DescriptorBinding& binding) { + return vectorBytes(binding.guestDescriptor) + vectorBytes(binding.samplerDepthCompare) + vectorBytes(binding.imageWritten) + + vectorBytes(binding.imageDepthCompare) + vectorBytes(binding.imageAtomic) + vectorBytes(binding.imageAtomic64) + + vectorBytes(binding.imageShapes) + vectorBytes(binding.bufferAtomic) + vectorBytes(binding.bufferWritten) + + vectorBytes(binding.samplerUnnormalized) + vectorBytes(binding.samplerIntegerBorder) + vectorBytes(binding.samplerCoordinatesIgnored) + + vectorBytes(binding.imageUnnormalized) + vectorBytes(binding.imageUnnormalizedBaseLevel) + vectorBytes(binding.imageSamplers) + + vectorBytes(binding.bufferRead) + vectorBytes(binding.bufferExtent); +} + +} + +std::uint64_t HeapWalkResultBytes(const ShaderRecompiler::RecompileResult& result) { + auto bytes = sizeof(ShaderRecompiler::RecompileResult) + vectorBytes(result.vertexInputPatches) + vectorBytes(result.runtimeImageResources) + + vectorBytes(result.vertexInputs) + vectorBytes(result.parameterExports) + vectorBytes(result.fragmentParameters) + + vectorBytes(result.stencilReferences) + vectorBytes(result.specialization) + vectorBytes(result.heapSlotWords) + + vectorBytes(result.bindings) + vectorBytes(result.pushConstants) + vectorBytes(result.vertexAttributes) + + vectorBytes(result.imageTablePoison) + vectorBytes(result.imageTableRanges) + vectorBytes(result.imageTableReadRanges) + + vectorBytes(result.heapUserWords) + vectorBytes(result.shaderDataUserWords); + if (result.meshBarycentrics.has_value()) bytes += vectorBytes(result.meshBarycentrics->parameters); + for (const auto& binding : result.bindings) bytes += descriptorBindingBytes(binding); + for (const auto& ranges : result.imageTableReadRanges) bytes += vectorBytes(ranges); + return bytes; +} + +std::uint64_t HeapWalkCaptureBytes(const ShaderRecompiler::ResourceCapture& capture) { + const auto& snapshot = capture.snapshot; + auto bytes = sizeof(ShaderRecompiler::ResourceCapture) + vectorBytes(snapshot.buffers) + vectorBytes(snapshot.images) + + vectorBytes(snapshot.samplers) + vectorBytes(snapshot.flattenedSrt) + vectorBytes(snapshot.userData) + + vectorBytes(snapshot.srtPoison) + vectorBytes(snapshot.tables.tables) + vectorBytes(snapshot.tables.ranges) + + vectorBytes(capture.readTrace.leaves) + vectorBytes(capture.readTrace.otherReads) + vectorBytes(capture.userDataUse.reads); + for (const auto& column : snapshot.tables.tables) + bytes += sharedBytes(column.codes) + sharedBytes(column.words) + sharedBytes(column.readRanges); + return bytes; +} + +std::uint64_t HeapWalkWordsBytes(const ShaderRecompiler::HeapStageWords& words) { + auto bytes = sizeof(ShaderRecompiler::HeapStageWords) + vectorBytes(words.userData) + vectorBytes(words.flattenedSrt) + + vectorBytes(words.vertexAttributes) + vectorBytes(words.bindings); + for (const auto& binding : words.bindings) bytes += vectorBytes(binding); + return bytes; +} + +std::uint64_t HeapWalkEntryOwnBytes(const HeapWalkEntry& entry) { + return sizeof(HeapWalkEntry) + vectorBytes(entry.userData) + vectorBytes(entry.ranges) + vectorBytes(entry.values) + + vectorBytes(entry.accessible) + vectorBytes(entry.userDataUse.reads); +} + +std::uint64_t HeapWalkEntryBytes(const HeapWalkEntry& entry) { + auto bytes = HeapWalkEntryOwnBytes(entry); + if (entry.capture != nullptr) bytes += HeapWalkCaptureBytes(*entry.capture); + if (entry.result != nullptr) bytes += HeapWalkResultBytes(*entry.result); + if (entry.words != nullptr) bytes += HeapWalkWordsBytes(*entry.words); + return bytes; +} + +std::uint64_t HeapWalkMemoBudget() { + static const std::uint64_t budget = [] { + const char* value = std::getenv("APS5_HEAP_WALK_MEMO_MB"); + const std::uint64_t megabytes = value == nullptr || *value == '\0' ? DefaultWalkMemoMiB : std::strtoull(value, nullptr, 10); + return megabytes << 20u; + }(); + return budget; +} + +void HeapWalkMemo::trim() { + retirePrevious(); + if (heldBytes <= budget) return; + std::vector> stamps; + for (const auto& [key, ways] : current) { + for (const auto& entry : ways) stamps.emplace_back(entry.stored, key); + } + std::sort(stamps.begin(), stamps.end()); + const auto target = budget / 4u * 3u; + for (const auto& [stored, key] : stamps) { + if (heldBytes <= target) break; + const auto ways = current.find(key); + const auto entry = std::find_if(ways->second.begin(), ways->second.end(), [&](const HeapWalkEntry& way) { return way.stored == stored; }); + drop(*entry); + ways->second.erase(entry); + if (ways->second.empty()) current.erase(ways); + } +} + +namespace { + +HeapWalkMemo& heapWalkMemo() { + struct WalkMemoStorage {}; + auto& memo = HostThreadLocal(); + return memo; +} + +} + +namespace { + +using AddressRanges = std::vector>; + +struct WalkReads { + std::vector words; + std::uint64_t page = ~std::uint64_t{0}; + AddressRanges accessible; + bool failed = false; + + void Reset() { + words.clear(); + accessible.clear(); + failed = false; + page = ~std::uint64_t{0}; + } +}; + +bool readWalkWord(void* context, std::uint64_t address, std::uint32_t* value) { + auto& reads = *static_cast(context); + const auto page = address & ~(WalkPageBytes - 1u); + const bool crossesPage = (address & (WalkPageBytes - 1u)) > WalkPageBytes - sizeof(*value); + if (page != reads.page || crossesPage) { + if (GuestMemory::Accessible(reinterpret_cast(page), WalkPageBytes)) reads.page = page; + else if (!GuestMemory::Accessible(reinterpret_cast(address), sizeof(*value))) { + reads.failed = true; + return false; + } + } + std::memcpy(value, reinterpret_cast(address), sizeof(*value)); + reads.words.push_back(address); + return true; +} + +bool walkAccessible(void* context, std::uint64_t address, std::uint64_t bytes) { + auto& reads = *static_cast(context); + const bool accessible = GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes)); + if (accessible) reads.accessible.emplace_back(address, bytes); + else reads.failed = true; + return accessible; +} + +void coalesceWords(std::vector& words, AddressRanges& ranges) { + std::sort(words.begin(), words.end()); + ranges.clear(); + for (const auto address : words) { + if (!ranges.empty() && address <= ranges.back().second) { + ranges.back().second = std::max(ranges.back().second, address + WordBytes); + continue; + } + ranges.emplace_back(address, address + WordBytes); + } +} + +void copyRangeValues(const AddressRanges& ranges, std::vector& values) { + values.clear(); + for (const auto& [begin, end] : ranges) { + const auto offset = values.size(); + values.resize(offset + static_cast((end - begin) / WordBytes)); + std::memcpy(values.data() + offset, reinterpret_cast(begin), static_cast(end - begin)); + } +} + +bool rangesSettled(const AddressRanges& ranges, const Graphics::Recorder::WriteRanges* before, std::uint64_t generation) { + const bool queuedWrite = DrawPipeline::Active() && DrawPipeline::Queue0().Busy() && DrawPipeline::Queue0().OverlapsAny(ranges); + if (queuedWrite || !Graphics::Recorder::ReadsSettled(ranges, before)) return false; + for (const auto& [begin, end] : ranges) { + if (GuestMemory::NewestStamp(begin, static_cast(end - begin)) > generation) return false; + } + return true; +} + +void guardRanges(const AddressRanges& ranges) { + for (const auto& [begin, end] : ranges) GuestMemory::GuardRead(begin, static_cast(end - begin)); +} + +std::uint64_t walkMemoKey(const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span userData, + std::uint64_t shaderBase) { + auto key = (invocation.VariantId() * 0x9e3779b97f4a7c15ull ^ shaderBase) + invocation.Request().layout.pushConstantOffsetBytes; + for (const auto word : userData) key = (key ^ word) * 0x100000001b3ull; + return key; +} + +bool sameWalkInputs(const HeapWalkEntry& entry, const ShaderRecompiler::PreparedShaderInvocation& invocation, + std::span userData, std::uint64_t shaderBase) { + const bool sameStage = entry.variant == invocation.VariantId() && entry.shaderBase == shaderBase && + entry.pushOffset == invocation.Request().layout.pushConstantOffsetBytes; + if (!sameStage || !std::equal(entry.userData.begin(), entry.userData.end(), userData.begin(), userData.end())) return false; + std::size_t cursor = 0; + for (const auto& [begin, end] : entry.ranges) { + const auto count = static_cast((end - begin) / WordBytes); + if (GuestMemory::CompareMapped(begin, std::as_bytes(std::span(entry.values.data() + cursor, count))) != GuestMemory::Compare::Equal) + return false; + cursor += count; + } + for (const auto& [address, bytes] : entry.accessible) { + if (!GuestMemory::Accessible(reinterpret_cast(address), static_cast(bytes))) return false; + } + return true; +} + +void noteMemoBytes(const HeapWalkMemo& memo, std::uint64_t heldBefore) { + Graphics::NoteHeapWalkMemoBytes(static_cast(memo.heldBytes) - static_cast(heldBefore)); +} + +void rememberWalk(std::uint64_t memoKey, const ShaderRecompiler::PreparedShaderInvocation& invocation, + std::span userData, std::uint64_t shaderBase, const AddressRanges& ranges, + const std::vector& values, const AddressRanges& accessible, + const std::shared_ptr& capture, + const std::shared_ptr& result, const ShaderRecompiler::HeapStageWords& words) { + HeapWalkEntry entry; + entry.variant = invocation.VariantId(); + entry.pushOffset = invocation.Request().layout.pushConstantOffsetBytes; + entry.shaderBase = shaderBase; + entry.userData.assign(userData.begin(), userData.end()); + entry.ranges = ranges; + entry.values = values; + entry.accessible = accessible; + entry.userDataUse = capture->userDataUse; + entry.capture = capture; + entry.result = result; + entry.words = std::make_shared(words); + auto& memo = heapWalkMemo(); + const auto heldBefore = memo.heldBytes; + memo.remember(memoKey, std::move(entry)); + noteMemoBytes(memo, heldBefore); +} + +bool adoptWalk(std::uint64_t memoKey, const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span userData, + std::uint64_t shaderBase, const Graphics::Recorder::WriteRanges* before, std::uint64_t generation, + HeapStageMaterialized& adopted, bool& memoKnown) { + auto& memo = heapWalkMemo(); + const auto heldBefore = memo.heldBytes; + auto* found = memo.find(memoKey); + noteMemoBytes(memo, heldBefore); + memoKnown = found != nullptr; + for (std::size_t way = 0; found != nullptr && way < found->size(); ++way) { + auto& entry = (*found)[way]; + if (!sameWalkInputs(entry, invocation, userData, shaderBase)) continue; + if (!rangesSettled(entry.ranges, before, generation)) break; + memo.touch(entry); + guardRanges(entry.ranges); + adopted.words = entry.words; + adopted.reads = {entry.ranges, entry.values, entry.userDataUse, entry.capture, true}; + adopted.result = entry.result; + if (way != 0) + std::rotate(found->begin(), found->begin() + static_cast(way), + found->begin() + static_cast(way) + 1); + return true; + } + return false; +} + +std::shared_ptr +captureWalk(const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span userData, std::uint64_t shaderBase, + WalkReads& reads) { + ShaderRecompiler::SrtRuntime runtime; + runtime.userData = userData; + runtime.shaderBase = shaderBase; + runtime.userContext = &reads; + runtime.readMemory = readWalkWord; + runtime.accessible = walkAccessible; + return invocation.Capture(runtime); +} + +} + +HeapStageMaterialized MaterializeHeapStage(const ShaderRecompiler::PreparedShaderInvocation& invocation, std::span userData, + std::uint64_t shaderBase) { + struct ReadsStorage {}; + struct RangesStorage {}; + struct ValuesStorage {}; + auto& reads = HostThreadLocal(); + auto& ranges = HostThreadLocal(); + reads.Reset(); + Graphics::HeapCostScope snapshotCost(Graphics::HeapCost::Guards); + const auto before = Graphics::Recorder::PendingWriteSnapshot(); + snapshotCost.Stop(); + const auto generation = GuestMemory::TrackerGeneration(); + const bool memoOn = !invocation.Request().context.vertex.has_value(); + std::uint64_t memoKey = 0; + bool memoKnown = false; + + Graphics::HeapCostScope memoCost(Graphics::HeapCost::WalkMemo); + if (memoOn) { + memoKey = walkMemoKey(invocation, userData, shaderBase); + HeapStageMaterialized adopted; + if (adoptWalk(memoKey, invocation, userData, shaderBase, before.get(), generation, adopted, memoKnown)) + return adopted; + } + memoCost.Stop(); + + Graphics::HeapCostScope walkCost(Graphics::HeapCost::Walk); + auto capture = captureWalk(invocation, userData, shaderBase, reads); + coalesceWords(reads.words, ranges); + auto& values = HostThreadLocal, ValuesStorage>(); + if (!reads.failed) copyRangeValues(ranges, values); + const bool settled = rangesSettled(ranges, before.get(), generation); + walkCost.Stop(); + if (!settled) return {}; + + Graphics::HeapCostScope guardCost(Graphics::HeapCost::Guards); + guardRanges(ranges); + guardCost.Stop(); + Graphics::HeapCostScope materializeCost(Graphics::HeapCost::Materialize); + auto words = std::make_shared(); + auto result = invocation.MaterializeHeap(*capture, *words); + if (result == nullptr) result = invocation.Materialize(*capture); + materializeCost.Stop(); + const bool complete = !reads.failed && result != nullptr; + HeapStageMaterialized materialized; + if (complete) materialized.reads = {ranges, values, capture->userDataUse, capture, true}; + Graphics::HeapCostScope insertCost(Graphics::HeapCost::MemoInsert); + if (memoOn && complete && (memoKnown || heapWalkMemo().admit(memoKey))) + rememberWalk(memoKey, invocation, userData, shaderBase, ranges, values, reads.accessible, capture, result, *words); + materialized.result = std::move(result); + materialized.words = std::move(words); + return materialized; +} + +namespace { + +std::atomic unmappedStage{-1}; + +} + +void SetHeapStageUnmapped(std::optional stage) { + unmappedStage.store(stage.has_value() ? static_cast(*stage) : -1, std::memory_order_relaxed); +} + +bool HeapDrawStageMappable(const ShaderRecompiler::RecompileResult& result, ShaderRecompiler::ShaderStage stage) { + return unmappedStage.load(std::memory_order_relaxed) != static_cast(stage) && !HeapStageExcluded(stage) && HeapStageMappable(result); +} + +std::uint64_t HeapStagesTaken() { + return heapStages().load(); +} + +std::uint64_t HeapRingHitsVerified() { + return verifyCounts().ring.load(); +} + +std::uint64_t HeapRingHitsDiffering() { + return verifyCounts().ringDiffer.load(); +} + +std::shared_ptr PatchedHeapResult(const ShaderRecompiler::RecompileResult& shared, + const ShaderRecompiler::HeapStageWords& words) { + auto result = std::make_shared(shared); + for (auto& binding : result->bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt && !words.flattenedSrt.empty()) + binding.guestDescriptor = words.flattenedSrt; + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) + std::copy_n(words.userData.begin(), std::min(words.userData.size(), binding.guestDescriptor.size()), + binding.guestDescriptor.begin()); + } + if (!words.bindings.empty()) { + if (words.bindings.size() != result->bindings.size()) + throw std::runtime_error("a heap stage's per-draw bindings differ in number from its template"); + for (std::size_t index = 0; index < words.bindings.size(); ++index) { + if (words.bindings[index].empty()) continue; + if (result->bindings[index].role != ShaderRecompiler::DescriptorRole::FlattenedSrt && + words.bindings[index].size() != result->bindings[index].guestDescriptor.size()) + throw std::runtime_error("a heap stage's per-draw descriptor words differ in size from its template"); + result->bindings[index].guestDescriptor = words.bindings[index]; + } + } + if (!words.vertexAttributes.empty()) result->vertexAttributes = words.vertexAttributes; + return result; +} + +void NoteHeapDirect() { + heapStages().fetch_add(1, std::memory_order_relaxed); +} + +void NoteHeapDirectVerify(const ShaderRecompiler::RecompileResult& direct, const ShaderRecompiler::RecompileResult& expected, + const ShaderRecompiler::ResourceCapture* expectedCapture) { + auto& counts = verifyCounts(); + if (expectedCapture != nullptr) { + const auto partial = [](const std::vector& values) { + return std::ranges::any_of(values, [](const ShaderRecompiler::DescriptorValue& value) { + return value.availableDwordMask != static_cast((1u << value.dwordCount) - 1u); + }); + }; + const auto& snapshot = expectedCapture->snapshot; + if (partial(snapshot.buffers) || partial(snapshot.images) || partial(snapshot.samplers)) { + counts.notComparable.fetch_add(1, std::memory_order_relaxed); + return; + } + } + counts.verified.fetch_add(1, std::memory_order_relaxed); + std::size_t index = 0; + if (const auto* what = resultDifference(direct, expected, index)) { + counts.differ.fetch_add(1, std::memory_order_relaxed); + reportHeapMismatch("heap stage vs full capture", what, expected, index); + } +} + +void VerifyHeapAccessor(const Graphics::CompiledShader& stage) { + if (stage.heap == nullptr || stage.program == nullptr) return; + const auto patched = PatchedHeapResult(*stage.program, *stage.heap); + auto& counts = verifyCounts(); + counts.accessor.fetch_add(1, std::memory_order_relaxed); + for (std::size_t index = 0; index < stage.program->bindings.size(); ++index) { + if (Graphics::DrawGuestWords(stage, stage.program->bindings[index]) == patched->bindings[index].guestDescriptor) continue; + counts.accessorDiffer.fetch_add(1, std::memory_order_relaxed); + reportHeapMismatch("accessor vs patched result", "binding words", *patched, index); + } + const auto& attributes = Graphics::DrawVertexAttributes(stage); + bool sameAttributes = attributes.size() == patched->vertexAttributes.size(); + for (std::size_t index = 0; sameAttributes && index < attributes.size(); ++index) + sameAttributes = attributes[index].resource.fields == patched->vertexAttributes[index].resource.fields && + attributes[index].fetchIndex == patched->vertexAttributes[index].fetchIndex; + if (!sameAttributes) { + counts.accessorDiffer.fetch_add(1, std::memory_order_relaxed); + reportHeapMismatch("accessor vs patched result", "vertex attributes", *patched, 0); + } +} + +void VerifyRingWords(const ShaderRecompiler::RecompileResult& kept, const ShaderRecompiler::HeapStageWords& ring, + const ShaderRecompiler::RecompileResult& fresh, const ShaderRecompiler::HeapStageWords* freshWords) { + const auto reused = PatchedHeapResult(kept, ring); + const auto expected = + freshWords != nullptr ? PatchedHeapResult(fresh, *freshWords) : std::make_shared(fresh); + auto& counts = verifyCounts(); + counts.ring.fetch_add(1, std::memory_order_relaxed); + std::size_t index = 0; + if (const auto* what = resultDifference(*reused, *expected, index)) { + counts.ringDiffer.fetch_add(1, std::memory_order_relaxed); + reportHeapMismatch("ring hit vs fresh build", what, *expected, index); + } +} + +namespace { + +using ExpectedWords = std::vector>; + +void syncPipelinedWrites(std::uint64_t address, std::size_t bytes) { + if (!DrawPipeline::Active()) return; + auto& pipeline = DrawPipeline::Queue0(); + if (pipeline.Busy() && pipeline.Overlaps(address, bytes)) pipeline.Drain(DrawPipeline::DrainReason::Capture); +} + +bool readExpectedWord(void* context, std::uint64_t address, std::uint32_t* value) { + if (!GuestMemory::Accessible(reinterpret_cast(address), sizeof(*value))) return false; + syncPipelinedWrites(address, sizeof(*value)); + GuestMemory::Read(address, std::as_writable_bytes(std::span(value, 1)), sizeof(*value)); + static_cast(context)->emplace_back(address, *value); + return true; +} + +struct ExpectationState { + std::uint64_t epoch = 0; + std::uint64_t unwatch = 0; + std::uint64_t ownWrites = 0; + std::uint64_t pending = 0; + std::uint64_t published = 0; + std::uint64_t forgotten = 0; + std::uint64_t allocations = 0; + + bool operator==(const ExpectationState& other) const = default; +}; + +ExpectationState expectationState() { + return {GuestMemory::ThreadCollectEpoch(), GuestMemory::UnwatchSerial(), GuestMemory::OwnWriteSerial(), Graphics::StorageTexture::PendingSerial(), Graphics::Recorder::PublishGeneration(), GuestMemory::ForgetSerial(), GuestAllocations::GuestAllocationsGeneration_nid_postfix()}; +} + +struct ExpectationMemo { + std::shared_ptr plan; + std::uint64_t codeAddress = 0; + std::vector userData; + std::vector images; + ExpectedWords words; + ExpectationState state; +}; + +constexpr std::size_t ExpectationMemoEntries = 16384; +constexpr std::size_t ExpectedRunWords = 64; + +std::unordered_map& expectationMemos() { + struct ExpectationMemoStorage {}; + return HostThreadLocal, ExpectationMemoStorage>(); +} + +std::uint64_t expectationKey(const ShaderRecompiler::IrResourcePlan* plan, std::uint64_t codeAddress, std::span userData) { + std::uint64_t hash = 0xcbf29ce484222325ull; + const auto mix = [&](std::uint64_t value) { hash = (hash ^ value) * 0x100000001b3ull; }; + mix(static_cast(reinterpret_cast(plan))); + mix(codeAddress); + for (const auto word : userData) mix(word); + return hash; +} + +template +bool forEachExpectedRun(const ExpectedWords& words, Visit visit) { + for (std::size_t first = 0; first < words.size();) { + auto last = first + 1u; + while (last < words.size() && last - first < ExpectedRunWords && words[last].first == words[last - 1u].first + sizeof(std::uint32_t)) ++last; + if (!visit(first, last)) return false; + first = last; + } + return true; +} + +bool pipelineWritesExpectedWords(const ExpectedWords& words) { + if (!DrawPipeline::Active() || !DrawPipeline::Queue0().Busy()) return false; + auto& pipeline = DrawPipeline::Queue0(); + return !forEachExpectedRun(words, [&](std::size_t first, std::size_t last) { return !pipeline.Overlaps(words[first].first, (last - first) * sizeof(std::uint32_t)); }); +} + +bool expectedWordsUnchanged(const ExpectedWords& words) { + std::array current{}; + return forEachExpectedRun(words, [&](std::size_t first, std::size_t last) { + const auto address = words[first].first; + const auto bytes = (last - first) * sizeof(std::uint32_t); + if (!GuestMemory::Accessible(reinterpret_cast(address), bytes)) return false; + syncPipelinedWrites(address, bytes); + GuestMemory::Read(address, std::as_writable_bytes(std::span(current.data(), last - first)), sizeof(std::uint32_t)); + for (auto index = first; index < last; ++index) { + if (current[index - first] != words[index].second) return false; + } + return true; + }); +} + +} + +bool ExpectDeviceImages(const VulkanDevice& device, std::span userData, std::uint64_t codeAddress, const ShaderRecompiler::RecompileResult& result, bool dispatch) { + namespace Abi = ShaderRecompiler::DeviceImageTableAbi; + if (!result.deviceImageTables || !Graphics::DeviceImageTable::ExpectsKeys()) return true; + device.NoteDeviceImageProgram(result, codeAddress); + if (result.deviceImagePlan == nullptr) { + device.FallBackDeviceImages(result, codeAddress, "its result carries no plan to expect its device images from"); + return false; + } + auto& memos = expectationMemos(); + const auto key = expectationKey(result.deviceImagePlan.get(), codeAddress, userData); + const auto state = expectationState(); + std::vector images; + bool memoHit = false; + if (const auto found = memos.find(key); found != memos.end() && found->second.plan == result.deviceImagePlan && found->second.codeAddress == codeAddress && std::ranges::equal(found->second.userData, userData)) { + auto& memo = found->second; + if ((state.epoch != 0u && memo.state == state && !pipelineWritesExpectedWords(memo.words)) || expectedWordsUnchanged(memo.words)) { + memo.state = state; + images = memo.images; + memoHit = true; + } + } + if (!memoHit) { + ExpectedWords words; + ShaderRecompiler::SrtRuntime runtime; + runtime.userData = userData; + runtime.shaderBase = codeAddress; + runtime.readMemory = &readExpectedWord; + runtime.userContext = &words; + bool complete = false; + images = ShaderRecompiler::ResourceMaterializer::ExpectedDeviceImages(*result.deviceImagePlan, runtime, complete); + if (!complete) { + device.FallBackDeviceImages(result, codeAddress, "a device image use whose T# cannot be evaluated before the work is recorded (a key from a lane, from memory the CPU cannot read, or more than " + std::to_string(ShaderRecompiler::ResourceMaterializer::MaxDeviceImageCandidates) + " candidates)"); + return false; + } + std::ranges::sort(words); + words.erase(std::unique(words.begin(), words.end()), words.end()); + if (memos.size() >= ExpectationMemoEntries) memos.clear(); + memos.insert_or_assign(key, ExpectationMemo{result.deviceImagePlan, codeAddress, {userData.begin(), userData.end()}, images, std::move(words), state}); + } + Abi::NameInterfaces(images, result.deviceImageInterfaces); + Abi::NameSamplerClasses(images, result.deviceSamplerClasses); + std::optional unserved; + { + GuestMemory::TagGpuLockSite(dispatch ? GuestMemory::GpuLockSite::Dispatch : GuestMemory::GpuLockSite::Draw); + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + unserved = device.ExpectDeviceImages(images); + } + if (!unserved.has_value()) return true; + device.FallBackDeviceImages(result, codeAddress, *unserved); + return false; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp index fa1676e68c33..f53baccda375 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp @@ -22,7 +22,7 @@ void Driver::FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, co if (result.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(program, result.instanceOffsetSgpr); } -std::optional Driver::ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) { +std::optional Driver::ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect, const ShaderRecompiler::HeapStageWords* heap) { std::optional indirectCpu; auto& indirect = *drawParameters.indirect; using Rule = Pm4::DrawParameters::IndirectDraw::Rule; @@ -57,7 +57,7 @@ std::optional Driver::ClassifyIndirectDraw(const Sha if ((indirect.vertexRule == Rule::Constant || indirect.instanceRule == Rule::Constant) && indirect.count > 256) fail(Path::FeatureGap); static const std::uint64_t vertexCap = [] { const char* text = std::getenv("APS5_INDIRECT_VERTEX_MIB"); return (text ? std::strtoull(text, nullptr, 10) : 64ull) << 20u; }(); - for (const auto& attribute : result.vertexAttributes) { + for (const auto& attribute : heap != nullptr && !heap->vertexAttributes.empty() ? heap->vertexAttributes : result.vertexAttributes) { const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; const auto extent = stride == 0 ? static_cast(attribute.resource.fields[2]) : static_cast(attribute.resource.fields[2]) * stride; if (extent > vertexCap) fail(Path::VertexRange); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index 93598b3b93ea..b4ec0db2592a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DrawWriteRanges.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -17,13 +18,14 @@ namespace AgcDriver::DriverDetail { namespace { template -void visitWrittenBuffers(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit) { +void visitWrittenBuffers(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit, const ShaderRecompiler::HeapStageWords* heap = nullptr) { for (const auto& binding : compiled.bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { + const auto& guestWords = Graphics::DrawGuestWords(compiled, heap, binding); const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; - if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 4) continue; - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4); + if (!written || guestWords.size() < (static_cast(element) + 1) * 4) continue; + const auto words = std::span(guestWords).subspan(static_cast(element) * 4, 4); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; const auto base = descriptor.Base48(); const auto size = descriptor.GetSize(); @@ -86,7 +88,7 @@ void Driver::noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint void Driver::noteDrawWriters(std::span stages, std::uint32_t queue) { for (const auto& stage : stages) { - forEachWrittenBuffer(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { noteForeignWriter(begin, end, queue); }); + visitWrittenBuffers(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { noteForeignWriter(begin, end, queue); }, stage.heap); } } @@ -104,13 +106,14 @@ std::vector> DrawWriteRanges(const Graph }; for (const auto& stage : stages) { if (stage.program == nullptr) continue; - visitWrittenBuffers(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); + visitWrittenBuffers(*stage.program, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }, stage.heap); for (const auto& binding : stage.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages || binding.kind != ShaderRecompiler::DescriptorKind::StorageImage) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { const bool written = element >= binding.imageWritten.size() || binding.imageWritten[element]; - if (!written || binding.guestDescriptor.size() < (static_cast(element) + 1) * 8) continue; - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8, 8); + const auto& guestWords = Graphics::DrawGuestWords(stage, binding); + if (!written || guestWords.size() < (static_cast(element) + 1) * 8) continue; + const auto words = std::span(guestWords).subspan(static_cast(element) * 8, 8); if (Graphics::IsNullTextureResource(words)) continue; const auto resource = Graphics::DecodeTextureResource(words); if (resource.baseAddress == 0) continue; @@ -169,16 +172,18 @@ std::vector> QueuedDrawReadRanges(const for (const auto& stage : stages) { if (stage.program == nullptr) continue; for (const auto& binding : stage.program->bindings) { - if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers || binding.guestDescriptor.size() < static_cast(binding.count) * 4u) continue; + if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + const auto& guest = Graphics::DrawGuestWords(stage, binding); + if (guest.size() < static_cast(binding.count) * 4u) continue; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; + const auto* words = guest.data() + static_cast(element) * 4u; const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; add(descriptor.Base48(), descriptor.GetSize()); } } } if (stages.empty() || stages.front().program == nullptr) return ranges; - for (const auto& attribute : stages.front().program->vertexAttributes) { + for (const auto& attribute : Graphics::DrawVertexAttributes(stages.front())) { if (Graphics::NullVertexDescriptor(attribute)) continue; const auto address = attribute.resource.fields[0] | (static_cast(attribute.resource.fields[1] & 0xffffu) << 32u); std::uint64_t bytes = 0; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 5c42933ab942..af303c221309 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -6,6 +6,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawPipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "CacheKey.hpp" @@ -1129,6 +1130,12 @@ std::shared_ptr LinkUserDataCallees(const S } +bool PreparedHeapStage(const ShaderSnapshot& snapshot, std::size_t codeOffset) { + if (snapshot.prepared == nullptr) return false; + std::lock_guard lock(snapshot.prepared->mutex); + return std::ranges::any_of(snapshot.prepared->entries, [&](const auto& entry) { return entry.codeOffset == codeOffset && entry.handle != nullptr && entry.handle->artifact != nullptr && ShaderRecompiler::GetPreparedArtifact(*entry.handle).gpuResolvedImages; }); +} + ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); const auto code = std::span(snapshot.code).subspan(codeOffset); @@ -1156,7 +1163,7 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s if (entry.codeOffset != codeOffset) continue; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); if (!linked.code.empty() && !std::ranges::equal(invocationRequest.shader.code, linked.code)) continue; - if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) { + if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(std::move(invocationRequest), entry.handle, key)) { NoteMatchedEntry(entry, preparedAtUse); return std::move(*invocation); } @@ -1166,7 +1173,7 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s auto handle = PrepareShaderWithDiagnostics(source); invocationRequest = source; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle); - auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key); + auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(std::move(invocationRequest), handle, key); if (!invocation.has_value()) throw std::runtime_error("AGC driver: an artifact prepared at use does not match its invocation"); snapshot.prepared->entries.push_back({codeOffset, std::move(handle)}); return std::move(*invocation); @@ -1402,6 +1409,7 @@ std::unique_ptr PlanRegistered(const ShaderSnapshot& snap } plan->memory = {{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; auto& request = plan->request.emplace(ShaderRecompiler::RecompileRequest{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, plan->userData, compute, pixel, vertex, plan->memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics}); + if (request.target.deviceImageTables) request.target.deviceImageTables = device.DeviceImagesServe(address); if (graphics && graphics->mesh) request.layout.pushConstantSizeBytes = ShaderRecompiler::MeshDrawPushOffsetBytes; if (graphics && graphics->tessellation) request.layout.pushConstantSizeBytes = ShaderRecompiler::TessellationLevelPushOffsetBytes; if (pixel && device.GraphicsPipelineLibraries()) request.layout.pushConstantOffsetBytes = Graphics::FixedPushOffset(Stage::Fragment); @@ -1418,6 +1426,7 @@ bool LibraryPrebuildDisabled() { void PrebuildStageLibrary(const std::shared_ptr& device, const Graphics::ShaderStages& stages, ShaderRecompiler::ShaderStage stage, std::shared_ptr result, std::uint32_t pushOffset, std::uint64_t program) { if (LibraryPrebuildDisabled() || result == nullptr || (stage != ShaderRecompiler::ShaderStage::Vertex && stage != ShaderRecompiler::ShaderStage::Fragment)) return; + if (result->gpuResolvedImages && !HeapStageMappable(*result)) return; SubmitLibraryPrebuild([device, stages, stage, result = std::move(result), pushOffset] { const std::array shaders{{{stage, result.get(), pushOffset}}}; try { @@ -1437,13 +1446,30 @@ std::vector PrepareRegistered(RegisteredPreparation& pla entries.push_back({plan.codeOffset, PrepareShaderWithDiagnostics(request, &preparation)}); }; append(); - if (plan.prebuildDevice != nullptr && request.target.registrationModules && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment && !LibraryPrebuildDisabled()) { + const bool fragment = request.shader.stage == ShaderRecompiler::ShaderStage::Fragment; + const bool heapVertex = HeapPreRasterStage(request.shader.stage) && request.target.descriptorMemory; + const bool vertex = request.shader.stage == ShaderRecompiler::ShaderStage::Vertex; + const auto prebuild = [&](const ShaderRecompiler::RecompileRequest& stageRequest, const ShaderRecompiler::SourceHandle& handle) { + if (plan.prebuildDevice == nullptr || !stageRequest.target.registrationModules || !(fragment || (vertex && heapVertex && stageRequest.target.gpuResolvedImages)) || LibraryPrebuildDisabled()) return; try { - PrebuildStageLibrary(plan.prebuildDevice, {Graphics::ShaderPath::Vertex, 0u, 32u, request.context.waveSize, {}, {}}, request.shader.stage, ShaderRecompiler::RegistrationResult(request, *entries.front().handle), request.layout.pushConstantOffsetBytes, plan.program); + PrebuildStageLibrary(plan.prebuildDevice, {Graphics::ShaderPath::Vertex, 0u, 32u, stageRequest.context.waveSize, {}, {}}, stageRequest.shader.stage, ShaderRecompiler::RegistrationResult(stageRequest, handle), stageRequest.layout.pushConstantOffsetBytes, plan.program); } catch (const std::exception& error) { APS5_LOG_OUT("shader library prebuild skipped: %s", error.what()); } + }; + bool heapPrebuilt = false; + if ((fragment || heapVertex) && !HeapStageExcluded(request.shader.stage) && HeapFragmentsEnabled()) { + const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(request, entries.front().handle); + if (invocation && HeapStageEligible(request.shader.stage, invocation->Plan().get(), heapVertex)) { + auto heapRequest = request; + heapRequest.target.gpuResolvedImages = true; + PerformanceTimer timing("Shader.PrepareArtifact"); + entries.push_back({plan.codeOffset, PrepareShaderWithDiagnostics(heapRequest, &preparation)}); + prebuild(heapRequest, *entries.back().handle); + heapPrebuilt = request.target.descriptorMemory; + } } + if (!heapPrebuilt) prebuild(request, *entries.front().handle); if (request.context.compute) { request.context.compute->partialThreads = {1, 1, 1}; append(); @@ -1922,6 +1948,24 @@ std::optional RegistrationPreparationsPending() { std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target, bool deferUnprepared) { ShaderPreparationTransaction transaction; + const auto preparedHandle = [&](const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { + std::vector key; + ShaderRecompiler::BuildPreparedShaderKey(request, key); + for (const auto& entry : transaction.Read(snapshot).entries) { + if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle; + } + PerformanceTimer timing("Shader.PrepareArtifact"); + return PrepareShaderWithDiagnostics(request); + }; + const auto registrationFor = [&](const ShaderRecompiler::RecompileRequest& request, const ShaderRecompiler::SourceHandle& handle) -> std::shared_ptr { + if (!target.registrationModules || LibraryPrebuildDisabled()) return nullptr; + try { + return ShaderRecompiler::RegistrationResult(request, handle); + } catch (const std::exception& error) { + APS5_LOG_OUT("shader library prebuild skipped: %s", error.what()); + return nullptr; + } + }; require(decoded.programs.size() == decoded.roles.size(), "graphics ABI program roles are incomplete"); for (const auto& program : decoded.programs) require(program.snapshot != nullptr, "graphics ABI has no registered shader snapshot"); std::vector expected; @@ -1946,6 +1990,7 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod linked.push_back({decoded.roles[index], program.binary, program.userDataBase, program.firstUserSgpr, program.userData}); } std::vector prepared; + std::vector> requests; std::uint32_t pushOffset = 0; const bool fixedSlots = target.fixedPushSlots; const auto capacity = Graphics::StagePushLimit(decoded.state.stages, 0, decoded.state.depthRemap.mode != 0u); @@ -1962,6 +2007,7 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod } if (deferUnprepared && DeferFailedRegistration(*program.snapshot)) { prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + requests.emplace_back(); continue; } std::optional vertex; @@ -1971,6 +2017,7 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod if (!deferUnprepared) throw; DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what()); prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + requests.emplace_back(); continue; } if (vertex) { @@ -1982,36 +2029,43 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod std::vector key; ShaderRecompiler::BuildPreparedShaderKey(request, key); std::shared_ptr handle; - for (const auto& entry : transaction.Read(*program.snapshot).entries) { - if (entry.codeOffset == program.codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) { - handle = entry.handle; - break; - } - } - if (handle == nullptr) { - PerformanceTimer timing("Shader.PrepareArtifact"); - try { - handle = PrepareShaderWithDiagnostics(request); - } catch (const std::exception& error) { - if (!deferUnprepared) throw; - DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what()); - prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); - continue; - } + try { + handle = preparedHandle(*program.snapshot, program.codeOffset, request); + } catch (const std::exception& error) { + if (!deferUnprepared) throw; + DeferPipelineStage(transaction.Edit(*program.snapshot), *program.snapshot, error.what()); + prepared.push_back({program.snapshot, {program.codeOffset, nullptr}, program.binary.stage, offset, nullptr}); + requests.emplace_back(); + continue; } const auto bytes = handle->artifact->bindings.pushConstantSizeBytes; require(bytes <= room, "prepared graphics stages exceed the push constant block"); - std::shared_ptr registration; - if (target.registrationModules && !LibraryPrebuildDisabled()) { - try { - registration = ShaderRecompiler::RegistrationResult(request, *handle); - } catch (const std::exception& error) { - APS5_LOG_OUT("shader library prebuild skipped: %s", error.what()); - } - } - prepared.push_back({program.snapshot, {program.codeOffset, handle}, program.binary.stage, offset, std::move(registration)}); + prepared.push_back({program.snapshot, {program.codeOffset, handle}, program.binary.stage, offset, registrationFor(request, *handle)}); + requests.push_back(std::move(request)); pushOffset += bytes; } + if (HeapFragmentsEnabled()) { + std::vector heap; + const auto addHeap = [&](std::size_t index, bool heapVertex) { + const auto& stage = prepared[index]; + if (stage.entry.handle == nullptr || !requests[index] || HeapStageExcluded(stage.stage)) return false; + const auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(*requests[index], stage.entry.handle); + if (!invocation || !HeapStageEligible(stage.stage, invocation->Plan().get(), heapVertex)) return false; + auto request = *requests[index]; + request.target.gpuResolvedImages = true; + const auto handle = preparedHandle(*stage.snapshot, stage.entry.codeOffset, request); + heap.push_back({stage.snapshot, {stage.entry.codeOffset, handle}, stage.stage, stage.pushOffset, registrationFor(request, *handle), true}); + return true; + }; + bool fragmentOnHeap = true; + for (std::size_t index = 0; index < prepared.size(); ++index) { + if (prepared[index].stage == ShaderRecompiler::ShaderStage::Fragment) fragmentOnHeap = addHeap(index, false); + } + for (std::size_t index = 0; index < prepared.size(); ++index) { + if (HeapPreRasterStage(prepared[index].stage)) static_cast(addHeap(index, target.descriptorMemory && fragmentOnHeap)); + } + prepared.insert(prepared.begin(), std::make_move_iterator(heap.begin()), std::make_move_iterator(heap.end())); + } transaction.Commit(); return prepared; } @@ -2114,11 +2168,13 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std if (!duplicate) transaction.Edit(*stage.snapshot).entries.push_back(std::move(stage.entry)); } for (const auto& stage : prepared) { - if (stage.snapshot->type == 1) continue; + if (stage.snapshot->type == 1 || stage.heapVariant) continue; ResolvePreparedGraphics(*stage.snapshot, {}, primitiveType == state.userConfig.end() ? 0u : primitiveType->second, localDevice->Target(), true); } PreparedShaderState::GraphicsAbi abi{key, registry, {}}; - for (const auto& stage : prepared) abi.stages.push_back(stage.snapshot); + for (const auto& stage : prepared) { + if (!stage.heapVariant) abi.stages.push_back(stage.snapshot); + } auto& graphicsAbis = transaction.Edit(*owner).graphicsAbis; std::erase_if(graphicsAbis, [](const auto& entry) { return entry.registry.expired(); }); graphicsAbis.push_back(std::move(abi)); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index 73071a0ce58c..1671325f0925 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -120,6 +120,7 @@ std::atomic forgetBytes{0}; std::atomic collectMemoHits{0}; std::atomic collectEpochBumps{0}; std::atomic unwatchSerial{0}; +std::atomic ownWriteSerial{0}; // Walks that reported at least one written page (the ones the probe pass used to double), and the // tracker mutex acquisitions that had to wait (APS5_PROFILE_DRAW; see lockTracker). std::atomic collectDirty{0}; @@ -1913,6 +1914,10 @@ std::uint64_t UnwatchSerial() { return unwatchSerial.load(std::memory_order_acquire); } +std::uint64_t OwnWriteSerial() { + return ownWriteSerial.load(std::memory_order_acquire); +} + void AdoptLatestCollectEpoch() { const auto latest = lastBumpedEpoch.load(std::memory_order_acquire); if (latest > HostThreadSlot::Get()) HostThreadSlot::Set(latest); @@ -2215,6 +2220,7 @@ std::uint64_t storeOwn(std::uint64_t address, std::size_t bytes, const std::func BreakProofs(); const auto stampStored = [complete](WriteTracker& tracker, std::pair stored) -> std::uint64_t { if (stored.second <= stored.first) return 0; + ownWriteSerial.fetch_add(1, std::memory_order_release); ++tracker.generation; if (!tracker.watched || !tracker.covers(stored.first, static_cast(stored.second - stored.first))) { tracker.noteUnstampedStore(stored.first, stored.second); @@ -2264,6 +2270,7 @@ static std::uint64_t markWritten(std::uint64_t address, std::size_t bytes, bool auto lock = lockTracker(tracker); tracker.initialize(); if (bytes == 0) return 0; + ownWriteSerial.fetch_add(1, std::memory_order_release); ++tracker.generation; if (!tracker.watched || !tracker.covers(address, bytes)) { tracker.noteUnstampedStore(address, address + bytes); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp index 7b505eb70a85..f45abe66db70 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/PerformanceOverlay.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceOverlay.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -1172,7 +1173,7 @@ OverlayRect PerformanceOverlay::RecordRects(VkCommandBuffer commands, std::size_ context.Function("vkCmdSetViewport")(commands, 0, 1, &viewport); context.Function("vkCmdSetScissor")(commands, 0, 1, &area); const OverlayParameters parameters{OverlayPalette(format, colorSpace), {static_cast(extent.width), static_cast(extent.height)}}; - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(parameters), ¶meters); + Graphics::CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(parameters), ¶meters); const VkBuffer buffer = slot.rects->Handle(); const VkDeviceSize offset = 0; context.Function("vkCmdBindVertexBuffers")(commands, 0, 1, &buffer, &offset); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp index b7e57f9e77e6..8c072d5f8425 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/PresentationPass.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationPass.hpp" #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp" @@ -8,8 +9,11 @@ #include "prx/libSceAgcDriver/Graphics/shaders/PresentationPassVert_spv.h" #include #include +#include #include +#include #include +#include namespace AgcDriver { namespace { @@ -19,13 +23,24 @@ constexpr std::uint32_t ModeRedLow = 2; constexpr std::uint32_t ModeKeepTenBit = 4; constexpr std::uint32_t ModeSubsample = 8; constexpr std::uint32_t ModePq = 16; +constexpr std::uint32_t PresentHeapKind = ShaderRecompiler::RuntimeAbi::FirstImageBinding + 9u; + +struct HeapPush { + std::int32_t parameters[9]; + std::uint32_t pad; + std::uint64_t pqNits; + std::uint32_t slot; + std::uint32_t pad2; +}; + +static_assert(offsetof(HeapPush, pqNits) == 40 && offsetof(HeapPush, slot) == 48); } PresentationPass::PresentationPass(const Graphics::Context& context, std::size_t slotCount) : context(context), slots(slotCount) { Graphics::Require(slotCount != 0, "presentation pass needs at least one slot"); try { - VkSamplerCreateInfo samplerInfo{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + heap = Graphics::DescriptorHeapEnabled() && Graphics::BindlessHeap::For(context).DescriptorHeapMode(); samplerInfo.magFilter = VK_FILTER_NEAREST; samplerInfo.minFilter = VK_FILTER_NEAREST; samplerInfo.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; @@ -59,7 +74,7 @@ PresentationPass::PresentationPass(const Graphics::Context& context, std::size_t Graphics::Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &slot.set), "vkAllocateDescriptorSets presentation pass"); } const auto& luminance = PqEotfNits(); - pqNits = std::make_unique(context, sizeof(luminance), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + pqNits = std::make_unique(context, sizeof(luminance), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (heap ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u)); std::memcpy(pqNits->Bytes().data(), luminance.data(), sizeof(luminance)); const VkDescriptorBufferInfo tableInfo{pqNits->Handle(), 0, sizeof(luminance)}; std::vector tableWrites(slots.size(), VkWriteDescriptorSet{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}); @@ -91,6 +106,7 @@ void PresentationPass::release() noexcept { } for (const auto& [key, entry] : targets) { if (entry.pipeline) context.Function("vkDestroyPipeline")(context.device, entry.pipeline, nullptr); + if (entry.heapPipeline) context.Function("vkDestroyPipeline")(context.device, entry.heapPipeline, nullptr); if (entry.renderPass) context.Function("vkDestroyRenderPass")(context.device, entry.renderPass, nullptr); } targets.clear(); @@ -125,6 +141,7 @@ void PresentationPass::Release(std::size_t index) noexcept { if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); } slot.transient.clear(); + if (slot.source && heap) slot.ReleaseHeapSlots(context); if (slot.source) context.Function("vkDestroyImageView")(context.device, slot.source, nullptr); slot.source = VK_NULL_HANDLE; } @@ -246,7 +263,33 @@ const PresentationPass::Target& PresentationPass::target(VkFormat format, VkImag pipelineInfo.layout = pipelineLayout; pipelineInfo.renderPass = created.renderPass; Graphics::Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &created.pipeline), "vkCreateGraphicsPipelines presentation pass"); + if (heap) { + const auto& bindless = Graphics::BindlessHeap::For(context); + std::array mappings{{{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}, {VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}}}; + mappings[0].firstBinding = 0; + mappings[0].bindingCount = 1; + mappings[0].resourceMask = VK_SPIRV_RESOURCE_TYPE_COMBINED_SAMPLED_IMAGE_BIT_EXT; + mappings[0].source = VK_DESCRIPTOR_MAPPING_SOURCE_HEAP_WITH_PUSH_INDEX_EXT; + mappings[0].sourceData.pushIndex.pushOffset = offsetof(HeapPush, slot); + mappings[0].sourceData.pushIndex.heapIndexStride = static_cast(bindless.ImageHeapStride()); + mappings[0].sourceData.pushIndex.heapArrayStride = static_cast(bindless.ImageHeapStride()); + mappings[0].sourceData.pushIndex.pEmbeddedSampler = &samplerInfo; + mappings[1].firstBinding = 1; + mappings[1].bindingCount = 1; + mappings[1].resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT | VK_SPIRV_RESOURCE_TYPE_READ_ONLY_STORAGE_BUFFER_BIT_EXT; + mappings[1].source = VK_DESCRIPTOR_MAPPING_SOURCE_PUSH_ADDRESS_EXT; + mappings[1].sourceData.pushAddressOffset = offsetof(HeapPush, pqNits); + VkShaderDescriptorSetAndBindingMappingInfoEXT mappingInfo{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}; + mappingInfo.mappingCount = static_cast(mappings.size()); + mappingInfo.pMappings = mappings.data(); + stages[1].pNext = &mappingInfo; + auto heapFlags = Graphics::DescriptorHeapFlags(); + pipelineInfo.pNext = &heapFlags; + pipelineInfo.layout = VK_NULL_HANDLE; + Graphics::Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &created.heapPipeline), "vkCreateGraphicsPipelines presentation pass heap"); + } } catch (...) { + if (created.pipeline) context.Function("vkDestroyPipeline")(context.device, created.pipeline, nullptr); if (created.renderPass) context.Function("vkDestroyRenderPass")(context.device, created.renderPass, nullptr); throw; } @@ -343,18 +386,27 @@ void PresentationPass::BindTexels(std::size_t index, VkCommandBuffer commands, s Bind(index, slot.uploadImage, width, height, pixelFormat, whiteNits); } -void PresentationPass::draw(VkCommandBuffer commands, const Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (¶meters)[9]) const { +void PresentationPass::draw(VkCommandBuffer commands, Slot& slot, const Target& entry, VkFramebuffer framebuffer, VkExtent2D extent, VkRect2D scissor, const std::int32_t (¶meters)[9]) const { VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; begin.renderPass = entry.renderPass; begin.framebuffer = framebuffer; begin.renderArea = {{0, 0}, extent}; context.Function("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, entry.pipeline); + const auto heapSlot = entry.heapPipeline != VK_NULL_HANDLE ? Graphics::BindlessHeap::For(context).ViewSlot(PresentHeapKind, &slot, slot.source) : std::nullopt; + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, heapSlot ? entry.heapPipeline : entry.pipeline); const VkViewport viewport{0.0f, 0.0f, static_cast(extent.width), static_cast(extent.height), 0.0f, 1.0f}; context.Function("vkCmdSetViewport")(commands, 0, 1, &viewport); context.Function("vkCmdSetScissor")(commands, 0, 1, &scissor); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &slot.set, 0, nullptr); - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(parameters), parameters); + if (heapSlot) { + Graphics::BindlessHeap::For(context).BindDescriptorHeaps(commands); + HeapPush pushed{{}, 0u, pqNits->DeviceAddress(), *heapSlot, 0u}; + std::copy(std::begin(parameters), std::end(parameters), pushed.parameters); + const VkPushDataInfoEXT data{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {&pushed, sizeof(pushed)}}; + context.Function("vkCmdPushDataEXT")(commands, &data); + } else { + Graphics::CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &slot.set, Graphics::SetBindSite::Presentation); + Graphics::CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(parameters), parameters); + } context.Function("vkCmdDraw")(commands, 3, 1, 0, 0); context.Function("vkCmdEndRenderPass")(commands); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 8ffb1adcb3c7..f7226a1ea5d0 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -13,6 +13,8 @@ #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Execution/include/FloatFlushProbe.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" @@ -148,6 +150,45 @@ APS5_RUN_REPORT_COMPONENT("libSceAgcDriver") namespace { +std::optional ActiveValidationLayerVersion(PFN_vkGetInstanceProcAddr instanceProc) { + const auto requested = [](const char* variable) { + const char* value = std::getenv(variable); + return value != nullptr && (std::strstr(value, "validation") != nullptr || std::strchr(value, '*') != nullptr); + }; + if (!requested("VK_INSTANCE_LAYERS") && !requested("VK_LOADER_LAYERS_ENABLE")) return std::nullopt; + const auto enumerate = reinterpret_cast(instanceProc(VK_NULL_HANDLE, "vkEnumerateInstanceLayerProperties")); + if (enumerate == nullptr) return std::nullopt; + std::uint32_t count = 0; + if (enumerate(&count, nullptr) != VK_SUCCESS) return std::nullopt; + std::vector layers(count); + if (enumerate(&count, layers.data()) != VK_SUCCESS) return std::nullopt; + for (const auto& layer : layers) { + if (std::strcmp(layer.layerName, "VK_LAYER_KHRONOS_validation") == 0) return layer.specVersion; + } + return std::nullopt; +} + +constexpr std::uint32_t MinHeapValidationLayerVersion = VK_MAKE_API_VERSION(0, 1, 4, 363); +constexpr std::uint32_t HeapDataUniformDwordCap = 16384u; + +bool DescriptorHeapAvailable(const std::vector& availableExtensions, PFN_vkGetInstanceProcAddr instanceProc) { + if (!Graphics::GpuHeapEnabled()) return false; + const auto heapExtension = std::find_if(availableExtensions.begin(), availableExtensions.end(), [](const auto& item) { return std::strcmp(item.extensionName, VK_EXT_DESCRIPTOR_HEAP_EXTENSION_NAME) == 0; }); + if (heapExtension == availableExtensions.end()) return false; + if (heapExtension->specVersion < VK_EXT_DESCRIPTOR_HEAP_SPEC_VERSION) { + std::fprintf(stderr, "[gpu-heap] the driver's VK_EXT_descriptor_heap is revision %u, older than the released revision %u, so the GPU heap stays off\n", + heapExtension->specVersion, static_cast(VK_EXT_DESCRIPTOR_HEAP_SPEC_VERSION)); + return false; + } + const auto validationVersion = ActiveValidationLayerVersion(instanceProc); + if (validationVersion.has_value() && *validationVersion < MinHeapValidationLayerVersion) { + std::fprintf(stderr, "[gpu-heap] VK_LAYER_KHRONOS_validation %u.%u.%u predates its descriptor heap rules (1.4.363 is the oldest layer that validates heap push data), so the GPU heap stays off\n", + VK_API_VERSION_MAJOR(*validationVersion), VK_API_VERSION_MINOR(*validationVersion), VK_API_VERSION_PATCH(*validationVersion)); + return false; + } + return true; +} + std::atomic descriptorUse[7 * 4]; void CountDescriptor(int type, int, int usage) { @@ -164,7 +205,6 @@ const ShaderRecompiler::DiagnosticSink runReportSink{ }; [[maybe_unused]] const bool runReportSinkInstalled = (LibcRunReportSetDescriptorCounters_nid_no_patch(descriptorUse), ShaderRecompiler::SetDiagnosticSink(&runReportSink), true); - std::uint64_t fillCompletionSyncs = 0; void check(VkResult result, const char* operation) { @@ -178,6 +218,11 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("Vulkan presentation: ") + reason); } +void reportDeviceImagePathRefusal(const std::string& reason) { + static std::atomic reported{false}; + if (!reported.exchange(true)) std::fprintf(stderr, "[device-images] APS5_GPU_PATH stays off on this device: %s\n", reason.c_str()); +} + void checkHostRequirements(const HostGpuQuery& query, bool printReport) { HostGpuFacts gpu; try { @@ -249,6 +294,31 @@ const wchar_t* ComputePipelineDumpDirectory() { return _wgetenv(L"APS5_DUMP_COMP const char* ComputePipelineDumpDirectory() { return std::getenv("APS5_DUMP_COMPUTE_PIPELINES"); } #endif +static_assert(Graphics::ComputePushConstantBytes + sizeof(VkDeviceAddress) == ShaderRecompiler::GpuScratchPoolAddressPushByte && + ShaderRecompiler::GpuScratchPoolAddressPushByte + sizeof(VkDeviceAddress) <= ShaderRecompiler::GpuScratchPoolDwordsPushByte && + (ShaderRecompiler::StagePushFeatures(ShaderRecompiler::ShaderStage::Compute, 0u) & ShaderRecompiler::PushBytesMask(Graphics::ComputePushConstantBytes, ShaderRecompiler::GpuScratchPoolDwordsPushByte + sizeof(std::uint32_t) - Graphics::ComputePushConstantBytes)) == ShaderRecompiler::PushBytesMask(Graphics::ComputePushConstantBytes, ShaderRecompiler::GpuScratchPoolDwordsPushByte + sizeof(std::uint32_t) - Graphics::ComputePushConstantBytes)); + +void CreateComputeHeapPipeline(VkDevice device, PFN_vkCreateComputePipelines create, VkPipelineCache cache, VkShaderModule module, VkPipelineCreateFlags flags, VkPipelineShaderStageCreateFlags stageFlags, const VkSpecializationInfo* specialization, std::uint32_t subgroupSize, std::span mappings, VkPipeline* pipeline, Graphics::PipelineCacheSynchronization* synchronization) { + auto flags2 = Graphics::DescriptorHeapFlags(static_cast(flags)); + VkComputePipelineCreateInfo info{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO, &flags2}; + info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + info.stage.flags = stageFlags; + info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + info.stage.module = module; + info.stage.pName = "main"; + info.stage.pSpecializationInfo = specialization; + VkShaderDescriptorSetAndBindingMappingInfoEXT mappingInfo{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}; + mappingInfo.mappingCount = static_cast(mappings.size()); + mappingInfo.pMappings = mappings.data(); + VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT subgroup{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT}; + subgroup.requiredSubgroupSize = subgroupSize; + info.stage.pNext = &mappingInfo; + if (subgroupSize != 0) mappingInfo.pNext = &subgroup; + info.layout = VK_NULL_HANDLE; + auto cacheLock = synchronization != nullptr ? synchronization->LockForPipelineUse() : Graphics::PipelineCacheSynchronization::PipelineUseLock{}; + check(create(device, cache, 1, &info, nullptr, pipeline), "vkCreateComputePipelines (descriptor heap)"); +} + bool CaptureComputePipeline(std::size_t words) { const char* minimum = std::getenv("APS5_DUMP_COMPUTE_MIN_WORDS"); if (minimum == nullptr) return true; @@ -359,6 +429,8 @@ struct ComputePipelineObjects { VkDescriptorSetLayout resourcesLayout = VK_NULL_HANDLE; std::mutex splitMutex; std::map, VkPipeline> splitPipelines; + std::vector pushPlacements; + bool descriptorHeap = false; ~ComputePipelineObjects() { for (const auto& [base, split] : splitPipelines) destroyPipeline(device, split, nullptr); if (pipeline != VK_NULL_HANDLE) destroyPipeline(device, pipeline, nullptr); @@ -385,6 +457,7 @@ struct WorkgroupMemory { std::unique_ptr buffer; VkDescriptorPool pool = VK_NULL_HANDLE; VkDescriptorSet set = VK_NULL_HANDLE; + VkDeviceAddress address = 0; ~WorkgroupMemory() { buffer.reset(); if (pool != VK_NULL_HANDLE) destroyPool(device, pool, nullptr); @@ -397,6 +470,7 @@ struct ScratchPool { std::unique_ptr buffer; VkDescriptorPool pool = VK_NULL_HANDLE; VkDescriptorSet set = VK_NULL_HANDLE; + VkDeviceAddress address = 0; std::uint64_t slots = 0; std::uint64_t headerBytes = 0; std::uint64_t bytes = 0; @@ -600,10 +674,16 @@ struct VulkanDevice::State { // built once after setup (see graphicsContext); the context's pool reference is dropped before // the pool at teardown. Graphics::DeviceFunctions deviceFunctions; + std::unique_ptr functionCache = std::make_unique(); bool functionsReady = false; Graphics::Context context; bool contextReady = false; std::unique_ptr detiler; + bool descriptorHeap = false; + bool deviceImagePath = false; + VkPhysicalDeviceDescriptorHeapPropertiesEXT descriptorHeapProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_HEAP_PROPERTIES_EXT}; + std::uint32_t heapDataUniformDwords = 0; + bool heapBufferDescriptors = false; std::unique_ptr recorder; // Device-local buffers of a repeated 16-byte fill pattern (FillBuffer), most recently used // last; each is filled once by a doubling chain in device memory. @@ -631,6 +711,7 @@ struct VulkanDevice::State { std::shared_ptr journalReplay; std::atomic journalReplayPending{false}; std::atomic computePipelinesCreated{0}; + std::atomic descriptorHeapComputePipelines{0}; std::unique_ptr colorTransfer; std::shared_ptr bufferPool; std::unique_ptr emptyBuffer; @@ -732,12 +813,14 @@ struct VulkanDevice::State { template TFunction DeviceFunction(const char* name) const { + if (const auto cached = functionCache->Find(name, deviceProc); cached != nullptr) return reinterpret_cast(cached); ++Graphics::DeviceProcLookups(); - auto function = reinterpret_cast(deviceProc(device, name)); + const auto function = deviceProc(device, name); if (function == nullptr) { throw std::runtime_error(std::string("Vulkan device function missing: ") + name); } - return function; + functionCache->Keep(name, deviceProc, function); + return reinterpret_cast(function); } void CreateWorkgroupMemoryLayout() { @@ -764,7 +847,7 @@ struct VulkanDevice::State { auto memory = std::make_shared(); memory->device = device; memory->destroyPool = DeviceFunction("vkDestroyDescriptorPool"); - memory->buffer = std::make_unique(context, static_cast(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + memory->buffer = std::make_unique(context, static_cast(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | (context.descriptorHeap ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u)); const VkDescriptorPoolSize poolSize{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; poolInfo.maxSets = 1; @@ -784,6 +867,7 @@ struct VulkanDevice::State { write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; write.pBufferInfo = &described; DeviceFunction("vkUpdateDescriptorSets")(device, 1, &write, 0, nullptr); + if (context.descriptorHeap) memory->address = Graphics::BufferDeviceAddress(context, memory->buffer->Handle()); workgroupMemory = std::move(memory); return workgroupMemory; } @@ -811,7 +895,7 @@ struct VulkanDevice::State { created->headerBytes = wanted * 2u * 4u; created->bytes = size; created->lastUse = use; - created->buffer = std::make_unique(context, static_cast(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + created->buffer = std::make_unique(context, static_cast(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (context.descriptorHeap ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u)); const VkDescriptorPoolSize poolSize{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; poolInfo.maxSets = 1; @@ -831,6 +915,7 @@ struct VulkanDevice::State { write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; write.pBufferInfo = &described; DeviceFunction("vkUpdateDescriptorSets")(device, 1, &write, 0, nullptr); + if (context.descriptorHeap) created->address = Graphics::BufferDeviceAddress(context, created->buffer->Handle()); ZeroScratchPoolHeader(context, *created); scratchPools[identity] = created; return created; @@ -1106,6 +1191,8 @@ struct VulkanDevice::State { Graphics::ClearRetainedSpaces(device); Graphics::ClearCachedTextures(device); Graphics::ClearSnapshotCopies(device); + Graphics::DeviceImageTable::Release(device); + Graphics::BindlessHeap::Release(device); Graphics::ClearImageMirrors(device); Graphics::ClearHostImports(device); patternBuffers.clear(); @@ -2111,6 +2198,29 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); } } + VkPhysicalDeviceDescriptorHeapFeaturesEXT descriptorHeapFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_HEAP_FEATURES_EXT}; + if (DescriptorHeapAvailable(availableExtensions, state->instanceProc) && hasExtension(VK_KHR_MAINTENANCE_5_EXTENSION_NAME) && hasExtension(VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME) && hasExtension(VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME) && hasExtension(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME)) { + VkPhysicalDeviceMaintenance5FeaturesKHR heapMaintenance5{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MAINTENANCE_5_FEATURES_KHR, &descriptorHeapFeatures}; + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &heapMaintenance5}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + if (descriptorHeapFeatures.descriptorHeap == VK_TRUE && heapMaintenance5.maintenance5 == VK_TRUE) { + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &state->descriptorHeapProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + } + if (descriptorHeapFeatures.descriptorHeap == VK_TRUE && heapMaintenance5.maintenance5 == VK_TRUE && Graphics::BindlessHeap::Fits(state->descriptorHeapProperties)) { + state->descriptorHeap = true; + state->maintenance5 = true; + state->heapDataUniformDwords = std::min(HeapDataUniformDwordCap, state->properties.limits.maxUniformBufferRange / 4u); + if (Graphics::BindlessHeap::RingInResourceHeap(state->descriptorHeapProperties, state->properties.limits.minUniformBufferOffsetAlignment)) + state->heapBufferDescriptors = state->descriptorHeapProperties.bufferDescriptorSize == Graphics::BindlessHeap::BufferSlotBytes; + deviceExtensions.push_back(VK_EXT_DESCRIPTOR_HEAP_EXTENSION_NAME); + for (const char* required : {VK_KHR_MAINTENANCE_5_EXTENSION_NAME, VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME, VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME, VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME}) { + if (std::find_if(deviceExtensions.begin(), deviceExtensions.end(), [&](const char* name) { return std::strcmp(name, required) == 0; }) == deviceExtensions.end()) deviceExtensions.push_back(required); + } + } + descriptorHeapFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_HEAP_FEATURES_EXT}; + descriptorHeapFeatures.descriptorHeap = state->descriptorHeap ? VK_TRUE : VK_FALSE; + } descriptorIndexingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; if (state->descriptorIndexing || state->storageBufferUpdateAfterBind || state->sampledImageUpdateAfterBind || state->descriptorBindingPartiallyBound || state->descriptorBindingVariableDescriptorCount || state->runtimeDescriptorArray) deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME); if (state->descriptorIndexing) { @@ -2141,6 +2251,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : meshFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &meshFeatures; } + if (state->descriptorHeap) { + descriptorHeapFeatures.pNext = const_cast(deviceInfo.pNext); + deviceInfo.pNext = &descriptorHeapFeatures; + } if (state->depthClipControl) { depthClipFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &depthClipFeatures; @@ -2438,11 +2552,19 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->functionsReady = true; } Graphics::PrepareImportWatch(graphicsContext()); + Graphics::SetDescriptorHeapDevice(state->descriptorHeap && state->descriptorHeapProperties.maxPushDataSize >= Graphics::PipelinePushConstantBytes); + Graphics::BufferPool::SetForceAddresses(Graphics::DescriptorHeapEnabled()); + if (Graphics::GpuPathEnabled()) { + const auto refusal = Graphics::DeviceImagePathRefusal({Graphics::GpuHeapEnabled(), Graphics::DescriptorHeapEnabled()}); + state->deviceImagePath = !refusal.has_value(); + if (refusal.has_value()) reportDeviceImagePathRefusal(*refusal); + } state->bufferPool = std::make_shared(graphicsContext()); state->emptyBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); std::ranges::fill(state->emptyBuffer->Bytes(), std::byte{0}); state->oversizedBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); std::ranges::fill(state->oversizedBuffer->Bytes(), std::byte{0}); + if (state->descriptorHeap && Graphics::DescriptorHeapEnabled()) APS5_LOG_OUT("Vulkan descriptor heap: %s", "GPU-mode draws and dispatches bind from VK_EXT_descriptor_heap"); if (!compileOnly && Pm4::GdsImportable() && Graphics::HostImportFor(graphicsContext(), Pm4::GdsAddress(), Pm4::GdsBytes, Graphics::ImportRequester::Map) == nullptr) Pm4::RejectGdsImport(); static const bool headlessPipelineCache = std::getenv("APS5_HEADLESS_PIPELINE_CACHE") != nullptr; state->pipelineCache = std::make_unique(graphicsContext(), state->properties, window != nullptr || compileOnly || headlessPipelineCache); @@ -2462,6 +2584,11 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->recorder->Activate(); state->context = buildContext(); state->contextReady = true; + if (state->deviceImagePath) { + std::unique_lock gpuLock(GuestMemory::GpuMutex(), std::defer_lock); + if (!GuestMemory::GpuMutex().HeldByThisThread()) gpuLock.lock(); + Graphics::DeviceImageTable::Create(state->context, ShaderRecompiler::ResourceMaterializer::FindDeviceImageInterface); + } if (window != nullptr) { require(window->getDrawableSize != nullptr, "missing window drawable size query"); std::uint32_t drawableWidth = 0; @@ -2592,6 +2719,10 @@ std::uint64_t VulkanDevice::ComputePipelinesCreated() const { return state->computePipelinesCreated.load(std::memory_order_relaxed); } +std::uint64_t VulkanDevice::DescriptorHeapComputePipelines() const { + return state->descriptorHeapComputePipelines.load(std::memory_order_relaxed); +} + bool VulkanDevice::ReplayComputePipeline(const ShaderRecompiler::RecompileResult& shader, bool pushConstants) { ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); const auto key = ComputePipelineKey::Of(shader, pushConstants); @@ -3988,6 +4119,7 @@ bool VulkanDevice::present(std::uint32_t width, std::uint32_t height, bool opaqu check(state->DeviceFunction("vkResetCommandBuffer")(commands, 0), "vkResetCommandBuffer"); VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + Graphics::NoteCommandBufferBegun(commands); check(state->DeviceFunction("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); APS5_LOG_OUT_DEBUG("Presentation command buffer begin commands=%p image=%p", reinterpret_cast(commands), reinterpret_cast(state->images[index])); using CommandClass = Graphics::Recorder::CommandClass; @@ -4317,9 +4449,37 @@ bool VulkanDevice::PresentsOnOwnQueue() const { return state->presentQueue != VK_NULL_HANDLE; } +bool VulkanDevice::DeviceImagePath() const { + return state->deviceImagePath; +} + +bool VulkanDevice::DeviceImagesServe(std::uint64_t codeAddress) const { + if (!state->deviceImagePath) return false; + const auto* table = Graphics::DeviceImageTable::Existing(state->context); + return table != nullptr && table->ServesProgram(codeAddress); +} + +void VulkanDevice::NoteDeviceImageProgram(const ShaderRecompiler::RecompileResult& result, std::uint64_t codeAddress) const { + if (!result.deviceImageTables) return; + if (auto* table = Graphics::DeviceImageTable::Existing(state->context)) table->NoteProgram(result.Identity(), codeAddress); +} + +std::optional VulkanDevice::ExpectDeviceImages(std::span images) const { + auto* table = Graphics::DeviceImageTable::Existing(state->context); + require(table != nullptr && state->recorder != nullptr, "expected device images need the device's table and its recorder"); + return table->Expect(graphicsContext(), *state->recorder, images); +} + +void VulkanDevice::FallBackDeviceImages(const ShaderRecompiler::RecompileResult& result, std::uint64_t codeAddress, const std::string& reason) const { + auto* table = Graphics::DeviceImageTable::Existing(state->context); + require(table != nullptr, "a device image fallback needs the device's table"); + table->FallBackProgram(result.Identity(), codeAddress, reason); +} + ShaderRecompiler::SpirvTarget VulkanDevice::ComputeTarget(std::uint32_t waveSize) const { auto target = Target(); if (waveSize == 32u && state->computeWave32) target.subgroupSize = 32u; + target.gpuResolvedImages = Graphics::DescriptorHeapEnabled(); return target; } @@ -4384,6 +4544,10 @@ ShaderRecompiler::SpirvTarget VulkanDevice::Target() const { target.fusedExactArithmetic = state->fusedExactArithmetic; target.hostInterpolationFirst = state->hostInterpolationFirst; target.float32DenormPreserve = state->float32DenormPreserve; + target.descriptorMemory = Graphics::DescriptorHeapEnabled(); + target.deviceImageTables = state->deviceImagePath; + target.heapDataUniformDwords = target.descriptorMemory ? state->heapDataUniformDwords : 0u; + target.heapBufferDescriptors = target.descriptorMemory && state->heapBufferDescriptors; return target; } @@ -4475,10 +4639,15 @@ std::pair VulkanDevice::scratchPoolHeader(const ShaderR return {found->second->buffer->Handle(), found->second->headerBytes}; } -Graphics::Context VulkanDevice::graphicsContext() const { +const Graphics::Context& VulkanDevice::contextRef(std::optional& scratch) const { static const bool noCache = std::getenv("APS5_NO_CONTEXT_CACHE") != nullptr; if (state->contextReady && !noCache) return state->context; - return buildContext(); + return scratch.emplace(buildContext()); +} + +Graphics::Context VulkanDevice::graphicsContext() const { + std::optional scratch; + return contextRef(scratch); } Graphics::Context VulkanDevice::buildContext() const { @@ -4526,6 +4695,9 @@ Graphics::Context VulkanDevice::buildContext() const { context.settleFamily = state->settleFamily; context.settleQueueMutex = &state->settleQueueMutex; context.recorder = state->recorder.get(); + context.descriptorHeap = state->descriptorHeap; + context.descriptorHeapProperties = state->descriptorHeapProperties; + context.heapBufferDescriptors = state->heapBufferDescriptors; context.descriptorCache = state->descriptorCache.get(); context.vertexInputs = state->vertexInputs.get(); context.vertexFormats = &state->vertexFormats; @@ -4557,6 +4729,7 @@ Graphics::Context VulkanDevice::buildContext() const { context.exactRobustLoads = state->exactRobustLoads; context.copiedWriters = state->copiedWriters.get(); context.functions = state->functionsReady ? &state->deviceFunctions : nullptr; + context.functionCache = state->functionsReady ? state->functionCache.get() : nullptr; context.descriptorIndexing = state->descriptorIndexing; context.descriptorIndexingLimits = state->descriptorIndexingProperties; context.imageInt64Atomics = state->imageInt64Atomics; @@ -4610,7 +4783,8 @@ VulkanDevice::IndirectDrawSupport VulkanDevice::DrawIndirectSupport() const { } std::optional VulkanDevice::KnownDrawRejection(const Graphics::State& graphics, std::span shaders) const { - return Graphics::KnownValidationFailure(graphicsContext(), shaders, graphics); + std::optional scratch; + return Graphics::KnownValidationFailure(contextRef(scratch), shaders, graphics); } void VulkanDevice::ColorMetadataPass(const Graphics::ColorMetadataPass& pass) { @@ -5371,7 +5545,7 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco if (built.Layout() != VK_NULL_HANDLE) { const auto layouts = built.SetLayouts(); prepared->objects = computePipeline(context, shader, layouts.main, layouts.mainKey, pushStages, programAddress); - } else if (context.descriptorCache != nullptr) { + } else if (context.descriptorCache != nullptr && !shader.gpuResolvedImages) { const auto described = Graphics::DescribeSetLayouts(context, shaders); prepared->describedLayoutKey = described.mainKey; prepared->objects = computePipeline(context, shader, described.layouts.main, described.mainKey, pushStages, programAddress); @@ -5571,6 +5745,11 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { for (const auto& snapshot : record.bindings->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size()); } recordStep(PhaseRecordKeeps); + if (resources.HeapStage()) { + if (record.threadGroup != std::array{}) resources.forgetShaderDataWords(*record.shader); + resources.RefreshHeapStage(std::span(record.shader, 1)); + if (record.bindings != nullptr) resources.RefreshMovedAddresses(*record.bindings); + } if (record.dataRefresh != RecordedDispatch::DataRefresh::None) { // The template's data buffers take this dispatch's words: a transfer write the pre-dispatch // barrier makes visible (so it is recorded whatever the previous command covered), ordered @@ -5590,10 +5769,10 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { } std::shared_ptr threadArguments; if (record.threadGroup != std::array{}) { - const auto data = resources.shaderDataBuffer(*record.shader); + const auto [data, dataOffset] = resources.shaderDataLocation(*record.shader, record.bindings.get()); const auto limitDword = record.shader->program->imageMetadataDword; Graphics::Require(data != VK_NULL_HANDLE && limitDword >= 3, "Vulkan dispatch: a thread-dimension dispatch has no ShaderData thread limit"); - const VkDeviceSize limitByte = static_cast(limitDword - 3) * sizeof(std::uint32_t); + const VkDeviceSize limitByte = dataOffset + static_cast(limitDword - 3) * sizeof(std::uint32_t); Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); if (argumentImport != nullptr) { threadArguments = std::make_shared(context, 24, VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); @@ -5638,7 +5817,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { } else { const auto timing = recorder.BeginGpuTiming(CommandClass::DispatchLeading); const VkPipelineStageFlags leadingStages = overlapCapable ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT : VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT; - const VkAccessFlags leadingAccess = overlapCapable ? VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT : VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; + const VkAccessFlags leadingAccess = overlapCapable ? VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT : VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT; Graphics::RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, leadingStages, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, leadingAccess); Graphics::Recorder::CountBarriers(CommandClass::DispatchLeading); recorder.EndGpuTiming(timing); @@ -5659,20 +5838,58 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { } } context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline); - if (record.bindings != nullptr) Graphics::Require(record.bindings->allocation.set != VK_NULL_HANDLE, "dispatch overlay has no descriptor set"); - if (record.bindings != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, 0, 1, &record.bindings->allocation.set, 0, nullptr); - else resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); + Graphics::NoteHeapRecord(true, record.objects->descriptorHeap); + if (record.objects->descriptorHeap) { + resources.BindDescriptorHeaps(commands); + } else if (record.bindings != nullptr) { + Graphics::Require(record.bindings->allocation.set != VK_NULL_HANDLE && !resources.HeapStage(), "dispatch overlay has no descriptor set"); + Graphics::CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, 0, 1, &record.bindings->allocation.set, Graphics::SetBindSite::Dispatch); + } else { + Graphics::Require(!resources.HeapStage(), "a GPU-mode dispatch needs a VK_EXT_descriptor_heap pipeline"); + resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); + } + VkDeviceAddress workgroupAddress = 0; if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) { auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords); - context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr); + const auto workgroupSet = record.shader->program->gpuResolvedImages ? ShaderRecompiler::GpuWorkgroupMemoryDescriptorSet : ShaderRecompiler::WorkgroupMemoryDescriptorSet; + if (record.objects->descriptorHeap) { + workgroupAddress = memory->address; + } else { + Graphics::CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, workgroupSet, 1, &memory->set, Graphics::SetBindSite::Dispatch); + } recorder.Keep(std::move(memory)); } + std::array stageAddresses{}; + std::memcpy(stageAddresses.data(), &workgroupAddress, sizeof(workgroupAddress)); if (pool != nullptr) { - context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::ScratchPoolDescriptorSet, 1, &pool->set, 0, nullptr); + if (record.objects->descriptorHeap) { + const auto dwords = static_cast(pool->bytes / sizeof(std::uint32_t)); + std::memcpy(stageAddresses.data() + (ShaderRecompiler::GpuScratchPoolAddressPushByte - Graphics::ComputePushConstantBytes), &pool->address, sizeof(pool->address)); + std::memcpy(stageAddresses.data() + (ShaderRecompiler::GpuScratchPoolDwordsPushByte - Graphics::ComputePushConstantBytes), &dwords, sizeof(dwords)); + } else { + Graphics::CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::ScratchPoolDescriptorSet, 1, &pool->set, Graphics::SetBindSite::Dispatch); + } recorder.Keep(std::move(pool)); } + if (record.objects->descriptorHeap && record.pushStages == 0 && (workgroupAddress != 0 || record.shader->program->scratchPoolSlotDwords != 0)) { + const VkPushDataInfoEXT push{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, Graphics::ComputePushConstantBytes, {stageAddresses.data(), stageAddresses.size()}}; + context.Function("vkCmdPushDataEXT")(commands, &push); + } if (record.pushStages != 0) { - context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes, record.pushBytes->data()); + std::array patchedBytes; + const std::byte* pushed = record.pushBytes->data(); + if (resources.HeapStage() || record.objects->descriptorHeap) { + patchedBytes = *record.pushBytes; + resources.PatchDataAddresses(patchedBytes, std::span(record.shader, 1), record.objects->pushPlacements, record.bindings.get()); + pushed = patchedBytes.data(); + } + if (record.objects->descriptorHeap) { + std::memcpy(patchedBytes.data() + Graphics::ComputePushConstantBytes, stageAddresses.data(), stageAddresses.size()); + const VkPushDataInfoEXT push{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {patchedBytes.data(), patchedBytes.size()}}; + context.Function("vkCmdPushDataEXT")(commands, &push); + } else { + Graphics::CmdPushConstants(context, commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes, pushed); + } } static const bool reportingDispatch = LibcRunReportEnabled_nid_no_patch(); if (reportingDispatch) recorder.NoteGuestWork(commands); @@ -5757,9 +5974,6 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { recordStep(PhaseRecordCompletion); } -// The module, pipeline layout and pipeline of a compute result over `setLayout` (the set layout -// its bindings describe): what a dispatch or its preparation builds at the first use of a module, -// and what the journal replay builds on its own threads before that use. std::shared_ptr VulkanDevice::createComputePipeline(const Graphics::Context& context, const ShaderRecompiler::RecompileResult& shader, VkDescriptorSetLayout setLayout, std::span layoutKey, VkShaderStageFlags pushStages, std::uint64_t programAddress) { PerformanceTimer timing("Vulkan.ComputePipeline"); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; @@ -5771,17 +5985,24 @@ std::shared_ptr VulkanDevice::createComputePipeline(cons objects->destroyPipeline = state->DeviceFunction("vkDestroyPipeline"); objects->module = Graphics::CreateShaderModule(state->DeviceFunction("vkCreateShaderModule"), state->device, state->graphicsPipelineLibrary, shader.spirv.data(), shader.spirv.size() * sizeof(std::uint32_t), "vkCreateShaderModule"); objects->resourcesLayout = setLayout; + const bool gpuMode = shader.gpuResolvedImages; const bool scratchPool = shader.scratchPoolSlotDwords != 0; - const std::array setLayouts{setLayout, shader.workgroupMemoryDwords != 0 || scratchPool ? state->workgroupMemoryLayout : VK_NULL_HANDLE, scratchPool ? state->workgroupMemoryLayout : VK_NULL_HANDLE}; const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes}; - VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - layoutInfo.setLayoutCount = scratchPool ? ShaderRecompiler::ScratchPoolDescriptorSet + 1 : shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1; - layoutInfo.pSetLayouts = setLayouts.data(); - layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; - layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; - check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); + if (!gpuMode) { + const std::array setLayouts{setLayout, shader.workgroupMemoryDwords != 0 || scratchPool ? state->workgroupMemoryLayout : VK_NULL_HANDLE, scratchPool ? state->workgroupMemoryLayout : VK_NULL_HANDLE}; + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + layoutInfo.setLayoutCount = scratchPool ? ShaderRecompiler::ScratchPoolDescriptorSet + 1 : shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1; + layoutInfo.pSetLayouts = setLayouts.data(); + layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; + layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; + check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); + } const VkPipelineCreateFlags pipelineFlags = computePipelineFlags(); - const Graphics::PipelineSpecialization specialization(shader); + const Graphics::CompiledShader placed{ShaderRecompiler::ShaderStage::Compute, &shader, 0u}; + objects->pushPlacements = Graphics::PushDataPlacements(std::span(&placed, 1)); + Graphics::PipelineSpecialization specialization(shader); + specialization.PlacePushData(objects->pushPlacements.front()); + if (gpuMode && context.descriptorHeap && Graphics::DescriptorHeapEnabled()) specialization.PlaceHeapImageSlots(Graphics::BindlessHeap::For(context).ImageSlots()); const auto subgroupSize = computeSubgroupSize(shader); const auto stageFlags = computeStageFlags(shader); if (const auto* directory = ComputePipelineDumpDirectory(); directory != nullptr && *directory != 0 && !state->probingFloatControls && ComputePipelineCapture::OptionalEnabled() && CaptureComputePipeline(shader.spirv.size())) { @@ -5812,12 +6033,22 @@ std::shared_ptr VulkanDevice::createComputePipeline(cons if (!path.empty()) std::fprintf(stderr, "[compute-capture] %s\n", path.string().c_str()); } if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); - CreateComputePipeline(state->device, state->DeviceFunction("vkCreateComputePipelines"), context.pipelineCache, objects->module, objects->layout, pipelineFlags, stageFlags, specialization.Info(), subgroupSize, &objects->pipeline, context.pipelineCacheSynchronization); + std::vector heapMappings; + const bool workgroupAddress = shader.workgroupMemoryDwords != 0; + const bool heapDevice = gpuMode && context.descriptorHeap && Graphics::DescriptorHeapEnabled(); + const auto workgroupAddressOffset = workgroupAddress ? Graphics::ComputePushConstantBytes : ~0u; + const std::span spirv(shader.spirv.data(), shader.spirv.size()); + objects->descriptorHeap = heapDevice && Graphics::BindlessHeap::For(context).DescriptorHeapMappings(spirv, heapMappings, workgroupAddressOffset, shader.dataAddressPush, false, false, false, scratchPool, &objects->pushPlacements.front()); + require(!gpuMode || objects->descriptorHeap, "a GPU-mode compute stage does not map onto VK_EXT_descriptor_heap"); + const auto createPipelines = state->DeviceFunction("vkCreateComputePipelines"); + if (objects->descriptorHeap) CreateComputeHeapPipeline(state->device, createPipelines, context.pipelineCache, objects->module, pipelineFlags, stageFlags, specialization.Info(), subgroupSize, heapMappings, &objects->pipeline, context.pipelineCacheSynchronization); + else CreateComputePipeline(state->device, createPipelines, context.pipelineCache, objects->module, objects->layout, pipelineFlags, stageFlags, specialization.Info(), subgroupSize, &objects->pipeline, context.pipelineCacheSynchronization); objects->destroyModule(objects->device, objects->module, nullptr); objects->module = VK_NULL_HANDLE; if (!state->probingFloatControls) { computePipelinesCreated.fetch_add(1, std::memory_order_relaxed); state->computePipelinesCreated.fetch_add(1, std::memory_order_relaxed); + if (objects->descriptorHeap) state->descriptorHeapComputePipelines.fetch_add(1, std::memory_order_relaxed); if (GuestMemory::GpuMutex().HeldByThisThread()) computePipelinesCreatedLocked.fetch_add(1, std::memory_order_relaxed); } Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline); @@ -5830,7 +6061,9 @@ VkPipeline VulkanDevice::splitComputePipeline(const Graphics::Context& context, if (const auto found = objects.splitPipelines.find(base); found != objects.splitPipelines.end()) return found->second; std::array offsets{}; for (std::uint32_t component = 0; component < offsets.size(); ++component) offsets[component] = {ShaderRecompiler::PipelineSpecialization::WorkgroupOffsetBase + component, base[component]}; - const Graphics::PipelineSpecialization specialization(shader, offsets); + Graphics::PipelineSpecialization specialization(shader, offsets); + specialization.PlacePushData(objects.pushPlacements.front()); + if (objects.descriptorHeap) specialization.PlaceHeapImageSlots(Graphics::BindlessHeap::For(context).ImageSlots()); VkShaderModule module = Graphics::CreateShaderModule(state->DeviceFunction("vkCreateShaderModule"), state->device, state->graphicsPipelineLibrary, shader.spirv.data(), shader.spirv.size() * sizeof(std::uint32_t), "vkCreateShaderModule"); VkPipeline pipeline = VK_NULL_HANDLE; try { @@ -5895,7 +6128,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec if (base != 0 && bytes != 0) static_cast(Graphics::FlushDepthPlanes(base, static_cast(bytes))); } } - state->recorder->BoundKeptBytes(); + state->recorder->BoundDrawBatch(); ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); PerformanceTimer timing("Vulkan.Dispatch"); if (recipeOut != nullptr) *recipeOut = nullptr; @@ -6037,7 +6270,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } if (moved.has_value() && !moved->empty()) { overlay = cached->PrepareDrawBindings(*state->recorder, *moved, true); - if (overlay != nullptr && overlay->allocation.set == VK_NULL_HANDLE) overlay.reset(); + if (overlay != nullptr && overlay->allocation.set == VK_NULL_HANDLE && !cached->HeapStage()) overlay.reset(); if (overlay == nullptr) { proof.refusal = Graphics::ShaderResources::MoveRefusal::Bindings; moved.reset(); @@ -6123,7 +6356,8 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec if (objects == nullptr) { if (verifySetLayouts && context.descriptorCache != nullptr) { const auto described = Graphics::DescribeSetLayouts(context, shaders); - require(described.layouts.main == resources->Layout() && !described.layouts.split, "the set layout described from the compute shader differs from the dispatch's"); + const auto live = resources->SetLayouts(); + require(described.layouts.main == resources->Layout() && described.layouts.split == live.split && described.layouts.fragment == live.fragment, "the set layout described from the compute shader differs from the dispatch's"); } const auto layouts = resources->SetLayouts(); objects = computePipeline(context, shader, layouts.main, layouts.mainKey, pushStages, programAddress); @@ -6234,7 +6468,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::recordCompletedDispatch(const Shader } RecipeOutcome VulkanDevice::DispatchRecipe(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint64_t arguments, std::uint64_t programAddress, const std::shared_ptr& hit, IndirectOutcome& outcome, const std::shared_ptr& verify, bool refreshByWords, const std::array& threadGroup) { - state->recorder->BoundKeptBytes(); + state->recorder->BoundDrawBatch(); ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); PerformanceTimer timing("Vulkan.DispatchRecipe"); outcome = {0, 0}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp new file mode 100644 index 000000000000..c32528f3c6f5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp @@ -0,0 +1,356 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_BINDLESSHEAP_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_BINDLESSHEAP_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +struct StagePushPlacement; +} + +namespace AgcDriver::Graphics { + +class Buffer; +class Recorder; +class HeapSlotOwner; +class Sampler; +class StorageTexture; +class Texture; + +class BindlessHeap { +public: + explicit BindlessHeap(const Context& context); + ~BindlessHeap(); + BindlessHeap(const BindlessHeap&) = delete; + BindlessHeap& operator=(const BindlessHeap&) = delete; + + static BindlessHeap& For(const Context& context); + static BindlessHeap* Existing(const Context& context); + static void Release(VkDevice device); + static bool Fits(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties); + static bool RingInResourceHeap(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties, VkDeviceSize uniformAlignment); + + bool DescriptorHeapMode() const { return resourceHeap != nullptr; } + VkDeviceSize ImageHeapStride() const { return imageHeapStride; } + void BindDescriptorHeaps(VkCommandBuffer commands) const; + bool DescriptorHeapMappings(std::span spirv, std::vector& mappings, + std::uint32_t workgroupAddressOffset = ~0u, bool rootGds = false, bool rootFault = false, + bool pixelLds = false, bool meshWindow = false, bool scratchPool = false, + const ShaderRecompiler::StagePushPlacement* placement = nullptr) const; + void WritePixelLdsBuffer(VkDeviceAddress address, VkDeviceSize bytes); + void WriteMeshWindowBuffer(VkDeviceAddress address, VkDeviceSize bytes); + std::pair DataLocation(VkDeviceAddress address) const; + + std::optional TextureSlot(std::uint32_t kind, Texture* texture, VkImageView view, VkImageLayout layout); + std::optional StorageSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view); + std::optional ViewSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view); + void ReleaseSlots(const HeapSlotOwner* owner); + std::uint32_t SlotGeneration(std::uint32_t slot) const; + const Texture* SlotTexture(std::uint32_t kind, std::uint32_t slot) const; + bool SlotHoldsStorage(std::uint32_t kind, std::uint32_t slot, const HeapSlotOwner* owner, VkImageView view) const; + std::optional SamplerSlot(const std::shared_ptr& sampler); + std::uint32_t SamplerGeneration(std::uint32_t slot) const; + std::uint64_t SlotSerial() const { return slotSerial.load(std::memory_order_acquire); } + bool GenerationsCurrent(std::span> slots, std::span slotGenerations, + std::span samplerSlots, std::span samplerGenerations) const; + std::uint32_t AbsentSlot(std::uint32_t kind, VkImageView view); + static bool NearlyFull(); + static std::uint32_t ImageSlotCapacity(); + std::uint32_t ImageSlots() const { return slotCap; } + std::uint32_t TouchSlots(std::span> slots, std::uint32_t touched) const; + + struct DataSpan { + std::uint32_t offset = 0; + std::span words; + VkDeviceAddress address = 0; + }; + DataSpan Allocate(std::size_t words); + struct SpanMark { + std::uint32_t offset = 0; + std::uint32_t words = 0; + std::uint64_t entry = 0; + std::uint32_t block = 0; + VkDeviceAddress address = 0; + }; + DataSpan Allocate(std::size_t words, SpanMark& mark); + DataSpan RingAllocate(std::size_t words, SpanMark& mark); + bool Held(const SpanMark& mark); + std::uint32_t RingBufferDescriptors(std::span buffers, SpanMark& mark); + static constexpr std::uint32_t BufferSlotBytes = 16u; + + static constexpr std::uint32_t SamplerKind = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Samplers); + void Submitted(Recorder& recorder, std::uint64_t serial); + bool TakeRingPressure(); + + struct Counts { + std::uint64_t ringWraps = 0, encodes = 0, encodedCopies = 0; + std::uint64_t slotsFreed = 0, slotsReused = 0, slotsEvicted = 0, evictionWaits = 0; + std::uint64_t ringWaits = 0, ringBatchPeakBytes = 0, ringSpanPeakBytes = 0, ringBytes = 0, imageSlotsHeld = 0; + std::uint64_t spansWritten = 0, spansKept = 0, spansExpired = 0, keptChunksSkipped = 0, busyChunksSkipped = 0, bufferDescriptors = 0; + std::uint64_t ringPressureSubmits = 0; + std::uint64_t poolBytes = 0, poolBatchPeakBytes = 0, poolBlocksMade = 0, poolBlocksRecycled = 0, poolBlocksFreed = 0, poolBlocksHeld = 0, poolBlockBytes = 0; + }; + Counts Statistics() const; + std::size_t RingChunkCount() const { return ringChunks.size(); } + + static constexpr std::size_t RingBytes = 64u << 20u; + static constexpr std::size_t DefaultRingBytes = 16u << 20u; + static constexpr std::size_t PoolBlockBytes = 2u << 20u; + static constexpr std::size_t PoolIdleBytes = 32u << 20u; + static constexpr std::size_t RingChunkBytes = 2u << 20u; + +private: + Context context; + std::byte* ringData = nullptr; + std::size_t ringBytes = 0; + VkDeviceSize ringHeapOffset = 0; + bool ringInHeap = false; + std::unique_ptr resourceHeap; + std::unique_ptr samplerHeap; + VkDeviceSize imageHeapStride = 0; + VkDeviceSize samplerHeapStride = 0; + VkDeviceSize resourceHeapUsed = 0; + VkDeviceSize pixelLdsHeapOffset = 0; + VkDeviceSize meshWindowHeapOffset = 0; + VkDeviceSize samplerHeapUsed = 0; + PFN_vkWriteResourceDescriptorsEXT writeResourceDescriptors = nullptr; + PFN_vkWriteSamplerDescriptorsEXT writeSamplerDescriptors = nullptr; + std::vector encodeScratch; + void writeImageSlot(std::uint32_t kind, std::uint32_t slot, VkImageView view, VkImageLayout layout); + void writeSamplerSlot(std::uint32_t slot, VkSampler sampler); + void writeBufferDescriptor(VkDeviceSize offset, VkDeviceAddress address, VkDeviceSize bytes, const char* what); + struct Chunk { + Recorder* recorder = nullptr; + std::uint64_t serial = 0; + bool open = false; + std::uint64_t entered = 0; + }; + std::vector ringChunks; + std::uint64_t ringEntries = 0; + std::size_t ringChunk = ~std::size_t{0}; + void enterRingChunk(std::size_t index); + std::size_t wrapRingOffset(std::size_t offset, std::size_t bytes); + std::size_t ringCursor = 0; + struct PoolBlock { + std::unique_ptr buffer; + std::byte* data = nullptr; + VkDeviceAddress address = 0; + std::size_t bytes = 0; + std::size_t cursor = 0; + std::uint64_t generation = 0; + Recorder* recorder = nullptr; + std::uint64_t serial = 0; + bool open = false; + }; + std::vector poolBlocks; + std::size_t poolCurrent = ~std::size_t{0}; + std::uint32_t poolAlignment = 256; + bool poolDeviceLocal = true; + std::size_t poolBatchBytes = 0; + std::size_t takePoolBlockLocked(std::size_t bytes); + bool poolBlockDoneLocked(PoolBlock& block); + void trimPoolLocked(); + std::uint32_t ringAlignment = 256; + mutable std::mutex mutex; + std::atomic slotSerial{1}; + struct ImageSlotKey { + std::uint32_t kind; + const HeapSlotOwner* owner; + VkImageView view; + VkImageLayout layout; + bool operator==(const ImageSlotKey&) const = default; + }; + struct ImageSlotKeyHash { + std::size_t operator()(const ImageSlotKey& key) const { + constexpr std::size_t HashSalt = 0x9e3779b97f4a7c15ull; + std::size_t seed = 0; + const auto mix = [&seed](std::size_t value) { seed ^= value + HashSalt + (seed << 6u) + (seed >> 2u); }; + mix(key.kind); + mix(static_cast(key.layout)); + mix(std::hash{}(key.owner)); + mix(std::hash{}(reinterpret_cast(key.view))); + return seed; + } + }; + std::unordered_map imageSlots; + std::unordered_map> ownerSlots; + struct SlotPool { + std::uint32_t next = 0; + std::uint32_t limit = 0; + std::vector free; + std::vector generations; + std::optional Take(bool& reused); + void Free(std::uint32_t slot); + std::size_t Unused() const; + std::uint32_t Generation(std::uint32_t slot) const; + }; + struct SlotOwner { + const Texture* texture = nullptr; + ImageSlotKey key{}; + }; + std::vector owners; + SlotPool imagePool; + SlotPool samplerPool; + std::uint32_t slotCap = 0; + std::unique_ptr[]> slotUses; + std::atomic useEpoch{1}; + struct RetiringSlot { + std::uint32_t slot = 0; + std::uint64_t serial = 0; + }; + std::deque retiring; + std::size_t ringBatchBytes = 0; + std::atomic ringOpenChunks{0}; + std::size_t retiringAssigned = 0; + std::uint32_t evictScanEpoch = 0; + Recorder* submitter = nullptr; + void touchLocked(std::uint32_t slot) const; + void evictLocked(); + bool reclaimLocked(bool wait); + std::optional imageSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view, VkImageLayout layout, Texture* texture); + SlotOwner& claimSlotLocked(const ImageSlotKey& key, std::uint32_t slot); + void freeSlotLocked(std::uint32_t slot); + const SlotOwner* ownerLocked(std::uint32_t slot) const; + std::unordered_map samplerSlots; + std::unordered_set absentSlots; + bool refused = false; + void noteRefusedLocked(const char* what, std::uint32_t margin); + void noteTakenLocked(); + void noteSamplerTakenLocked(); + void noteReleasedLocked(); + std::uint32_t imageMargin() const; + bool roomLocked() const; + Counts counts; +}; + +class HeapSlotOwner { +public: + HeapSlotOwner(const HeapSlotOwner&) = delete; + HeapSlotOwner& operator=(const HeapSlotOwner&) = delete; + +protected: + HeapSlotOwner() = default; + ~HeapSlotOwner() = default; + void ReleaseHeapSlots(const Context& context) noexcept { + if (heapSlotted.exchange(false, std::memory_order_acq_rel)) { + if (auto* heap = BindlessHeap::Existing(context)) heap->ReleaseSlots(this); + } + } + +private: + friend class BindlessHeap; + mutable std::atomic heapSlotted{false}; +}; + +bool GpuHeapEnabled(); +void SetDescriptorHeapDevice(bool available); +bool DescriptorHeapEnabled(); +bool DescriptorHeapMappable(std::span spirv, bool workgroupAddress, bool gdsAddress, bool pixelLds = false, bool meshWindow = false, bool scratchPool = false); +void NoteHeapComputeStage(); +std::uint64_t HeapComputeStages(); +void NoteImageTableBuild(); +std::uint64_t ImageTableBuilds(); +void NoteDataAddressPush(); +void NoteEmptyResourceSet(); +std::uint64_t EmptyResourceSets(); +std::uint64_t DataAddressPushes(); +enum class SetBindSite : std::uint8_t { Draw, Dispatch, Detiler, ColorTransfer, Presentation, DepthTransfer, StagedMerge, FirstSampleCopy, Count }; +enum class HeapFallback : std::uint8_t { DrawUnmapped, DrawMixedRetry, DrawRefusedPair, DrawMixedReused, DispatchUnmapped, Count }; +enum class HeapStageReason : std::uint8_t { + Heap, + Disabled, + PairRefused, + FragmentOffHeap, + StageKind, + NoPlan, + PlanStage, + UnresolvedImageSource, + KnownIneligible, + Count +}; +void NoteHeapStageReason(HeapStageReason reason); +void NoteHeapRecord(bool compute, bool heap); +void NoteHeapFallback(HeapFallback reason); +std::uint64_t HeapFallbacks(HeapFallback reason); +void NoteCpuBufferCopy(std::uint64_t bytes); +enum class HeapCost : std::uint8_t { + StagePrepare, + HeapStageCall, + StageFinish, + WalkMemo, + Walk, + Materialize, + TemplateFind, + TemplateBuild, + Slots, + StageData, + BufferAddresses, + MemoInsert, + Guards, + Count +}; +bool HeapCostTimed(); + +void NoteHeapWalkMemoBytes(std::int64_t delta); +void NoteHeapCost(HeapCost cost, std::uint64_t nanoseconds); +class HeapCostScope { +public: + explicit HeapCostScope(HeapCost cost) : + cost(cost), start(HeapCostTimed() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}) {} + ~HeapCostScope() { Stop(); } + HeapCostScope(const HeapCostScope&) = delete; + HeapCostScope& operator=(const HeapCostScope&) = delete; + void Stop() { + if (start == std::chrono::steady_clock::time_point{}) return; + NoteHeapCost(cost, static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count())); + start = {}; + } + +private: + HeapCost cost; + std::chrono::steady_clock::time_point start; +}; +void NoteCommandBufferBegun(VkCommandBuffer commands); +void CmdPushConstants(const Context& context, VkCommandBuffer commands, VkPipelineLayout layout, VkShaderStageFlags stages, + std::uint32_t offset, std::uint32_t size, const void* values); +void CmdBindDescriptorSets(const Context& context, VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout, + std::uint32_t firstSet, std::uint32_t setCount, const VkDescriptorSet* sets, SetBindSite site); +void CmdPushDescriptorSet(PFN_vkCmdPushDescriptorSetKHR push, VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout, std::uint32_t set, + std::uint32_t writeCount, const VkWriteDescriptorSet* writes, SetBindSite site); +std::uint64_t DescriptorHeapBinds(); +inline constexpr std::uint32_t HeapPassSampledKind = ShaderRecompiler::RuntimeAbi::FirstImageBinding; +inline constexpr std::uint32_t HeapPassStorageKind = ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding; +inline constexpr VkSpirvResourceTypeFlagsEXT HeapPassStorageImages = VK_SPIRV_RESOURCE_TYPE_READ_ONLY_IMAGE_BIT_EXT | VK_SPIRV_RESOURCE_TYPE_READ_WRITE_IMAGE_BIT_EXT; +class HeapPassSlots : public HeapSlotOwner { +public: + explicit HeapPassSlots(const Context& context) : context(context) {} + ~HeapPassSlots() { ReleaseHeapSlots(context); } + void Release() noexcept { ReleaseHeapSlots(context); } + +private: + Context context; +}; +bool HeapPassesActive(const Context& context); +VkDescriptorSetAndBindingMappingEXT HeapPushAddressMapping(std::uint32_t set, std::uint32_t binding, std::uint32_t pushOffset); +VkDescriptorSetAndBindingMappingEXT HeapPushIndexMapping(const BindlessHeap& heap, std::uint32_t set, std::uint32_t binding, VkSpirvResourceTypeFlagsEXT resources, + std::uint32_t pushOffset); +VkPipeline CreateHeapComputePipeline(const Context& context, VkComputePipelineCreateInfo info, std::span mappings); +VkPipeline CreateHeapGraphicsPipeline(const Context& context, VkGraphicsPipelineCreateInfo info, std::span mappings); +void CmdPushData(const Context& context, VkCommandBuffer commands, std::uint32_t bytes, const void* values); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 8372333ee4a2..8dc408e6fd1d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -69,6 +69,8 @@ class BufferPool { // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. static std::size_t Capacity(std::size_t bytes); static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties); + static bool ForceAddresses(); + static void SetForceAddresses(bool force); VkDeviceMemory AllocateMemory(const VkMemoryAllocateInfo& allocation, const char* operation, std::optional hostType = std::nullopt, std::uint32_t* allocatedType = nullptr, bool newImage = false); ImageAllocation AllocateImage(const Context& context, VkImage image, const VkImageCreateInfo& imageInfo, const char* operation, bool newImage, bool budgeted); std::size_t ImageSlabBlocks(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 1f9f7e982b65..92d559f4201d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -8,6 +8,7 @@ #include #include "prx/libc/include/PrimedSharedMutex.hpp" #include +#include "prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp" #include #include #include @@ -148,8 +149,52 @@ struct DeviceFunctions { // Per-thread count of vkGetDeviceProcAddr lookups made through Context::Function (the [vk] line). std::uint64_t& DeviceProcLookups(); +class DeviceFunctionCache { +public: + PFN_vkVoidFunction Find(const char* name, PFN_vkGetDeviceProcAddr proc) const { + for (std::size_t probe = 0, slot = slotOf(name); probe < MaxProbes; ++probe, slot = (slot + 1u) & (Slots - 1u)) { + const auto& entry = entries[slot]; + const auto* stored = entry.name.load(std::memory_order_acquire); + if (stored == nullptr) return nullptr; + if (stored != name) continue; + const auto function = entry.function.load(std::memory_order_acquire); + return function != nullptr && entry.proc.load(std::memory_order_relaxed) == proc ? function : nullptr; + } + return nullptr; + } + + void Keep(const char* name, PFN_vkGetDeviceProcAddr proc, PFN_vkVoidFunction function) { + for (std::size_t probe = 0, slot = slotOf(name); probe < MaxProbes; ++probe, slot = (slot + 1u) & (Slots - 1u)) { + auto& entry = entries[slot]; + const char* expected = nullptr; + if (!entry.name.compare_exchange_strong(expected, name, std::memory_order_acq_rel) && expected != name) continue; + if (expected == name) return; + entry.proc.store(proc, std::memory_order_relaxed); + entry.function.store(function, std::memory_order_release); + return; + } + } + +private: + static constexpr std::size_t Slots = 2048; + static constexpr std::size_t MaxProbes = 32; + struct Entry { + std::atomic name{nullptr}; + std::atomic proc{nullptr}; + std::atomic function{nullptr}; + }; + static std::size_t slotOf(const char* name) { + return static_cast((reinterpret_cast(name) >> 3u) * 0x9e3779b97f4a7c15ull >> 40u) & (Slots - 1u); + } + std::array entries{}; +}; + inline constexpr std::size_t EmptyBufferBytes = 16; +inline VkPipelineCreateFlags2CreateInfoKHR DescriptorHeapFlags(VkPipelineCreateFlags2KHR flags = 0, const void* next = nullptr) { + return {VK_STRUCTURE_TYPE_PIPELINE_CREATE_FLAGS_2_CREATE_INFO_KHR, next, flags | VK_PIPELINE_CREATE_2_DESCRIPTOR_HEAP_BIT_EXT}; +} + struct Context { VkDevice device; VkPhysicalDevice physical; @@ -199,6 +244,9 @@ struct Context { GraphicsPipelineCache* graphicsPipelines = nullptr; // Batches GPU work across guest commands (see Recorder); null before the device finished setup. Recorder* recorder = nullptr; + bool descriptorHeap = false; + bool heapBufferDescriptors = false; + VkPhysicalDeviceDescriptorHeapPropertiesEXT descriptorHeapProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_HEAP_PROPERTIES_EXT}; // Per-device caches of the immutable descriptor objects a ShaderResources build needs (set // layouts, descriptor pools, samplers); null (tests) means every build makes and destroys its own. DescriptorCache* descriptorCache = nullptr; @@ -230,6 +278,7 @@ struct Context { const std::vector>* copiedWriters = nullptr; // The device's resolved entry points (see DeviceFunctions); null until the device set up. const DeviceFunctions* functions = nullptr; + DeviceFunctionCache* functionCache = nullptr; // VK_EXT_descriptor_indexing with non-uniform sampled/storage image array indexing enabled // (bindless image tables in graphics stages). bool descriptorIndexing = false; @@ -292,10 +341,21 @@ struct Context { template TFunction Function(const char* name) const { Require(deviceProc != nullptr, "missing Vulkan device function resolver"); + const auto* cache = functions != nullptr ? functionCache : nullptr; + if (cache != nullptr) { + if (const auto cached = cache->Find(name, deviceProc); cached != nullptr) return reinterpret_cast(cached); + } + if (descriptorHeap) { + if (const auto hooked = DescriptorHeapObjectFunction(device, deviceProc, name); hooked != nullptr) { + if (cache != nullptr) functionCache->Keep(name, deviceProc, hooked); + return reinterpret_cast(hooked); + } + } ++DeviceProcLookups(); - const auto function = reinterpret_cast(deviceProc(device, name)); + const auto function = deviceProc(device, name); if (function == nullptr) throw std::runtime_error(std::string("AGC graphics: missing Vulkan function: ") + name); - return function; + if (cache != nullptr) functionCache->Keep(name, deviceProc, function); + return reinterpret_cast(function); } // The table's entry point, or a per-call lookup while the table is absent or lacks it. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp new file mode 100644 index 000000000000..ac2b8a2ff592 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp @@ -0,0 +1,38 @@ +#ifndef AGC_GRAPHICS_DESCRIPTOR_HEAP_OBJECTS_HPP +#define AGC_GRAPHICS_DESCRIPTOR_HEAP_OBJECTS_HPP + +#include + +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct ImageViewRecord { + VkImageViewCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; + VkImageViewMinLodCreateInfoEXT minLod{VK_STRUCTURE_TYPE_IMAGE_VIEW_MIN_LOD_CREATE_INFO_EXT}; + bool hasUsage = false; + bool hasMinLod = false; + std::uint64_t generation = 0; + const VkImageViewCreateInfo* Chain(); +}; + +struct SamplerRecord { + VkSamplerCreateInfo info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; + VkSamplerReductionModeCreateInfo reduction{VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO}; + bool hasReduction = false; + const VkSamplerCreateInfo* Chain(); +}; + +PFN_vkVoidFunction DescriptorHeapObjectFunction(VkDevice device, PFN_vkGetDeviceProcAddr deviceProc, const char* name); +std::optional FindImageView(VkImageView view); +std::optional FindSampler(VkSampler sampler); +bool CopyEncodedImageView(VkImageView view, std::uint32_t key, std::span target); +void KeepEncodedImageView(VkImageView view, std::uint64_t generation, std::uint32_t key, std::span bytes); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp new file mode 100644 index 000000000000..ce428791b0cc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp @@ -0,0 +1,81 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEVICEIMAGEKEYS_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEVICEIMAGEKEYS_HPP + +#include "DeviceImageTableAbi.hpp" +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct DeviceImageValue { + std::uint32_t slot = 0; + std::uint32_t mode = 0; + std::uint32_t fault = 0; + std::uint32_t samplerSlot = 0; + + bool operator==(const DeviceImageValue& other) const = default; +}; + +class DeviceImageKeys { +public: + using Key = ShaderRecompiler::DeviceImageTableAbi::Key; + static constexpr std::uint32_t InitialBuckets = 4096u; + static constexpr std::uint32_t MaxBuckets = 1u << 17u; + + explicit DeviceImageKeys(std::uint32_t buckets = InitialBuckets, std::uint32_t maxBuckets = MaxBuckets); + + std::optional Find(const Key& key) const; + enum class Insertion { Inserted, Updated, Full }; + Insertion Insert(const Key& key, const DeviceImageValue& value); + void Clear(); + std::uint32_t Retain(const std::function& keep); + + std::uint32_t Buckets() const { return static_cast(words.size() / ShaderRecompiler::DeviceImageTableAbi::BucketWords); } + std::uint32_t Entries() const { return entries; } + std::span Words() const { return words; } + bool Rebuilt() const { return rebuilt; } + std::vector TakeChanged(); + + template + void ForEach(TVisit&& visit) const { + namespace Abi = ShaderRecompiler::DeviceImageTableAbi; + for (std::uint32_t bucket = 0; bucket < Buckets(); ++bucket) { + const auto* at = words.data() + static_cast(bucket) * Abi::BucketWords; + if (at[Abi::StateWord] != Abi::LiveState) continue; + Key key{}; + for (std::uint32_t word = 0; word < Abi::KeyWords; ++word) key[word] = at[word]; + visit(key, DeviceImageValue{at[Abi::SlotWord], at[Abi::ModeWord], at[Abi::FaultWord], at[Abi::SamplerSlotWord]}); + } + } + +private: + std::optional place(std::vector& target, const Key& key, bool& found) const; + bool grow(); + + std::vector words; + std::uint32_t maxBuckets; + std::uint32_t entries = 0; + bool rebuilt = true; + std::vector changed; +}; + +struct DeviceImageMiss { + DeviceImageKeys::Key key{}; + std::uint32_t pc = 0; +}; + +struct DeviceImageRing { + std::vector misses; + std::uint64_t written = 0; + std::uint64_t dropped = 0; +}; + +DeviceImageRing DecodeDeviceImageRing(std::span ring, std::uint32_t capacity); +std::vector DistinctDeviceImageMisses(std::span misses); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp new file mode 100644 index 000000000000..fc70e0fa837e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp @@ -0,0 +1,254 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEVICEIMAGETABLE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_DEVICEIMAGETABLE_HPP + +#ifndef VK_NO_PROTOTYPES +#define VK_NO_PROTOTYPES +#endif +#include +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp" +#include "Recompiler.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "IntermediateRepresentation/IrMetadata/Resources.hpp" +#include "DeviceImageTableAbi.hpp" +#include "ImageTableAbi.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +class Buffer; +class Recorder; +class Texture; + +struct DeviceTableView { + ShaderRecompiler::DescriptorImageShape shape = ShaderRecompiler::DescriptorImageShape::Image2D; + bool depthCompare = false; + bool identitySwizzle = false; +}; + +struct DeviceTableTexture { + std::shared_ptr texture; + VkImageView view = VK_NULL_HANDLE; + VkImageLayout layout = VK_IMAGE_LAYOUT_UNDEFINED; + std::uint64_t address = 0; + std::uint64_t guestBytes = 0; + std::uint64_t generation = 0; + std::string unservable; + std::array words{}; + bool depthCompare = false; +}; + +DeviceTableTexture LookupDeviceTableTexture(const Context& context, std::span descriptor, const DeviceTableView& view, std::uint32_t samplerClass); +bool DeviceTableTextureCurrent(const Context& context, const std::array& words, bool depthCompare, const Texture& texture); + +bool GpuPathEnabled(); + +struct DeviceImagePathFeatures { + bool gpuHeap = false; + bool descriptorHeap = false; +}; + +std::optional DeviceImagePathRefusal(const DeviceImagePathFeatures& features); + +class DeviceImageTable { +public: + using Key = DeviceImageKeys::Key; + using Interface = ShaderRecompiler::DeviceImageInterface; + using InterfaceLookup = std::function(std::uint64_t id)>; + struct Slots { + std::function(std::uint32_t kind, const std::shared_ptr& texture, VkImageView view, VkImageLayout layout)> take; + std::function generation; + std::function serial; + std::function nearlyFull; + std::function> slots, std::uint32_t touched)> touch; + }; + struct Budget { + std::uint32_t entries = 16384u; + std::uint64_t textureBytes = 0; + std::uint64_t checkPresents = 60u; + std::uint64_t idlePresents = 120u; + std::uint32_t maxBuckets = DeviceImageKeys::MaxBuckets; + }; + static constexpr std::uint32_t OverflowBatches = 3u; + + static constexpr std::uint32_t RingCapacity = 4096u; + static constexpr std::uint32_t RingChunks = 16u; + static constexpr std::uint32_t RingWords = ShaderRecompiler::DeviceImageTableAbi::RingRecordsWord + RingCapacity * ShaderRecompiler::DeviceImageTableAbi::RecordWords; + static constexpr std::uint32_t TableWords = ShaderRecompiler::DeviceImageTableAbi::HeaderWords + DeviceImageKeys::MaxBuckets * ShaderRecompiler::DeviceImageTableAbi::BucketWords; + + static Slots HeapSlots(const Context& context); + + DeviceImageTable(const Context& context, InterfaceLookup lookup, Slots slots); + DeviceImageTable(const Context& context, InterfaceLookup lookup, Slots slots, Budget budget); + ~DeviceImageTable(); + DeviceImageTable(const DeviceImageTable&) = delete; + DeviceImageTable& operator=(const DeviceImageTable&) = delete; + + static DeviceImageTable& Create(const Context& context, InterfaceLookup lookup); + static DeviceImageTable* Existing(const Context& context); + static bool ExpectsKeys(); + static void Release(VkDevice device); + + void Prepare(const Context& build, Recorder& recorder, std::span programs = {}); + std::optional Expect(const Context& build, Recorder& recorder, std::span images); + void FallBackProgram(const ShaderRecompiler::PipelineIdentity& identity, std::uint64_t codeAddress, const std::string& reason); + void SetInterfaceLookup(InterfaceLookup lookup); + void NoteProgram(const ShaderRecompiler::PipelineIdentity& identity, std::uint64_t codeAddress); + bool ServesProgram(std::uint64_t codeAddress) const; + bool ServesIdentity(const ShaderRecompiler::PipelineIdentity& identity) const; + ShaderRecompiler::ImageTableAbi::Trailer Trailer(Recorder& recorder); + std::span RingChunk(Recorder& recorder); + VkBuffer TableBuffer() const; + std::optional Find(const Key& key) const; + std::vector KeyWords() const; + std::string LastFault() const; + std::string LastFallback() const; + std::uint32_t OverflowStreak(const ShaderRecompiler::PipelineIdentity& identity) const; + std::shared_ptr EntryTexture(const Key& key) const; + + struct Counts { + std::uint64_t prepares = 0; + std::uint64_t drained = 0; + std::uint64_t distinct = 0; + std::uint64_t dropped = 0; + std::uint64_t inserted = 0; + std::uint64_t faults = 0; + std::uint64_t rejected = 0; + std::uint64_t uploads = 0; + std::uint64_t uploadBytes = 0; + std::uint64_t chunkWaits = 0; + std::uint64_t unserved = 0; + std::uint64_t heapFull = 0; + std::uint64_t tableFull = 0; + std::uint64_t scans = 0; + std::uint64_t checked = 0; + std::uint64_t refreshed = 0; + std::uint64_t evicted = 0; + std::uint64_t swept = 0; + std::uint64_t fallbacks = 0; + std::uint64_t expected = 0; + std::uint64_t expectedInserted = 0; + std::uint32_t entries = 0; + std::uint32_t buckets = 0; + std::array peakTextures{}; + }; + Counts Totals() const; + static std::vector DeviceTotals(); + +private: + struct Entry { + std::shared_ptr texture; + std::uint32_t kind = 0; + std::uint32_t slot = 0; + std::uint32_t slotGeneration = 0; + std::uint64_t address = 0; + std::uint64_t guestBytes = 0; + std::uint64_t generation = 0; + std::uint64_t stamp = 0; + std::uint64_t used = 0; + std::uint64_t usedPresent = 0; + std::array words{}; + bool depthCompare = false; + }; + struct ValidationState { + std::uint64_t epoch = 0; + std::uint64_t unwatch = 0; + std::uint64_t depthSurfaces = 0; + std::uint64_t pending = 0; + std::uint64_t ownWrites = 0; + bool operator==(const ValidationState&) const = default; + }; + struct Block { + std::uint64_t generation = 0; + std::set keys; + }; + struct Rejection { + ShaderRecompiler::ImageTableAbi::PoisonReason reason = ShaderRecompiler::ImageTableAbi::PoisonReason::DriverRejected; + std::string message; + }; + struct Chunk { + std::uint64_t serial = 0; + std::uint64_t generation = 0; + bool busy = false; + std::set programs; + }; + using Programs = std::shared_ptr>; + struct PendingMiss { + DeviceImageMiss miss; + Programs programs; + }; + + void resolve(const Context& build, Recorder& recorder); + std::optional lookupKey(const Context& build, const Key& key, Entry& entry, std::string* failure = nullptr); + DeviceImageValue reject(const Key& key, ShaderRecompiler::ImageTableAbi::PoisonReason reason, const std::string& message); + void unserved(const Key& key, const std::string& why); + void heapFull(const Key& key, std::uint32_t kind); + bool insert(const Key& key, const DeviceImageValue& value); + void fallBack(const std::set& programs, const std::string& reason); + void hold(const Key& key, Entry entry); + void release(const Key& key, Recorder& recorder); + void retainSlots(Recorder& recorder); + void touchSlots(); + void refresh(const Context& build, Recorder& recorder); + void evict(const Context& build, Recorder& recorder); + void upload(Recorder& recorder); + VkDeviceSize writeHeader(VkCommandBuffer commands); + std::uint32_t chunkFor(Recorder& recorder, bool mayWait); + void drain(std::uint32_t chunk); + std::string faultText(const Key& key, std::uint32_t pc) const; + + Context context; + InterfaceLookup lookup; + Slots slots; + mutable std::recursive_mutex mutex; + DeviceImageKeys keys; + std::map entries; + std::map rejections; + std::vector pending; + std::unique_ptr table; + std::unique_ptr ring; + VkDeviceSize chunkStride = 0; + std::vector chunks; + std::uint32_t uploadedBuckets = 0; + std::array liveTextures{}; + std::map, std::uint32_t> textureUses; + std::set unservedKeys; + std::set fullKinds; + Budget budget; + std::map blocks; + std::optional validated; + std::uint64_t scannedStamp = 0; + std::uint64_t slotSerial = 0; + std::vector> heldSlots; + bool heldSlotsStale = true; + std::uint32_t touchedEpoch = 0; + std::uint64_t useSerial = 0; + std::uint64_t evictionCheck = 0; + std::map> identityPrograms; + std::set failedPrograms; + std::set fallbackPrograms; + std::map overflowStreaks; + std::map overflowFrom; + std::uint64_t resolveGeneration = 0; + bool tableFullReported = false; + std::string lastFault; + std::string lastFallback; + std::shared_ptr alive = std::make_shared(true); + Counts totals; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp index 4f973ff75e27..6b384ab00c81 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp @@ -6,6 +6,7 @@ namespace AgcDriver::Graphics { class Recorder; +class Texture; class FirstSampleCopy { public: @@ -14,7 +15,7 @@ class FirstSampleCopy { FirstSampleCopy(const FirstSampleCopy&) = delete; FirstSampleCopy& operator=(const FirstSampleCopy&) = delete; - void Record(Recorder& recorder, VkImageView source, VkImageLayout layout); + void Record(Recorder& recorder, Texture& source); VkImage Image() const { return image; } private: @@ -33,6 +34,7 @@ class FirstSampleCopy { VkShaderModule vertex = VK_NULL_HANDLE; VkShaderModule fragment = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline heapPipeline = VK_NULL_HANDLE; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp index 672a39649356..69d20e25bce0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp @@ -32,6 +32,8 @@ class GpuColorTransfer { VkDescriptorSet descriptorSet = VK_NULL_HANDLE; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline heapPipeline = VK_NULL_HANDLE; + VkBufferUsageFlags addressUsage = 0; // Guest bytes in host memory, and the device-local copies the conversion shader works on. std::unique_ptr tiled; std::unique_ptr tiledDevice; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 8e21355bd9cf..cdddf910540b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -379,6 +379,7 @@ class GuestBufferMemory { void UploadPrepare(bool addressable); void UploadFinish(bool addressable, std::optional> htileClear = std::nullopt); VkDescriptorBufferInfo Descriptor(std::uint64_t address, std::size_t bytes, std::uint32_t& adjustment) const; + std::optional TryDescriptor(std::uint64_t address, std::size_t bytes, std::uint32_t& adjustment, std::string* failure = nullptr) const; static std::uint64_t ViewBytes(std::uint64_t bytes, std::uint32_t adjustment); std::vector AddressRanges() const; // The BDA table of the cached address space when it serves this upload alone (an address-based diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp index c94ee8c8d878..cd0cedad85d5 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp @@ -20,6 +20,14 @@ namespace AgcDriver::Graphics { +class HeapSlotOwner; + +struct MultisampleStorage { + VkImageView view = VK_NULL_HANDLE; + const HeapSlotOwner* owner = nullptr; + std::shared_ptr keep; +}; + class StorageTexture; class Texture; struct GuestTextureResource; @@ -104,7 +112,7 @@ VkImageView MultisampleTargetView(const Context& context, const ColorTarget& tar VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target, std::shared_ptr& owner, VkImage* image = nullptr); void KeepMultisampleTarget_nid_no_patch(const std::shared_ptr& owner); void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget& target); -VkImageView MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource, bool uintStorage = false); +MultisampleStorage MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource, bool uintStorage = false); bool MultisampleTargetStorable(const Context& context, const ColorTarget& target); void ResolveMultisampleTarget(const Context& context, const ColorTarget& source, StorageTexture& destination, VkImageView destinationView); bool MultisampledColorFillAbsorbs(std::uint64_t address, std::size_t bytes, std::uint32_t pattern); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 9977db556e6d..0c42e343878f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -47,6 +47,7 @@ class Pipeline { Pipeline(const Pipeline&) = delete; Pipeline& operator=(const Pipeline&) = delete; VkPipelineLayout Layout() const; + bool DescriptorHeap() const { return descriptorHeap; } std::shared_ptr AcquireFramebuffer(std::span targets, std::span> owners, VkExtent2D extent, std::uint32_t layers = 1); // Begins the render pass on the framebuffer, binds the pipeline and sets its dynamic state. void Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, VkExtent2D extent, const State& state) const; @@ -63,6 +64,7 @@ class Pipeline { bool SampleLocationsEnabled() const { return sampleLocationsEnabled; } VkExtent2D HostSampleLocationGrid() const { return sampleLocationGrid; } void PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const; + std::span PushPlacements() const { return pushPlacements; } // Forgets the Vulkan objects without destroying them: for entries of a device that is already gone. void Abandon() noexcept; @@ -80,6 +82,7 @@ class Pipeline { void setLibraryState(VkCommandBuffer commands, const State& state) const; Context context; std::vector _modules; + std::vector pushPlacements; VkPipelineLayout layout = VK_NULL_HANDLE; VkRenderPass renderPass = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; @@ -94,6 +97,7 @@ class Pipeline { bool libraries = false; std::shared_ptr optimized; bool dynamicRendering = false; + bool descriptorHeap = false; VkFormat depthFormat = VK_FORMAT_UNDEFINED; VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_UNDEFINED; std::vector colorSlots; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp index fcd61045697d..18ff01097e70 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp @@ -42,7 +42,7 @@ struct OptimizedPipeline { void Schedule(); }; -VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized = nullptr, bool partsOnly = false, std::span modules = {}); +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, bool descriptorHeap, std::shared_ptr* optimized = nullptr, bool partsOnly = false, std::span modules = {}); void WaitForOptimizedPipelines(VkDevice device); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp index d7628355bbed..3d9a7b194390 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp @@ -79,6 +79,24 @@ class PipelineSpecialization { [[nodiscard]] const VkSpecializationInfo* Info() const { return entries.empty() ? nullptr : &info; } + void PlaceHeapImageSlots(std::uint32_t slots) { + for (auto& constant : values) { + if (constant.id == ShaderRecompiler::PipelineSpecialization::HeapImageSlots) constant.value = slots; + } + } + + void PlacePushData(const ShaderRecompiler::StagePushPlacement& placement) { + namespace Ids = ShaderRecompiler::PipelineSpecialization; + for (auto& constant : values) { + if (constant.id == Ids::PushedDataBase) constant.value = placement.pushed.base; + if (constant.id == Ids::PushedDataCount) constant.value = placement.pushed.count; + if (constant.id >= Ids::RootItemPositionBase && constant.id < Ids::RootItemPositionBase + ShaderRecompiler::RuntimeAbi::RootItemCount) { + constant.value = placement.items[constant.id - Ids::RootItemPositionBase]; + Require(constant.value != ShaderRecompiler::RuntimeAbi::NoPushPosition, "a stage reads a root item its pipeline's push layout does not hold"); + } + } + } + private: void describe(std::span constants) { entries.reserve(constants.size()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index e4dc5db5229b..9d37d07e9834 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -121,6 +121,7 @@ class Recorder { void FlushDeferredReleases(); static constexpr std::size_t KeptBytesBudget = std::size_t{512} << 20u; void BoundKeptBytes(); + void BoundDrawBatch(); std::size_t InFlightKeptBytes() const { return inFlightKeptBytes; } enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32, Index16Restart, Index32Restart, PaddedVertex }; static constexpr std::size_t DrawSnapshotBudget = std::size_t{1024} << 20u; @@ -405,6 +406,7 @@ class Recorder { // validation instead of one per run, and every test sees the same snapshot (design13 R1's p0). using WriteRanges = std::vector>; static std::shared_ptr PendingWriteSnapshot(); + static bool ReadsSettled(std::span> ranges, const WriteRanges* before); static bool SnapshotOverlaps(const WriteRanges* snapshot, std::uint64_t address, std::size_t bytes); // A label behind pending completions (CPU write-backs the label must not precede): stored to // guest memory by a completion action appended to the open batch, or to the newest in-flight @@ -784,9 +786,14 @@ class Recorder { int meshArgumentState = 0; VkPipelineLayout meshArgumentLayout = VK_NULL_HANDLE; VkPipeline meshArgumentPipeline = VK_NULL_HANDLE; + VkPipeline meshArgumentHeapPipeline = VK_NULL_HANDLE; int changedCopyState = 0; VkPipelineLayout changedCopyLayout = VK_NULL_HANDLE; VkPipeline changedCopyPipeline = VK_NULL_HANDLE; + VkPipeline changedCopyHeapPipeline = VK_NULL_HANDLE; + VkPipeline createPushOnlyHeapPipeline(const VkComputePipelineCreateInfo& info) const; + void bindPushOnlyPipeline(VkCommandBuffer commands, VkPipeline pipeline, VkPipeline heapPipeline) const; + void pushOnlyParameters(VkCommandBuffer commands, VkPipeline heapPipeline, VkPipelineLayout layout, std::uint32_t bytes, const void* values) const; int sampleCounterState = 0; std::unique_ptr sampleCounter; VkPipelineLayout sampleLayout = VK_NULL_HANDLE; @@ -798,6 +805,7 @@ class Recorder { int dispatchArgumentState = 0; VkPipelineLayout dispatchArgumentLayout = VK_NULL_HANDLE; VkPipeline dispatchArgumentPipeline = VK_NULL_HANDLE; + VkPipeline dispatchArgumentHeapPipeline = VK_NULL_HANDLE; // BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which // Commands() would close). std::uint32_t beginTiming(std::uint64_t key); @@ -818,7 +826,6 @@ class Recorder { PFN_vkCmdResetQueryPool cmdResetQueryPool = nullptr; PFN_vkCmdCopyQueryPoolResults cmdCopyQueryPoolResults = nullptr; PFN_vkCmdBindPipeline cmdBindPipeline = nullptr; - PFN_vkCmdPushConstants cmdPushConstants = nullptr; PFN_vkCmdDispatch cmdDispatch = nullptr; template TFunction function(TFunction resolved, const char* name) const { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index 2de6f3010068..87762d9fc4b8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -6,6 +6,8 @@ namespace AgcDriver::Graphics { +VkDeviceAddress BufferDeviceAddress(const Context& context, VkBuffer buffer); + class Buffer { public: Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/SampleOrder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/SampleOrder.hpp index dc1a089fc536..2479210a4249 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/SampleOrder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/SampleOrder.hpp @@ -37,6 +37,8 @@ struct SampleOrderPass { bool located = false; }; +class HeapSlotOwner; + struct SampleOrderTarget { VkImage image = VK_NULL_HANDLE; VkImageView view = VK_NULL_HANDLE; @@ -45,6 +47,7 @@ struct SampleOrderTarget { std::uint32_t samples = 1; SampleOrderMethod method = SampleOrderMethod::Storage; std::shared_ptr keep; + const HeapSlotOwner* owner = nullptr; }; struct DepthSampleLocations { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp index 65384895fd60..7616def6496a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp @@ -1,19 +1,22 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SAMPLER_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_SAMPLER_HPP +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp" #include "prx/libSceAgcDriver/Execution/include/Mutex.hpp" #include +#include #include #include #include #include +#include #include namespace AgcDriver::Graphics { -class Sampler { +class Sampler : public HeapSlotOwner { public: Sampler(const Context& context, const GuestSamplerResource& descriptor); ~Sampler(); @@ -28,6 +31,8 @@ class Sampler { static void Validate(const Context& context, const GuestSamplerResource& descriptor); static bool ShaderAppliesLodBias(const GuestSamplerResource& descriptor); static bool ReadsOpaqueBlackBorder(const GuestSamplerResource& descriptor); + const std::array* CacheKey() const { return cacheKey ? &*cacheKey : nullptr; } + void SetCacheKey(const std::array& key) { cacheKey = key; } private: void release() noexcept; @@ -38,6 +43,7 @@ class Sampler { bool requiresFilterMinmax = false; bool opaqueBlackBorder = false; bool requiresNonSeamlessCube = false; + std::optional> cacheKey; }; // One VkSampler per distinct S# (its 4 words plus the shader's depth-compare use and the diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index f32b1d1f8c7c..3429ecb960f8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -3,6 +3,8 @@ #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" #include "prx/libSceAgcDriver/Execution/include/Mutex.hpp" @@ -28,6 +30,7 @@ namespace AgcDriver { class VulkanDevice; } namespace AgcDriver::Graphics { + class Recorder; void FlushCachedTextures(VkDevice device); @@ -55,9 +58,7 @@ std::shared_ptr CachedSampledTexture(const Context& context, std::span< bool SampledTexturesShareEntry(std::span first, std::span second); bool AnyPendingCpuWrites(const Context& context); bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std::size_t bytes); -bool SpirvStoresThroughAddresses(std::span words); bool SpirvFaultReportOnly(std::span words, std::uint32_t set, std::uint32_t binding); -bool ShaderStoresThroughAddresses(const ShaderRecompiler::SharedSpirv& spirv); void LastRecordIndices(std::span logged, std::span objects, std::vector& indices); // The cached storage image of a surface (render targets use it as their resident image); brought up @@ -74,6 +75,11 @@ std::shared_ptr RepeatStorageSurface(const Context& context, con // a storage image other than `except` has results pending in [address, address + bytes). bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except); +inline std::size_t HeapStageIndex(ShaderRecompiler::ShaderStage stage) { + using ShaderRecompiler::ShaderStage; + return static_cast(ShaderRecompiler::RuntimeAbi::StageGroup(stage == ShaderStage::Fragment, stage == ShaderStage::TessellationControl, stage == ShaderStage::TessellationEvaluation)); +} + struct PipelineSetLayouts { VkDescriptorSetLayout main = VK_NULL_HANDLE; VkDescriptorSetLayout fragment = VK_NULL_HANDLE; @@ -164,6 +170,7 @@ ImageTableReuseCounts ImageTableReuses(); std::uint64_t SampledTexturesCleared(); std::uint64_t SampledLookupKeyScans(); std::uint64_t SampledLookupKeyProofs(); +std::uint64_t DrawInputCopies(); struct PresyncSurface { std::uint64_t address; @@ -193,9 +200,12 @@ class ShaderResources { std::span LayoutBindingFlags() const { return layoutBindingFlags; } VkDescriptorSetLayout FragmentLayout() const { return fragment.layout; } bool SplitSets() const { return splitSets; } - PipelineSetLayouts SetLayouts() const { return {_layout, fragment.layout, splitSets, layoutKey, fragmentLayoutKey}; } + PipelineSetLayouts SetLayouts() const; + bool HeapStage() const { return heapStage; } + void RefreshHeapStage(std::span shaders); void RequireGuestBuffers() const { requireGuestBuffers(); } void Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const; + void BindDescriptorHeaps(VkCommandBuffer commands) const; struct DrawBindings { DrawBindings() = default; DrawBindings(const DrawBindings&) = delete; @@ -213,8 +223,12 @@ class ShaderResources { DescriptorCache::SetAllocation fragmentAllocation; DescriptorCache::SetAllocation allocation; std::vector snapshots; + bool addressOffsets = false; + std::array heapOffsets{}; + std::vector addressOverrides; ~DrawBindings(); }; + void RefreshMovedAddresses(DrawBindings& rewritten) const; struct MovedBuffer { std::size_t allocation; std::uint64_t address; @@ -354,6 +368,8 @@ class ShaderResources { void PatchPushConstants(std::span bytes) const { for (const auto& [position, adjustment] : pushPatches) bytes[position] = static_cast(adjustment); } + void PatchDataAddresses(std::span bytes, std::span shaders, std::span placements, const DrawBindings* rewritten) const; + void writePushedData(std::span bytes, const CompiledShader& shader, ShaderRecompiler::RuntimeAbi::PushedPlacement placement) const; // Whether RefreshData would record anything for `shader` (the per-word compare; verification). bool DataWordsDiffer(const CompiledShader& shader) const; std::size_t BindingCount() const { return bindings.size(); } @@ -401,6 +417,7 @@ class ShaderResources { bool refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder); std::optional shaderDataAllocation(const CompiledShader& shader) const; VkBuffer shaderDataBuffer(const CompiledShader& shader) const; + std::pair shaderDataLocation(const CompiledShader& shader, const DrawBindings* rewritten) const; void forgetShaderDataWords(const CompiledShader& shader); struct DescribedRange { const char* kind; @@ -444,12 +461,18 @@ class ShaderResources { std::vector imageAllocations; VkDescriptorBindingFlags flags = 0; std::int32_t set = -1; + bool heap = false; + bool addressed = false; + std::uint32_t dataStage = 0; + std::uint32_t addressDword = 0; + ShaderRecompiler::DescriptorRole role = ShaderRecompiler::DescriptorRole::GuestBuffers; }; struct TableStage { ShaderRecompiler::ShaderStage stage = ShaderRecompiler::ShaderStage::Compute; const ShaderRecompiler::RecompileResult* program = nullptr; const ShaderRecompiler::DescriptorBinding* images = nullptr; + std::span imageWords; std::size_t mapAllocation = std::numeric_limits::max(); std::vector map; std::vector poison; @@ -462,15 +485,18 @@ class ShaderResources { bool storage = false; std::vector elementTextures; std::vector> samplerRejections; + bool heap = false; + std::vector elementStorage; + std::vector tableSamplers; }; std::vector tableStages; std::array tableBindings{std::numeric_limits::max(), std::numeric_limits::max()}; std::array tableElements{}; std::array storageTableBindings{std::numeric_limits::max(), std::numeric_limits::max()}; TableStage& tableStage(const CompiledShader& shader); - static const ShaderRecompiler::DescriptorBinding* tableImages(std::span shaders, ShaderRecompiler::ShaderStage stage); + static const ShaderRecompiler::DescriptorBinding* tableImages(std::span shaders, ShaderRecompiler::ShaderStage stage, std::span& words); std::uint32_t rejectTableElement(TableStage& table, std::span words, const std::string& message); - void addSamplerTable(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, TableStage& table); + void addSamplerTable(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, VkShaderStageFlags flags, TableStage& table); void resolveImageTables(); void finishImageTables(); @@ -503,7 +529,7 @@ class ShaderResources { void requireGuestBuffers(std::span moved = {}) const; // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). - void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags); + void addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, VkShaderStageFlags flags); // Stage A: one record per planned image element (ImageRecord), in plan order: the decoded // descriptor and its surface size (computed once for the build), the write watch walked so stage // B's collects are memo hits (APS5_NO_PRECOLLECT=1 skips the pass entirely), and for a sampled @@ -546,12 +572,13 @@ class ShaderResources { // Stage B: the record's texture when the fastRevalidate predicate proves it current under the // lock and the cache still holds it; null sends the element to cachedTexture. std::shared_ptr fastTexture(ImageRecord& record); - void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item, std::span> shaderSamplers); + void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, Binding& item, std::span> shaderSamplers); void applySamplerVariants(); void forgetDeferredInputs(); void release() noexcept; void prepareAddressBindings(std::span shaders, std::span snapshots); - VkDescriptorBufferInfo descriptor(Allocation& allocation); + VkDescriptorBufferInfo descriptor(Allocation& allocation) { return descriptor(allocation, allocation.adjustment); } + VkDescriptorBufferInfo descriptor(const Allocation& allocation, std::uint32_t& adjustment) const; VkDescriptorBufferInfo wholeDwordRange(VkDescriptorBufferInfo info) const; void noteReusable(); void reportDescriptorCaches() const; @@ -641,6 +668,48 @@ class ShaderResources { DescriptorCache::SetAllocation cacheAllocation; }; bool splitSets = false; + bool heapStage = false; + std::array heapOffsets{}; + void writeHeapStage(const CompiledShader& shader); + struct RingDescriptors { + std::vector ranges; + BindlessHeap::SpanMark mark; + std::uint32_t first = 0; + }; + struct HeapStageState { + std::vector key; + std::array, static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)> slots; + std::vector slotGenerations; + std::vector samplerSlots; + std::vector samplerGenerations; + std::uint64_t slotSerial = 0; + std::uint32_t touchedEpoch = 0; + std::vector> dataPatches; + struct KeptSpan { + std::vector words; + BindlessHeap::SpanMark mark; + }; + KeptSpan data, flat, map; + std::vector dataInput; + RingDescriptors descriptors; + }; + std::array heapStages; + std::uint64_t heapSpan(HeapStageState::KeptSpan& kept, std::span words); + bool ownsHeapBinding(const Binding& binding, VkShaderStageFlags stageBit) const; + VkImageView heapPlaceholderView(std::uint32_t kind) const; + VkImageView heapSampledView(std::size_t index) const; + VkImageView heapStorageView(std::size_t index, std::uint32_t binding) const; + VkImageView heapMultisampleStorageView(std::size_t index) const; + const HeapSlotOwner* heapMultisampleStorageOwner(std::size_t index) const; + void appendHeapSlotKey(VkShaderStageFlags stageBit, std::vector& key) const; + void takeHeapSlots(BindlessHeap& heap, HeapStageState& state, VkShaderStageFlags stageBit, std::span key); + static void noteHeapSlotGenerations(const BindlessHeap& heap, HeapStageState& state); + void verifyHeapSlots(const BindlessHeap& heap, const HeapStageState& state, VkShaderStageFlags stageBit) const; + void patchHeapSlotWords(const CompiledShader& shader, const HeapStageState& stageState, std::vector& patched, bool current) const; + static constexpr std::int64_t HeapRingDataFirst = -2; + static constexpr std::int64_t HeapRingData(std::size_t stage) { return HeapRingDataFirst - static_cast(stage); } + static constexpr bool IsHeapRingData(std::int64_t allocation) { return allocation <= HeapRingDataFirst && allocation > HeapRingDataFirst - static_cast(ShaderRecompiler::RuntimeAbi::StageCount); } + static constexpr std::size_t HeapRingStage(std::int64_t allocation) { return static_cast(HeapRingDataFirst - allocation); } std::vector fragmentLayoutKey; SetObjects fragment; struct TableWrites { @@ -658,6 +727,21 @@ class ShaderResources { mutable std::vector descriptorBuffers; std::vector descriptorImages; std::vector descriptorWrites; + bool addressedBuffers = false; + std::size_t gdsAllocation = std::numeric_limits::max(); + std::array, ShaderRecompiler::RuntimeAbi::StageCount> stageDataWords; + std::array stageDataIndex{}; + std::array heapSrtDwords{}; + std::array heapTableMaps{}; + std::array heapTableMapSet{}; + void writeHeapTableMap(const CompiledShader& shader, std::size_t stageIndex); + void writeBufferAddresses(std::span words, std::uint32_t dataStage, const std::vector* overrides, + std::vector* ranges = nullptr) const; + VkDeviceAddressRangeEXT srtRange(std::size_t stage) const; + std::uint32_t ringDescriptors(std::size_t stage, const std::vector* overrides, RingDescriptors& kept) const; + std::array heapSrtOffsets{}; + std::array ringDescriptorStages{}; + std::array addressedOffsets(const std::vector* overrides) const; struct FaultDescriptor { std::size_t write; std::size_t buffer; @@ -665,7 +749,7 @@ class ShaderResources { }; std::vector faultDescriptors; void prepareFaultSlot() const; - void writeDrawSets(DrawBindings& result, std::span selected, const BdaResources* faultRecord) const; + void writeDrawSets(DrawBindings& result, std::span selected, const BdaResources* faultRecord, std::span> inPlace = {}) const; std::shared_ptr> ownRecordBusy = std::make_shared>(false); mutable std::vector drawPoolSizes; // Guest buffer elements bound read-only: each use of this object skips that many pending-write @@ -682,7 +766,7 @@ class ShaderResources { std::vector textureBaseLevel; std::vector textureSrgb; std::vector> storageTextures; - std::vector multisampleStorageViews; + std::vector multisampleStorage; std::vector storageMips; std::vector storageKeys; std::vector storageFirstLayer; @@ -732,6 +816,7 @@ class ShaderResources { std::vector refreshResourceKey; struct DeferredImages { const ShaderRecompiler::DescriptorBinding* binding; + std::span words; std::size_t index; std::size_t firstSampler = 0; std::size_t samplerCount = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp index e2f279e8e4b0..eab75a0ccf7d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp @@ -4,6 +4,8 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "IntermediateRepresentation/IrMetadata/ResourcePlan.hpp" #include "Recompiler.hpp" +#include "PushDataLayout.hpp" +#include #include #include #include @@ -14,6 +16,12 @@ inline constexpr std::uint32_t PipelinePushConstantBytes = 256; inline constexpr std::uint32_t PipelinePushSlotBytes = 128; inline constexpr std::uint32_t ComputePushConstantBytes = 128; +inline constexpr std::uint32_t HeapLayoutKeyTag = 0x48454150u; +inline constexpr std::uint32_t DataSetKeyTag = 0x44415441u; +inline constexpr std::uint32_t MainSetBit = 1u; +inline constexpr std::uint32_t HeapSetBit = 2u; +inline constexpr std::uint32_t DataSetBit = 4u; + inline std::uint32_t PushBlockBytes(bool libraries) { return libraries ? PipelinePushConstantBytes : PipelinePushSlotBytes; } @@ -39,8 +47,61 @@ struct CompiledShader { const ShaderRecompiler::RecompileResult* program; std::uint32_t pushConstantOffset; const std::vector* registeredImageViews = nullptr; + const ShaderRecompiler::HeapStageWords* heap = nullptr; }; +inline bool HeapShader(const CompiledShader& shader) { + return shader.program != nullptr && shader.program->gpuResolvedImages; +} + +inline bool AnyHeapShader(std::span shaders) { + return std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return HeapShader(shader); }); +} + +inline bool RootedPushData(std::span shaders) { + return std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->dataAddressPush; }); +} + +inline bool PushesGeneratedFaultAddress(const CompiledShader& shader, bool rooted) { + if (!rooted || shader.program == nullptr || shader.program->dataAddressPush || HeapShader(shader)) return false; + return std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer; }); +} + +inline std::vector PushDataPlacements(std::span shaders) { + namespace Abi = ShaderRecompiler::RuntimeAbi; + std::vector stages(shaders.size()); + std::uint64_t occupied = 0; + const bool rooted = RootedPushData(shaders); + for (std::size_t index = 0; index < shaders.size(); ++index) { + const auto* program = shaders[index].program; + stages[index].part = ShaderRecompiler::PushPartOf(shaders[index].stage); + if (program == nullptr) continue; + occupied |= ShaderRecompiler::StagePushFeatures(shaders[index].stage, program->pushItems); + if (program->dataAddressPush) { + stages[index].items = program->pushItems & ShaderRecompiler::RootItemMask; + stages[index].pushedWanted = static_cast(program->pushedData.size()); + continue; + } + if (!program->pushConstants.empty()) occupied |= ShaderRecompiler::PushBytesMask(shaders[index].pushConstantOffset, static_cast(program->pushConstants.size())); + if (PushesGeneratedFaultAddress(shaders[index], rooted)) stages[index].items = Abi::RootItemBit(Abi::RootItem::FaultAddress); + } + return ShaderRecompiler::PlacePushData(stages, occupied); +} + +inline const std::vector& DrawGuestWords(const ShaderRecompiler::RecompileResult& program, const ShaderRecompiler::HeapStageWords* heap, const ShaderRecompiler::DescriptorBinding& binding) { + if (heap == nullptr || heap->bindings.empty()) return binding.guestDescriptor; + const auto index = static_cast(&binding - program.bindings.data()); + return index < heap->bindings.size() && !heap->bindings[index].empty() ? heap->bindings[index] : binding.guestDescriptor; +} + +inline const std::vector& DrawGuestWords(const CompiledShader& shader, const ShaderRecompiler::DescriptorBinding& binding) { + return DrawGuestWords(*shader.program, shader.heap, binding); +} + +inline const std::vector& DrawVertexAttributes(const CompiledShader& shader) { + return shader.heap != nullptr && !shader.heap->vertexAttributes.empty() ? shader.heap->vertexAttributes : shader.program->vertexAttributes; +} + inline VkShaderStageFlagBits VulkanStage(ShaderRecompiler::ShaderStage stage) { switch (stage) { case ShaderRecompiler::ShaderStage::Vertex: @@ -75,7 +136,7 @@ inline VkShaderStageFlags PushConstantStages(std::span sha VkShaderStageFlags result = 0; for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); - if (!shader.program->pushConstants.empty() || shader.program->readsDepthRemapPush || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage); + if (!shader.program->pushConstants.empty() || shader.program->readsDepthRemapPush || shader.program->dataAddressPush || shader.stage == ShaderRecompiler::ShaderStage::Mesh || shader.stage == ShaderRecompiler::ShaderStage::TessellationControl) result |= VulkanStage(shader.stage); } return result; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/StagedMerge.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/StagedMerge.hpp index 60e2a764f7d2..3add8e5d57a7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/StagedMerge.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/StagedMerge.hpp @@ -15,7 +15,9 @@ struct StagedTemporary { std::shared_ptr AcquireStagedTemporary(const Context& context, VkExtent2D extent, std::uint32_t samples); void RecordStagedClear(const Context& context, VkCommandBuffer commands, StagedTemporary& temporary); -void RecordStagedMerge(const Context& context, VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area); +class HeapSlotOwner; +void RecordStagedMerge(const Context& context, VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area, + const HeapSlotOwner* targetOwner = nullptr); void ClearStagedMerges(VkDevice device); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index e56c8ca1b170..715dfc07108e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURE_HPP #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_TEXTURE_HPP +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" @@ -60,7 +61,7 @@ struct OwnedImage { ImageAllocation allocation; }; -class Texture { +class Texture : public HeapSlotOwner { public: Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot, bool depthCompare = false, bool layerRows = false, const VkClearColorValue* clear = nullptr); Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components); @@ -107,6 +108,7 @@ class Texture { ViewRange SampledViewRange(bool firstLayer) const { return firstLayer ? firstLayerRange : viewRange; } // The layout the image is kept in while sampled. VkImageLayout Layout() const { return layout; } + VkImage Image() const { return image; } VkDeviceSize AllocationBytes() const { return allocationBytes; } VkFormat ViewFormat() const { return viewFormat; } // Whether this texture is a view of a storage image (no snapshot of its own). @@ -255,7 +257,7 @@ class UnitFlags { // either at once (WriteBack) or deferred: MarkDirty keeps them on the GPU until something reads that // memory (FlushPending, through the GuestMemory flush hook), the image is refreshed after a CPU write, // or it leaves the cache. Dispatches reusing the image meanwhile skip the round trip entirely. -class StorageTexture : public std::enable_shared_from_this { +class StorageTexture : public std::enable_shared_from_this, public HeapSlotOwner { public: struct CreationHandOver { DccKeys keys = DccKeys::Uncompressed; @@ -280,6 +282,7 @@ class StorageTexture : public std::enable_shared_from_this { VkImageView View(std::uint32_t mip); VkImageView FirstLayerView(std::uint32_t mip); VkImageView StorageView(std::uint32_t mip, bool firstLayer, bool uintStorage = false); + VkImageView UintStorageViewOrDefault(std::uint32_t mip, bool firstLayer); VkImageView ElementView(); VkImageView AtomicView(std::uint32_t mip, bool firstLayer); VkImageView Atomic64View(std::uint32_t mip, bool firstLayer); @@ -485,6 +488,7 @@ class StorageTexture : public std::enable_shared_from_this { DccKeyProof& KeyProof() const { return keyProof; } DccRangeProof& TargetKeyProof() const { return targetKeyProof; } DccKeys ProvedKeys() const; + bool UntrackedUnchanged() const; bool ServesKeysAt(std::uint64_t dccAddress) const; // Brings the image up to date with guest memory before another use; returns whether its content // was still current (nothing uploaded). diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index 0914660be154..4de881e0b9b8 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -13,6 +13,8 @@ namespace AgcDriver::Graphics { +class HeapSlotOwner; + class Buffer; // The part of a mip one Dispatch moves: only elements whose tiled offset (relative to the mip's @@ -48,20 +50,21 @@ namespace AgcDriver::Graphics { // restricts the move to part of the mip (a partial upload or write-back, StorageTexture), // leaving the other elements of both buffers alone. void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); - void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {}); + void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {}, const HeapSlotOwner* owner = nullptr); void DispatchRepack(VkCommandBuffer commands, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, std::uint64_t bytes); static VkFormat ImageElementFormat(std::uint32_t elementBytes); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); - void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write); + void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write, const HeapSlotOwner* owner = nullptr); std::uint32_t CmaskErrors(); - void DispatchMultisampleReadback(VkCommandBuffer commands, VkFormat format, VkBuffer destination, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements); - void DispatchMultisampleImage(VkCommandBuffer commands, std::uint32_t elementBytes, VkBuffer source, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements); + void DispatchMultisampleReadback(VkCommandBuffer commands, VkFormat format, VkBuffer destination, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements, const HeapSlotOwner* owner = nullptr); + void DispatchMultisampleImage(VkCommandBuffer commands, std::uint32_t elementBytes, VkBuffer source, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements, const HeapSlotOwner* owner = nullptr); private: - VkPipeline multisampleReadbackPipeline(std::uint32_t elementBytes, bool depth); - VkPipeline multisampleUploadPipeline(std::uint32_t elementBytes); - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false); + VkPipeline multisampleReadbackPipeline(std::uint32_t elementBytes, bool depth, bool heap); + VkPipeline multisampleUploadPipeline(std::uint32_t elementBytes, bool heap); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false, bool addressed = false, bool heap = false); + VkDeviceAddress bufferAddress(VkBuffer buffer) const; void release() noexcept; struct DescriptorRing { std::vector pools; @@ -85,6 +88,7 @@ namespace AgcDriver::Graphics { VkDescriptorSetLayout cmaskDescriptorLayout = VK_NULL_HANDLE; VkPipelineLayout cmaskPipelineLayout = VK_NULL_HANDLE; VkPipeline cmaskPipeline = VK_NULL_HANDLE; + VkPipeline cmaskHeapPipeline = VK_NULL_HANDLE; DescriptorRing cmaskDescriptorSets; std::unique_ptr cmaskErrors; VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE; @@ -92,6 +96,12 @@ namespace AgcDriver::Graphics { VkShaderModule imageModule = VK_NULL_HANDLE; VkShaderModule repackModule = VK_NULL_HANDLE; VkPipeline repackPipeline = VK_NULL_HANDLE; + VkPipeline heapRepackPipeline = VK_NULL_HANDLE; + VkPipelineLayout addressedPipelineLayout = VK_NULL_HANDLE; + VkPipelineLayout addressedRepackLayout = VK_NULL_HANDLE; + VkShaderModule addressedModule = VK_NULL_HANDLE; + VkShaderModule addressedRepackModule = VK_NULL_HANDLE; + VkPipeline addressedRepackPipeline = VK_NULL_HANDLE; VkShaderModule imageWordModule = VK_NULL_HANDLE; VkShaderModule wordModule = VK_NULL_HANDLE; DescriptorRing imageDescriptorSets; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 6c8e6b6a7821..6a960b9b8b25 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -39,6 +39,11 @@ layout(local_size_x = 8, local_size_y = 8) in; #ifdef LINEAR_IMAGE layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer; layout(set = 0, binding = 1) uniform uimage2D linearImage; +#elif defined(ADDRESSED) +#extension GL_EXT_buffer_reference : require +#extension GL_EXT_buffer_reference_uvec2 : require +layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer Input { uint data[]; }; +layout(buffer_reference, std430, buffer_reference_align = 4) buffer Output { uint data[]; }; #else layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer; @@ -69,8 +74,17 @@ layout(push_constant) uniform Push { uint rowBegin; uint pipeBankXor; uint splitStride; +#ifdef ADDRESSED + uvec2 inputAddress; + uvec2 outputAddress; +#endif } params; +#ifdef ADDRESSED +#define inputBuffer Input(params.inputAddress) +#define outputBuffer Output(params.outputAddress) +#endif + uint equationBit(uint mask, uvec2 p) { uint selected = (p.x & (mask & 0xfffu)) ^ ((p.y << 12) & (mask & 0xfff000u)) ^ ((params.slice << 24) & (mask & 0xff000000u)); return bitCount(selected) & 1u; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureRepack.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureRepack.comp index 6de2f05e4a82..2febfe6ae06b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureRepack.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureRepack.comp @@ -2,16 +2,32 @@ layout(local_size_x = 64) in; +#ifdef ADDRESSED +#extension GL_EXT_buffer_reference : require +#extension GL_EXT_buffer_reference_uvec2 : require +layout(buffer_reference, std430, buffer_reference_align = 4) readonly buffer Source { uint data[]; }; +layout(buffer_reference, std430, buffer_reference_align = 4) buffer Destination { uint data[]; }; +#else layout(std430, set = 0, binding = 0) readonly buffer Source { uint source[]; }; layout(std430, set = 0, binding = 1) buffer Destination { uint destination[]; }; +#endif layout(push_constant) uniform Push { uint sourceBase; uint destinationBase; uint words; uint rowWords; +#ifdef ADDRESSED + uvec2 sourceAddress; + uvec2 destinationAddress; +#endif } push; +#ifdef ADDRESSED +#define source Source(push.sourceAddress).data +#define destination Destination(push.destinationAddress).data +#endif + void main() { const uint index = gl_GlobalInvocationID.y * push.rowWords + gl_GlobalInvocationID.x; if (gl_GlobalInvocationID.x >= push.rowWords || index >= push.words) return; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp index ec6e17783514..5919a6f94115 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp new file mode 100644 index 000000000000..9b951d08074d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp @@ -0,0 +1,1523 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "PushDataLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "Recompiler.hpp" +#include "RuntimeAbi.hpp" +#include "SpirvBackend/SpirvAnalysis.hpp" +#include "prx/libc/include/HostThreadLocal.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +std::atomic& walkMemoHeld() { + static std::atomic held{0}; + return held; +} + +std::atomic& walkMemoPeak() { + static std::atomic peak{0}; + return peak; +} + +} + +void NoteHeapWalkMemoBytes(std::int64_t delta) { + if (delta == 0) return; + const auto now = walkMemoHeld().fetch_add(delta, std::memory_order_relaxed) + delta; + auto peak = walkMemoPeak().load(std::memory_order_relaxed); + while (now > peak && !walkMemoPeak().compare_exchange_weak(peak, now, std::memory_order_relaxed)) {} +} + +namespace { + +namespace Abi = ShaderRecompiler::RuntimeAbi; + +constexpr std::size_t RingSpanAlignment = 16u; + +bool pipelineStats() { + static const bool enabled = std::getenv("APS5_PIPELINE_STATS") != nullptr; + return enabled; +} + +bool isSampledKind(std::uint32_t kind) { + return kind >= Abi::FirstImageBinding && kind < Abi::FirstStorageImageBinding; +} + +bool isImageKind(std::uint32_t kind) { + return kind >= Abi::FirstImageBinding && kind < BindlessHeap::SamplerKind; +} + +std::size_t chunkOf(std::size_t offset) { + return offset / BindlessHeap::RingChunkBytes; +} + +std::size_t lastChunkOf(std::size_t offset, std::size_t bytes) { + return chunkOf(offset + std::max(bytes, 1u) - 1u); +} + +std::atomic heapComputeStages{0}; +std::atomic descriptorHeapDevice{false}; +std::atomic imageTableBuilds{0}; +std::atomic dataAddressPushes{0}; +std::atomic emptyResourceSets{0}; + +} + +bool GpuHeapEnabled() { + static const bool enabled = std::getenv("APS5_GPU_HEAP") != nullptr; + return enabled; +} + +void NoteImageTableBuild() { + imageTableBuilds.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t ImageTableBuilds() { + return imageTableBuilds.load(std::memory_order_relaxed); +} + +void NoteDataAddressPush() { + dataAddressPushes.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t DataAddressPushes() { + return dataAddressPushes.load(std::memory_order_relaxed); +} + +void NoteEmptyResourceSet() { + emptyResourceSets.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t EmptyResourceSets() { + return emptyResourceSets.load(std::memory_order_relaxed); +} + +void NoteHeapComputeStage() { + heapComputeStages.fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t HeapComputeStages() { + return heapComputeStages.load(std::memory_order_relaxed); +} + +void SetDescriptorHeapDevice(bool available) { + descriptorHeapDevice.store(available, std::memory_order_relaxed); +} + +bool DescriptorHeapEnabled() { + return GpuHeapEnabled() && descriptorHeapDevice.load(std::memory_order_relaxed); +} + +namespace { + +struct HeapSizes { + std::uint32_t imageSlots = 0; + VkDeviceSize imageStride = 0; + VkDeviceSize samplerStride = 0; + VkDeviceSize pixelLdsOffset = 0; + VkDeviceSize meshWindowOffset = 0; + VkDeviceSize resourceUsed = 0; + VkDeviceSize samplerUsed = 0; + VkDeviceSize ringOffset = 0; + std::size_t ringBytes = 0; +}; + +std::size_t ringTargetBytes() { + static const std::size_t bytes = [] { + std::size_t target = BindlessHeap::DefaultRingBytes; + if (const char* value = std::getenv("APS5_HEAP_RING_MIB")) { + const auto mib = std::strtoull(value, nullptr, 10); + if (mib != 0u) target = static_cast(std::clamp(mib, (2u * BindlessHeap::RingChunkBytes) >> 20u, BindlessHeap::RingBytes >> 20u)) << 20u; + } + return target / BindlessHeap::RingChunkBytes * BindlessHeap::RingChunkBytes; + }(); + return bytes; +} + +VkDeviceSize alignUp(VkDeviceSize value, VkDeviceSize alignment) { + return alignment == 0 ? value : (value + alignment - 1) / alignment * alignment; +} + +constexpr std::uint32_t MaximumImageSlots = 1u << 19u; + +HeapSizes heapLayout(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties, std::uint32_t slots, VkDeviceSize uniformAlignment) { + HeapSizes sizes; + sizes.imageSlots = slots; + sizes.imageStride = alignUp(properties.imageDescriptorSize, properties.imageDescriptorAlignment); + sizes.samplerStride = alignUp(properties.samplerDescriptorSize, properties.samplerDescriptorAlignment); + const auto imageBytes = static_cast(slots) * sizes.imageStride; + sizes.pixelLdsOffset = alignUp(imageBytes, properties.bufferDescriptorAlignment); + sizes.meshWindowOffset = alignUp(sizes.pixelLdsOffset + properties.bufferDescriptorSize, properties.bufferDescriptorAlignment); + sizes.resourceUsed = alignUp(sizes.meshWindowOffset + properties.bufferDescriptorSize, properties.resourceHeapAlignment); + sizes.samplerUsed = alignUp(static_cast(Abi::GpuSamplerHeapSlots) * sizes.samplerStride, properties.samplerHeapAlignment); + const auto offset = alignUp(sizes.resourceUsed + properties.minResourceHeapReservedRange, std::max(uniformAlignment, properties.resourceHeapAlignment)); + if (offset >= properties.maxResourceHeapSize) return sizes; + const auto room = static_cast(properties.maxResourceHeapSize - offset) / BindlessHeap::RingChunkBytes * BindlessHeap::RingChunkBytes; + const auto bytes = std::min(room, ringTargetBytes()); + if (bytes < 2u * BindlessHeap::RingChunkBytes) return sizes; + sizes.ringOffset = offset; + sizes.ringBytes = bytes; + return sizes; +} + +VkDeviceSize layoutEnd(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties, const HeapSizes& sizes) { + return sizes.ringBytes != 0u ? sizes.ringOffset + sizes.ringBytes : sizes.resourceUsed + properties.minResourceHeapReservedRange; +} + +HeapSizes heapSizes(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties, VkDeviceSize uniformAlignment) { + const auto base = heapLayout(properties, Abi::GpuImageHeapSlots, uniformAlignment); + const auto end = layoutEnd(properties, base); + if (end >= properties.maxResourceHeapSize || base.imageStride == 0u) return base; + auto slots = static_cast(std::min(MaximumImageSlots, Abi::GpuImageHeapSlots + (properties.maxResourceHeapSize - end) / base.imageStride)); + while (slots > Abi::GpuImageHeapSlots) { + const auto grown = heapLayout(properties, slots, uniformAlignment); + const auto grownEnd = layoutEnd(properties, grown); + if (grown.ringBytes == base.ringBytes && grownEnd <= properties.maxResourceHeapSize) return grown; + const auto over = grown.ringBytes == base.ringBytes ? grownEnd - properties.maxResourceHeapSize : static_cast(BindlessHeap::RingChunkBytes); + const auto step = static_cast(std::min(slots, over / base.imageStride + 1u)); + slots = slots - step > Abi::GpuImageHeapSlots ? slots - step : Abi::GpuImageHeapSlots; + } + return base; +} + +} + +bool BindlessHeap::RingInResourceHeap(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties, VkDeviceSize uniformAlignment) { + return heapSizes(properties, uniformAlignment).ringBytes != 0u; +} + +bool BindlessHeap::Fits(const VkPhysicalDeviceDescriptorHeapPropertiesEXT& properties) { + const auto sizes = heapLayout(properties, Abi::GpuImageHeapSlots, 0u); + const bool resourcesFit = sizes.resourceUsed + properties.minResourceHeapReservedRange <= properties.maxResourceHeapSize; + const bool samplersFit = sizes.samplerUsed + properties.minSamplerHeapReservedRangeWithEmbedded <= properties.maxSamplerHeapSize; + return resourcesFit && samplersFit; +} + + +namespace { + +std::uint32_t configuredSlotCap(std::uint32_t slots) { + static const std::uint32_t requested = [] { + const char* value = std::getenv("APS5_HEAP_SLOT_CAP"); + return value != nullptr ? static_cast(std::strtoul(value, nullptr, 10)) : 0u; + }(); + if (requested == 0u || slots <= BindlessHeap::SamplerKind) return slots; + return BindlessHeap::SamplerKind + std::min(requested, slots - BindlessHeap::SamplerKind); +} + +std::atomic deviceImageSlots{Abi::GpuImageHeapSlots}; + +} + +BindlessHeap::BindlessHeap(const Context& context) : context(context) { + Require(context.descriptorHeap, "the GPU heap needs VK_EXT_descriptor_heap"); + const auto& heapProperties = context.descriptorHeapProperties; + const auto sizes = heapSizes(heapProperties, context.limits.minUniformBufferOffsetAlignment); + imageHeapStride = sizes.imageStride; + samplerHeapStride = sizes.samplerStride; + resourceHeapUsed = sizes.resourceUsed; + pixelLdsHeapOffset = sizes.pixelLdsOffset; + meshWindowHeapOffset = sizes.meshWindowOffset; + samplerHeapUsed = sizes.samplerUsed; + Require(Fits(heapProperties), "the GPU heap's slots exceed the device's descriptor heap sizes"); + static const bool hostDescriptors = std::getenv("APS5_NO_HEAP_DEVICE_DESCRIPTORS") != nullptr; + constexpr VkMemoryPropertyFlags deviceVisible = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + const auto makeBuffer = [&](std::size_t bytes, VkBufferUsageFlags usage, bool deviceLocal) { + if (deviceLocal) { + try { + return std::make_unique(context, bytes, usage, deviceVisible); + } catch (const std::exception&) { + } + } + return std::make_unique(context, bytes, usage); + }; + constexpr VkBufferUsageFlags ringUsage = + VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + constexpr VkBufferUsageFlags heapUsage = VK_BUFFER_USAGE_DESCRIPTOR_HEAP_BIT_EXT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + ringInHeap = sizes.ringBytes != 0u; + const auto resourceBytes = ringInHeap ? static_cast(sizes.ringOffset) + sizes.ringBytes + : static_cast(resourceHeapUsed + heapProperties.minResourceHeapReservedRange); + resourceHeap = makeBuffer(resourceBytes, ringInHeap ? heapUsage | ringUsage : heapUsage, ringInHeap || !hostDescriptors); + samplerHeap = makeBuffer(static_cast(samplerHeapUsed + heapProperties.minSamplerHeapReservedRangeWithEmbedded), heapUsage, !hostDescriptors); + std::fill(resourceHeap->Bytes().begin(), resourceHeap->Bytes().end(), std::byte{0}); + std::fill(samplerHeap->Bytes().begin(), samplerHeap->Bytes().end(), std::byte{0}); + if (ringInHeap) { + ringHeapOffset = sizes.ringOffset; + ringBytes = sizes.ringBytes; + ringData = resourceHeap->Bytes().data() + ringHeapOffset; + } else { + ringBytes = 0; + } + ringAlignment = static_cast(std::max(BufferSlotBytes, heapProperties.bufferDescriptorAlignment)); + poolAlignment = static_cast(std::max({context.limits.minStorageBufferOffsetAlignment, context.limits.minUniformBufferOffsetAlignment, RingSpanAlignment})); + poolDeviceLocal = !hostDescriptors; + ringChunks.resize(ringBytes / RingChunkBytes); + slotCap = configuredSlotCap(sizes.imageSlots); + deviceImageSlots.store(slotCap, std::memory_order_relaxed); + slotUses = std::make_unique[]>(slotCap); + imagePool.next = SamplerKind; + imagePool.limit = slotCap; + samplerPool.limit = Abi::GpuSamplerHeapSlots; +} + +namespace { + +constexpr std::uint32_t ImageSlotMargin = 1024u; +constexpr std::uint32_t EvictIdleSubmissions = 2u; +constexpr std::uint32_t SamplerSlotMargin = Abi::GpuSamplerHeapSlots / 4u; +std::atomic refusedHeaps{0}; + +} + +BindlessHeap::~BindlessHeap() { + if (refused) refusedHeaps.fetch_sub(1, std::memory_order_relaxed); +} + +namespace { + +std::mutex& heapsMutex() { + static auto& mutex = *new std::mutex(); + return mutex; +} + +std::unordered_map>& heaps() { + static auto& map = *new std::unordered_map>(); + return map; +} + +std::atomic heapsEpoch{0}; + +} + +BindlessHeap& BindlessHeap::For(const Context& context) { + struct Cached { + VkDevice device = VK_NULL_HANDLE; + BindlessHeap* heap = nullptr; + std::uint64_t epoch = ~std::uint64_t{0}; + }; + thread_local Cached cached; + if (cached.heap != nullptr && cached.device == context.device && cached.epoch == heapsEpoch.load(std::memory_order_acquire)) { + return *cached.heap; + } + std::lock_guard lock(heapsMutex()); + auto& heap = heaps()[context.device]; + if (heap == nullptr) { + heap = std::make_unique(context); + heapsEpoch.fetch_add(1, std::memory_order_acq_rel); + } + cached = {context.device, heap.get(), heapsEpoch.load(std::memory_order_acquire)}; + return *heap; +} + +void BindlessHeap::Release(VkDevice device) { + std::unique_ptr released; + { + std::lock_guard lock(heapsMutex()); + const auto found = heaps().find(device); + if (found == heaps().end()) return; + released = std::move(found->second); + heaps().erase(found); + heapsEpoch.fetch_add(1, std::memory_order_acq_rel); + } +} + +BindlessHeap* BindlessHeap::Existing(const Context& context) { + std::lock_guard lock(heapsMutex()); + const auto found = heaps().find(context.device); + return found == heaps().end() ? nullptr : found->second.get(); +} + +std::optional BindlessHeap::TextureSlot(std::uint32_t kind, Texture* texture, VkImageView view, VkImageLayout layout) { + Require(isSampledKind(kind) && texture != nullptr && view != VK_NULL_HANDLE, "bindless heap: invalid texture slot request"); + const HeapSlotOwner* owner = texture; + return imageSlot(kind, owner, view, layout, texture); +} + +std::optional BindlessHeap::StorageSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view) { + Require(Abi::GpuHeapStorageKind(kind) && owner != nullptr && view != VK_NULL_HANDLE, "bindless heap: invalid storage slot request"); + return imageSlot(kind, owner, view, VK_IMAGE_LAYOUT_GENERAL, nullptr); +} + +std::optional BindlessHeap::ViewSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view) { + Require(isImageKind(kind) && owner != nullptr && view != VK_NULL_HANDLE, "bindless heap: invalid view slot request"); + return imageSlot(kind, owner, view, VK_IMAGE_LAYOUT_GENERAL, nullptr); +} + +std::optional BindlessHeap::imageSlot(std::uint32_t kind, const HeapSlotOwner* owner, VkImageView view, VkImageLayout layout, Texture* texture) { + std::lock_guard lock(mutex); + const ImageSlotKey key{kind, owner, view, layout}; + if (const auto found = imageSlots.find(key); found != imageSlots.end()) { + touchLocked(found->second); + return found->second; + } + bool reused = false; + auto slot = imagePool.Take(reused); + if (!slot && reclaimLocked(true)) slot = imagePool.Take(reused); + if (!slot) return std::nullopt; + if (reused) ++counts.slotsReused; + writeImageSlot(kind, *slot, view, layout); + claimSlotLocked(key, *slot).texture = texture; + touchLocked(*slot); + owner->heapSlotted.store(true, std::memory_order_release); + noteTakenLocked(); + return slot; +} + +void BindlessHeap::ReleaseSlots(const HeapSlotOwner* owner) { + std::lock_guard lock(mutex); + bool released = false; + if (const auto found = ownerSlots.find(owner); found != ownerSlots.end()) { + const auto slots = std::move(found->second); + ownerSlots.erase(found); + for (const auto slot : slots) freeSlotLocked(slot); + released = true; + } + if (const auto found = samplerSlots.find(owner); found != samplerSlots.end()) { + samplerPool.Free(found->second); + samplerSlots.erase(found); + ++counts.slotsFreed; + released = true; + } + if (!released) return; + slotSerial.fetch_add(1, std::memory_order_release); + noteReleasedLocked(); +} + +BindlessHeap::SlotOwner& BindlessHeap::claimSlotLocked(const ImageSlotKey& key, std::uint32_t slot) { + imageSlots.emplace(key, slot); + ownerSlots[key.owner].push_back(slot); + if (owners.size() <= slot) owners.resize(slot + 1u); + owners[slot].key = key; + return owners[slot]; +} + +void BindlessHeap::freeSlotLocked(std::uint32_t slot) { + auto& owner = owners[slot]; + imageSlots.erase(owner.key); + owner.texture = nullptr; + owner.key = {}; + imagePool.Free(slot); + ++counts.slotsFreed; +} + +const BindlessHeap::SlotOwner* BindlessHeap::ownerLocked(std::uint32_t slot) const { + return slot < owners.size() ? &owners[slot] : nullptr; +} + +std::optional BindlessHeap::SlotPool::Take(bool& reused) { + reused = false; + if (!free.empty()) { + const auto slot = free.back(); + free.pop_back(); + reused = true; + return slot; + } + if (next >= limit) return std::nullopt; + const auto slot = next++; + if (generations.size() <= slot) generations.resize(static_cast(slot) + 1u, 0u); + return slot; +} + +void BindlessHeap::SlotPool::Free(std::uint32_t slot) { + ++generations[slot]; + free.push_back(slot); +} + +std::size_t BindlessHeap::SlotPool::Unused() const { + return (next < limit ? limit - next : 0u) + free.size(); +} + +std::uint32_t BindlessHeap::SlotPool::Generation(std::uint32_t slot) const { + return slot < generations.size() ? generations[slot] : 0u; +} + +bool BindlessHeap::NearlyFull() { + return refusedHeaps.load(std::memory_order_relaxed) != 0u; +} + +std::uint32_t BindlessHeap::ImageSlotCapacity() { + return deviceImageSlots.load(std::memory_order_relaxed) - SamplerKind; +} + +std::uint32_t BindlessHeap::imageMargin() const { + return std::max(1u, std::min(ImageSlotMargin, (slotCap - SamplerKind) / 4u)); +} + +bool BindlessHeap::roomLocked() const { + if (imagePool.Unused() + retiringAssigned < 2u * static_cast(imageMargin())) return false; + return samplerPool.Unused() >= 2u * SamplerSlotMargin; +} + +void BindlessHeap::noteRefusedLocked(const char* what, std::uint32_t margin) { + if (refused) return; + refused = true; + refusedHeaps.fetch_add(1, std::memory_order_relaxed); + std::fprintf(stderr, "[gpu-heap] fewer than %u %s slots are free; draws and dispatches take the per-draw path until released slots leave twice that many free\n", margin, what); +} + +void BindlessHeap::noteTakenLocked() { + const auto margin = imageMargin(); + if (imagePool.Unused() >= 2u * static_cast(margin)) return; + if (imagePool.Unused() + retiring.size() < 2u * static_cast(margin)) evictLocked(); + if (!refused && imagePool.Unused() + retiringAssigned < margin) noteRefusedLocked("image", margin); +} + +void BindlessHeap::touchLocked(std::uint32_t slot) const { + if (slot < slotCap) slotUses[slot].store(useEpoch.load(std::memory_order_relaxed), std::memory_order_relaxed); +} + +std::uint32_t BindlessHeap::TouchSlots(std::span> slots, std::uint32_t touched) const { + const auto epoch = useEpoch.load(std::memory_order_relaxed); + if (touched == epoch) return epoch; + for (const auto& kindSlots : slots) { + for (const auto slot : kindSlots) { + if (slot < slotCap) slotUses[slot].store(epoch, std::memory_order_relaxed); + } + } + return epoch; +} + +void BindlessHeap::evictLocked() { + static const bool disabled = std::getenv("APS5_NO_HEAP_EVICTION") != nullptr; + const auto epoch = useEpoch.load(std::memory_order_relaxed); + if (disabled || submitter == nullptr || !submitter->HasTimeline() || evictScanEpoch == epoch) return; + GuestMemory::AssertGpuLockHeld("BindlessHeap::evictLocked"); + evictScanEpoch = epoch; + std::vector> idle; + for (std::uint32_t slot = SamplerKind; slot < imagePool.next && slot < owners.size(); ++slot) { + if (owners[slot].key.owner == nullptr) continue; + const auto use = slotUses[slot].load(std::memory_order_relaxed); + if (use + EvictIdleSubmissions <= epoch) idle.emplace_back(use, slot); + } + const auto wanted = std::min(idle.size(), 4u * static_cast(imageMargin())); + if (wanted == 0u) return; + std::nth_element(idle.begin(), idle.begin() + static_cast(wanted - 1u), idle.end()); + for (std::size_t index = 0; index < wanted; ++index) { + const auto slot = idle[index].second; + auto& owner = owners[slot]; + imageSlots.erase(owner.key); + if (const auto found = ownerSlots.find(owner.key.owner); found != ownerSlots.end()) { + auto& held = found->second; + held.erase(std::remove(held.begin(), held.end(), slot), held.end()); + if (held.empty()) ownerSlots.erase(found); + } + owner.texture = nullptr; + owner.key = {}; + ++imagePool.generations[slot]; + retiring.push_back({slot, 0u}); + ++counts.slotsEvicted; + } + slotSerial.fetch_add(1, std::memory_order_release); +} + +bool BindlessHeap::reclaimLocked(bool wait) { + bool reclaimed = false; + while (!retiring.empty() && retiring.front().serial != 0u && submitter != nullptr && submitter->HasTimeline()) { + const auto serial = retiring.front().serial; + if (!submitter->WaitSerialFor(serial, 0u)) { + if (!wait || reclaimed || serial > submitter->Submissions()) break; + submitter->WaitSerial(serial); + ++counts.evictionWaits; + } + while (!retiring.empty() && retiring.front().serial != 0u && retiring.front().serial <= serial) { + imagePool.free.push_back(retiring.front().slot); + retiring.pop_front(); + --retiringAssigned; + reclaimed = true; + } + } + if (reclaimed) noteReleasedLocked(); + return reclaimed; +} + +void BindlessHeap::noteSamplerTakenLocked() { + if (!refused && samplerPool.Unused() < SamplerSlotMargin) noteRefusedLocked("sampler", SamplerSlotMargin); +} + +void BindlessHeap::noteReleasedLocked() { + if (!refused || !roomLocked()) return; + refused = false; + refusedHeaps.fetch_sub(1, std::memory_order_relaxed); + std::fprintf(stderr, "[gpu-heap] released slots left twice the margin free; draws take the heap again\n"); +} + +std::uint32_t BindlessHeap::SamplerGeneration(std::uint32_t slot) const { + std::lock_guard lock(mutex); + return samplerPool.Generation(slot); +} + +bool BindlessHeap::GenerationsCurrent(std::span> slots, std::span slotGenerations, + std::span samplerSlots, + std::span samplerGenerations) const { + std::lock_guard lock(mutex); + std::size_t generation = 0; + for (std::uint32_t kind = 0; kind < slots.size(); ++kind) { + for (const auto slot : slots[kind]) { + if (generation >= slotGenerations.size() || imagePool.Generation(slot) != slotGenerations[generation++]) return false; + } + } + for (std::size_t i = 0; i < samplerSlots.size(); ++i) { + if (i >= samplerGenerations.size() || samplerPool.Generation(samplerSlots[i]) != samplerGenerations[i]) return false; + } + return true; +} + +std::uint32_t BindlessHeap::SlotGeneration(std::uint32_t slot) const { + std::lock_guard lock(mutex); + return imagePool.Generation(slot); +} + +const Texture* BindlessHeap::SlotTexture(std::uint32_t kind, std::uint32_t slot) const { + std::lock_guard lock(mutex); + const auto* owner = ownerLocked(slot); + return owner != nullptr && owner->key.kind == kind ? owner->texture : nullptr; +} + +bool BindlessHeap::SlotHoldsStorage(std::uint32_t kind, std::uint32_t slot, const HeapSlotOwner* owner, VkImageView view) const { + std::lock_guard lock(mutex); + const auto* held = ownerLocked(slot); + return held != nullptr && held->key == ImageSlotKey{kind, owner, view, VK_IMAGE_LAYOUT_GENERAL}; +} + +std::uint32_t BindlessHeap::AbsentSlot(std::uint32_t kind, VkImageView view) { + Require(isImageKind(kind), "bindless heap: invalid absent slot request"); + std::lock_guard lock(mutex); + if (absentSlots.contains(kind)) return kind; + writeImageSlot(kind, kind, view, VK_IMAGE_LAYOUT_GENERAL); + absentSlots.insert(kind); + return kind; +} + +std::optional BindlessHeap::SamplerSlot(const std::shared_ptr& sampler) { + Require(sampler != nullptr, "bindless heap: invalid sampler slot request"); + std::lock_guard lock(mutex); + if (const auto found = samplerSlots.find(static_cast(sampler.get())); found != samplerSlots.end()) return found->second; + bool reused = false; + const auto slot = samplerPool.Take(reused); + if (!slot) return std::nullopt; + if (reused) ++counts.slotsReused; + writeSamplerSlot(*slot, sampler->Handle()); + const HeapSlotOwner* owner = sampler.get(); + samplerSlots.emplace(owner, *slot); + owner->heapSlotted.store(true, std::memory_order_release); + noteSamplerTakenLocked(); + return slot; +} + +BindlessHeap::DataSpan BindlessHeap::Allocate(std::size_t words) { + SpanMark mark; + return Allocate(words, mark); +} + +bool BindlessHeap::Held(const SpanMark& mark) { + if (mark.entry == 0u || mark.words == 0u) return false; + if (mark.block != 0u) { + std::lock_guard lock(mutex); + const auto index = mark.block - 1u; + if (index >= poolBlocks.size() || poolBlocks[index].generation != mark.entry) { + ++counts.spansExpired; + return false; + } + poolBlocks[index].open = true; + ++counts.spansKept; + return true; + } + const auto first = chunkOf(mark.offset); + const auto last = lastChunkOf(mark.offset, static_cast(mark.words) * sizeof(std::uint32_t)); + std::lock_guard lock(mutex); + for (auto chunk = first; chunk <= last; ++chunk) { + if (ringChunks[chunk].entered > mark.entry) { + ++counts.spansExpired; + return false; + } + } + for (auto chunk = first; chunk <= last; ++chunk) { + if (!ringChunks[chunk].open) ringOpenChunks.fetch_add(1u, std::memory_order_relaxed); + ringChunks[chunk].open = true; + } + ++counts.spansKept; + return true; +} + +BindlessHeap::DataSpan BindlessHeap::Allocate(std::size_t words, SpanMark& mark) { + const auto bytes = alignUp(std::max(words, 1u) * sizeof(std::uint32_t), RingSpanAlignment); + std::lock_guard lock(mutex); + if (poolCurrent >= poolBlocks.size() || alignUp(poolBlocks[poolCurrent].cursor, poolAlignment) + bytes > poolBlocks[poolCurrent].bytes) poolCurrent = takePoolBlockLocked(bytes); + auto& block = poolBlocks[poolCurrent]; + const auto offset = static_cast(alignUp(block.cursor, poolAlignment)); + block.cursor = offset + bytes; + block.open = true; + ++counts.spansWritten; + counts.poolBytes += bytes; + counts.ringSpanPeakBytes = std::max(counts.ringSpanPeakBytes, bytes); + poolBatchBytes += bytes; + mark = {static_cast(offset), static_cast(words), block.generation, static_cast(poolCurrent + 1u), block.address + offset}; + return {static_cast(offset), std::span(reinterpret_cast(block.data + offset), words), mark.address}; +} + +bool BindlessHeap::poolBlockDoneLocked(PoolBlock& block) { + if (block.serial == 0u) return true; + if (block.recorder == nullptr || !block.recorder->HasTimeline() || !block.recorder->WaitSerialFor(block.serial, 0u)) return false; + block.serial = 0u; + return true; +} + +std::size_t BindlessHeap::takePoolBlockLocked(std::size_t bytes) { + const auto need = std::max(bytes, PoolBlockBytes); + auto best = poolBlocks.size(); + std::size_t vacant = poolBlocks.size(); + for (std::size_t index = 0; index < poolBlocks.size(); ++index) { + auto& block = poolBlocks[index]; + if (block.buffer == nullptr) { + vacant = std::min(vacant, index); + continue; + } + if (index == poolCurrent || block.open || block.bytes < need) continue; + if (best < poolBlocks.size() && poolBlocks[best].bytes <= block.bytes) continue; + if (poolBlockDoneLocked(block)) best = index; + } + if (best < poolBlocks.size()) { + auto& block = poolBlocks[best]; + block.cursor = 0; + ++block.generation; + ++counts.poolBlocksRecycled; + return best; + } + const auto size = need == PoolBlockBytes ? PoolBlockBytes : static_cast(alignUp(need, PoolBlockBytes)); + constexpr VkBufferUsageFlags usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + constexpr VkMemoryPropertyFlags deviceVisible = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT | VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + std::unique_ptr buffer; + if (poolDeviceLocal) { + try { + buffer = std::make_unique(context, size, usage, deviceVisible); + } catch (const std::exception&) { + } + } + if (buffer == nullptr) buffer = std::make_unique(context, size, usage); + if (vacant == poolBlocks.size()) poolBlocks.emplace_back(); + auto& block = poolBlocks[vacant]; + block.data = buffer->Bytes().data(); + block.address = buffer->DeviceAddress(); + block.buffer = std::move(buffer); + block.bytes = size; + block.cursor = 0; + ++block.generation; + block.recorder = nullptr; + block.serial = 0u; + block.open = false; + ++counts.poolBlocksMade; + return vacant; +} + +void BindlessHeap::trimPoolLocked() { + std::size_t idle = 0; + for (std::size_t index = 0; index < poolBlocks.size(); ++index) { + const auto& block = poolBlocks[index]; + if (block.buffer != nullptr && index != poolCurrent && !block.open) idle += block.bytes; + } + for (std::size_t index = poolBlocks.size(); index-- > 0u && idle > PoolIdleBytes;) { + auto& block = poolBlocks[index]; + if (block.buffer == nullptr || index == poolCurrent || block.open || !poolBlockDoneLocked(block)) continue; + idle -= block.bytes; + block.buffer.reset(); + block.data = nullptr; + block.address = 0; + block.bytes = 0; + ++block.generation; + ++counts.poolBlocksFreed; + } +} + +bool BindlessHeap::TakeRingPressure() { + if (ringChunks.empty() || ringOpenChunks.load(std::memory_order_relaxed) < std::max(2u, ringChunks.size() / 2u)) return false; + std::lock_guard lock(mutex); + ++counts.ringPressureSubmits; + return true; +} + +std::pair BindlessHeap::DataLocation(VkDeviceAddress address) const { + std::lock_guard lock(mutex); + for (const auto& block : poolBlocks) { + if (block.buffer != nullptr && address >= block.address && address < block.address + block.bytes) return {block.buffer->Handle(), address - block.address}; + } + throw std::runtime_error("AGC bindless heap: shader data lies outside every data pool block"); +} + +BindlessHeap::DataSpan BindlessHeap::RingAllocate(std::size_t words, SpanMark& mark) { + Require(ringInHeap, "the GPU heap has no descriptor ring"); + const auto bytes = alignUp(words * sizeof(std::uint32_t), RingSpanAlignment); + if (bytes > ringBytes / 4u) throw std::runtime_error("AGC bindless heap: one draw's buffer descriptors exceed a quarter of the descriptor ring"); + std::lock_guard lock(mutex); + auto offset = wrapRingOffset(alignUp(ringCursor, ringAlignment), bytes); + enum class Taken { No, Kept, Busy }; + const auto taken = [&](bool skipBusy) { + for (auto chunk = chunkOf(offset); chunk <= lastChunkOf(offset, bytes); ++chunk) { + if (chunk == ringChunk ? offset < ringCursor : ringChunks[chunk].open) return Taken::Kept; + } + if (!skipBusy) return Taken::No; + for (auto chunk = chunkOf(offset); chunk <= lastChunkOf(offset, bytes); ++chunk) { + auto& entry = ringChunks[chunk]; + if (chunk == ringChunk || entry.serial == 0u || entry.recorder == nullptr || !entry.recorder->HasTimeline()) continue; + if (!entry.recorder->WaitSerialFor(entry.serial, 0u)) return Taken::Busy; + entry.serial = 0u; + } + return Taken::No; + }; + const auto start = offset; + const auto wraps = counts.ringWraps; + const auto keptSkips = counts.keptChunksSkipped; + const auto busySkips = counts.busyChunksSkipped; + auto reason = taken(true); + for (std::size_t skipped = 0; reason != Taken::No && skipped < ringChunks.size(); ++skipped) { + offset = wrapRingOffset((chunkOf(offset) + 1u) * RingChunkBytes, bytes); + ++(reason == Taken::Busy ? counts.busyChunksSkipped : counts.keptChunksSkipped); + reason = taken(true); + } + if (reason != Taken::No) { + counts.ringWraps = wraps; + counts.keptChunksSkipped = keptSkips; + counts.busyChunksSkipped = busySkips; + std::optional oldest; + std::uint64_t oldestSerial = ~std::uint64_t{0}; + offset = start; + for (std::size_t step = 0; step < ringChunks.size(); ++step) { + if (taken(false) == Taken::No) { + std::uint64_t serial = 0; + for (auto chunk = chunkOf(offset); chunk <= lastChunkOf(offset, bytes); ++chunk) { + if (chunk != ringChunk) serial = std::max(serial, ringChunks[chunk].serial); + } + if (serial < oldestSerial) { + oldest = offset; + oldestSerial = serial; + } + } + offset = wrapRingOffset((chunkOf(offset) + 1u) * RingChunkBytes, bytes); + } + counts.ringWraps = wraps; + if (!oldest) throw std::runtime_error("AGC bindless heap: one batch's buffer descriptors exceed the descriptor ring"); + offset = *oldest; + if (offset < start) ++counts.ringWraps; + } + const auto consumed = bytes + (offset >= ringCursor ? offset - ringCursor : ringBytes - ringCursor + offset); + ringBatchBytes += consumed; + counts.ringBytes += consumed; + ringCursor = offset + bytes; + for (auto chunk = chunkOf(offset); chunk <= lastChunkOf(offset, bytes); ++chunk) enterRingChunk(chunk); + auto* base = reinterpret_cast(ringData + offset); + ++counts.spansWritten; + mark = {static_cast(offset), static_cast(words), ringEntries, 0u, resourceHeap->DeviceAddress() + ringHeapOffset + offset}; + return {static_cast(offset), std::span(base, words), mark.address}; +} + +std::size_t BindlessHeap::wrapRingOffset(std::size_t offset, std::size_t bytes) { + if (offset + bytes <= ringBytes) return offset; + ++counts.ringWraps; + return 0; +} + +void BindlessHeap::enterRingChunk(std::size_t index) { + if (index == ringChunk) return; + ringChunk = index; + auto& chunk = ringChunks[index]; + if (chunk.open) throw std::runtime_error("AGC bindless heap: one batch's buffer descriptors exceed the descriptor ring"); + if (chunk.serial != 0u) { + if (!chunk.recorder->HasTimeline()) + throw std::runtime_error("AGC bindless heap: the descriptor ring needs the recorder timeline to wait for a batch"); + if (!chunk.recorder->WaitSerialFor(chunk.serial, 0u)) { + chunk.recorder->WaitSerial(chunk.serial); + ++counts.ringWaits; + } + chunk.serial = 0u; + } + chunk.open = true; + ringOpenChunks.fetch_add(1u, std::memory_order_relaxed); + chunk.entered = ++ringEntries; +} + +void BindlessHeap::writeSamplerSlot(std::uint32_t slot, VkSampler sampler) { + auto record = FindSampler(sampler); + Require(record.has_value(), "a GPU heap sampler has no recorded create info"); + const auto offset = static_cast(slot) * samplerHeapStride; + Require(offset + samplerHeapStride <= samplerHeapUsed, "a GPU heap sampler slot lies outside the sampler heap"); + const VkHostAddressRangeEXT target{samplerHeap->Bytes().data() + offset, static_cast(samplerHeapStride)}; + if (writeSamplerDescriptors == nullptr) writeSamplerDescriptors = context.Function("vkWriteSamplerDescriptorsEXT"); + Check(writeSamplerDescriptors(context.device, 1, record->Chain(), &target), "vkWriteSamplerDescriptorsEXT"); +} + +void BindlessHeap::writeImageSlot(std::uint32_t kind, std::uint32_t slot, VkImageView view, VkImageLayout layout) { + const auto offset = static_cast(slot) * imageHeapStride; + Require(offset + imageHeapStride <= resourceHeapUsed, "a GPU heap image slot lies outside the resource heap"); + auto* target = resourceHeap->Bytes().data() + offset; + if (view == VK_NULL_HANDLE) { + std::memset(target, 0, static_cast(imageHeapStride)); + return; + } + const bool storage = Abi::GpuHeapStorageKind(kind); + const auto key = static_cast(layout) | (storage ? 0x80000000u : 0u); + const auto stride = static_cast(imageHeapStride); + if (CopyEncodedImageView(view, key, std::span(target, stride))) { + ++counts.encodedCopies; + return; + } + auto record = FindImageView(view); + Require(record.has_value(), "a GPU heap image view has no recorded create info"); + const VkImageDescriptorInfoEXT imageInfo{VK_STRUCTURE_TYPE_IMAGE_DESCRIPTOR_INFO_EXT, nullptr, record->Chain(), layout}; + VkResourceDescriptorInfoEXT resource{VK_STRUCTURE_TYPE_RESOURCE_DESCRIPTOR_INFO_EXT}; + resource.type = storage ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + resource.data.pImage = &imageInfo; + encodeScratch.resize(stride); + const VkHostAddressRangeEXT range{encodeScratch.data(), stride}; + if (writeResourceDescriptors == nullptr) writeResourceDescriptors = context.Function("vkWriteResourceDescriptorsEXT"); + Check(writeResourceDescriptors(context.device, 1, &resource, &range), "vkWriteResourceDescriptorsEXT"); + std::memcpy(target, encodeScratch.data(), stride); + KeepEncodedImageView(view, record->generation, key, encodeScratch); + ++counts.encodes; +} + +namespace { + +const std::uint32_t GeneratedFaultBinding = Abi::BindingNumber(Abi::Stage::TessellationControl, Abi::Binding::FaultBuffer); + +struct HeapBindingRoots { + bool workgroupAddress = false; + bool gdsAddress = false; + bool faultAddress = false; + bool pixelLds = false; + bool meshWindow = false; + bool scratchPool = false; +}; + +using SetAndBinding = std::pair; + +struct HeapBindingUse { + std::vector imageBindings; + bool workgroupMemory = false; + std::uint32_t gdsBinding = ~0u; + bool faultMemory = false; + bool pixelLdsMemory = false; + bool meshWindowMemory = false; + bool scratchPoolMemory = false; + std::vector dataUniforms; + std::vector ringDescriptors; + std::pair refused{~0u, ~0u}; +}; + +std::optional> decoratedVariables(std::span spirv) { + constexpr std::size_t HeaderWords = 5u; + constexpr std::uint32_t DecorateWords = 4u; + if (spirv.size() < HeaderWords) return std::nullopt; + std::unordered_map decorated; + for (std::size_t at = HeaderWords; at < spirv.size();) { + const auto count = spirv[at] >> 16u; + if (count == 0u || at + count > spirv.size()) return std::nullopt; + if ((spirv[at] & 0xffffu) == static_cast(spv::OpDecorate) && count >= DecorateWords) { + auto& entry = decorated.try_emplace(spirv[at + 1u], SetAndBinding{~0u, ~0u}).first->second; + if (spirv[at + 2u] == static_cast(spv::DecorationDescriptorSet)) entry.first = spirv[at + 3u]; + if (spirv[at + 2u] == static_cast(spv::DecorationBinding)) entry.second = spirv[at + 3u]; + } + at += count; + } + return decorated; +} + +bool heapBindings(std::span spirv, const HeapBindingRoots& roots, HeapBindingUse& use) { + use = {}; + const auto decorated = decoratedVariables(spirv); + if (!decorated) return false; + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + constexpr auto firstImage = bindingCount + Abi::FirstImageBinding; + constexpr auto samplers = bindingCount + BindlessHeap::SamplerKind; + for (const auto& [id, entry] : *decorated) { + const auto [set, binding] = entry; + if (set == ~0u && binding == ~0u) continue; + if (roots.workgroupAddress && set == ShaderRecompiler::GpuWorkgroupMemoryDescriptorSet && binding == 0u) { + use.workgroupMemory = true; + continue; + } + if (roots.gdsAddress && set == Abi::GpuGdsSet && use.gdsBinding == ~0u) { + use.gdsBinding = binding; + continue; + } + if (roots.faultAddress && set == Abi::DescriptorSet && binding == GeneratedFaultBinding) { + use.faultMemory = true; + continue; + } + if (roots.pixelLds && !use.pixelLdsMemory && set == ShaderRecompiler::PixelFunctionLdsDescriptorSet && binding == 0u) { + use.pixelLdsMemory = true; + continue; + } + if (roots.meshWindow && !use.meshWindowMemory && set == ShaderRecompiler::MeshWindowDescriptorSet && binding == 0u) { + use.meshWindowMemory = true; + continue; + } + if (roots.scratchPool && set == ShaderRecompiler::GpuScratchPoolDescriptorSet && binding == 0u) { + use.scratchPoolMemory = true; + continue; + } + if (set == Abi::GpuDataSet && binding % bindingCount == static_cast(Abi::Binding::ShaderData) && binding / bindingCount < Abi::StageCount) { + use.dataUniforms.push_back(binding); + continue; + } + const bool stageBinding = binding / bindingCount < Abi::StageCount; + if (roots.gdsAddress && stageBinding && ((set == Abi::DescriptorSet && binding % bindingCount == static_cast(Abi::Binding::Buffers)) || + (set == Abi::GpuDataSet && binding % bindingCount == static_cast(Abi::Binding::FlattenedSrt)))) { + if (std::find(use.ringDescriptors.begin(), use.ringDescriptors.end(), entry) == use.ringDescriptors.end()) use.ringDescriptors.push_back(entry); + continue; + } + const bool heapImageOrSampler = set == Abi::FragmentDescriptorSet && binding >= firstImage && binding <= samplers; + if (!heapImageOrSampler) { + use.refused = entry; + return false; + } + if (std::find(use.imageBindings.begin(), use.imageBindings.end(), binding) == use.imageBindings.end()) + use.imageBindings.push_back(binding); + } + return (!roots.workgroupAddress || use.workgroupMemory) && (!roots.faultAddress || use.faultMemory) && + (!roots.pixelLds || use.pixelLdsMemory) && (!roots.meshWindow || use.meshWindowMemory) && (!roots.scratchPool || use.scratchPoolMemory); +} + +VkDescriptorSetAndBindingMappingEXT heapMapping(std::uint32_t set, std::uint32_t binding, VkSpirvResourceTypeFlagsEXT resources, + VkDeviceSize heapOffset, VkDeviceSize stride) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = set; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = resources; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_HEAP_WITH_CONSTANT_OFFSET_EXT; + mapping.sourceData.constantOffset.heapOffset = static_cast(heapOffset); + mapping.sourceData.constantOffset.heapArrayStride = static_cast(stride); + return mapping; +} + +VkDescriptorSetAndBindingMappingEXT pushAddressMapping(std::uint32_t set, std::uint32_t binding, std::uint32_t pushOffset) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = set; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_PUSH_ADDRESS_EXT; + mapping.sourceData.pushAddressOffset = pushOffset; + return mapping; +} + +std::uint32_t rootItemOffset(const ShaderRecompiler::StagePushPlacement* placement, Abi::RootItem item) { + Require(placement != nullptr && placement->Item(item) != Abi::NoPushPosition, "a heap mapping reads a root item its pipeline's push layout does not hold"); + return placement->Item(item) * static_cast(sizeof(std::uint32_t)); +} + +VkDescriptorSetAndBindingMappingEXT ringDescriptorMapping(std::uint32_t set, std::uint32_t binding, VkDeviceSize ringOffset, const ShaderRecompiler::StagePushPlacement* placement) { + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + const bool buffers = binding % bindingCount == static_cast(Abi::Binding::Buffers); + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = set; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_ONLY_STORAGE_BUFFER_BIT_EXT | VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_HEAP_WITH_PUSH_INDEX_EXT; + mapping.sourceData.pushIndex.heapOffset = static_cast(ringOffset + (buffers ? BindlessHeap::BufferSlotBytes : 0u)); + mapping.sourceData.pushIndex.pushOffset = rootItemOffset(placement, Abi::RootItem::FlattenedSrt); + mapping.sourceData.pushIndex.heapIndexStride = BindlessHeap::BufferSlotBytes; + mapping.sourceData.pushIndex.heapArrayStride = BindlessHeap::BufferSlotBytes; + return mapping; +} + +VkDescriptorSetAndBindingMappingEXT dataUniformMapping(std::uint32_t binding, const ShaderRecompiler::StagePushPlacement* placement) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = Abi::GpuDataSet; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_UNIFORM_BUFFER_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_PUSH_ADDRESS_EXT; + mapping.sourceData.pushAddressOffset = rootItemOffset(placement, Abi::RootItem::ShaderData); + return mapping; +} + +} + +bool DescriptorHeapMappable(std::span spirv, bool workgroupAddress, bool gdsAddress, bool pixelLds, bool meshWindow, bool scratchPool) { + HeapBindingUse use; + const bool mappable = heapBindings(spirv, {.workgroupAddress = workgroupAddress, .gdsAddress = gdsAddress, .pixelLds = pixelLds, .meshWindow = meshWindow, .scratchPool = scratchPool}, use); + if (!mappable && pipelineStats()) { + static std::mutex causesMutex; + static std::map causes; + std::lock_guard lock(causesMutex); + const auto count = ++causes[use.refused]; + if (std::has_single_bit(count)) { + std::fprintf( + stderr, + "[gpu-heap] unmappable stage: descriptor set %d binding %d (workgroup address %d, gds address %d); %llu stage variants with this cause\n", + static_cast(use.refused.first), static_cast(use.refused.second), workgroupAddress ? 1 : 0, gdsAddress ? 1 : 0, + static_cast(count)); + } + } + return mappable; +} + +bool BindlessHeap::DescriptorHeapMappings(std::span spirv, std::vector& mappings, + std::uint32_t workgroupAddressOffset, bool rootGds, bool rootFault, bool pixelLds, bool meshWindow, + bool scratchPool, const ShaderRecompiler::StagePushPlacement* placement) const { + mappings.clear(); + if (resourceHeap == nullptr) return false; + HeapBindingUse use; + const HeapBindingRoots roots{ + .workgroupAddress = workgroupAddressOffset != ~0u, .gdsAddress = rootGds, .faultAddress = rootFault, .pixelLds = pixelLds, .meshWindow = meshWindow, + .scratchPool = scratchPool}; + if (!heapBindings(spirv, roots, use)) return false; + if (!use.ringDescriptors.empty() && !ringInHeap) return false; + if (!use.ringDescriptors.empty() && context.descriptorHeapProperties.bufferDescriptorSize != BufferSlotBytes) return false; + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + constexpr VkSpirvResourceTypeFlagsEXT imageResources = VK_SPIRV_RESOURCE_TYPE_SAMPLED_IMAGE_BIT_EXT | + VK_SPIRV_RESOURCE_TYPE_READ_ONLY_IMAGE_BIT_EXT | + VK_SPIRV_RESOURCE_TYPE_READ_WRITE_IMAGE_BIT_EXT; + std::sort(use.imageBindings.begin(), use.imageBindings.end()); + for (const auto binding : use.imageBindings) { + const auto kind = binding - bindingCount; + if (kind == SamplerKind) + mappings.push_back( + heapMapping(Abi::FragmentDescriptorSet, binding, VK_SPIRV_RESOURCE_TYPE_SAMPLER_BIT_EXT, 0, samplerHeapStride)); + else mappings.push_back(heapMapping(Abi::FragmentDescriptorSet, binding, imageResources, 0, imageHeapStride)); + } + for (const auto binding : use.dataUniforms) mappings.push_back(dataUniformMapping(binding, placement)); + std::sort(use.ringDescriptors.begin(), use.ringDescriptors.end()); + for (const auto& [set, binding] : use.ringDescriptors) mappings.push_back(ringDescriptorMapping(set, binding, ringHeapOffset, placement)); + if (use.workgroupMemory) + mappings.push_back(pushAddressMapping(ShaderRecompiler::GpuWorkgroupMemoryDescriptorSet, 0, workgroupAddressOffset)); + if (use.scratchPoolMemory) + mappings.push_back(pushAddressMapping(ShaderRecompiler::GpuScratchPoolDescriptorSet, 0, ShaderRecompiler::GpuScratchPoolAddressPushByte)); + if (use.gdsBinding != ~0u) + mappings.push_back(pushAddressMapping(Abi::GpuGdsSet, use.gdsBinding, rootItemOffset(placement, Abi::RootItem::GdsAddress))); + if (use.faultMemory) + mappings.push_back(pushAddressMapping(Abi::DescriptorSet, GeneratedFaultBinding, rootItemOffset(placement, Abi::RootItem::FaultAddress))); + if (use.pixelLdsMemory) + mappings.push_back(heapMapping(ShaderRecompiler::PixelFunctionLdsDescriptorSet, 0, + VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT, pixelLdsHeapOffset, 0)); + if (use.meshWindowMemory) + mappings.push_back(heapMapping(ShaderRecompiler::MeshWindowDescriptorSet, 0, + VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT, meshWindowHeapOffset, 0)); + return true; +} + +std::uint32_t BindlessHeap::RingBufferDescriptors(std::span buffers, SpanMark& mark) { + Require(ringInHeap && context.descriptorHeapProperties.bufferDescriptorSize == BufferSlotBytes, "ring buffer descriptors need the descriptor ring inside the resource heap"); + const auto span = RingAllocate(buffers.size() * (BufferSlotBytes / sizeof(std::uint32_t)), mark); + Require(span.offset % BufferSlotBytes == 0u, "a ring buffer descriptor span is misaligned"); + struct ResourcesStorage {}; + struct RangesStorage {}; + auto& resources = HostThreadLocal, ResourcesStorage>(); + auto& ranges = HostThreadLocal, RangesStorage>(); + resources.assign(buffers.size(), {VK_STRUCTURE_TYPE_RESOURCE_DESCRIPTOR_INFO_EXT}); + ranges.resize(buffers.size()); + auto* bytes = reinterpret_cast(span.words.data()); + for (std::size_t index = 0; index < buffers.size(); ++index) { + resources[index].type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + resources[index].data.pAddressRange = &buffers[index]; + ranges[index] = {bytes + index * BufferSlotBytes, BufferSlotBytes}; + } + PFN_vkWriteResourceDescriptorsEXT write = nullptr; + { + std::lock_guard lock(mutex); + if (writeResourceDescriptors == nullptr) writeResourceDescriptors = context.Function("vkWriteResourceDescriptorsEXT"); + write = writeResourceDescriptors; + counts.bufferDescriptors += buffers.size(); + } + Check(write(context.device, static_cast(buffers.size()), resources.data(), ranges.data()), "vkWriteResourceDescriptorsEXT ring buffers"); + return static_cast(span.offset / BufferSlotBytes); +} + +void BindlessHeap::writeBufferDescriptor(VkDeviceSize offset, VkDeviceAddress address, VkDeviceSize bytes, const char* what) { + const auto& properties = context.descriptorHeapProperties; + Require(offset + properties.bufferDescriptorSize <= resourceHeapUsed, std::string("the ") + what + " descriptor lies outside the resource heap"); + const VkDeviceAddressRangeEXT buffer{address, bytes}; + VkResourceDescriptorInfoEXT resource{VK_STRUCTURE_TYPE_RESOURCE_DESCRIPTOR_INFO_EXT}; + resource.type = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + resource.data.pAddressRange = &buffer; + const VkHostAddressRangeEXT range{resourceHeap->Bytes().data() + offset, static_cast(properties.bufferDescriptorSize)}; + Check(context.Function("vkWriteResourceDescriptorsEXT")(context.device, 1, &resource, &range), + (std::string("vkWriteResourceDescriptorsEXT ") + what).c_str()); +} + +void BindlessHeap::WritePixelLdsBuffer(VkDeviceAddress address, VkDeviceSize bytes) { + writeBufferDescriptor(pixelLdsHeapOffset, address, bytes, "pixel function LDS"); +} + +void BindlessHeap::WriteMeshWindowBuffer(VkDeviceAddress address, VkDeviceSize bytes) { + writeBufferDescriptor(meshWindowHeapOffset, address, bytes, "mesh window"); +} + +namespace { + +std::mutex boundHeapsMutex; +std::unordered_set boundHeaps; +std::atomic heapBinds{0}; +std::atomic heapBindWindow{0}; +std::atomic commandBuffersBegun{0}; +std::atomic cpuBufferCopies{0}; +std::atomic cpuBufferCopyBytes{0}; +std::array, static_cast(SetBindSite::Count)> setBinds{}; +std::array, static_cast(HeapFallback::Count)> heapFallbacks{}; +std::array, 4> heapRecords{}; + +std::size_t heapRecordIndex(bool compute, bool heap) { + return (compute ? 2u : 0u) + (heap ? 1u : 0u); +} +std::array, static_cast(HeapStageReason::Count)> heapStageReasons{}; +constexpr std::array(HeapStageReason::Count)> HeapStageReasonNames{"heap", + "disabled", + "pair refused", + "fragment off heap", + "stage kind", + "no plan", + "plan stage", + "unresolved image source", + "known ineligible"}; +constexpr std::array(SetBindSite::Count)> SetBindSiteNames{"draw", "dispatch", "detiler", + "color transfer", "presentation", "depth transfer", "staged merge", "first-sample copy"}; +constexpr std::array(HeapFallback::Count)> HeapFallbackNames{ + "draw unmapped", "draw mixed retry", "draw refused pair", "draw mixed reused", "dispatch unmapped"}; + +std::uint64_t takeCount(std::atomic& counter) { + return counter.exchange(0, std::memory_order_relaxed); +} + +template +std::string takeNamedCounts(std::array, N>& counters, const std::array& names) { + std::string text; + for (std::size_t i = 0; i < N; ++i) text += " " + std::string(names[i]) + " " + std::to_string(takeCount(counters[i])); + return text; +} + +unsigned long long printed(std::uint64_t value) { + return static_cast(value); +} + +std::array, static_cast(HeapCost::Count)> heapCostNs{}; +std::array, static_cast(HeapCost::Count)> heapCostCalls{}; +constexpr std::array(HeapCost::Count)> HeapCostNames{ + "stage prepare", "heap stage call", "stage finish", "walk memo", "walk", "materialize", "template find", + "template build", "slots", "stage data", "buffer addresses", "memo insert", "guards"}; + +void reportHeapStatistics() { + if (!pipelineStats()) return; + static std::atomic calls{0}; + if ((calls.fetch_add(1, std::memory_order_relaxed) & 1023u) != 0u) return; + static std::mutex reportMutex; + static auto last = std::chrono::steady_clock::now(); + std::unique_lock lock(reportMutex, std::try_to_lock); + if (!lock.owns_lock()) return; + const auto now = std::chrono::steady_clock::now(); + if (now - last < std::chrono::seconds(10)) return; + last = now; + const auto binds = takeNamedCounts(setBinds, SetBindSiteNames); + const auto fallbacks = takeNamedCounts(heapFallbacks, HeapFallbackNames); + const auto stageReasons = takeNamedCounts(heapStageReasons, HeapStageReasonNames); + std::string costs; + for (std::size_t i = 0; i < heapCostNs.size(); ++i) { + const auto calls = takeCount(heapCostCalls[i]); + const auto ns = static_cast(takeCount(heapCostNs[i])); + const auto averageUs = calls != 0 ? ns / static_cast(calls) / 1000.0 : 0.0; + char cost[96]; + std::snprintf(cost, sizeof(cost), " %s %llu x %.2f us = %.0f ms", HeapCostNames[i], printed(calls), averageUs, ns / 1e6); + costs += cost; + } + if (HeapCostTimed()) { + const auto held = static_cast(std::max(walkMemoHeld().load(std::memory_order_relaxed), 0)) / 1048576.0; + const auto peak = static_cast(walkMemoPeak().load(std::memory_order_relaxed)) / 1048576.0; + std::fprintf(stderr, "[gpu-heap] cost (10 s):%s; walk memos hold %.1f MiB on all threads (peak %.1f MiB)\n", costs.c_str(), held, peak); + } + const auto heapDraws = takeCount(heapRecords[heapRecordIndex(false, true)]); + const auto plainDraws = takeCount(heapRecords[heapRecordIndex(false, false)]); + const auto heapDispatches = takeCount(heapRecords[heapRecordIndex(true, true)]); + const auto plainDispatches = takeCount(heapRecords[heapRecordIndex(true, false)]); + const auto heapBindsSeen = takeCount(heapBindWindow); + const auto commandBuffers = takeCount(commandBuffersBegun); + const auto copies = takeCount(cpuBufferCopies); + const auto copiedMiB = takeCount(cpuBufferCopyBytes) >> 20u; + static BindlessHeap::Counts lastTotals; + BindlessHeap::Counts totals; + { + std::lock_guard heapsLock(heapsMutex()); + for (const auto& entry : heaps()) { + const auto counts = entry.second->Statistics(); + totals.spansWritten += counts.spansWritten; + totals.spansKept += counts.spansKept; + totals.spansExpired += counts.spansExpired; + totals.encodes += counts.encodes; + totals.encodedCopies += counts.encodedCopies; + totals.bufferDescriptors += counts.bufferDescriptors; + totals.slotsEvicted += counts.slotsEvicted; + totals.evictionWaits += counts.evictionWaits; + totals.ringWaits += counts.ringWaits; + totals.busyChunksSkipped += counts.busyChunksSkipped; + totals.ringBytes += counts.ringBytes; + totals.imageSlotsHeld += counts.imageSlotsHeld; + totals.ringBatchPeakBytes = std::max(totals.ringBatchPeakBytes, counts.ringBatchPeakBytes); + totals.ringSpanPeakBytes = std::max(totals.ringSpanPeakBytes, counts.ringSpanPeakBytes); + totals.poolBytes += counts.poolBytes; + totals.ringPressureSubmits += counts.ringPressureSubmits; + totals.poolBlocksMade += counts.poolBlocksMade; + totals.poolBlocksRecycled += counts.poolBlocksRecycled; + totals.poolBlocksFreed += counts.poolBlocksFreed; + totals.poolBlocksHeld += counts.poolBlocksHeld; + totals.poolBlockBytes += counts.poolBlockBytes; + totals.poolBatchPeakBytes = std::max(totals.poolBatchPeakBytes, counts.poolBatchPeakBytes); + } + } + const auto windowDelta = [](std::uint64_t now, std::uint64_t before) { return now >= before ? now - before : now; }; + const auto spansWritten = windowDelta(totals.spansWritten, lastTotals.spansWritten); + const auto spansKept = windowDelta(totals.spansKept, lastTotals.spansKept); + const auto spansExpired = windowDelta(totals.spansExpired, lastTotals.spansExpired); + const auto imageEncodes = windowDelta(totals.encodes, lastTotals.encodes); + const auto imageCopies = windowDelta(totals.encodedCopies, lastTotals.encodedCopies); + const auto bufferDescriptors = windowDelta(totals.bufferDescriptors, lastTotals.bufferDescriptors); + const auto slotsEvicted = windowDelta(totals.slotsEvicted, lastTotals.slotsEvicted); + const auto evictionWaits = windowDelta(totals.evictionWaits, lastTotals.evictionWaits); + const auto ringWaits = windowDelta(totals.ringWaits, lastTotals.ringWaits); + const auto busySkips = windowDelta(totals.busyChunksSkipped, lastTotals.busyChunksSkipped); + const auto ringMiB = windowDelta(totals.ringBytes, lastTotals.ringBytes) >> 20u; + const auto poolMiB = windowDelta(totals.poolBytes, lastTotals.poolBytes) >> 20u; + const auto pressureSubmits = windowDelta(totals.ringPressureSubmits, lastTotals.ringPressureSubmits); + const auto poolMade = windowDelta(totals.poolBlocksMade, lastTotals.poolBlocksMade); + const auto poolRecycled = windowDelta(totals.poolBlocksRecycled, lastTotals.poolBlocksRecycled); + const auto poolFreed = windowDelta(totals.poolBlocksFreed, lastTotals.poolBlocksFreed); + lastTotals = totals; + const auto templates = ShaderRecompiler::HeapTemplateCounts(); + std::fprintf(stderr, + "[gpu-heap] (10 s): draws %llu heap / %llu plain, dispatches %llu heap / %llu plain; " + "heap binds %llu over %llu command buffers; set binds:%s; fallbacks:%s; draw stages:%s; " + "CPU buffer copies %llu (%llu MiB); spans %llu written / %llu kept / %llu expired; data pool %llu MiB written, %llu blocks (%llu MiB), %llu made / %llu recycled / %llu freed, since start at most %llu KiB per batch and %llu KiB per span; descriptor ring %llu MiB, %llu waits, %llu busy chunks skipped, %llu early submits, since start at most %llu KiB per batch; image descriptors %llu encoded / %llu copied; buffer descriptors %llu; image slots %llu of %u held, %llu evicted (%llu waits); templates %llu (%llu MiB, %llu evicted)\n", + printed(heapDraws), printed(plainDraws), printed(heapDispatches), printed(plainDispatches), printed(heapBindsSeen), + printed(commandBuffers), binds.c_str(), fallbacks.c_str(), stageReasons.c_str(), printed(copies), printed(copiedMiB), + printed(spansWritten), printed(spansKept), printed(spansExpired), printed(poolMiB), printed(totals.poolBlocksHeld), printed(totals.poolBlockBytes >> 20u), printed(poolMade), printed(poolRecycled), printed(poolFreed), printed(totals.poolBatchPeakBytes >> 10u), printed(totals.ringSpanPeakBytes >> 10u), printed(ringMiB), printed(ringWaits), printed(busySkips), printed(pressureSubmits), printed(totals.ringBatchPeakBytes >> 10u), printed(imageEncodes), printed(imageCopies), printed(bufferDescriptors), printed(totals.imageSlotsHeld), BindlessHeap::ImageSlotCapacity(), printed(slotsEvicted), printed(evictionWaits), printed(templates.entries), printed(templates.bytes >> 20u), printed(templates.evicted)); +} + +void forgetHeapBinding(VkCommandBuffer commands) { + if (!DescriptorHeapEnabled()) return; + std::lock_guard lock(boundHeapsMutex); + boundHeaps.erase(commands); +} + +} + +void NoteHeapRecord(bool compute, bool heap) { + if (!pipelineStats()) return; + heapRecords[heapRecordIndex(compute, heap)].fetch_add(1, std::memory_order_relaxed); + reportHeapStatistics(); +} + +void NoteHeapStageReason(HeapStageReason reason) { + heapStageReasons[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +bool HeapCostTimed() { + return pipelineStats(); +} + +void NoteHeapCost(HeapCost cost, std::uint64_t nanoseconds) { + heapCostNs[static_cast(cost)].fetch_add(nanoseconds, std::memory_order_relaxed); + heapCostCalls[static_cast(cost)].fetch_add(1, std::memory_order_relaxed); +} + +void NoteCpuBufferCopy(std::uint64_t bytes) { + cpuBufferCopies.fetch_add(1, std::memory_order_relaxed); + cpuBufferCopyBytes.fetch_add(bytes, std::memory_order_relaxed); +} + +void NoteHeapFallback(HeapFallback reason) { + heapFallbacks[static_cast(reason)].fetch_add(1, std::memory_order_relaxed); +} + +std::uint64_t HeapFallbacks(HeapFallback reason) { + return heapFallbacks[static_cast(reason)].load(std::memory_order_relaxed); +} + +void NoteCommandBufferBegun(VkCommandBuffer commands) { + commandBuffersBegun.fetch_add(1, std::memory_order_relaxed); + forgetHeapBinding(commands); +} + +void CmdPushConstants(const Context& context, VkCommandBuffer commands, VkPipelineLayout layout, VkShaderStageFlags stages, + std::uint32_t offset, std::uint32_t size, const void* values) { + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, offset, size, values); + forgetHeapBinding(commands); +} + +void CmdBindDescriptorSets(const Context& context, VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout, + std::uint32_t firstSet, std::uint32_t setCount, const VkDescriptorSet* sets, SetBindSite site) { + context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, firstSet, setCount, + sets, 0, nullptr); + if (pipelineStats()) setBinds[static_cast(site)].fetch_add(1, std::memory_order_relaxed); + forgetHeapBinding(commands); +} + +void CmdPushDescriptorSet(PFN_vkCmdPushDescriptorSetKHR push, VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout, std::uint32_t set, + std::uint32_t writeCount, const VkWriteDescriptorSet* writes, SetBindSite site) { + push(commands, bindPoint, layout, set, writeCount, writes); + if (pipelineStats()) setBinds[static_cast(site)].fetch_add(1, std::memory_order_relaxed); + forgetHeapBinding(commands); +} + +std::uint64_t DescriptorHeapBinds() { + return heapBinds.load(std::memory_order_relaxed); +} + +bool HeapPassesActive(const Context& context) { + return DescriptorHeapEnabled() && context.descriptorHeap && BindlessHeap::For(context).DescriptorHeapMode(); +} + +VkDescriptorSetAndBindingMappingEXT HeapPushAddressMapping(std::uint32_t set, std::uint32_t binding, std::uint32_t pushOffset) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = set; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT | VK_SPIRV_RESOURCE_TYPE_READ_ONLY_STORAGE_BUFFER_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_PUSH_ADDRESS_EXT; + mapping.sourceData.pushAddressOffset = pushOffset; + return mapping; +} + +VkDescriptorSetAndBindingMappingEXT HeapPushIndexMapping(const BindlessHeap& heap, std::uint32_t set, std::uint32_t binding, VkSpirvResourceTypeFlagsEXT resources, + std::uint32_t pushOffset) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = set; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = resources; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_HEAP_WITH_PUSH_INDEX_EXT; + mapping.sourceData.pushIndex.pushOffset = pushOffset; + mapping.sourceData.pushIndex.heapIndexStride = static_cast(heap.ImageHeapStride()); + mapping.sourceData.pushIndex.heapArrayStride = static_cast(heap.ImageHeapStride()); + return mapping; +} + +VkPipeline CreateHeapComputePipeline(const Context& context, VkComputePipelineCreateInfo info, std::span mappings) { + if (!HeapPassesActive(context)) return VK_NULL_HANDLE; + VkShaderDescriptorSetAndBindingMappingInfoEXT mappingInfo{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}; + mappingInfo.pNext = info.stage.pNext; + mappingInfo.mappingCount = static_cast(mappings.size()); + mappingInfo.pMappings = mappings.data(); + info.stage.pNext = &mappingInfo; + auto heapFlags = DescriptorHeapFlags(); + heapFlags.pNext = info.pNext; + info.pNext = &heapFlags; + info.layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + if (context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &pipeline) != VK_SUCCESS) + return VK_NULL_HANDLE; + return pipeline; +} + +VkPipeline CreateHeapGraphicsPipeline(const Context& context, VkGraphicsPipelineCreateInfo info, std::span mappings) { + if (!HeapPassesActive(context)) return VK_NULL_HANDLE; + std::vector stages(info.pStages, info.pStages + info.stageCount); + std::vector mappingInfos(stages.size(), {VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}); + for (std::size_t index = 0; index < stages.size(); ++index) { + mappingInfos[index].pNext = stages[index].pNext; + mappingInfos[index].mappingCount = static_cast(mappings.size()); + mappingInfos[index].pMappings = mappings.data(); + stages[index].pNext = &mappingInfos[index]; + } + info.pStages = stages.data(); + auto heapFlags = DescriptorHeapFlags(); + heapFlags.pNext = info.pNext; + info.pNext = &heapFlags; + info.layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + if (context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &pipeline) != VK_SUCCESS) + return VK_NULL_HANDLE; + return pipeline; +} + +void CmdPushData(const Context& context, VkCommandBuffer commands, std::uint32_t bytes, const void* values) { + const VkPushDataInfoEXT data{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {values, bytes}}; + context.Function("vkCmdPushDataEXT")(commands, &data); +} + +void BindlessHeap::BindDescriptorHeaps(VkCommandBuffer commands) const { + if (resourceHeap == nullptr) return; + { + std::lock_guard lock(boundHeapsMutex); + if (!boundHeaps.insert(commands).second) return; + } + heapBinds.fetch_add(1, std::memory_order_relaxed); + heapBindWindow.fetch_add(1, std::memory_order_relaxed); + VkBindHeapInfoEXT info{VK_STRUCTURE_TYPE_BIND_HEAP_INFO_EXT}; + info.heapRange = {resourceHeap->DeviceAddress(), ringInHeap ? ringHeapOffset + ringBytes : resourceHeapUsed + context.descriptorHeapProperties.minResourceHeapReservedRange}; + info.reservedRangeOffset = resourceHeapUsed; + info.reservedRangeSize = context.descriptorHeapProperties.minResourceHeapReservedRange; + context.Function("vkCmdBindResourceHeapEXT")(commands, &info); + info.heapRange = {samplerHeap->DeviceAddress(), + samplerHeapUsed + context.descriptorHeapProperties.minSamplerHeapReservedRangeWithEmbedded}; + info.reservedRangeOffset = samplerHeapUsed; + info.reservedRangeSize = context.descriptorHeapProperties.minSamplerHeapReservedRangeWithEmbedded; + context.Function("vkCmdBindSamplerHeapEXT")(commands, &info); +} + +void BindlessHeap::Submitted(Recorder& recorder, std::uint64_t serial) { + std::lock_guard lock(mutex); + submitter = &recorder; + useEpoch.fetch_add(1, std::memory_order_relaxed); + counts.ringBatchPeakBytes = std::max(counts.ringBatchPeakBytes, ringBatchBytes); + ringBatchBytes = 0; + if (refused && imagePool.Unused() + retiring.size() < 2u * static_cast(imageMargin())) evictLocked(); + for (auto entry = retiring.rbegin(); entry != retiring.rend() && entry->serial == 0u; ++entry) { + entry->serial = serial; + ++retiringAssigned; + } + if (!retiring.empty()) reclaimLocked(false); + noteReleasedLocked(); + for (std::size_t index = 0; index < ringChunks.size(); ++index) { + if (!ringChunks[index].open && index != ringChunk) continue; + ringChunks[index].recorder = &recorder; + ringChunks[index].serial = serial; + ringChunks[index].open = index == ringChunk; + } + ringOpenChunks.store(ringChunk < ringChunks.size() && ringChunks[ringChunk].open ? 1u : 0u, std::memory_order_relaxed); + counts.poolBatchPeakBytes = std::max(counts.poolBatchPeakBytes, poolBatchBytes); + poolBatchBytes = 0; + for (std::size_t index = 0; index < poolBlocks.size(); ++index) { + auto& block = poolBlocks[index]; + if (block.buffer == nullptr || (!block.open && index != poolCurrent)) continue; + block.recorder = &recorder; + block.serial = serial; + block.open = index == poolCurrent; + } + trimPoolLocked(); +} + +BindlessHeap::Counts BindlessHeap::Statistics() const { + std::lock_guard lock(mutex); + auto snapshot = counts; + for (const auto& block : poolBlocks) { + if (block.buffer == nullptr) continue; + ++snapshot.poolBlocksHeld; + snapshot.poolBlockBytes += block.bytes; + } + snapshot.imageSlotsHeld = imagePool.next - std::min(imagePool.next, SamplerKind + imagePool.free.size() + retiring.size()); + return snapshot; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferAddress.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferAddress.cpp index 857cce2cc2f8..f788d9722265 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferAddress.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferAddress.cpp @@ -7,10 +7,14 @@ VkDeviceAddress Buffer::DeviceAddress() const { return deviceAddress; } +VkDeviceAddress BufferDeviceAddress(const Context& context, VkBuffer buffer) { + const VkBufferDeviceAddressInfo info{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; + return context.Function("vkGetBufferDeviceAddressKHR")(context.device, &info); +} + void Buffer::initializeAddress(VkBufferUsageFlags usage) { if ((usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) == 0) return; - const VkBufferDeviceAddressInfo info{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; - deviceAddress = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &info); + deviceAddress = BufferDeviceAddress(context, buffer); Require(deviceAddress != 0, "vkGetBufferDeviceAddressKHR returned a null address"); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index cc90299ad703..6e3e23a7186e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -409,7 +409,22 @@ std::size_t BufferPool::NextCapacity(std::size_t capacity) { return capacity < classLimit ? capacity * 2 : capacity + (std::bit_floor(capacity) >> deviceClassBits); } +namespace { + +std::atomic forceAddresses{false}; + +} + +bool BufferPool::ForceAddresses() { + return forceAddresses.load(std::memory_order_relaxed); +} + +void BufferPool::SetForceAddresses(bool force) { + forceAddresses.store(force, std::memory_order_relaxed); +} + VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { + if (ForceAddresses() && (usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0) usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; return DeviceTiered(properties) ? usage | deviceUsage : usage; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index a0c0d60ee37f..9c32e4326e4e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -21,6 +22,7 @@ #include "RdnaDecoder/include/RdnaDecoder/RdnaDescriptorFormat.hpp" #include #include +#include #include #include #include @@ -93,6 +95,14 @@ struct RectPush { std::array extent; }; +struct HeapRectPush { + RectPush rect; + std::uint32_t depth; + std::uint32_t texels; +}; + +static_assert(offsetof(HeapRectPush, depth) == 16 && offsetof(HeapRectPush, texels) == 20); + class DepthToStoragePass { public: explicit DepthToStoragePass(const Context& context) : context(context) { @@ -121,6 +131,13 @@ class DepthToStoragePass { auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &pipeline); } + if (created == VK_SUCCESS && HeapPassesActive(context)) { + const auto& heap = BindlessHeap::For(context); + const std::array mappings{ + HeapPushIndexMapping(heap, 0, 0, VK_SPIRV_RESOURCE_TYPE_SAMPLED_IMAGE_BIT_EXT, offsetof(HeapRectPush, depth)), + HeapPushIndexMapping(heap, 0, 1, HeapPassStorageImages, offsetof(HeapRectPush, texels))}; + heapPipeline = CreateHeapComputePipeline(context, info, mappings); + } context.Function("vkDestroyShaderModule")(context.device, compute, nullptr); Check(created, "vkCreateComputePipelines depth to storage"); } catch (...) { @@ -134,17 +151,31 @@ class DepthToStoragePass { VkDevice Device() const { return context.device; } - void Record(VkCommandBuffer commands, VkImageView depthPlane, VkImageView texels, std::span rects) const { + void Record(VkCommandBuffer commands, VkImageView depthPlane, const HeapSlotOwner* depthOwner, VkImageView texels, const HeapSlotOwner* texelOwner, std::span rects) const { + const auto dispatch = context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch"); + if (heapPipeline != VK_NULL_HANDLE) { + auto& heap = BindlessHeap::For(context); + const auto depthSlot = heap.ViewSlot(HeapPassSampledKind, depthOwner, depthPlane); + const auto texelSlot = depthSlot ? heap.ViewSlot(HeapPassStorageKind, texelOwner, texels) : std::nullopt; + if (texelSlot) { + heap.BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + for (const auto& rect : rects) { + const HeapRectPush push{{{static_cast(rect.offset.x), static_cast(rect.offset.y)}, {rect.extent.width, rect.extent.height}}, *depthSlot, *texelSlot}; + CmdPushData(context, commands, sizeof(push), &push); + dispatch(commands, (rect.extent.width + 7u) / 8u, (rect.extent.height + 7u) / 8u, 1); + } + return; + } + } const VkDescriptorImageInfo depthInfo{VK_NULL_HANDLE, depthPlane, VK_IMAGE_LAYOUT_GENERAL}; const VkDescriptorImageInfo texelInfo{VK_NULL_HANDLE, texels, VK_IMAGE_LAYOUT_GENERAL}; const std::array writes{imageWrite(0, VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, depthInfo), imageWrite(1, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, texelInfo)}; context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - pushSet(commands, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, static_cast(writes.size()), writes.data()); - const auto pushConstants = context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants"); - const auto dispatch = context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch"); + CmdPushDescriptorSet(pushSet, commands, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, static_cast(writes.size()), writes.data(), SetBindSite::DepthTransfer); for (const auto& rect : rects) { const RectPush push{{static_cast(rect.offset.x), static_cast(rect.offset.y)}, {rect.extent.width, rect.extent.height}}; - pushConstants(commands, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); + CmdPushConstants(context, commands, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); dispatch(commands, (rect.extent.width + 7u) / 8u, (rect.extent.height + 7u) / 8u, 1); } } @@ -161,9 +192,11 @@ class DepthToStoragePass { void release() noexcept { context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + if (heapPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, heapPipeline, nullptr); context.Function("vkDestroyPipelineLayout")(context.device, layout, nullptr); context.Function("vkDestroyDescriptorSetLayout")(context.device, setLayout, nullptr); pipeline = VK_NULL_HANDLE; + heapPipeline = VK_NULL_HANDLE; layout = VK_NULL_HANDLE; setLayout = VK_NULL_HANDLE; } @@ -173,6 +206,7 @@ class DepthToStoragePass { VkDescriptorSetLayout setLayout = VK_NULL_HANDLE; VkPipelineLayout layout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline heapPipeline = VK_NULL_HANDLE; }; std::mutex& depthToStoragePassesMutex() { @@ -602,7 +636,7 @@ class DepthSurface { allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory depth target"); Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory depth"); - if (target.samples != 1u) owned = std::make_shared(this->context, image, memory); + owned = std::make_shared(this->context, image, memory); VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; viewInfo.image = image; viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; @@ -858,13 +892,13 @@ class DepthSurface { return texture; } - std::shared_ptr FirstSample(const Texture& source, VkComponentMapping components) { + std::shared_ptr FirstSample(Texture& source, VkComponentMapping components) { auto* recorder = RecorderFor(context); Require(recorder != nullptr, "a native multisampled depth target is sampled only by recorded draws and dispatches"); const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT; const auto format = d16 ? VK_FORMAT_R16_UNORM : VK_FORMAT_R32_SFLOAT; if (firstSample == nullptr) firstSample = std::make_shared(context, format, target.extent); - firstSample->copy.Record(*recorder, source.View(), source.Layout()); + firstSample->copy.Record(*recorder, source); const std::array key{static_cast(components.r), static_cast(components.g), static_cast(components.b), static_cast(components.a)}; auto& texture = firstSample->textures[key]; if (texture == nullptr) { @@ -931,7 +965,8 @@ class DepthSurface { } bool SampledElsewhere() const { - return std::any_of(textures.begin(), textures.end(), [](const auto& entry) { return entry.second.use_count() > 1; }); + const auto held = [](const auto& entry) { return entry.second.use_count() > 1; }; + return std::any_of(textures.begin(), textures.end(), held) || std::any_of(stencilCopy.textures.begin(), stencilCopy.textures.end(), held); } void NoteHostWrite() { @@ -1575,6 +1610,7 @@ class DepthSurface { struct StencilCopy { VkImage image = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; + std::shared_ptr owner; std::unique_ptr staging; std::map, std::shared_ptr> textures; std::uint64_t written = 0; @@ -1596,6 +1632,7 @@ class DepthSurface { std::uint64_t stencilMemory = 0; StencilCopy stencilCopy; VkImageView transferView = VK_NULL_HANDLE; + HeapPassSlots transferSlots{context}; void createSampleShadow() { VkImage image = VK_NULL_HANDLE; @@ -1688,6 +1725,7 @@ class DepthSurface { allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); stencilCopy.memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory stencil copy"); Check(context.Function("vkBindImageMemory")(context.device, stencilCopy.image, stencilCopy.memory, 0), "vkBindImageMemory stencil copy"); + stencilCopy.owner = std::make_shared(context, stencilCopy.image, stencilCopy.memory); stencilCopy.staging = std::make_unique(context, static_cast(target.extent.width) * target.extent.height, VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); } catch (...) { releaseStencilCopy(); @@ -1700,7 +1738,7 @@ class DepthSurface { } auto& texture = stencilCopy.textures[key]; if (texture == nullptr) { - texture = std::make_shared(context, stencilCopy.image, VK_FORMAT_R8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT, components, viewType); + texture = std::make_shared(context, stencilCopy.image, VK_FORMAT_R8_UNORM, VK_IMAGE_ASPECT_COLOR_BIT, components, viewType, 0, stencilCopy.owner); texture->MarkRefreshedPerUse(); } return texture; @@ -1742,8 +1780,11 @@ class DepthSurface { void releaseStencilCopy() noexcept { stencilCopy.textures.clear(); stencilCopy.staging.reset(); - if (stencilCopy.image) context.Function("vkDestroyImage")(context.device, stencilCopy.image, nullptr); - if (stencilCopy.memory) MemoryAccounting::Free(context, stencilCopy.memory); + if (stencilCopy.owner != nullptr) stencilCopy.owner.reset(); + else { + if (stencilCopy.image) context.Function("vkDestroyImage")(context.device, stencilCopy.image, nullptr); + if (stencilCopy.memory) MemoryAccounting::Free(context, stencilCopy.memory); + } stencilCopy = {}; } @@ -1806,7 +1847,7 @@ class DepthSurface { const auto& areas = rects != nullptr ? *rects : whole; const auto texels = storage->AtomicView(0, storage->Descriptor().dimension == TextureDimension::k2DArray); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - depthToStoragePassFor(context).Record(commands, depthTransferView(), texels, areas); + depthToStoragePassFor(context).Record(commands, depthTransferView(), &transferSlots, texels, storage.get(), areas); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT); depthTransferPassCount.fetch_add(1, std::memory_order_relaxed); if (batch) { @@ -1872,6 +1913,7 @@ class DepthSurface { mirrorPlane = {}; stencilMirror = {}; textures.clear(); + transferSlots.Release(); if (transferView) context.Function("vkDestroyImageView")(context.device, transferView, nullptr); transferView = VK_NULL_HANDLE; if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); @@ -2424,10 +2466,11 @@ void DepthSurface::PublishToGuest() { info.format = hostFormat; info.subresourceRange = {static_cast(aspect), 0u, 1u, 0u, 1u}; Check(context.Function("vkCreateImageView")(context.device, &info, nullptr, &arrayView), "vkCreateImageView depth publication"); + HeapPassSlots passSlots(context); CommandBatch batch(context); constexpr VkAccessFlags writes = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT; RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, writes, VK_ACCESS_SHADER_READ_BIT); - context.detiler->DispatchMultisampleReadback(batch.Handle(), format, linear.Handle(), 0u, linearBytes, arrayView, target.extent.width, target.extent.height, 1u, target.samples, 0u, static_cast(layout.LinearBytes() / elementBytes)); + context.detiler->DispatchMultisampleReadback(batch.Handle(), format, linear.Handle(), 0u, linearBytes, arrayView, target.extent.width, target.extent.height, 1u, target.samples, 0u, static_cast(layout.LinearBytes() / elementBytes), &passSlots); RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); batch.SubmitAndWait(); } catch (...) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorHeapObjects.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorHeapObjects.cpp new file mode 100644 index 000000000000..fd62cf8ec388 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DescriptorHeapObjects.cpp @@ -0,0 +1,212 @@ +#include "prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp" + +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +std::atomic realCreateImageView{nullptr}; +std::atomic realDestroyImageView{nullptr}; +std::atomic realCreateSampler{nullptr}; +std::atomic realDestroySampler{nullptr}; + +std::mutex& recordsMutex() { + static auto& mutex = *new std::mutex(); + return mutex; +} + +struct EncodedImageView { + std::uint32_t key = 0; + std::vector bytes; +}; + +struct ViewEntry { + ImageViewRecord record; + std::vector encoded; +}; + +std::unordered_map& views() { + static auto& map = *new std::unordered_map(); + return map; +} + +std::unordered_map& samplers() { + static auto& map = *new std::unordered_map(); + return map; +} + +ImageViewRecord recordOf(const VkImageViewCreateInfo& info) { + ImageViewRecord record; + record.info = info; + record.info.pNext = nullptr; + for (auto* next = static_cast(info.pNext); next != nullptr; next = next->pNext) { + if (next->sType == VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO) { + record.usage = *reinterpret_cast(next); + record.hasUsage = true; + } else if (next->sType == VK_STRUCTURE_TYPE_IMAGE_VIEW_MIN_LOD_CREATE_INFO_EXT) { + record.minLod = *reinterpret_cast(next); + record.hasMinLod = true; + } + } + return record; +} + +SamplerRecord recordOf(const VkSamplerCreateInfo& info) { + SamplerRecord record; + record.info = info; + record.info.pNext = nullptr; + for (auto* next = static_cast(info.pNext); next != nullptr; next = next->pNext) { + if (next->sType == VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO) { + record.reduction = *reinterpret_cast(next); + record.hasReduction = true; + } + } + return record; +} + +VKAPI_ATTR VkResult VKAPI_CALL createImageView(VkDevice device, const VkImageViewCreateInfo* info, const VkAllocationCallbacks* allocator, + VkImageView* view) { + const auto result = realCreateImageView.load(std::memory_order_acquire)(device, info, allocator, view); + if (result == VK_SUCCESS) { + auto record = recordOf(*info); + static std::atomic generations{0}; + record.generation = generations.fetch_add(1, std::memory_order_relaxed) + 1u; + std::lock_guard lock(recordsMutex()); + views()[*view] = ViewEntry{std::move(record), {}}; + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL destroyImageView(VkDevice device, VkImageView view, const VkAllocationCallbacks* allocator) { + { + std::lock_guard lock(recordsMutex()); + views().erase(view); + } + realDestroyImageView.load(std::memory_order_acquire)(device, view, allocator); +} + +VKAPI_ATTR VkResult VKAPI_CALL createSampler(VkDevice device, const VkSamplerCreateInfo* info, const VkAllocationCallbacks* allocator, + VkSampler* sampler) { + const auto result = realCreateSampler.load(std::memory_order_acquire)(device, info, allocator, sampler); + if (result == VK_SUCCESS) { + auto record = recordOf(*info); + std::lock_guard lock(recordsMutex()); + samplers()[*sampler] = std::move(record); + } + return result; +} + +VKAPI_ATTR void VKAPI_CALL destroySampler(VkDevice device, VkSampler sampler, const VkAllocationCallbacks* allocator) { + { + std::lock_guard lock(recordsMutex()); + samplers().erase(sampler); + } + realDestroySampler.load(std::memory_order_acquire)(device, sampler, allocator); +} + +template +bool resolve(std::atomic& real, VkDevice device, PFN_vkGetDeviceProcAddr deviceProc, const char* name) { + const auto function = reinterpret_cast(deviceProc(device, name)); + if (function == nullptr) return false; + real.store(function, std::memory_order_release); + return true; +} + +struct ObjectHook { + const char* name; + bool (*resolveReal)(VkDevice, PFN_vkGetDeviceProcAddr, const char*); + PFN_vkVoidFunction hook; +}; + +template & Real> +bool resolveRealFunction(VkDevice device, PFN_vkGetDeviceProcAddr deviceProc, const char* name) { + return resolve(Real, device, deviceProc, name); +} + +const std::array ObjectHooks{{ + {"vkCreateImageView", &resolveRealFunction, reinterpret_cast(&createImageView)}, + {"vkDestroyImageView", &resolveRealFunction, reinterpret_cast(&destroyImageView)}, + {"vkCreateSampler", &resolveRealFunction, reinterpret_cast(&createSampler)}, + {"vkDestroySampler", &resolveRealFunction, reinterpret_cast(&destroySampler)}, +}}; + +} + +const VkImageViewCreateInfo* ImageViewRecord::Chain() { + info.pNext = nullptr; + usage.pNext = nullptr; + minLod.pNext = nullptr; + if (hasMinLod) { + minLod.pNext = info.pNext; + info.pNext = &minLod; + } + if (hasUsage) { + usage.pNext = info.pNext; + info.pNext = &usage; + } + return &info; +} + +const VkSamplerCreateInfo* SamplerRecord::Chain() { + info.pNext = nullptr; + reduction.pNext = nullptr; + if (hasReduction) info.pNext = &reduction; + return &info; +} + +PFN_vkVoidFunction DescriptorHeapObjectFunction(VkDevice device, PFN_vkGetDeviceProcAddr deviceProc, const char* name) { + if (device == VK_NULL_HANDLE) return nullptr; + for (const auto& entry : ObjectHooks) { + if (std::strcmp(name, entry.name) != 0) continue; + return entry.resolveReal(device, deviceProc, name) ? entry.hook : nullptr; + } + return nullptr; +} + +bool CopyEncodedImageView(VkImageView view, std::uint32_t key, std::span target) { + std::lock_guard lock(recordsMutex()); + const auto found = views().find(view); + if (found == views().end()) return false; + for (const auto& encoded : found->second.encoded) { + if (encoded.key != key || encoded.bytes.size() != target.size()) continue; + std::memcpy(target.data(), encoded.bytes.data(), target.size()); + return true; + } + return false; +} + +void KeepEncodedImageView(VkImageView view, std::uint64_t generation, std::uint32_t key, std::span bytes) { + std::lock_guard lock(recordsMutex()); + const auto record = views().find(view); + if (record == views().end() || record->second.record.generation != generation) return; + auto& entries = record->second.encoded; + for (auto& encoded : entries) { + if (encoded.key == key) { + encoded.bytes.assign(bytes.begin(), bytes.end()); + return; + } + } + entries.push_back({key, std::vector(bytes.begin(), bytes.end())}); +} + +std::optional FindImageView(VkImageView view) { + std::lock_guard lock(recordsMutex()); + const auto found = views().find(view); + if (found == views().end()) return std::nullopt; + return found->second.record; +} + +std::optional FindSampler(VkSampler sampler) { + std::lock_guard lock(recordsMutex()); + const auto found = samplers().find(sampler); + if (found == samplers().end()) return std::nullopt; + return found->second; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageKeys.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageKeys.cpp new file mode 100644 index 000000000000..1878eed98e4f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageKeys.cpp @@ -0,0 +1,163 @@ +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace Abi = ShaderRecompiler::DeviceImageTableAbi; + +DeviceImageKeys::DeviceImageKeys(std::uint32_t buckets, std::uint32_t maxBuckets) : maxBuckets(maxBuckets) { + if (!std::has_single_bit(buckets) || buckets < Abi::ProbeSteps || maxBuckets < buckets || !std::has_single_bit(maxBuckets)) throw std::invalid_argument("device image keys: the bucket counts must be powers of two of at least " + std::to_string(Abi::ProbeSteps) + " buckets, the maximum not below the start"); + words.assign(static_cast(buckets) * Abi::BucketWords, 0u); +} + +std::optional DeviceImageKeys::place(std::vector& target, const Key& key, bool& found) const { + const auto mask = static_cast(target.size() / Abi::BucketWords) - 1u; + const auto home = Abi::Hash(key) & mask; + found = false; + for (std::uint32_t step = 0; step < Abi::ProbeSteps; ++step) { + const auto bucket = (home + step) & mask; + const auto* at = target.data() + static_cast(bucket) * Abi::BucketWords; + if (at[Abi::StateWord] != Abi::LiveState) return bucket; + if (std::equal(key.begin(), key.end(), at)) { + found = true; + return bucket; + } + } + return std::nullopt; +} + +std::optional DeviceImageKeys::Find(const Key& key) const { + const auto mask = Buckets() - 1u; + const auto home = Abi::Hash(key) & mask; + for (std::uint32_t step = 0; step < Abi::ProbeSteps; ++step) { + const auto* at = words.data() + static_cast((home + step) & mask) * Abi::BucketWords; + if (at[Abi::StateWord] != Abi::LiveState) return std::nullopt; + if (std::equal(key.begin(), key.end(), at)) return DeviceImageValue{at[Abi::SlotWord], at[Abi::ModeWord], at[Abi::FaultWord], at[Abi::SamplerSlotWord]}; + } + return std::nullopt; +} + +std::uint32_t DeviceImageKeys::Retain(const std::function& keep) { + std::vector kept(words.size(), 0u); + std::uint32_t dropped = 0; + std::uint32_t live = 0; + for (std::uint32_t bucket = 0; bucket < Buckets(); ++bucket) { + const auto* at = words.data() + static_cast(bucket) * Abi::BucketWords; + if (at[Abi::StateWord] != Abi::LiveState) continue; + Key key{}; + std::copy_n(at, Abi::KeyWords, key.begin()); + if (!keep(key)) { + ++dropped; + continue; + } + bool found = false; + const auto destination = place(kept, key, found); + if (!destination.has_value()) throw std::runtime_error("device image keys: a retained key found no bucket in a table of the same size"); + std::copy_n(at, Abi::BucketWords, kept.begin() + static_cast(*destination) * Abi::BucketWords); + ++live; + } + if (dropped == 0u) return 0u; + words = std::move(kept); + entries = live; + rebuilt = true; + changed.clear(); + return dropped; +} + +bool DeviceImageKeys::grow() { + for (auto buckets = Buckets() * 2u; buckets != 0u && buckets <= maxBuckets; buckets *= 2u) { + std::vector moved(static_cast(buckets) * Abi::BucketWords, 0u); + bool complete = true; + for (std::uint32_t bucket = 0; bucket < Buckets() && complete; ++bucket) { + const auto* at = words.data() + static_cast(bucket) * Abi::BucketWords; + if (at[Abi::StateWord] != Abi::LiveState) continue; + Key key{}; + std::copy_n(at, Abi::KeyWords, key.begin()); + bool found = false; + const auto destination = place(moved, key, found); + complete = destination.has_value(); + if (complete) std::copy_n(at, Abi::BucketWords, moved.begin() + static_cast(*destination) * Abi::BucketWords); + } + if (!complete) continue; + words = std::move(moved); + rebuilt = true; + changed.clear(); + return true; + } + return false; +} + +DeviceImageKeys::Insertion DeviceImageKeys::Insert(const Key& key, const DeviceImageValue& value) { + const auto store = [&](std::uint32_t bucket) { + auto* at = words.data() + static_cast(bucket) * Abi::BucketWords; + at[Abi::SlotWord] = value.slot; + at[Abi::ModeWord] = value.mode; + at[Abi::FaultWord] = value.fault; + at[Abi::SamplerSlotWord] = value.samplerSlot; + changed.push_back(bucket); + }; + bool found = false; + if (const auto bucket = place(words, key, found); bucket.has_value() && found) { + store(*bucket); + return Insertion::Updated; + } + if ((entries + 1u) * 2u > Buckets() && !grow()) return Insertion::Full; + for (;;) { + if (const auto bucket = place(words, key, found); bucket.has_value()) { + auto* at = words.data() + static_cast(*bucket) * Abi::BucketWords; + std::copy(key.begin(), key.end(), at); + at[Abi::StateWord] = Abi::LiveState; + ++entries; + store(*bucket); + return Insertion::Inserted; + } + if (!grow()) return Insertion::Full; + } +} + +void DeviceImageKeys::Clear() { + std::fill(words.begin(), words.end(), 0u); + entries = 0; + rebuilt = true; + changed.clear(); +} + +std::vector DeviceImageKeys::TakeChanged() { + std::ranges::sort(changed); + changed.erase(std::unique(changed.begin(), changed.end()), changed.end()); + rebuilt = false; + return std::exchange(changed, {}); +} + +DeviceImageRing DecodeDeviceImageRing(std::span ring, std::uint32_t capacity) { + if (ring.size() < Abi::RingRecordsWord + static_cast(capacity) * Abi::RecordWords) throw std::invalid_argument("device image ring: the buffer is smaller than its capacity"); + DeviceImageRing result; + result.written = ring[Abi::RingCountWord]; + const auto records = std::min(result.written, capacity); + result.dropped = result.written - records; + result.misses.reserve(static_cast(records)); + for (std::uint64_t record = 0; record < records; ++record) { + const auto* at = ring.data() + Abi::RingRecordsWord + record * Abi::RecordWords; + DeviceImageMiss miss; + std::copy_n(at, Abi::KeyWords, miss.key.begin()); + miss.pc = at[Abi::RecordPcWord]; + result.misses.push_back(miss); + } + return result; +} + +std::vector DistinctDeviceImageMisses(std::span misses) { + std::vector distinct; + std::set seen; + for (const auto& miss : misses) { + if (seen.insert(miss.key).second) distinct.push_back(miss); + } + return distinct; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp new file mode 100644 index 000000000000..40d2b8487f5d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp @@ -0,0 +1,731 @@ +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "IntermediateRepresentation/IrProgram.hpp" +#include "DeviceImageTableAbi.hpp" +#include "ImageTableAbi.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +namespace Abi = ShaderRecompiler::DeviceImageTableAbi; +namespace Runtime = ShaderRecompiler::RuntimeAbi; +using Reason = ShaderRecompiler::ImageTableAbi::PoisonReason; + +constexpr VkDeviceSize UpdateBytes = 65536u; +constexpr VkDeviceSize TableBytes = static_cast(DeviceImageTable::TableWords) * sizeof(std::uint32_t); +constexpr VkDeviceSize RingBytes = static_cast(DeviceImageTable::RingWords) * sizeof(std::uint32_t); +constexpr VkDeviceSize BucketsOffset = static_cast(Abi::HeaderWords) * sizeof(std::uint32_t); +constexpr std::uint32_t BlockShift = 16u; + +std::string hexWords(std::span words) { + std::ostringstream text; + text << std::hex << std::setfill('0'); + for (std::size_t index = 0; index < words.size(); ++index) text << (index == 0 ? "" : " ") << std::setw(8) << words[index]; + return text.str(); +} + +struct Tables { + std::mutex mutex; + std::unordered_map> tables; +}; + +Tables& tables() { + static auto& registry = *new Tables(); + return registry; +} + +std::string hexText(std::uint64_t value) { + std::ostringstream text; + text << std::hex << value; + return text.str(); +} + +bool pointSampled(const ShaderRecompiler::ImageResource& mode) { + return mode.numericClass == ShaderRecompiler::IrTextureNumericClass::Uint || mode.numericClass == ShaderRecompiler::IrTextureNumericClass::Sint || mode.conversionFormat != ShaderRecompiler::IrBufferFormat::Invalid || mode.depthBits; +} + +bool multisampled(const ShaderRecompiler::ImageResource& mode) { + return mode.dimension == ShaderRecompiler::RdnaImageDimension::Dim2DMsaa || mode.dimension == ShaderRecompiler::RdnaImageDimension::Dim2DMsaaArray; +} + +ShaderRecompiler::DescriptorImageShape shapeOf(const ShaderRecompiler::ImageResource& mode) { + using Shape = ShaderRecompiler::DescriptorImageShape; + using Dimension = ShaderRecompiler::RdnaImageDimension; + switch (mode.dimension) { + case Dimension::Dim1D: return Shape::Image1D; + case Dimension::Dim1DArray: return Shape::Image1DArray; + case Dimension::Dim2D: return Shape::Image2D; + case Dimension::Dim2DArray: return Shape::Image2DArray; + case Dimension::Dim3D: return Shape::Image3D; + default: break; + } + throw std::runtime_error("AGC device image table: an image mode of dimension " + std::to_string(static_cast(mode.dimension)) + " has no sampled descriptor image shape"); +} + +} + +DeviceImageTable::Slots DeviceImageTable::HeapSlots(const Context& context) { + auto* heap = &BindlessHeap::For(context); + Slots slots; + slots.take = [heap](std::uint32_t kind, const std::shared_ptr& texture, VkImageView view, VkImageLayout layout) { return heap->TextureSlot(kind, texture.get(), view, layout); }; + slots.generation = [heap](std::uint32_t, std::uint32_t slot) { return heap->SlotGeneration(slot); }; + slots.serial = [heap] { return heap->SlotSerial(); }; + slots.nearlyFull = [] { return BindlessHeap::NearlyFull(); }; + slots.touch = [heap](std::span> held, std::uint32_t touched) { return heap->TouchSlots(held, touched); }; + return slots; +} + +DeviceImageTable::DeviceImageTable(const Context& context, InterfaceLookup lookup, Slots slots) : DeviceImageTable(context, std::move(lookup), std::move(slots), Budget{}) {} + +DeviceImageTable::DeviceImageTable(const Context& context, InterfaceLookup lookup, Slots slots, Budget budget) : context(context), lookup(std::move(lookup)), slots(std::move(slots)), keys(std::min(DeviceImageKeys::InitialBuckets, budget.maxBuckets), budget.maxBuckets), budget(budget) { + if (!this->slots.take || !this->slots.generation || !this->slots.serial || !this->slots.nearlyFull || !this->slots.touch) throw std::runtime_error("AGC device image table: its heap slots are incomplete"); + table = std::make_unique(context, static_cast(TableBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 16u); + chunkStride = (RingBytes + alignment - 1u) / alignment * alignment; + ring = std::make_unique(context, static_cast(chunkStride * RingChunks), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + if (table->DeviceAddress() == 0 || ring->DeviceAddress() == 0) throw std::runtime_error("AGC device image table: its table or miss ring has no device address"); + std::memset(ring->Bytes().data(), 0, ring->Bytes().size()); + CommandBatch batch(context); + context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(batch.Handle(), table->Handle(), 0, VK_WHOLE_SIZE, 0u); + RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + writeHeader(batch.Handle()); + RecordMemoryBarrier(context, batch.Handle(), VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + batch.SubmitAndWait(); + chunks.resize(RingChunks); + slotSerial = this->slots.serial(); +} + +DeviceImageTable::~DeviceImageTable() = default; + +DeviceImageTable& DeviceImageTable::Create(const Context& context, InterfaceLookup lookup) { + auto& registry = tables(); + std::lock_guard lock(registry.mutex); + auto& table = registry.tables[context.device]; + if (table != nullptr) throw std::runtime_error("AGC device image table: the device already has one"); + table = std::make_unique(context, std::move(lookup), HeapSlots(context)); + return *table; +} + +bool GpuPathEnabled() { + static const bool enabled = std::getenv("APS5_GPU_PATH") != nullptr; + return enabled; +} + +std::optional DeviceImagePathRefusal(const DeviceImagePathFeatures& features) { + if (!features.gpuHeap) return std::string("APS5_GPU_HEAP is unset, and the device image table takes its slots from the GPU heap and is read only by GPU-mode stages"); + if (!features.descriptorHeap) return std::string("the device lacks VK_EXT_descriptor_heap, which the GPU heap the device image table takes its slots from needs"); + return std::nullopt; +} + +bool DeviceImageTable::ExpectsKeys() { + static const bool expects = std::getenv("APS5_NO_DEVICE_IMAGE_EXPECT") == nullptr; + return expects; +} + +DeviceImageTable* DeviceImageTable::Existing(const Context& context) { + auto& registry = tables(); + std::lock_guard lock(registry.mutex); + const auto found = registry.tables.find(context.device); + return found != registry.tables.end() ? found->second.get() : nullptr; +} + +void DeviceImageTable::Release(VkDevice device) { + std::unique_ptr released; + auto& registry = tables(); + std::lock_guard lock(registry.mutex); + const auto found = registry.tables.find(device); + if (found == registry.tables.end()) return; + released = std::move(found->second); + registry.tables.erase(found); +} + +void DeviceImageTable::Prepare(const Context& build, Recorder& recorder, std::span programs) { + std::lock_guard lock(mutex); + ++totals.prepares; + retainSlots(recorder); + resolve(build, recorder); + refresh(build, recorder); + evict(build, recorder); + auto& chunk = chunks[chunkFor(recorder, true)]; + chunk.programs.insert(programs.begin(), programs.end()); + retainSlots(recorder); + upload(recorder); + touchSlots(); +} + +std::optional DeviceImageTable::Expect(const Context& build, Recorder& recorder, std::span images) { + if (!ExpectsKeys()) return std::nullopt; + std::lock_guard lock(mutex); + retainSlots(recorder); + std::optional unserved; + for (const auto& image : images) { + ++totals.expected; + if (keys.Find(image.key).has_value()) continue; + Entry entry; + std::string failure; + const auto value = lookupKey(build, image.key, entry, &failure); + if (!value.has_value()) { + if (!image.candidate && !unserved.has_value()) unserved = "the expected descriptor (words " + hexWords(std::span(image.key).first(Abi::DescriptorWords)) + ") at pc 0x" + hexText(image.pc) + " stays unresolved: " + failure; + continue; + } + if (!insert(image.key, *value)) { + if (entry.texture != nullptr) recorder.Keep(std::move(entry.texture)); + if (!image.candidate && !unserved.has_value()) unserved = "the key table is full at " + std::to_string(keys.Entries()) + " entries in " + std::to_string(keys.Buckets()) + " buckets"; + continue; + } + if (entry.texture == nullptr) continue; + hold(image.key, std::move(entry)); + ++totals.inserted; + ++totals.expectedInserted; + } + return unserved; +} + +void DeviceImageTable::FallBackProgram(const ShaderRecompiler::PipelineIdentity& identity, std::uint64_t codeAddress, const std::string& reason) { + std::lock_guard lock(mutex); + identityPrograms[identity].insert(codeAddress); + if (failedPrograms.contains(identity)) { + fallbackPrograms.insert(codeAddress); + return; + } + fallBack({identity}, reason); +} + +void DeviceImageTable::SetInterfaceLookup(InterfaceLookup interfaces) { + std::lock_guard lock(mutex); + lookup = std::move(interfaces); +} + +void DeviceImageTable::NoteProgram(const ShaderRecompiler::PipelineIdentity& identity, std::uint64_t codeAddress) { + std::lock_guard lock(mutex); + identityPrograms[identity].insert(codeAddress); + if (failedPrograms.contains(identity)) fallbackPrograms.insert(codeAddress); +} + +bool DeviceImageTable::ServesProgram(std::uint64_t codeAddress) const { + std::lock_guard lock(mutex); + return !fallbackPrograms.contains(codeAddress); +} + +bool DeviceImageTable::ServesIdentity(const ShaderRecompiler::PipelineIdentity& identity) const { + std::lock_guard lock(mutex); + return !failedPrograms.contains(identity); +} + +void DeviceImageTable::fallBack(const std::set& programs, const std::string& reason) { + for (const auto& identity : programs) { + if (!failedPrograms.insert(identity).second) continue; + ++totals.fallbacks; + lastFallback = reason; + std::string addresses; + if (const auto found = identityPrograms.find(identity); found != identityPrograms.end()) { + for (const auto address : found->second) { + fallbackPrograms.insert(address); + addresses += (addresses.empty() ? "0x" : ", 0x") + hexText(address); + } + } + std::fprintf(stderr, "[device-images] the program %s (identity %016llx%016llx) takes the CPU table path from its next recompile: %s\n", addresses.empty() ? "of no noted address" : addresses.c_str(), static_cast(identity.high), static_cast(identity.low), reason.c_str()); + } +} + +ShaderRecompiler::ImageTableAbi::Trailer DeviceImageTable::Trailer(Recorder& recorder) { + std::lock_guard lock(mutex); + const auto chunk = chunkFor(recorder, false); + return {table->DeviceAddress(), ring->DeviceAddress() + static_cast(chunk) * chunkStride}; +} + +std::span DeviceImageTable::RingChunk(Recorder& recorder) { + std::lock_guard lock(mutex); + const auto chunk = chunkFor(recorder, true); + auto bytes = ring->Bytes().subspan(static_cast(chunk * chunkStride), static_cast(RingBytes)); + return {reinterpret_cast(bytes.data()), RingWords}; +} + +VkBuffer DeviceImageTable::TableBuffer() const { + return table->Handle(); +} + +std::optional DeviceImageTable::Find(const Key& key) const { + std::lock_guard lock(mutex); + return keys.Find(key); +} + +std::vector DeviceImageTable::KeyWords() const { + std::lock_guard lock(mutex); + const auto words = keys.Words(); + return {words.begin(), words.end()}; +} + +std::string DeviceImageTable::LastFault() const { + std::lock_guard lock(mutex); + return lastFault; +} + +std::string DeviceImageTable::LastFallback() const { + std::lock_guard lock(mutex); + return lastFallback; +} + +std::uint32_t DeviceImageTable::OverflowStreak(const ShaderRecompiler::PipelineIdentity& identity) const { + std::lock_guard lock(mutex); + const auto found = overflowStreaks.find(identity); + return found != overflowStreaks.end() ? found->second : 0u; +} + +std::shared_ptr DeviceImageTable::EntryTexture(const Key& key) const { + std::lock_guard lock(mutex); + const auto found = entries.find(key); + return found != entries.end() ? found->second.texture : nullptr; +} + +void DeviceImageTable::resolve(const Context& build, Recorder& recorder) { + if (pending.empty()) return; + ++resolveGeneration; + std::vector order; + std::map> missed; + const std::set* resolved = nullptr; + for (const auto& [miss, programs] : std::exchange(pending, {})) { + const auto [found, inserted] = missed.try_emplace(miss.key); + if (inserted) order.push_back(miss.key); + if (programs == nullptr) continue; + found->second.insert(programs->begin(), programs->end()); + if (programs.get() == resolved) continue; + resolved = programs.get(); + for (const auto& identity : *programs) overflowFrom.try_emplace(identity, resolveGeneration); + } + for (const auto& key : order) { + const auto& programs = missed.at(key); + if (keys.Find(key).has_value()) continue; + ++totals.distinct; + Entry entry; + std::string failure; + const auto value = lookupKey(build, key, entry, &failure); + if (!value.has_value()) { + if (!failure.empty()) fallBack(programs, failure); + continue; + } + if (!insert(key, *value)) { + if (entry.texture != nullptr) recorder.Keep(std::move(entry.texture)); + fallBack(programs, "the key table is full at " + std::to_string(keys.Entries()) + " entries in " + std::to_string(keys.Buckets()) + " buckets, even after its budget evicted"); + continue; + } + if (entry.texture == nullptr) continue; + hold(key, std::move(entry)); + ++totals.inserted; + } +} + +std::optional DeviceImageTable::lookupKey(const Context& build, const Key& key, Entry& entry, std::string* failure) { + const auto id = static_cast(key[Abi::InterfaceWord]) | (static_cast(key[Abi::InterfaceWord + 1u]) << 32u); + const auto interface = lookup ? lookup(id) : nullptr; + if (interface == nullptr) throw std::runtime_error("AGC device image table: a missed key names the image interface 0x" + hexText(id) + ", which no compiled program registered"); + ShaderRecompiler::DescriptorValue descriptor; + descriptor.dwordCount = Abi::DescriptorWords; + descriptor.availableDwordMask = 0xffu; + std::copy_n(key.begin(), Abi::DescriptorWords, descriptor.dwords.begin()); + const auto match = ShaderRecompiler::ResourceMaterializer::TryRuntimeImageMode(interface->image, descriptor, interface->modes); + if (!match.mode.has_value()) return reject(key, match.reason, match.detail); + if (*match.mode >= interface->modes.size()) throw std::runtime_error("AGC device image table: the runtime image mode check chose a mode the interface does not have"); + const auto& mode = interface->modes[*match.mode]; + if (multisampled(mode)) return reject(key, Reason::Multisampled, "an image table entry is multisampled"); + const auto samplerClass = key[Abi::SamplerClassWord]; + if (pointSampled(mode) && (samplerClass & Abi::SamplerReduction) != 0u) return reject(key, Reason::Reduction, "a min or max reduction sampler that filters between texels or mip levels samples an image that needs point filtering (sint, converted or depth-bits format), which is not implemented"); + const auto kind = static_cast(ShaderRecompiler::DescriptorBindingForImage(mode)); + if (kind < Runtime::FirstImageBinding || kind >= Runtime::FirstStorageImageBinding) return reject(key, Reason::UnsupportedOperation, "the device image table serves sampled images only"); + DeviceTableTexture texture; + std::uint64_t stamp = 0; + try { + const DeviceTableView view{shapeOf(mode), mode.depthCompare, mode.conversionFormat != ShaderRecompiler::IrBufferFormat::Invalid || mode.depthBits}; + texture = LookupDeviceTableTexture(build, std::span(key.data(), Abi::DescriptorWords), view, samplerClass); + stamp = GuestMemory::TrackerGeneration(); + } catch (const std::runtime_error& error) { + return reject(key, Reason::DriverRejected, error.what()); + } + if (!texture.unservable.empty()) { + unserved(key, texture.unservable); + if (failure != nullptr) *failure = texture.unservable; + return std::nullopt; + } + const auto slot = slots.nearlyFull() ? std::nullopt : slots.take(Runtime::FirstImageBinding, texture.texture, texture.view, texture.layout); + if (!slot.has_value()) { + heapFull(key, Runtime::FirstImageBinding); + if (failure != nullptr) *failure = "the image heap has no free sampled slot outside its margin"; + return std::nullopt; + } + entry = {std::move(texture.texture), Runtime::FirstImageBinding, *slot, slots.generation(Runtime::FirstImageBinding, *slot), texture.address, texture.guestBytes, texture.generation, stamp, 0u, 0u, texture.words, texture.depthCompare}; + return DeviceImageValue{*slot, *match.mode, 0u, 0u}; +} + +DeviceImageValue DeviceImageTable::reject(const Key& key, Reason reason, const std::string& message) { + if (rejections.try_emplace(key, Rejection{reason, message}).second) { + ++totals.rejected; + const auto name = ShaderRecompiler::ImageTableAbi::PoisonReasonName(reason); + std::fprintf(stderr, "[device-images] the descriptor (words %s) is rejected: %.*s: %s\n", hexWords(std::span(key).first(Abi::DescriptorWords)).c_str(), static_cast(name.size()), name.data(), message.c_str()); + } + return {0u, 0u, 1u + static_cast(reason), 0u}; +} + +void DeviceImageTable::unserved(const Key& key, const std::string& why) { + ++totals.unserved; + if (unservedKeys.insert(key).second) std::fprintf(stderr, "[device-images] the descriptor (words %s) stays unresolved: %s\n", hexWords(std::span(key).first(Abi::DescriptorWords)).c_str(), why.c_str()); +} + +void DeviceImageTable::heapFull(const Key& key, std::uint32_t kind) { + ++totals.heapFull; + if (fullKinds.insert(kind).second) std::fprintf(stderr, "[device-images] the image heap has no free slot of kind %u outside its margin for the descriptor (words %s); it stays unresolved\n", kind, hexWords(std::span(key).first(Abi::DescriptorWords)).c_str()); +} + +bool DeviceImageTable::insert(const Key& key, const DeviceImageValue& value) { + if (keys.Insert(key, value) != DeviceImageKeys::Insertion::Full) return true; + ++totals.tableFull; + if (!tableFullReported) std::fprintf(stderr, "[device-images] the key table is full at %u entries in %u buckets; new keys stay unresolved\n", keys.Entries(), keys.Buckets()); + tableFullReported = true; + return false; +} + +void DeviceImageTable::hold(const Key& key, Entry entry) { + const std::pair use{entry.kind, static_cast(entry.texture.get())}; + if (textureUses[use]++ == 0u) { + auto& live = liveTextures[entry.kind]; + totals.peakTextures[entry.kind] = std::max(totals.peakTextures[entry.kind], ++live); + } + entry.used = ++useSerial; + entry.usedPresent = Recorder::Presents(); + const auto first = entry.address >> BlockShift; + const auto last = (entry.address + std::max(entry.guestBytes, 1u) - 1u) >> BlockShift; + for (auto block = first; block <= last; ++block) blocks[block].keys.insert(key); + entries[key] = std::move(entry); + heldSlotsStale = true; +} + +void DeviceImageTable::release(const Key& key, Recorder& recorder) { + const auto found = entries.find(key); + if (found == entries.end()) return; + auto& entry = found->second; + const auto first = entry.address >> BlockShift; + const auto last = (entry.address + std::max(entry.guestBytes, 1u) - 1u) >> BlockShift; + for (auto block = first; block <= last; ++block) { + const auto held = blocks.find(block); + if (held == blocks.end()) continue; + held->second.keys.erase(key); + if (held->second.keys.empty()) blocks.erase(held); + } + const std::pair use{entry.kind, static_cast(entry.texture.get())}; + if (const auto counted = textureUses.find(use); counted != textureUses.end() && --counted->second == 0u) { + textureUses.erase(counted); + --liveTextures[entry.kind]; + } + if (entry.texture != nullptr) recorder.Keep(std::move(entry.texture)); + entries.erase(found); + heldSlotsStale = true; +} + +void DeviceImageTable::retainSlots(Recorder& recorder) { + const auto serial = slots.serial(); + if (serial == slotSerial) return; + slotSerial = serial; + std::set swept; + for (const auto& [key, entry] : entries) { + if (slots.generation(entry.kind, entry.slot) != entry.slotGeneration) swept.insert(key); + } + if (swept.empty()) return; + for (const auto& key : swept) release(key, recorder); + totals.swept += keys.Retain([&](const Key& key) { return !swept.contains(key); }); +} + +void DeviceImageTable::touchSlots() { + if (heldSlotsStale) { + heldSlots.assign(1u, {}); + heldSlots.front().reserve(entries.size()); + for (const auto& [key, entry] : entries) heldSlots.front().push_back(entry.slot); + heldSlotsStale = false; + touchedEpoch = 0; + } + if (!heldSlots.front().empty()) touchedEpoch = slots.touch(heldSlots, touchedEpoch); +} + +void DeviceImageTable::refresh(const Context& build, Recorder& recorder) { + if (entries.empty()) return; + const ValidationState state{GuestMemory::ThreadCollectEpoch(), GuestMemory::UnwatchSerial(), DepthSurfacesSerial(), StorageTexture::PendingSerial(), GuestMemory::OwnWriteSerial()}; + if (!validated.has_value()) { + validated = state; + return; + } + if (state.epoch != 0u && *validated == state) return; + const bool everything = validated->unwatch != state.unwatch || validated->depthSurfaces != state.depthSurfaces; + const bool pendingMoved = validated->pending != state.pending; + const bool epochMoved = state.epoch == 0u || validated->epoch != state.epoch; + validated = state; + ++totals.scans; + const auto scanned = std::exchange(scannedStamp, GuestMemory::TrackerGeneration()); + std::set stale; + std::vector generations; + std::vector changed; + std::vector cpu; + for (auto run = blocks.begin(); run != blocks.end();) { + auto end = std::next(run); + while (end != blocks.end() && end->first == std::prev(end)->first + 1u) ++end; + const auto begin = run->first << BlockShift; + const auto count = static_cast(std::prev(end)->first - run->first + 1u); + const auto bytes = static_cast(count) << BlockShift; + static_cast(GuestMemory::CollectWrites(begin, bytes)); + if (!everything && !epochMoved && !pendingMoved && scanned != 0u && GuestMemory::NewestStamp(begin, bytes) <= scanned) { + run = end; + continue; + } + generations.clear(); + for (auto block = run; block != end; ++block) generations.push_back(everything ? 0u : block->second.generation); + changed.assign(count, GuestMemory::BlockWritten); + cpu.assign(count, 1u); + static_cast(GuestMemory::ChangedBlocks(begin, bytes, generations, changed, cpu)); + StorageTexture::PendingQuery runQuery{begin, begin + bytes, nullptr, nullptr, false}; + if (pendingMoved) static_cast(StorageTexture::ScanPending(std::span(&runQuery, 1))); + const auto stamp = GuestMemory::TrackerGeneration(); + std::size_t index = 0; + for (auto block = run; block != end; ++block, ++index) { + if (changed[index] != GuestMemory::BlockUnchanged || runQuery.overlaps) { + for (const auto& key : block->second.keys) { + if (stale.contains(key)) continue; + const auto& entry = entries.at(key); + const auto range = static_cast(entry.guestBytes); + const auto* image = entry.texture != nullptr ? entry.texture->StorageSource() : nullptr; + bool fresh = false; + if (everything || (image != nullptr && !image->Cached())) fresh = false; + else if (image != nullptr && entry.generation != 0u) fresh = cpu[index] == 0u || GuestMemory::UnchangedSinceCollected(entry.address, range, entry.generation); + else if (entry.generation != 0u) fresh = GuestMemory::UnchangedSince(entry.address, range, entry.generation); + else fresh = GuestMemory::NewestStamp(entry.address, range) <= entry.stamp && (!epochMoved || (image != nullptr ? image->UntrackedUnchanged() : entry.texture != nullptr && DeviceTableTextureCurrent(build, entry.words, entry.depthCompare, *entry.texture))); + if (fresh && runQuery.overlaps) { + StorageTexture::PendingQuery query{entry.address, entry.address + entry.guestBytes, image, nullptr, false}; + static_cast(StorageTexture::ScanPending(std::span(&query, 1))); + fresh = !query.overlaps; + } + if (!fresh) stale.insert(key); + } + } + block->second.generation = stamp; + } + run = end; + } + for (const auto& key : stale) { + ++totals.checked; + auto& held = entries.at(key); + Entry fresh; + const auto value = lookupKey(build, key, fresh); + const auto previous = keys.Find(key); + if (!value.has_value() || fresh.texture == nullptr) { + ++totals.refreshed; + release(key, recorder); + if (value.has_value()) static_cast(insert(key, *value)); + else static_cast(keys.Retain([&](const Key& kept) { return kept != key; })); + continue; + } + const bool replaced = fresh.texture != held.texture; + if (!previous.has_value() || *previous != *value) static_cast(insert(key, *value)); + if (!replaced) { + held.slot = fresh.slot; + held.slotGeneration = fresh.slotGeneration; + held.generation = fresh.generation; + held.stamp = fresh.stamp; + held.used = ++useSerial; + held.usedPresent = Recorder::Presents(); + continue; + } + ++totals.refreshed; + release(key, recorder); + hold(key, std::move(fresh)); + } +} + +void DeviceImageTable::evict(const Context& build, Recorder& recorder) { + const auto presents = Recorder::Presents(); + const bool overEntries = entries.size() > budget.entries; + if (!overEntries && presents < evictionCheck + budget.checkPresents) return; + evictionCheck = presents; + std::uint64_t bytes = 0; + for (const auto& [key, entry] : entries) bytes += entry.guestBytes; + const auto byteBudget = budget.textureBytes != 0u ? budget.textureBytes : SampledTextureCacheBudget(build) / 2u; + const bool overBytes = bytes > byteBudget; + if (!overEntries && !overBytes) return; + std::vector> order; + order.reserve(entries.size()); + for (const auto& [key, entry] : entries) order.emplace_back(entry.used, key); + std::ranges::sort(order, [](const auto& left, const auto& right) { return left.first < right.first; }); + const auto entryTarget = static_cast(budget.entries) / 4u * 3u; + const auto byteTarget = byteBudget / 4u * 3u; + std::set dropped; + for (const auto& [used, key] : order) { + const auto& entry = entries.at(key); + const bool count = overEntries && entries.size() - dropped.size() > entryTarget; + const bool idle = entry.usedPresent + budget.idlePresents <= presents; + const bool memory = overBytes && bytes > byteTarget && idle; + if (!count && !memory) { + if (!overBytes || bytes <= byteTarget) break; + continue; + } + bytes -= std::min(bytes, entry.guestBytes); + dropped.insert(key); + } + if (dropped.empty()) return; + for (const auto& key : dropped) release(key, recorder); + totals.evicted += keys.Retain([&](const Key& key) { return !dropped.contains(key); }); +} + +void DeviceImageTable::upload(Recorder& recorder) { + const bool header = uploadedBuckets != keys.Buckets(); + const bool rebuilt = keys.Rebuilt(); + const auto changed = keys.TakeChanged(); + if (!header && !rebuilt && changed.empty()) return; + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + const auto updateBuffer = context.Resolved(&DeviceFunctions::cmdUpdateBuffer, "vkCmdUpdateBuffer"); + const auto words = keys.Words(); + std::uint64_t bytes = 0; + if (rebuilt && keys.Entries() == 0u) { + context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, table->Handle(), BucketsOffset, words.size_bytes(), 0u); + } else if (rebuilt) { + const auto size = words.size_bytes(); + auto staging = std::make_shared(context, size, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging->Bytes().data(), words.data(), size); + const VkBufferCopy region{0, BucketsOffset, size}; + context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, staging->Handle(), table->Handle(), 1, ®ion); + recorder.Keep(std::move(staging), size); + bytes += size; + } else { + for (std::size_t first = 0; first < changed.size();) { + auto last = first; + while (last + 1u < changed.size() && changed[last + 1u] == changed[last] + 1u && static_cast(last + 2u - first) * Abi::BucketBytes <= UpdateBytes) ++last; + const auto size = static_cast(last + 1u - first) * Abi::BucketBytes; + updateBuffer(commands, table->Handle(), BucketsOffset + static_cast(changed[first]) * Abi::BucketBytes, size, words.data() + static_cast(changed[first]) * Abi::BucketWords); + bytes += size; + first = last + 1u; + } + } + if (header) bytes += writeHeader(commands); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + ++totals.uploads; + totals.uploadBytes += bytes; +} + +VkDeviceSize DeviceImageTable::writeHeader(VkCommandBuffer commands) { + std::array fields{}; + fields[Abi::HeaderVersion] = Abi::Version; + fields[Abi::HeaderMask] = keys.Buckets() - 1u; + fields[Abi::HeaderRingCapacity] = RingCapacity; + context.Resolved(&DeviceFunctions::cmdUpdateBuffer, "vkCmdUpdateBuffer")(commands, table->Handle(), 0, sizeof(fields), fields.data()); + uploadedBuckets = keys.Buckets(); + return sizeof(fields); +} + +std::uint32_t DeviceImageTable::chunkFor(Recorder& recorder, bool mayWait) { + const auto serial = recorder.OpenSerial(); + const auto held = std::find_if(chunks.begin(), chunks.end(), [&](const Chunk& chunk) { return chunk.busy && chunk.serial == serial; }); + if (held != chunks.end()) return static_cast(held - chunks.begin()); + const auto free = [&] { return std::find_if(chunks.begin(), chunks.end(), [](const Chunk& chunk) { return !chunk.busy; }); }; + auto found = free(); + if (found == chunks.end()) { + ++totals.chunkWaits; + static_cast(recorder.Reap()); + found = free(); + } + if (found == chunks.end() && mayWait) { + recorder.Sync(); + found = free(); + } + if (found == chunks.end()) throw std::runtime_error("AGC device image table: every miss ring chunk belongs to a batch still in flight, and a recording draw or dispatch cannot wait for one"); + const auto index = static_cast(found - chunks.begin()); + *found = {recorder.OpenSerial(), resolveGeneration, true, {}}; + std::memset(ring->Bytes().data() + static_cast(index * chunkStride), 0, Abi::RingRecordsWord * sizeof(std::uint32_t)); + recorder.OnComplete([token = std::weak_ptr(alive), this, index] { + if (token.lock() != nullptr) drain(index); + }); + return index; +} + +void DeviceImageTable::drain(std::uint32_t index) { + std::lock_guard lock(mutex); + const auto bytes = ring->Bytes().subspan(static_cast(index * chunkStride), static_cast(RingBytes)); + const auto decoded = DecodeDeviceImageRing(std::span(reinterpret_cast(bytes.data()), RingWords), RingCapacity); + auto& chunk = chunks[index]; + chunk.busy = false; + const auto taken = chunk.generation; + const auto programs = std::make_shared>(std::move(chunk.programs)); + chunk.programs.clear(); + totals.drained += decoded.misses.size(); + totals.dropped += decoded.dropped; + std::set overflowing; + for (const auto& identity : *programs) { + const auto from = overflowFrom.find(identity); + if (from == overflowFrom.end() || taken < from->second) continue; + if (decoded.dropped == 0u) { + overflowStreaks.erase(identity); + continue; + } + from->second = resolveGeneration + 1u; + if (++overflowStreaks[identity] >= OverflowBatches) overflowing.insert(identity); + } + if (!overflowing.empty()) fallBack(overflowing, "its misses overflowed the ring of " + std::to_string(RingCapacity) + " records in " + std::to_string(OverflowBatches) + " batches in a row"); + std::optional fault; + for (const auto& miss : decoded.misses) { + if ((miss.pc & Abi::FaultFlag) == 0u) { + pending.push_back({miss, programs}); + continue; + } + ++totals.faults; + if (!fault.has_value()) fault = faultText(miss.key, miss.pc & ~Abi::FaultFlag); + } + if (!fault.has_value()) return; + lastFault = *fault; + throw std::runtime_error(*fault); +} + +std::string DeviceImageTable::faultText(const Key& key, std::uint32_t pc) const { + std::ostringstream message; + message << "image table: pc 0x" << std::hex << pc << std::dec << " selected the descriptor (words " << hexWords(std::span(key).first(Abi::DescriptorWords)) << ")"; + if (const auto found = rejections.find(key); found != rejections.end()) { + message << ": " << ShaderRecompiler::ImageTableAbi::PoisonReasonName(found->second.reason); + if (!found->second.message.empty()) message << ": " << found->second.message; + } else if (const auto value = keys.Find(key); value.has_value() && value->fault != 0u && value->fault - 1u < static_cast(Reason::Count)) { + message << ": " << ShaderRecompiler::ImageTableAbi::PoisonReasonName(static_cast(value->fault - 1u)); + } else { + message << ": the device image table holds no rejection for it"; + } + return message.str(); +} + +std::vector DeviceImageTable::DeviceTotals() { + auto& registry = tables(); + std::lock_guard lock(registry.mutex); + std::vector totals; + for (const auto& [device, table] : registry.tables) { + if (table != nullptr) totals.push_back(table->Totals()); + } + return totals; +} + +DeviceImageTable::Counts DeviceImageTable::Totals() const { + std::lock_guard lock(mutex); + auto counts = totals; + counts.entries = keys.Entries(); + counts.buckets = keys.Buckets(); + return counts; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 502fa6dda05a..c92981977e00 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -1,3 +1,5 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libc/include/RunReport.hpp" @@ -62,6 +64,9 @@ struct PixelFunctionLdsPool { std::uint32_t* header = nullptr; std::uint32_t subgroupSize = 0; std::uint32_t slots = 0; + VkDeviceSize bytes = 0; + VkDeviceAddress address = 0; + const BindlessHeap* heapWritten = nullptr; bool announced = false; }; @@ -79,6 +84,9 @@ struct MeshWindowPool { Context context{}; VkBuffer buffer = VK_NULL_HANDLE; VkDeviceMemory memory = VK_NULL_HANDLE; + VkDeviceSize bytes = 0; + VkDeviceAddress address = 0; + const BindlessHeap* heapWritten = nullptr; VkDescriptorPool descriptorPool = VK_NULL_HANDLE; VkDescriptorSet set = VK_NULL_HANDLE; VkDescriptorSetLayout layout = VK_NULL_HANDLE; @@ -152,12 +160,15 @@ void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, s const auto bytes = 16ull + stride * pool.slots; VkBufferCreateInfo bufferInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bufferInfo.size = bytes; - bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + const bool addressed = context.descriptorHeap && context.bufferDeviceAddress; + bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (addressed ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &bufferInfo, nullptr, &pool.buffer), "vkCreateBuffer pixel function LDS"); VkMemoryRequirements requirements{}; context.Function("vkGetBufferMemoryRequirements")(context.device, pool.buffer, &requirements); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + const VkMemoryAllocateFlagsInfo addressFlags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + if (addressed) allocation.pNext = &addressFlags; allocation.allocationSize = requirements.size; constexpr auto hostCoherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; constexpr auto deviceLocalHost = hostCoherent | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; @@ -188,6 +199,10 @@ void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, s Check(allocated, "vkAllocateMemory pixel function LDS"); } Check(context.Function("vkBindBufferMemory")(context.device, pool.buffer, pool.memory, 0), "vkBindBufferMemory pixel function LDS"); + pool.bytes = bytes; + if (addressed) { + pool.address = BufferDeviceAddress(context, pool.buffer); + } Check(context.Function("vkMapMemory")(context.device, pool.memory, 0, 16, 0, reinterpret_cast(&pool.header)), "vkMapMemory pixel function LDS"); pool.header[0] = 0u; pool.header[1] = 0u; @@ -215,6 +230,14 @@ void PreparePixelFunctionLds(const Context& context, VkCommandBuffer commands, s pool.announced = true; } Require(pool.subgroupSize == subgroupSize, "pixel function LDS subgroup size does not match the pool"); + if (context.descriptorHeap && DescriptorHeapEnabled()) { + auto& heap = BindlessHeap::For(context); + if (pool.heapWritten != &heap) { + Require(pool.address != 0, "the pixel function LDS pool has no device address for the GPU heap"); + heap.WritePixelLdsBuffer(pool.address, pool.bytes); + pool.heapWritten = &heap; + } + } const VkMemoryBarrier beforeFill{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT}; context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1, &beforeFill, 0, nullptr, 0, nullptr); context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, pool.buffer, 0, 4, 0u); @@ -240,26 +263,45 @@ VkDescriptorSetLayout MeshWindowSetLayout(const Context& context) { return pool.layout; } +void WriteMeshWindowHeap(const Context& context, MeshWindowPool& pool) { + if (!context.descriptorHeap || !DescriptorHeapEnabled()) return; + auto& heap = BindlessHeap::For(context); + if (pool.heapWritten == &heap) return; + Require(pool.address != 0, "the mesh window pool has no device address for the GPU heap"); + heap.WriteMeshWindowBuffer(pool.address, pool.bytes); + pool.heapWritten = &heap; +} + void PrepareMeshWindow(const Context& context) { std::lock_guard lock(MeshWindowMutex()); auto& pool = MeshWindowPools()[context.device]; Require(pool.layout != VK_NULL_HANDLE, "mesh window descriptor set layout was not created"); - if (pool.set != VK_NULL_HANDLE) return; + if (pool.set != VK_NULL_HANDLE) { + WriteMeshWindowHeap(context, pool); + return; + } try { const auto bytes = (static_cast(ShaderRecompiler::MeshWindowHeaderDwords) + static_cast(ShaderRecompiler::MeshWindowSlots) * ShaderRecompiler::MeshWindowSlotDwords) * 4ull; Require(bytes <= context.limits.maxStorageBufferRange, "mesh window pool does not fit in the device storage buffer range"); VkBufferCreateInfo bufferInfo{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; bufferInfo.size = bytes; - bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + const bool addressed = context.descriptorHeap && context.bufferDeviceAddress; + bufferInfo.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | (addressed ? VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT : 0u); bufferInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &bufferInfo, nullptr, &pool.buffer), "vkCreateBuffer mesh window"); VkMemoryRequirements requirements{}; context.Function("vkGetBufferMemoryRequirements")(context.device, pool.buffer, &requirements); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + const VkMemoryAllocateFlagsInfo addressFlags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + if (addressed) allocation.pNext = &addressFlags; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); Check(MemoryAccounting::Allocate(context, allocation, "mesh window", &pool.memory), "vkAllocateMemory mesh window"); Check(context.Function("vkBindBufferMemory")(context.device, pool.buffer, pool.memory, 0), "vkBindBufferMemory mesh window"); + pool.bytes = bytes; + if (addressed) { + pool.address = BufferDeviceAddress(context, pool.buffer); + } const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1u}; VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; poolInfo.maxSets = 1u; @@ -287,6 +329,7 @@ void PrepareMeshWindow(const Context& context) { context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 1, &afterFill, 0, nullptr, 0, nullptr); batch.SubmitAndWait(); pool.set = set; + WriteMeshWindowHeap(context, pool); std::fprintf(stderr, "[gpu] mesh window pool: %u workgroup slots, %llu MiB\n", ShaderRecompiler::MeshWindowSlots, static_cast(bytes >> 20u)); } catch (...) { const auto layout = pool.layout; @@ -302,7 +345,7 @@ void BindMeshWindow(const Context& context, VkCommandBuffer commands, VkPipeline std::lock_guard lock(MeshWindowMutex()); const auto& pool = MeshWindowPools().at(context.device); Require(pool.set != VK_NULL_HANDLE, "mesh window buffer was not prepared"); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::MeshWindowDescriptorSet, 1, &pool.set, 0, nullptr); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::MeshWindowDescriptorSet, 1, &pool.set, SetBindSite::Draw); } void ReleaseMeshWindow(VkDevice device) { @@ -325,7 +368,7 @@ void BindPixelFunctionLds(const Context& context, VkCommandBuffer commands, VkPi std::lock_guard lock(PixelLdsMutex()); const auto& pool = PixelLdsPools().at(context.device); Require(pool.set != VK_NULL_HANDLE, "pixel function LDS buffer was not prepared"); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::PixelFunctionLdsDescriptorSet, 1, &pool.set, 0, nullptr); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::PixelFunctionLdsDescriptorSet, 1, &pool.set, SetBindSite::Draw); } void CheckPixelFunctionLds(const Context& context) { @@ -597,7 +640,7 @@ void recordCmaskClear(const Context& context, Recorder& recorder, const HostImpo const auto commands = recorder.Commands(); if (resident != nullptr && !resident->Cached()) recorder.KeepBytes(resident.get(), static_cast(resident->BudgetBytes())); recorder.Keep(resident); - context.detiler->DispatchCmaskClear(commands, import.buffer, HostImportOffset(import, address), bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0); + context.detiler->DispatchCmaskClear(commands, import.buffer, HostImportOffset(import, address), bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0, resident.get()); recorder.NotePendingWrite(address, bytes); proveCmask(address, bytes, GuestMemory::MarkWritten(address, bytes)); } @@ -1032,8 +1075,9 @@ bool ValidationKey(const Context& context, std::span shade add(shader.pushConstantOffset); add(program.pushConstants.size()); add(program.bdaAbiVersion); - add(program.vertexAttributes.size()); - for (const auto& attribute : program.vertexAttributes) { + const auto& attributes = DrawVertexAttributes(shader); + add(attributes.size()); + for (const auto& attribute : attributes) { add(attribute.location); add(attribute.components); add(attribute.fetchIndex); @@ -1191,11 +1235,11 @@ ResourceCache::Key DrawResourceKey(const Context& context, std::spandrawKeyMemo.words; - if (auto words = memo.load(std::memory_order_acquire); words != nullptr) { + if (auto words = memo.load(std::memory_order_acquire); words != nullptr && shader.heap == nullptr) { key.insert(key.end(), words->begin(), words->end()); } else { ShaderResources::AppendContentKey(shader, true, MovableBuffers(), key); - memo.store(std::make_shared>(key.begin() + static_cast(sizeAt + 1u), key.end()), std::memory_order_release); + if (shader.heap == nullptr) memo.store(std::make_shared>(key.begin() + static_cast(sizeAt + 1u), key.end()), std::memory_order_release); } key[sizeAt] = static_cast(key.size() - sizeAt - 1u); } @@ -1219,7 +1263,7 @@ void CheckBufferAliases(std::span shaders, const ColorTarg for (const auto& shader : shaders) { for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; - const auto& words = binding.guestDescriptor; + const auto& words = DrawGuestWords(shader, binding); for (std::size_t offset = 0; offset + 4 <= words.size(); offset += 4) { const ShaderRecompiler::ShaderBufferResource descriptor{{words[offset], words[offset + 1], words[offset + 2], words[offset + 3]}}; const auto address = descriptor.Base48(); @@ -1707,7 +1751,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P const bool restartLeft = draw.indirect.has_value(); auto highest = HighestDrawIndex(kept, draw.indexSize, restartLeft); constexpr std::uint32_t ZeroFetchSlack = 0x10000u; - if (const auto zeroFrom = highest && !shaders.front().program->vertexAttributes.empty() ? ZeroFetchVertex(shaders.front().program->vertexAttributes) : std::nullopt) { + if (const auto zeroFrom = highest && !DrawVertexAttributes(shaders.front()).empty() ? ZeroFetchVertex(DrawVertexAttributes(shaders.front())) : std::nullopt) { const auto limit = *zeroFrom > draw.firstVertex ? *zeroFrom - draw.firstVertex : 0u; if (*highest >= limit && *highest - limit >= ZeroFetchSlack) highest = ClampDrawIndices(kept, draw.indexSize, restartLeft, limit); } @@ -1746,7 +1790,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P return inputs; } constexpr std::uint32_t ZeroFetchSlack = 0x10000u; - if (const auto zeroFrom = state.stages.mesh || shaders.front().program->vertexAttributes.empty() ? std::nullopt : ZeroFetchVertex(shaders.front().program->vertexAttributes)) { + if (const auto zeroFrom = state.stages.mesh || DrawVertexAttributes(shaders.front()).empty() ? std::nullopt : ZeroFetchVertex(DrawVertexAttributes(shaders.front()))) { const auto limit = *zeroFrom > draw.firstVertex ? *zeroFrom - draw.firstVertex : 0u; if (*highest >= limit && *highest - limit >= ZeroFetchSlack) { GuestMemory::FlushGpuWrites(draw.indexAddress, static_cast(indexBytes)); @@ -1780,7 +1824,7 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P } APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); timing.Mark("index_upload"); - const auto& attributes = shaders.front().program->vertexAttributes; + const auto& attributes = DrawVertexAttributes(shaders.front()); // Validates the vertex descriptors; the layout also keys and builds the pipeline. if (recipe != nullptr) { inputs.vertexInput = recipe->vertexInput; @@ -1936,7 +1980,7 @@ AddressStages addressStages(std::span shaders) { auto stages = AddressStages::None; for (const auto& shader : shaders) { if (shader.program == nullptr || std::none_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable; })) continue; - if (ShaderStoresThroughAddresses(shader.program->spirv)) return AddressStages::Storing; + if (shader.program->bdaStores) return AddressStages::Storing; stages = AddressStages::ReadOnly; } return stages; @@ -2001,6 +2045,7 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; const bool candidate = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->PipelineVariantId() != 0; }); + HeapCostScope findCost(HeapCost::TemplateFind); resolved.cacheable = candidate && !ShaderResources::NeverReusable(shaders); const auto address = candidate && !ShaderResources::SpaceReuseDisabled() ? addressStages(shaders) : AddressStages::None; std::optional reuse; @@ -2052,6 +2097,8 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat static_cast(heldSpace.release()); resolved.resources = std::move(cached); resolved.moved = std::move(*moved); + if (resolved.resources->HeapStage()) resolved.resources->RefreshHeapStage(shaders); + outcome.addressBased = resolved.resources->HoldsLease(); outcome.kind = KindTemplateHit; countCache(&DrawProfile::cacheHits); if (counted) reuse = AddressDrawReuse::Reused; @@ -2068,9 +2115,12 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat } else { countCache(&DrawProfile::uncacheable); } + findCost.Stop(); timer.phase(PhaseLookup); if (resolved.resources == nullptr) { + HeapCostScope buildCost(HeapCost::TemplateBuild); resolved.resources = std::make_shared(context, shaders, state.color, static_cast(state.colors.size()), draw.indexAddress, static_cast(indexBytes), snapshots, recordable && drawStagingEnabled()); + buildCost.Stop(); resolved.built = &resolved.resources->Timing(); outcome.addressBased = resolved.resources->HoldsLease(); outcome.kind = outcome.addressBased ? KindBda : KindBuild; @@ -2414,6 +2464,7 @@ void keepRecordedDraw(Recorder& recorder, const std::shared_ptr struct StagedWrite { std::shared_ptr temporary; VkImageView target = VK_NULL_HANDLE; + const HeapSlotOwner* targetOwner = nullptr; std::uint16_t mask = 0; std::uint32_t samples = 1; VkRect2D area{}; @@ -2447,24 +2498,32 @@ struct RecordedDraw { // A remapped viewport depth range reads its scale and offset from the draw's push block (the vertex shader's // RemapPositionDepth), so draws that differ only in them share one pipeline; the draw's own state supplies them. -void patchDepthRemapPush(std::array& block, const State& state) { +void patchDepthRemapPush(std::array& block, const State& state, std::span shaders) { if (state.depthRemap.mode == 0u) return; + const auto reads = [](const CompiledShader& shader) { return shader.program != nullptr && (shader.program->pushItems & ShaderRecompiler::RuntimeAbi::DepthRemapPushBit) != 0u; }; + const auto rooted = [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->dataAddressPush; }; + if (std::ranges::any_of(shaders, rooted) && !std::ranges::any_of(shaders, reads)) return; static_assert(ShaderRecompiler::DepthRemapPushOffsetBytes + ShaderRecompiler::DepthRemapPushBytes <= PipelinePushSlotBytes); const std::array words{state.depthRemap.scale, state.depthRemap.offset}; std::memcpy(block.data() + ShaderRecompiler::DepthRemapPushOffsetBytes, words.data(), sizeof(words)); } -void recordStencilReplay(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, std::span>> replay, std::span shaders, const ShaderResources& resources, const std::array* bytes = nullptr, VkShaderStageFlags stages = 0) { +void recordStencilReplay(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, std::span>> replay, std::span shaders, const ShaderResources& resources, const std::array* bytes = nullptr, VkShaderStageFlags stages = 0, const ShaderResources::DrawBindings* drawBindings = nullptr) { State selected = state; auto block = bytes != nullptr ? *bytes : AssemblePushConstants(shaders); if (bytes == nullptr) { resources.PatchPushConstants(block); stages = PushConstantStages(shaders); } - patchDepthRemapPush(block, state); + patchDepthRemapPush(block, state, shaders); + Require(!replay.empty(), "stencil replay has no pipelines"); + const auto placements = replay.front().second->PushPlacements(); + resources.PatchDataAddresses(block, shaders, placements, drawBindings); const auto fragment = std::ranges::find_if(shaders, [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Fragment && !shader.program->stencilReferences.empty(); }); Require(fragment != shaders.end(), "stencil replay fragment stage is missing"); - const auto referenceOffset = fragment->pushConstantOffset + fragment->program->stencilReferenceDword * sizeof(std::uint32_t); + const auto stencilItem = fragment->program->dataAddressPush ? placements[static_cast(fragment - shaders.begin())].Item(ShaderRecompiler::RuntimeAbi::RootItem::StencilReference) : 0u; + Require(stencilItem != ShaderRecompiler::RuntimeAbi::NoPushPosition, "stencil replay fragment stage has no stencil reference in its push layout"); + const auto referenceOffset = fragment->program->dataAddressPush ? stencilItem * sizeof(std::uint32_t) : fragment->pushConstantOffset + fragment->program->stencilReferenceDword * sizeof(std::uint32_t); Require(referenceOffset <= block.size() - sizeof(std::uint32_t), "stencil replay reference exceeds the push block"); const auto run = [&](const Pm4::DrawParameters& piece, std::optional firstIndex) { std::uint32_t unusedSubdraw = 0; @@ -2495,13 +2554,14 @@ void recordStencilReplay(const Context& context, VkCommandBuffer commands, const } } -void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, std::span shaders, const ShaderResources& resources, const std::array* bytes, VkShaderStageFlags stages, VkDeviceAddress meshArguments = 0, bool restartTable = false, std::uint32_t meshLayerStride = 0) { +void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, std::span shaders, const ShaderResources& resources, const std::array* bytes, VkShaderStageFlags stages, VkDeviceAddress meshArguments = 0, bool restartTable = false, std::uint32_t meshLayerStride = 0, const ShaderResources::DrawBindings* drawBindings = nullptr) { auto block = bytes != nullptr ? *bytes : AssemblePushConstants(shaders); if (bytes == nullptr) { resources.PatchPushConstants(block); stages = PushConstantStages(shaders); } - patchDepthRemapPush(block, state); + patchDepthRemapPush(block, state, shaders); + resources.PatchDataAddresses(block, shaders, pipeline.PushPlacements(), drawBindings); if (state.stages.tessellation) { for (const auto& shader : shaders) Require(shader.program->pushConstants.empty() || shader.pushConstantOffset >= PipelinePushSlotBytes || shader.pushConstantOffset + shader.program->pushConstants.size() <= ShaderRecompiler::TessellationLevelPushOffsetBytes, "a stage's push constants overlap the tessellation levels"); const std::array levels{state.stages.minTessellationLevel, state.stages.maxTessellationLevel}; @@ -2783,18 +2843,25 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam record.pipeline->Begin(commands, *record.framebuffer, state.renderExtent, state); } APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK"); - if (drawBindings != nullptr) { + const bool heapPipeline = record.pipeline->DescriptorHeap(); + NoteHeapRecord(false, heapPipeline); + if (heapPipeline) { + resources.BindDescriptorHeaps(commands); + NoteEmptyResourceSet(); + } else if (drawBindings != nullptr) { + Require(!resources.HeapStage(), "a GPU-mode draw needs a VK_EXT_descriptor_heap pipeline"); const auto set = drawBindings->allocation.set; const auto fragmentSet = drawBindings->fragmentAllocation.set; - if (set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), 0, 1, &set, 0, nullptr); - if (fragmentSet != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragmentSet, 0, nullptr); + if (set != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), 0, 1, &set, SetBindSite::Draw); + if (fragmentSet != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, record.pipeline->Layout(), ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragmentSet, SetBindSite::Draw); } else { + Require(!resources.HeapStage(), "a GPU-mode draw needs a VK_EXT_descriptor_heap pipeline"); resources.BindDrawUse(commands, record.pipeline->Layout(), drawUse.get()); } - if (std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; })) BindPixelFunctionLds(context, commands, record.pipeline->Layout()); - if (UsesMeshWindow(shaders)) BindMeshWindow(context, commands, record.pipeline->Layout()); + if (!heapPipeline && std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; })) BindPixelFunctionLds(context, commands, record.pipeline->Layout()); + if (!heapPipeline && UsesMeshWindow(shaders)) BindMeshWindow(context, commands, record.pipeline->Layout()); APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); - pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty(), inputs.meshLayerStride); + pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty(), inputs.meshLayerStride, drawBindings.get()); APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); static const bool timeEachDraw = std::getenv("APS5_PROFILE_GPU_DRAWS") != nullptr; auto eachDrawTiming = Recorder::NoTiming; @@ -2807,7 +2874,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam } if (record.stencilReplay.empty()) recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset, trace, subdraw); else { - recordStencilReplay(context, commands, state, draw, inputs, record.stencilReplay, shaders, resources, record.pushBytes, record.pushStages); + recordStencilReplay(context, commands, state, draw, inputs, record.stencilReplay, shaders, resources, record.pushBytes, record.pushStages, drawBindings.get()); for (const auto& [reference, pipeline] : record.stencilReplay) recorder->Keep(pipeline); } if (record.pipeline->SplitsFaces()) { @@ -2853,7 +2920,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam if (!record.staged.empty()) { const auto merged = recorder->Commands(); for (auto& write : record.staged) { - RecordStagedMerge(context, merged, *write.temporary, write.target, write.mask, write.samples, write.area); + RecordStagedMerge(context, merged, *write.temporary, write.target, write.mask, write.samples, write.area, write.targetOwner); recorder->Keep(write.temporary); } } @@ -2941,6 +3008,8 @@ struct TargetBinding { std::shared_ptr multisampleOwner; std::shared_ptr stagedTemporary; VkImageView stagedTarget = VK_NULL_HANDLE; + const HeapSlotOwner* stagedTargetOwner = nullptr; + std::shared_ptr stagedTargetKeep; std::unique_ptr dump; }; @@ -3151,7 +3220,8 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara recipeOut = nullptr; } if (state.quadList || StripPatches(state, shaders)) recipeOut = nullptr; - if (context.recorder != nullptr) context.recorder->BoundKeptBytes(); + if (context.recorder != nullptr) context.recorder->BoundDrawBatch(); + if (AnyHeapShader(shaders)) recipeOut = nullptr; auto& timing = prepared.timing.emplace("Graphics.Draw"); // APS5_PROFILE_DRAW prints the time of each phase of the draw (microseconds) and the [draws] totals. static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; @@ -3191,7 +3261,10 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara const auto packed = StagedMergeTarget(color); binding.stagedTemporary = AcquireStagedTemporary(context, color.extent, color.samples); static_cast(MultisampleTargetView(context, packed, binding.multisampleOwner, &binding.multisampleImage)); - binding.stagedTarget = MultisampleTargetStorageView(context, TargetMergeSurface(packed)); + const auto storage = MultisampleTargetStorageView(context, TargetMergeSurface(packed)); + binding.stagedTarget = storage.view; + binding.stagedTargetOwner = storage.owner; + binding.stagedTargetKeep = storage.keep; Require(MultisampleTargetStorable(context, packed), "a partial write of a multisampled packed color target needs a multisample image of this form to allow storage"); targetViews.push_back(binding.stagedTemporary->View()); timer.phase(PhaseReadTarget); @@ -3343,8 +3416,9 @@ bool prepareDraw(const Context& context, const State& state, const Pm4::DrawPara for (const auto& shader : shaders) { for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages && binding.role != ShaderRecompiler::DescriptorRole::ImageTable) continue; - for (std::size_t offset = 0; offset + 8u <= binding.guestDescriptor.size(); offset += 8u) { - const auto words = std::span(binding.guestDescriptor).subspan(offset, 8u); + const auto& guestWords = DrawGuestWords(shader, binding); + for (std::size_t offset = 0; offset + 8u <= guestWords.size(); offset += 8u) { + const auto words = std::span(guestWords).subspan(offset, 8u); if (words[0] == 0u && (words[1] & 0xffu) == 0u) continue; const auto texture = DecodeTextureResource(words); Require(!DepthSurfaceAt(context, words, texture), "stencil replay refuses sampled depth-surface feedback"); @@ -3609,6 +3683,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D if (binding.multisampled) { record.multisampleImages.push_back(binding.multisampleImage); record.multisampleTargets.push_back(binding.multisampleOwner); + if (binding.stagedTargetKeep != nullptr && binding.stagedTargetKeep != binding.multisampleOwner) record.multisampleTargets.push_back(binding.stagedTargetKeep); } if (binding.stagedTemporary != nullptr) { record.multisampleImages.push_back(binding.stagedTemporary->Image()); @@ -3620,7 +3695,7 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D const auto right = std::min(static_cast(state.scissor.offset.x) + state.scissor.extent.width, color.extent.width); const auto bottom = std::min(static_cast(state.scissor.offset.y) + state.scissor.extent.height, color.extent.height); area.extent = {static_cast(std::max(right - area.offset.x, 0)), static_cast(std::max(bottom - area.offset.y, 0))}; - record.staged.push_back({binding.stagedTemporary, binding.stagedTarget, color.stagedMask, color.samples, area}); + record.staged.push_back({binding.stagedTemporary, binding.stagedTarget, binding.stagedTargetOwner, color.stagedMask, color.samples, area}); } } record.indirect = args != nullptr ? &indirect : nullptr; @@ -3776,10 +3851,14 @@ void recordPreparedDraw(const Context& context, const State& state, const Pm4::D APS5_LOG_OUT_DEBUG("Beginning pipeline renderExtent=%ux%u", state.renderExtent.width, state.renderExtent.height); pipeline->Begin(commands, *framebuffer, state.renderExtent, state); APS5_LOG_CHARS_OUT_DEBUG("Pipeline Begin OK"); - const auto drawUse = recorded ? resources->BeginDrawUse(*recorder) : nullptr; - resources->BindDrawUse(commands, pipeline->Layout(), drawUse.get()); - if (pixelLdsSubgroup != 0u) BindPixelFunctionLds(context, commands, pipeline->Layout()); - if (UsesMeshWindow(shaders)) BindMeshWindow(context, commands, pipeline->Layout()); + const bool heapPipeline = pipeline->DescriptorHeap(); + const auto drawUse = recorded && !heapPipeline ? resources->BeginDrawUse(*recorder) : nullptr; + if (heapPipeline) { + resources->BindDescriptorHeaps(commands); + NoteEmptyResourceSet(); + } else resources->BindDrawUse(commands, pipeline->Layout(), drawUse.get()); + if (pixelLdsSubgroup != 0u && !heapPipeline) BindPixelFunctionLds(context, commands, pipeline->Layout()); + if (!heapPipeline && UsesMeshWindow(shaders)) BindMeshWindow(context, commands, pipeline->Layout()); APS5_LOG_CHARS_OUT_DEBUG("Resources bound"); pushDrawConstants(*pipeline, commands, state, draw, shaders, *resources, nullptr, 0, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0, !inputs.restartTable.empty(), inputs.meshLayerStride); APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); @@ -4062,7 +4141,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& DrawRecipeOutcome DrawWithRecipe(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots, const DrawRecipe& recipe, const DrawTraceContext* trace) { const DepthCopySourceScope depthCopySource(state.depthCopySource); - if (context.recorder != nullptr) context.recorder->BoundKeptBytes(); + if (context.recorder != nullptr) context.recorder->BoundDrawBatch(); static_cast(snapshots); PerformanceTimer timing("Graphics.DrawWithRecipe"); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/FirstSampleCopy.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/FirstSampleCopy.cpp index 7fd0a5434bb5..9582ffbde8af 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/FirstSampleCopy.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/FirstSampleCopy.cpp @@ -1,4 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -179,6 +181,11 @@ FirstSampleCopy::FirstSampleCopy(const Context& context, VkFormat format, VkExte pipelineInfo.layout = pipelineLayout; pipelineInfo.renderPass = renderPass; Check(context.Function("vkCreateGraphicsPipelines")(context.device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines first sample"); + if (HeapPassesActive(context)) { + auto mapping = HeapPushIndexMapping(BindlessHeap::For(context), 0, 0, VK_SPIRV_RESOURCE_TYPE_COMBINED_SAMPLED_IMAGE_BIT_EXT, 0); + mapping.sourceData.pushIndex.pEmbeddedSampler = &samplerInfo; + heapPipeline = CreateHeapGraphicsPipeline(context, pipelineInfo, std::span(&mapping, 1)); + } } catch (...) { release(); throw; @@ -190,6 +197,7 @@ FirstSampleCopy::~FirstSampleCopy() { } void FirstSampleCopy::release() noexcept { + if (heapPipeline) context.Function("vkDestroyPipeline")(context.device, heapPipeline, nullptr); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); if (fragment) context.Function("vkDestroyShaderModule")(context.device, fragment, nullptr); if (vertex) context.Function("vkDestroyShaderModule")(context.device, vertex, nullptr); @@ -213,7 +221,26 @@ void FirstSampleCopy::release() noexcept { memory = VK_NULL_HANDLE; } -void FirstSampleCopy::Record(Recorder& recorder, VkImageView source, VkImageLayout layout) { +void FirstSampleCopy::Record(Recorder& recorder, Texture& texture) { + const auto source = texture.View(); + const auto layout = texture.Layout(); + const auto slot = heapPipeline != VK_NULL_HANDLE ? BindlessHeap::For(context).TextureSlot(ShaderRecompiler::RuntimeAbi::FirstImageBinding, &texture, source, layout) : std::nullopt; + if (slot) { + recorder.RecordUncounted([&](VkCommandBuffer commands) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + begin.renderPass = renderPass; + begin.framebuffer = framebuffer; + begin.renderArea = {{0, 0}, extent}; + context.Resolved(&DeviceFunctions::cmdBeginRenderPass, "vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, heapPipeline); + const std::uint32_t pushed = *slot; + CmdPushData(context, commands, sizeof(pushed), &pushed); + context.Function("vkCmdDraw")(commands, 3, 1, 0, 0); + context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); + }); + return; + } const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER, 1}; VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; poolInfo.maxSets = 1; @@ -244,7 +271,7 @@ void FirstSampleCopy::Record(Recorder& recorder, VkImageView source, VkImageLayo begin.renderArea = {{0, 0}, extent}; context.Resolved(&DeviceFunctions::cmdBeginRenderPass, "vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, 0, nullptr); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipelineLayout, 0, 1, &set, SetBindSite::FirstSampleCopy); context.Function("vkCmdDraw")(commands, 3, 1, 0, 0); context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); }); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp index 9881ba24732f..68be732d841d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GpuColorTransfer.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/GpuColorTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/ColorTransfer_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -6,10 +7,23 @@ #include #include #include +#include #include namespace AgcDriver::Graphics { +namespace { + +struct HeapPush { + std::array values; + std::uint32_t pad; + std::array buffers; +}; + +static_assert(offsetof(HeapPush, buffers) == 24 && sizeof(HeapPush) == 56); + +} + GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) { VkShaderModule module = VK_NULL_HANDLE; try { @@ -39,6 +53,10 @@ GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) { auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateComputePipelines color transfer"); } + std::array mappings{}; + for (std::uint32_t i = 0; i < mappings.size(); ++i) mappings[i] = HeapPushAddressMapping(0, i, static_cast(offsetof(HeapPush, buffers) + i * sizeof(VkDeviceAddress))); + heapPipeline = CreateHeapComputePipeline(context, pipelineInfo, mappings); + if (heapPipeline != VK_NULL_HANDLE) addressUsage = VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; context.Function("vkDestroyShaderModule")(context.device, module, nullptr); module = VK_NULL_HANDLE; const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4}; @@ -53,10 +71,10 @@ GpuColorTransfer::GpuColorTransfer(const Context& context) : context(context) { allocation.pSetLayouts = &descriptorLayout; Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &descriptorSet), "vkAllocateDescriptorSets color transfer"); const auto& table = HalfFloatScanoutThresholds(); - thresholds = std::make_unique(context, sizeof(table), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + thresholds = std::make_unique(context, sizeof(table), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | addressUsage); std::memcpy(thresholds->Bytes().data(), table.data(), sizeof(table)); const auto& luminance = AgcDriver::PqEotfNits(); - pqNits = std::make_unique(context, sizeof(luminance), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + pqNits = std::make_unique(context, sizeof(luminance), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | addressUsage); std::memcpy(pqNits->Bytes().data(), luminance.data(), sizeof(luminance)); const std::array tableInfo{{{thresholds->Handle(), 0, sizeof(table)}, {pqNits->Handle(), 0, sizeof(luminance)}}}; std::array tableWrites{}; @@ -83,6 +101,7 @@ GpuColorTransfer::~GpuColorTransfer() { void GpuColorTransfer::release() noexcept { if (descriptorPool) context.Function("vkDestroyDescriptorPool")(context.device, descriptorPool, nullptr); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + if (heapPipeline) context.Function("vkDestroyPipeline")(context.device, heapPipeline, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } @@ -96,8 +115,8 @@ void GpuColorTransfer::prepare(std::uint32_t newWidth, std::uint32_t newHeight, if (tiled && width == newWidth && height == newHeight && mode == newMode && elementBytes == newElementBytes) return; constexpr VkBufferUsageFlags copies = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; auto newTiled = std::make_unique(context, layout.Bytes(), copies); - auto newTiledDevice = std::make_unique(context, layout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); - auto newLinear = std::make_unique(context, linearBytes, copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + auto newTiledDevice = std::make_unique(context, layout.Bytes(), copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | addressUsage); + auto newLinear = std::make_unique(context, linearBytes, copies | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | addressUsage); const std::array buffers{{{newTiledDevice->Handle(), 0, layout.Bytes()}, {newLinear->Handle(), 0, linearBytes}}}; std::array writes{}; for (std::uint32_t i = 0; i < writes.size(); ++i) { @@ -146,9 +165,16 @@ void GpuColorTransfer::dispatch(VkCommandBuffer commands, bool toTiled, bool swa const auto barrier = context.Function("vkCmdPipelineBarrier"); barrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); const std::array push{width, height, (width + 127u) / 128u, (toTiled ? 1u : 0u) | (swapRedBlue ? 2u : 0u) | (tiledSource ? 4u : 0u) | (tenBit ? 8u : 0u) | (halfFloat ? 16u : 0u) | (pqWhiteNits ? 32u : 0u), std::bit_cast(pqWhiteNits.value_or(0.0f))}; - context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descriptorSet, 0, nullptr); - context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + if (heapPipeline != VK_NULL_HANDLE) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + const HeapPush pushed{push, 0u, {BufferDeviceAddress(context, tiledDevice->Handle()), BufferDeviceAddress(context, linear->Handle()), thresholds->DeviceAddress(), pqNits->DeviceAddress()}}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + } else { + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &descriptorSet, SetBindSite::ColorTransfer); + CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + } context.Function("vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); VkMemoryBarrier after{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; after.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index e74026aa63aa..037acc3a2218 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Graphics/include/MapImportGate.hpp" #include "prx/libSceAgcDriver/Graphics/include/AdaptiveStaging.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Execution/include/Mutex.hpp" #include "ThreadOwned.hpp" @@ -69,6 +70,7 @@ constexpr unsigned long UDMABUF_CREATE_LIST = 0; #include #include #include +#include #include #include #include @@ -676,8 +678,7 @@ const char* bindImport(const Context& context, HostImport& entry, VkExternalMemo allocated = true; entry.memoryType = allocation.memoryTypeIndex; if (const auto result = context.Function("vkBindBufferMemory")(context.device, entry.buffer, entry.memory, 0); result != VK_SUCCESS) return failed("vkBindBufferMemory", result); - const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, entry.buffer}; - entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + entry.address = BufferDeviceAddress(context, entry.buffer); if (entry.address == 0) return failed("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); return nullptr; } @@ -901,10 +902,9 @@ const char* createChunkedDmaBufImport(const Context& context, HostImport& entry, failure = result; return "vkBindBufferMemory"; } - const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; entry.buffer = buffer; entry.memory = VK_NULL_HANDLE; - entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + entry.address = BufferDeviceAddress(context, buffer); entry.chunk = std::move(chunk); entry.dmaBuf = true; return nullptr; @@ -1010,8 +1010,7 @@ bool createSparseDmaBufImport(const Context& context, HostImport& entry, int fil context.Function("vkDestroyFence")(context.device, fence, nullptr); if (bound != VK_SUCCESS) fail("vkQueueBindSparse", bound); if (waited != VK_SUCCESS) fail("vkWaitForFences", waited); - const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; - const auto address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + const auto address = BufferDeviceAddress(context, buffer); if (address == 0) fail("vkGetBufferDeviceAddressKHR", VK_ERROR_UNKNOWN); entry.buffer = buffer; entry.memory = VK_NULL_HANDLE; @@ -1176,6 +1175,7 @@ const char* settleImport(const Context& context, HostImports& state, const HostI if ((result = context.Function("vkResetCommandBuffer")(settle.settleCommands, 0)) != VK_SUCCESS) return "vkResetCommandBuffer (settle)"; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + NoteCommandBufferBegun(settle.settleCommands); if ((result = context.Function("vkBeginCommandBuffer")(settle.settleCommands, &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer (settle)"; const VkBufferCopy region{0, 0, 4}; context.Function("vkCmdCopyBuffer")(settle.settleCommands, source, settle.settleTarget, 1, ®ion); @@ -1270,6 +1270,7 @@ void submitSettleBatch(const Context& context, HostImports& state) { if ((result = context.Function("vkResetCommandBuffer")(batch.commands, 0)) != VK_SUCCESS) return fail("vkResetCommandBuffer (settle batch)"); VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + NoteCommandBufferBegun(batch.commands); if ((result = context.Function("vkBeginCommandBuffer")(batch.commands, &begin)) != VK_SUCCESS) return fail("vkBeginCommandBuffer (settle batch)"); const auto copy = context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer"); RecordMemoryBarrier(context, batch.commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_MEMORY_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); @@ -3315,6 +3316,13 @@ std::vector> refreshHeapRuns(const GuestBu return checked; } +void refreshWritableMirrors(const GuestBufferMemory::AddressSpace& space, std::vector& blocks) { + for (const auto index : space.writableMirrors) { + const auto& region = space.base[index]; + if (prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); + } +} + bool heapMirrorWatched(ImageMirror& mirror) { const auto serial = GuestMemory::UnwatchSerial(); if (mirror.watchedSerial.load(std::memory_order_relaxed) == serial) return true; @@ -4098,11 +4106,13 @@ void runImportProbe(const Context& context, std::uint64_t base, std::uint64_t by allocate.commandBufferCount = 3; if ((probe.result = context.Function("vkAllocateCommandBuffers")(context.device, &allocate, commands)) != VK_SUCCESS) return "vkAllocateCommandBuffers"; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; + NoteCommandBufferBegun(commands[0]); if ((probe.result = context.Function("vkBeginCommandBuffer")(commands[0], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; const VkBufferCopy region{0, 0, bytes}; context.Function("vkCmdCopyBuffer")(commands[0], import.buffer, destination, 1, ®ion); if ((probe.result = context.Function("vkEndCommandBuffer")(commands[0])) != VK_SUCCESS) return "vkEndCommandBuffer"; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + NoteCommandBufferBegun(commands[1]); if ((probe.result = context.Function("vkBeginCommandBuffer")(commands[1], &begin)) != VK_SUCCESS) return "vkBeginCommandBuffer"; context.Function("vkCmdFillBuffer")(commands[1], import.buffer, 0, bytes, 0x02020202u); if ((probe.result = context.Function("vkEndCommandBuffer")(commands[1])) != VK_SUCCESS) return "vkEndCommandBuffer"; @@ -4869,10 +4879,7 @@ void GuestBufferMemory::AcquireRegistered() { // before any of them is dereferenced. served = true; mirrored = space->mirrored; - for (const auto index : space->writableMirrors) { - const auto& region = space->base[index]; - if (prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); - } + refreshWritableMirrors(*space, blocks); lap(timing.mirrorsUs, at); } } @@ -5478,10 +5485,7 @@ bool GuestBufferMemory::RetakeSpace() { if (reuse) { noteEpochMirrors(true, *space); } else { - for (const auto index : space->writableMirrors) { - const auto& region = space->base[index]; - if (prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); - } + refreshWritableMirrors(*space, blocks); if (epochMirrors()) noteEpochMirrors(false, *space); } const auto checked = refreshHeapRuns(*space, blocks); @@ -6444,6 +6448,7 @@ void GuestBufferMemory::copyRegion(Region& region, bool addressable) { readBytes += region.snapshot.size(); } region.snapshot.clear(); + NoteCpuBufferCopy(readBytes); if (profile) readUs.fetch_add(microsecondsSince(readStart), std::memory_order_relaxed); if (profile) { static AgcDriver::Mutex reasonsMutex; @@ -6718,34 +6723,47 @@ void GuestBufferMemory::RecordCopyBacks(Recorder& recorder) { } VkDescriptorBufferInfo GuestBufferMemory::Descriptor(std::uint64_t address, std::size_t bytes, std::uint32_t& adjustment) const { - Require(uploaded && !committed, "guest GPU memory is not available"); - Require(bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest buffer view"); + std::string failure; + const auto info = TryDescriptor(address, bytes, adjustment, &failure); + if (!info.has_value()) Require(false, failure); + return *info; +} + +std::optional GuestBufferMemory::TryDescriptor(std::uint64_t address, std::size_t bytes, std::uint32_t& adjustment, std::string* failure) const { + const auto fail = [failure](std::string text) -> std::optional { + if (failure != nullptr) *failure = std::move(text); + return std::nullopt; + }; + if (!uploaded || committed) return fail("guest GPU memory is not available"); + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return fail("invalid guest buffer view"); const auto* found = owner(address, address + bytes); - Require(found != nullptr, "guest buffer has no GPU owner"); + if (found == nullptr) return fail("guest buffer has no GPU owner"); const auto& region = *found; const auto describe = [&](const char* what) { + if (failure == nullptr) return std::string(); char text[256]; std::snprintf(text, sizeof(text), "guest buffer view 0x%llx+0x%llx %s its GPU owner 0x%llx-0x%llx (buffer %d, direct %d, mirror %d)", static_cast(address), static_cast(bytes), what, static_cast(region.begin), static_cast(region.end), region.buffer != nullptr, region.direct != nullptr, region.mirror != nullptr); return std::string(text); }; - if (!(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr))) Require(false, describe("exceeds")); + if (!(address >= region.begin && address + bytes <= region.end && (region.buffer != nullptr || region.direct != nullptr || region.mirror != nullptr))) return fail(describe("exceeds")); const auto base = region.direct != nullptr ? region.direct->base - region.direct->lead : region.mirror != nullptr ? mirrorStart(*region.mirror) : region.begin; const auto offset = address - base; Require(context.limits.minStorageBufferOffsetAlignment != 0, "no storage buffer offset alignment"); - Require(base % 4 == 0, "a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented"); + if (base % 4 != 0) return fail("a guest buffer view in a GPU owner that does not start at a DWORD boundary is not implemented"); adjustment = static_cast(offset % std::max(context.limits.minStorageBufferOffsetAlignment, 4)); + if (bytes / 4 + adjustment / 4 + 2 > std::numeric_limits::max() / 4) return fail("guest buffer view range overflow"); auto range = ViewBytes(bytes, adjustment); const auto end = region.direct != nullptr ? region.direct->base + region.direct->bytes : servedEnd(region); const bool cut = (region.direct == nullptr || region.runCut) && end < region.end; if (cut) { - Require(address - adjustment < end, "guest buffer view starts past its copy"); + if (address - adjustment >= end) return fail("guest buffer view starts past its copy"); range = std::min(range, end - (address - adjustment)); } if (bytes <= context.limits.maxStorageBufferRange) range = std::min(range, context.limits.maxStorageBufferRange & ~std::uint64_t{3}); - if (address - adjustment + range > end) Require(false, describe("rounded up to its view range exceeds")); - Require(range <= context.limits.maxStorageBufferRange, "guest buffer view exceeds descriptor range limit"); + if (address - adjustment + range > end) return fail(describe("rounded up to its view range exceeds")); + if (range > context.limits.maxStorageBufferRange) return fail("guest buffer view exceeds descriptor range limit"); const auto handle = region.direct != nullptr ? region.direct->buffer : region.mirror != nullptr ? mirrorHandle(*region.mirror) : region.buffer->Handle(); - return {handle, offset - adjustment, range}; + return VkDescriptorBufferInfo{handle, offset - adjustment, range}; } std::uint64_t GuestBufferMemory::ViewBytes(std::uint64_t bytes, std::uint32_t adjustment) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleReadback.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleReadback.cpp index fe92fa1b979a..42c47502769b 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleReadback.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleReadback.cpp @@ -1,14 +1,26 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MultisampleReadbackUint_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/MultisampleReadbackDepth_spv.h" #include #include +#include #include namespace AgcDriver::Graphics { namespace { +struct HeapReadbackPush { + std::array values; + std::uint32_t pad; + VkDeviceAddress buffer; + std::uint32_t slot; + std::uint32_t pad2; +}; + +static_assert(offsetof(HeapReadbackPush, buffer) == 32 && offsetof(HeapReadbackPush, slot) == 40 && sizeof(HeapReadbackPush) == 48); + std::uint32_t readbackElementBytes(VkFormat format) { switch (format) { case VK_FORMAT_R8_UINT: return 1u; @@ -22,8 +34,8 @@ std::uint32_t readbackElementBytes(VkFormat format) { } -VkPipeline TextureDetiler::multisampleReadbackPipeline(std::uint32_t elementBytes, bool depth) { - const auto key = elementBytes | (depth ? 32u : 0u); +VkPipeline TextureDetiler::multisampleReadbackPipeline(std::uint32_t elementBytes, bool depth, bool heap) { + const auto key = elementBytes | (depth ? 32u : 0u) | (heap ? 64u : 0u); for (const auto& entry : readbackPipelines) if (entry.first == key) return entry.second; if (readbackDescriptorLayout == VK_NULL_HANDLE) { const std::array bindings{{ @@ -60,7 +72,12 @@ VkPipeline TextureDetiler::multisampleReadbackPipeline(std::uint32_t elementByte info.layout = readbackPipelineLayout; readbackPipelines.reserve(readbackPipelines.size() + 1u); VkPipeline result = VK_NULL_HANDLE; - { + if (heap) { + const std::array mappings{ + HeapPushAddressMapping(0u, 0u, offsetof(HeapReadbackPush, buffer)), + HeapPushIndexMapping(BindlessHeap::For(context), 0u, 1u, VK_SPIRV_RESOURCE_TYPE_SAMPLED_IMAGE_BIT_EXT, offsetof(HeapReadbackPush, slot))}; + result = CreateHeapComputePipeline(context, info, mappings); + } else { auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1u, &info, nullptr, &result), "vkCreateComputePipelines multisample readback"); } @@ -68,7 +85,7 @@ VkPipeline TextureDetiler::multisampleReadbackPipeline(std::uint32_t elementByte return result; } -void TextureDetiler::DispatchMultisampleReadback(VkCommandBuffer commands, VkFormat format, VkBuffer destination, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements) { +void TextureDetiler::DispatchMultisampleReadback(VkCommandBuffer commands, VkFormat format, VkBuffer destination, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements, const HeapSlotOwner* owner) { GuestMemory::AssertGpuLockHeld("TextureDetiler::DispatchMultisampleReadback"); const auto elementBytes = readbackElementBytes(format); Require(commands != VK_NULL_HANDLE && destination != VK_NULL_HANDLE && arrayView != VK_NULL_HANDLE, "multisample readback requires commands, buffer and array view"); @@ -87,7 +104,19 @@ void TextureDetiler::DispatchMultisampleReadback(VkCommandBuffer commands, VkFor Require(groupsX != 0u, "device has no multisample readback workgroups"); const auto groupsY = (groups + groupsX - 1u) / groupsX; Require(groupsY <= context.limits.maxComputeWorkGroupCount[1] && groupsX * groupsY * 64u <= std::numeric_limits::max(), "multisample readback dispatch exceeds device limits"); - const auto pipeline = multisampleReadbackPipeline(elementBytes, format == VK_FORMAT_D32_SFLOAT); + const std::array push{width, height, layers, fragments, firstElement, elements, static_cast(groupsX * 64u)}; + const auto address = owner != nullptr && HeapPassesActive(context) ? bufferAddress(destination) : 0; + const auto heapPipeline = address != 0 ? multisampleReadbackPipeline(elementBytes, format == VK_FORMAT_D32_SFLOAT, true) : VK_NULL_HANDLE; + const auto slot = heapPipeline != VK_NULL_HANDLE ? BindlessHeap::For(context).ViewSlot(HeapPassSampledKind, owner, arrayView) : std::nullopt; + if (slot) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + const HeapReadbackPush pushed{push, 0u, address + offset, *slot, 0u}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + context.Function("vkCmdDispatch")(commands, static_cast(groupsX), static_cast(groupsY), 1u); + return; + } + const auto pipeline = multisampleReadbackPipeline(elementBytes, format == VK_FORMAT_D32_SFLOAT, false); const auto set = allocateFrom(readbackDescriptorSets, readbackDescriptorLayout, {{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 64u}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 64u}}, "vkCreateDescriptorPool multisample readback", "vkAllocateDescriptorSets multisample readback", "vkResetDescriptorPool multisample readback"); const VkDescriptorBufferInfo buffer{destination, offset, bytes}; const VkDescriptorImageInfo image{VK_NULL_HANDLE, arrayView, VK_IMAGE_LAYOUT_GENERAL}; @@ -97,9 +126,8 @@ void TextureDetiler::DispatchMultisampleReadback(VkCommandBuffer commands, VkFor }}; context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0u, nullptr); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, readbackPipelineLayout, 0u, 1u, &set, 0u, nullptr); - const std::array push{width, height, layers, fragments, firstElement, elements, static_cast(groupsX * 64u)}; - context.Function("vkCmdPushConstants")(commands, readbackPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0u, sizeof(push), push.data()); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, readbackPipelineLayout, 0u, 1u, &set, SetBindSite::ColorTransfer); + CmdPushConstants(context, commands, readbackPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0u, sizeof(push), push.data()); context.Function("vkCmdDispatch")(commands, static_cast(groupsX), static_cast(groupsY), 1u); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp index 75becb830cf2..713c74a9d03e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleTarget.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp" @@ -193,7 +194,7 @@ struct TargetUse { std::atomic claimed{false}; }; -class MultisampleTarget : public std::enable_shared_from_this { +class MultisampleTarget : public std::enable_shared_from_this, public HeapSlotOwner { public: MultisampleTarget(const Context& context, const ColorTarget& target, bool guestSamples = false) : context(context), target(target), cmask(target.cmaskAddress, target.cmaskBytes), colorFill(target) { this->context.bufferPool.reset(); @@ -279,7 +280,7 @@ class MultisampleTarget : public std::enable_shared_from_this const auto bytes = static_cast(target.extent.width) * target.extent.height * target.samples * target.elementBytes; for (std::uint64_t offset = 0; offset < bytes; offset += stride) { const auto count = std::min(stride, bytes - offset); - context.detiler->DispatchMultisampleImage(commands, target.elementBytes, upload->Handle(), offset, (count + 3u) & ~std::uint64_t{3}, uploadView, target.extent.width, target.extent.height, 1u, target.samples, static_cast(offset / target.elementBytes), static_cast(count / target.elementBytes)); + context.detiler->DispatchMultisampleImage(commands, target.elementBytes, upload->Handle(), offset, (count + 3u) & ~std::uint64_t{3}, uploadView, target.extent.width, target.extent.height, 1u, target.samples, static_cast(offset / target.elementBytes), static_cast(count / target.elementBytes), this); } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); if (recorder != nullptr) { @@ -564,7 +565,7 @@ class MultisampleTarget : public std::enable_shared_from_this std::shared_ptr FirstSample(VkComponentMapping components) { const auto source = Sampled({}); if (firstSampleCopy == nullptr) firstSampleCopy = std::make_unique(context, target.format, target.extent); - firstSampleCopy->Record(*Recorder::Active(), source->View(), source->Layout()); + firstSampleCopy->Record(*Recorder::Active(), *source); const std::array key{static_cast(components.r), static_cast(components.g), static_cast(components.b), static_cast(components.a)}; auto& texture = firstSampleTextures[key]; if (texture == nullptr) { @@ -894,6 +895,7 @@ class MultisampleTarget : public std::enable_shared_from_this firstSampleTextures.clear(); firstSampleCopy.reset(); for (const auto& [format, resolvePass] : resolvePasses) context.Function("vkDestroyRenderPass")(context.device, resolvePass, nullptr); + ReleaseHeapSlots(context); if (uploadView) context.Function("vkDestroyImageView")(context.device, uploadView, nullptr); if (orderView) context.Function("vkDestroyImageView")(context.device, orderView, nullptr); if (twinView) context.Function("vkDestroyImageView")(context.device, twinView, nullptr); @@ -1427,7 +1429,7 @@ SampleOrderTarget MultisampleTargetOrderTarget(const Context& context, const Col auto& found = acquire(context, target); found.drawn = true; ++found.contents; - return {found.image, found.OrderView(), found.orderFormat, target.extent, target.samples, SampleOrderMethod::Storage, found.shared_from_this()}; + return {found.image, found.OrderView(), found.orderFormat, target.extent, target.samples, SampleOrderMethod::Storage, found.shared_from_this(), &found}; } VkImageView MultisampleTargetView(const Context& context, const ColorTarget& target, VkImage* image) { @@ -1534,7 +1536,7 @@ void MaterializeMultisampleTargetClear(const Context& context, const ColorTarget acquire(context, target); } -VkImageView MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource, bool uintStorage) { +MultisampleStorage MultisampleTargetStorageView(const Context& context, const GuestTextureResource& resource, bool uintStorage) { TargetsLock lock; discardRetiredTargets(); const auto format = ResolveTextureFormat(resource.format); @@ -1546,13 +1548,14 @@ VkImageView MultisampleTargetStorageView(const Context& context, const GuestText if (rendered.eqaaFragments != 0) throw std::runtime_error(eqaaReadRefusal("storage image access to", rendered, resource)); if (sameStorageForm(context, rendered.format, format) && rendered.extent.width == resource.width && rendered.extent.height == resource.height && rendered.samples == resource.samples && rendered.mip == 0) { if (!holdHostOnly(*target)) continue; - return target->Storage(StorageFormatForGuest(context, resource.format), uintStorage); + return {target->Storage(StorageFormatForGuest(context, resource.format), uintStorage), target.get(), target}; } other = target.get(); } if (other == nullptr) { Require(Recorder::Active() != nullptr, "a multisampled color target is only written by recorded dispatches"); - return createFilled(context, resource, format, "writing", true).Storage(StorageFormatForGuest(context, resource.format), uintStorage); + auto& created = createFilled(context, resource, format, "writing", true); + return {created.Storage(StorageFormatForGuest(context, resource.format), uintStorage), &created, created.shared_from_this()}; } char text[320]; std::snprintf(text, sizeof(text), "AGC graphics: storage image access to multisampled color target 0x%llx (%ux%u, %u samples, vk format %d) as a %ux%u texture of %u samples and guest format %u (vk %d) is not implemented", static_cast(resource.baseAddress), other->target.extent.width, other->target.extent.height, other->target.samples, static_cast(other->target.format), resource.width, resource.height, resource.samples, resource.format, static_cast(format)); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleUpload.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleUpload.cpp index 419ed7648e38..46551bd95b31 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleUpload.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/MultisampleUpload.cpp @@ -1,14 +1,30 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MultisampleUpload_spv.h" #include #include #include +#include #include namespace AgcDriver::Graphics { -VkPipeline TextureDetiler::multisampleUploadPipeline(std::uint32_t elementBytes) { - const auto key = (1u << 19u) | elementBytes; +namespace { + +struct HeapMultisamplePush { + std::array values; + std::uint32_t pad; + VkDeviceAddress buffer; + std::uint32_t slot; + std::uint32_t pad2; +}; + +static_assert(offsetof(HeapMultisamplePush, buffer) == 32 && offsetof(HeapMultisamplePush, slot) == 40 && sizeof(HeapMultisamplePush) == 48); + +} + +VkPipeline TextureDetiler::multisampleUploadPipeline(std::uint32_t elementBytes, bool heap) { + const auto key = (1u << 19u) | (heap ? 1u << 18u : 0u) | elementBytes; for (const auto& entry : pipelines) if (entry.first == key) return entry.second; if (multisampleUploadModule == VK_NULL_HANDLE) { VkShaderModuleCreateInfo info{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; @@ -27,7 +43,12 @@ VkPipeline TextureDetiler::multisampleUploadPipeline(std::uint32_t elementBytes) info.layout = imagePipelineLayout; pipelines.reserve(pipelines.size() + 1u); VkPipeline result = VK_NULL_HANDLE; - { + if (heap) { + const std::array mappings{ + HeapPushAddressMapping(0u, 0u, offsetof(HeapMultisamplePush, buffer)), + HeapPushIndexMapping(BindlessHeap::For(context), 0u, 1u, HeapPassStorageImages, offsetof(HeapMultisamplePush, slot))}; + result = CreateHeapComputePipeline(context, info, mappings); + } else { auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1u, &info, nullptr, &result), "vkCreateComputePipelines multisample upload"); } @@ -35,7 +56,7 @@ VkPipeline TextureDetiler::multisampleUploadPipeline(std::uint32_t elementBytes) return result; } -void TextureDetiler::DispatchMultisampleImage(VkCommandBuffer commands, std::uint32_t elementBytes, VkBuffer source, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements) { +void TextureDetiler::DispatchMultisampleImage(VkCommandBuffer commands, std::uint32_t elementBytes, VkBuffer source, std::uint64_t offset, std::uint64_t bytes, VkImageView arrayView, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t firstElement, std::uint32_t elements, const HeapSlotOwner* owner) { Require(context.storageImageMultisample && context.storageImageWriteWithoutFormat, "device does not enable multisampled storage image writes without format"); Require(commands != VK_NULL_HANDLE && source != VK_NULL_HANDLE && arrayView != VK_NULL_HANDLE, "multisample upload requires commands, buffer and array view"); Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "multisample upload element size is unsupported"); @@ -52,16 +73,27 @@ void TextureDetiler::DispatchMultisampleImage(VkCommandBuffer commands, std::uin Require(groupsX != 0u, "device has no multisample upload workgroups"); const auto groupsY = (groups + groupsX - 1u) / groupsX; Require(groupsY <= context.limits.maxComputeWorkGroupCount[1] && groupsX * groupsY * 64u <= std::numeric_limits::max(), "multisample upload dispatch exceeds the device limits"); - const auto target = multisampleUploadPipeline(elementBytes); + const std::array push{width, height, layers, fragments, firstElement, elements, static_cast(groupsX * 64u)}; + const auto address = owner != nullptr && HeapPassesActive(context) ? bufferAddress(source) : 0; + const auto heapTarget = address != 0 ? multisampleUploadPipeline(elementBytes, true) : VK_NULL_HANDLE; + const auto slot = heapTarget != VK_NULL_HANDLE ? BindlessHeap::For(context).ViewSlot(HeapPassStorageKind, owner, arrayView) : std::nullopt; + if (slot) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapTarget); + const HeapMultisamplePush pushed{push, 0u, address + offset, *slot, 0u}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + context.Function("vkCmdDispatch")(commands, static_cast(groupsX), static_cast(groupsY), 1u); + return; + } + const auto target = multisampleUploadPipeline(elementBytes, false); const auto set = allocateSet(true); const VkDescriptorBufferInfo buffer{source, offset, bytes}; const VkDescriptorImageInfo image{VK_NULL_HANDLE, arrayView, VK_IMAGE_LAYOUT_GENERAL}; const std::array writes{{{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, set, 0u, 0u, 1u, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, nullptr, &buffer, nullptr}, {VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET, nullptr, set, 1u, 0u, 1u, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, &image, nullptr, nullptr}}}; context.Function("vkUpdateDescriptorSets")(context.device, 2u, writes.data(), 0u, nullptr); context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); - context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0u, 1u, &set, 0u, nullptr); - const std::array push{width, height, layers, fragments, firstElement, elements, static_cast(groupsX * 64u)}; - context.Function("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0u, sizeof(push), push.data()); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0u, 1u, &set, SetBindSite::ColorTransfer); + CmdPushConstants(context, commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0u, sizeof(push), push.data()); context.Function("vkCmdDispatch")(commands, static_cast(groupsX), static_cast(groupsY), 1u); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 144e2f4325aa..16bcd55fe47f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -333,7 +334,7 @@ void appendKey(std::vector& key, const TValue& value) { key.insert(key.end(), bytes.begin(), bytes.end()); } -bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span shaders) { +bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const PipelineSetLayouts& resources, std::span shaders, std::span pushPlacements) { using Stage = ShaderRecompiler::ShaderStage; if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || (state.stages.tessellation && !context.tessellationPipelineCache) || state.depthBiasPerFace || state.rectList || BarycentricPatches(state, shaders) || SplitsAlternateAlpha(state) || std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->functionLdsBuffer; }) || UsesMeshWindow(shaders) || FragmentInterlock(shaders)) return false; keys = {}; @@ -347,18 +348,30 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& appendKey(key, shader.pushConstantOffset); appendKey(key, state.meshLayerBase); appendKey(key, state.meshLayerCount); + appendKey(key, pushPlacements[static_cast(&shader - shaders.data())]); if (resources.split) { auto& sets = shader.stage == Stage::Fragment ? keys.fragmentSets : keys.preRasterizationSets; - for (const auto& binding : shader.program->bindings) { - Require(binding.descriptorSet < 2u, "split pipeline library descriptor set exceeds two slots"); - sets |= 1u << binding.descriptorSet; + if (HeapShader(shader)) { + sets |= HeapSetBit; + for (const auto& binding : shader.program->bindings) { + if (binding.descriptorSet == 0u) sets |= MainSetBit; + } + } else { + for (const auto& binding : shader.program->bindings) { + Require(binding.descriptorSet < 2u, "split pipeline library descriptor set exceeds two slots"); + sets |= 1u << binding.descriptorSet; + } } } } + if (resources.split && AnyHeapShader(shaders)) { + keys.preRasterizationSets |= HeapSetBit; + keys.fragmentSets |= HeapSetBit; + } if (resources.split) { const auto unused = 3u & ~(keys.preRasterizationSets | keys.fragmentSets); - keys.preRasterizationSets |= unused & 1u; - keys.fragmentSets |= unused & 2u; + keys.preRasterizationSets |= unused & MainSetBit; + keys.fragmentSets |= unused & HeapSetBit; } for (const auto& binding : input.bindings) { appendKey(keys.vertexInput, binding.binding); @@ -435,14 +448,15 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& appendKey(key, ~0u); appendKey(key, sets); appendKey(key, pushStagesOf(context)); - if ((sets & 1u) != 0u) { + if ((sets & MainSetBit) != 0u) { appendKey(key, resources.mainKey.size()); for (const auto word : resources.mainKey) appendKey(key, word); } - if ((sets & 2u) != 0u) { + if ((sets & HeapSetBit) != 0u) { appendKey(key, resources.fragmentKey.size()); for (const auto word : resources.fragmentKey) appendKey(key, word); } + if ((sets & DataSetBit) != 0u) appendKey(key, DataSetKeyTag); }; splitKey(keys.layout, 3u); splitKey(keys.preRasterizationLayout, keys.preRasterizationSets); @@ -454,7 +468,7 @@ bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) : Pipeline(context, state, vertexInput, resources.SetLayouts(), shaders, attachmentLayout) {} -Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) { +Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const PipelineSetLayouts& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), pushPlacements(PushDataPlacements(shaders)), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) { PerformanceTimer timing("Vulkan.GraphicsPipeline"); for (const auto& color : state.colors) boundAttachments.push_back(color.partialWriteMask == 0u); if (state.depth) boundAttachments.push_back(true); @@ -506,6 +520,17 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput std::vector stages(shaders.size()); std::vector specializations; specializations.reserve(shaders.size()); + std::vector> heapMappings(shaders.size()); + std::vector heapMappingInfos(shaders.size(), VkShaderDescriptorSetAndBindingMappingInfoEXT{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}); + if (context.descriptorHeap && DescriptorHeapEnabled() && AnyHeapShader(shaders)) { + auto& heap = BindlessHeap::For(context); + const bool rooted = RootedPushData(shaders); + descriptorHeap = std::all_of(shaders.begin(), shaders.end(), [&](const CompiledShader& shader) { + const auto index = static_cast(&shader - shaders.data()); + return heap.DescriptorHeapMappings(std::span(shader.program->spirv.data(), shader.program->spirv.size()), heapMappings[index], ~0u, shader.program->dataAddressPush, PushesGeneratedFaultAddress(shader, rooted), shader.program->functionLdsBuffer, shader.program->meshWindowBuffer, false, &pushPlacements[index]); + }); + } + Require(descriptorHeap || !AnyHeapShader(shaders), "a pipeline with GPU-mode stages does not map onto VK_EXT_descriptor_heap"); const VkPipelineShaderStageRequiredSubgroupSizeCreateInfoEXT meshSubgroup{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_REQUIRED_SUBGROUP_SIZE_CREATE_INFO_EXT, nullptr, 32u}; for (std::uint32_t i = 0; i < shaders.size(); ++i) { const auto& shader = *shaders[i].program; @@ -516,13 +541,21 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput stages[i].module = _modules[i]; stages[i].pName = "main"; if (shaders[i].stage == ShaderRecompiler::ShaderStage::Mesh && context.meshWave32 && shader.hostSubgroupSize == 32u) stages[i].pNext = &meshSubgroup; + if (descriptorHeap) { + heapMappingInfos[i].mappingCount = static_cast(heapMappings[i].size()); + heapMappingInfos[i].pMappings = heapMappings[i].data(); + heapMappingInfos[i].pNext = stages[i].pNext; + stages[i].pNext = &heapMappingInfos[i]; + } specializations.emplace_back(shader, state.meshLayerBase, state.meshLayerCount); + specializations.back().PlacePushData(pushPlacements[i]); + if (descriptorHeap) specializations.back().PlaceHeapImageSlots(BindlessHeap::For(context).ImageSlots()); stages[i].pSpecializationInfo = specializations.back().Info(); } if (std::ranges::any_of(shaders, [](const CompiledShader& shader) { return shader.program != nullptr && !shader.program->stencilReferences.empty(); })) { for (std::size_t i = 0; i < shaders.size(); ++i) { if (shaders[i].stage != ShaderRecompiler::ShaderStage::Vertex) continue; - Require(stages[i].flags == 0u && stages[i].pNext == nullptr, "stencil replay vertex stage allows varying subgroup size"); + Require(stages[i].flags == 0u && (stages[i].pNext == nullptr || stages[i].pNext == &heapMappingInfos[i]), "stencil replay vertex stage allows varying subgroup size"); } } // A descriptor set layout with the same bindings as this one is compatible with the pipeline @@ -534,12 +567,12 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput const VkPushConstantRange push{pushStages, 0, PipelinePushSlotBytes}; const std::array slots{{{pushStagesOf(context), 0, PipelinePushSlotBytes}, {VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes}}}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - layoutInfo.flags = pixelLds || meshWindow || !resources.split ? 0 : VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT; + layoutInfo.flags = !pixelLds && !meshWindow && resources.split && context.graphicsPipelineLibrary ? VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT : 0; layoutInfo.setLayoutCount = meshWindow ? 4u : pixelLds ? 3u : resources.split ? 2u : 1u; layoutInfo.pSetLayouts = setLayouts.data(); layoutInfo.pushConstantRangeCount = dynamicRendering ? static_cast(slots.size()) : pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = dynamicRendering ? slots.data() : pushStages != 0 ? &push : nullptr; - Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics"); + if (!descriptorHeap) Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics"); std::vector colors; std::vector references(state.blends.size(), VkAttachmentReference{VK_ATTACHMENT_UNUSED, attachmentLayout}); for (std::uint32_t index = 0; index < state.colors.size(); ++index) { @@ -625,7 +658,7 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput viewports.viewportCount = 1; viewports.scissorCount = 1; PipelineLibraryKeys keys; - libraries = libraryKeys(keys, context, state, vertexInput, resources, shaders); + libraries = libraryKeys(keys, context, state, vertexInput, resources, shaders, pushPlacements); std::vector dynamicStates{VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR}; if (state.stencilReplayReference) dynamicStates.push_back(VK_DYNAMIC_STATE_STENCIL_REFERENCE); if (depthBounds) dynamicStates.push_back(VK_DYNAMIC_STATE_DEPTH_BOUNDS); @@ -732,6 +765,13 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput pipelineInfo.layout = layout; pipelineInfo.renderPass = renderPass; if (dynamicRendering) pipelineInfo.pNext = &rendering; + VkPipelineCreateFlags2CreateInfoKHR heapFlags{}; + if (descriptorHeap) { + heapFlags = DescriptorHeapFlags(static_cast(pipelineInfo.flags), pipelineInfo.pNext); + pipelineInfo.pNext = &heapFlags; + pipelineInfo.flags = 0; + pipelineInfo.layout = VK_NULL_HANDLE; + } timing.Mark("modules_and_state"); if (prebuilding && !libraries) { for (const auto module : _modules) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); @@ -739,14 +779,14 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput return; } if (prebuilding) { - static_cast(LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, true, _modules)); + static_cast(LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, descriptorHeap, &optimized, true, _modules)); for (const auto module : _modules) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); _modules.clear(); prebuilt = true; return; } const VkPipelineCache pipelineCache = state.stages.tessellation && !context.tessellationPipelineCache ? VK_NULL_HANDLE : context.pipelineCache; - if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized, false, _modules); + if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, descriptorHeap, &optimized, false, _modules); else { auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateGraphicsPipelines")(context.device, pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); @@ -982,12 +1022,17 @@ void Pipeline::ContinueBackFaces(VkCommandBuffer commands, const State& state) c void Pipeline::PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const { if (stages == 0) return; + if (descriptorHeap) { + const VkPushDataInfoEXT push{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {bytes.data(), bytes.size()}}; + context.Function("vkCmdPushDataEXT")(commands, &push); + return; + } if (dynamicRendering) { - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, pushStagesOf(context), 0, PipelinePushSlotBytes, bytes.data()); - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes, bytes.data() + PipelinePushSlotBytes); + CmdPushConstants(context, commands, layout, pushStagesOf(context), 0, PipelinePushSlotBytes, bytes.data()); + CmdPushConstants(context, commands, layout, VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes, bytes.data() + PipelinePushSlotBytes); return; } - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushSlotBytes, bytes.data()); + CmdPushConstants(context, commands, layout, stages, 0, PipelinePushSlotBytes, bytes.data()); } namespace { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp index 5dc8aca34c64..706463c880c7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/PipelineJournal.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libc/include/HostThreadLocal.hpp" @@ -894,8 +895,9 @@ class Journal { if (error) throw std::filesystem::filesystem_error("cannot resolve the pipeline journal directory", directory, error); directory = native; #endif - path = directory / "pipelines.journal"; - lockPath = directory / "pipelines.journal.lock"; + const std::string name = GpuHeapEnabled() ? "pipelines-heap.journal" : "pipelines.journal"; + path = directory / name; + lockPath = directory / (name + ".lock"); maxBytes = configured("APS5_PIPELINE_JOURNAL_MAX_MIB", 1024u) << 20u; std::thread([this] { run(); }).detach(); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp index f034f09428ac..806a131d027c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp @@ -112,10 +112,10 @@ std::span PipelineLibraryDynamicStates() { return dynamicStates; } -VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized, bool partsOnly, std::span modules) { +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, bool descriptorHeap, std::shared_ptr* optimized, bool partsOnly, std::span modules) { Require(info.renderPass == VK_NULL_HANDLE && info.pDynamicState != nullptr, "pipeline library parts need dynamic rendering and the library dynamic states"); const bool split = (layout.flags & VK_PIPELINE_LAYOUT_CREATE_INDEPENDENT_SETS_BIT_EXT) != 0; - Require(!split || (layout.setLayoutCount == 2u && layout.pSetLayouts != nullptr && ((keys.preRasterizationSets | keys.fragmentSets) & ~3u) == 0u), "split pipeline library layout requires two descriptor set slots and a valid mask"); + Require(!split || ((layout.setLayoutCount == 2u || layout.setLayoutCount == 3u) && layout.pSetLayouts != nullptr && ((keys.preRasterizationSets | keys.fragmentSets) & ~((1u << layout.setLayoutCount) - 1u)) == 0u), "split pipeline library layout requires two or three descriptor set slots and a valid mask"); auto& store = Libraries(); std::unique_lock lock(store.mutex); ++store.activeBuilds[context.device]; @@ -138,9 +138,9 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip auto found = device.layouts.find(key); if (found != device.layouts.end()) return found->second; auto info = layout; - std::array sets{VK_NULL_HANDLE, VK_NULL_HANDLE}; + std::array sets{VK_NULL_HANDLE, VK_NULL_HANDLE, VK_NULL_HANDLE}; if (mask.has_value()) { - for (std::uint32_t set = 0; set < sets.size(); ++set) { + for (std::uint32_t set = 0; set < layout.setLayoutCount; ++set) { if ((*mask & (1u << set)) != 0u) sets[set] = layout.pSetLayouts[set]; } info.pSetLayouts = sets.data(); @@ -149,8 +149,10 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip Check(context.Function("vkCreatePipelineLayout")(context.device, &info, nullptr, &handle), "vkCreatePipelineLayout library"); return device.layouts.emplace(key, handle).first->second; }; - const auto fullLayout = layoutFor(keys.layout, std::nullopt); - const std::array partLayouts{VK_NULL_HANDLE, split ? layoutFor(keys.preRasterizationLayout, keys.preRasterizationSets) : fullLayout, split ? layoutFor(keys.fragmentLayout, keys.fragmentSets) : fullLayout, VK_NULL_HANDLE}; + const std::array memoryMarker{std::byte{descriptorHeap ? std::uint8_t{2} : std::uint8_t{0}}}; + const Key memoryKey(memoryMarker.begin(), memoryMarker.end()); + const auto fullLayout = descriptorHeap ? VkPipelineLayout{VK_NULL_HANDLE} : layoutFor(Combined(keys.layout, memoryKey), std::nullopt); + const std::array partLayouts{VK_NULL_HANDLE, split && !descriptorHeap ? layoutFor(Combined(keys.preRasterizationLayout, memoryKey), keys.preRasterizationSets) : fullLayout, split && !descriptorHeap ? layoutFor(Combined(keys.fragmentLayout, memoryKey), keys.fragmentSets) : fullLayout, VK_NULL_HANDLE}; std::vector preRasterization; const VkPipelineShaderStageCreateInfo* fragment = nullptr; bool mesh = false; @@ -167,7 +169,8 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT, VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT, }; - const std::array partKeys{&keys.vertexInput, &keys.preRasterization, &keys.fragmentShader, &keys.fragmentOutput}; + std::array markedParts{Combined(keys.vertexInput, memoryKey), Combined(keys.preRasterization, memoryKey), Combined(keys.fragmentShader, memoryKey), Combined(keys.fragmentOutput, memoryKey)}; + const std::array partKeys{&markedParts[0], &markedParts[1], &markedParts[2], &markedParts[3]}; const Key none; VkPipelineRenderingCreateInfoKHR shaderRendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR}; shaderRendering.viewMask = rendering.viewMask; @@ -197,6 +200,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip library.flags = flags[part]; VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library}; create.flags = optimize ? VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT : VK_PIPELINE_CREATE_LIBRARY_BIT_KHR; + auto partFlags = DescriptorHeapFlags(static_cast(create.flags), &library); create.pDynamicState = info.pDynamicState; create.layout = partLayouts[part]; switch (flags[part]) { @@ -224,6 +228,11 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip create.layout = VK_NULL_HANDLE; break; } + if (descriptorHeap) { + create.pNext = &partFlags; + create.flags = 0; + create.layout = VK_NULL_HANDLE; + } VkPipeline handle = VK_NULL_HANDLE; // Built without the store's mutex: a draw whose parts are resident is not held behind // another thread's build, and builds from several threads (the registration prebuild, the @@ -272,11 +281,18 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip linked.libraryCount = static_cast(libraries.size()); linked.pLibraries = libraries.data(); VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked}; + create.flags = 0; + create.layout = info.layout; + auto linkFlags = DescriptorHeapFlags(0, &linked); + if (descriptorHeap) { + create.pNext = &linkFlags; + create.flags = 0; + create.layout = VK_NULL_HANDLE; + } if (sampleCounts) { linkedSamples.pNext = create.pNext; create.pNext = &linkedSamples; } - create.layout = info.layout; VkPipeline pipeline = VK_NULL_HANDLE; lock.unlock(); { @@ -293,20 +309,25 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip const auto createPipelines = context.Function("vkCreateGraphicsPipelines"); const auto cache = context.pipelineCache; const auto cacheSynchronization = context.pipelineCacheSynchronization; - const auto libraryLayout = fullLayout; - std::function relink = [slot, createPipelines, cache, cacheSynchronization, libraryLayout, libraries, sampleCounts, attachmentSamples, &store, devicePointer = &device] { + const auto libraryLayout = descriptorHeap ? VkPipelineLayout{VK_NULL_HANDLE} : fullLayout; + std::function relink = [slot, createPipelines, cache, cacheSynchronization, libraryLayout, libraries, sampleCounts, attachmentSamples, &store, devicePointer = &device, descriptorHeap] { if (slot->released.load()) return; auto linkedSamples = sampleCounts ? attachmentSamples(*sampleCounts) : VkAttachmentSampleCountInfoNV{}; VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR}; linked.libraryCount = static_cast(libraries.size()); linked.pLibraries = libraries.data(); VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked}; + create.flags = VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT; + create.layout = libraryLayout; + auto relinkFlags = DescriptorHeapFlags(static_cast(VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT), &linked); + if (descriptorHeap) { + create.pNext = &relinkFlags; + create.flags = 0; + } if (sampleCounts) { linkedSamples.pNext = create.pNext; create.pNext = &linkedSamples; } - create.flags = VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT; - create.layout = libraryLayout; VkPipeline result = VK_NULL_HANDLE; const auto pipelineCreationResult = [&] { auto pipelineCacheLock = cacheSynchronization != nullptr ? cacheSynchronization->LockForPipelineUse() : PipelineCacheSynchronization::PipelineUseLock{}; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index c1955ce0c14e..65fbc4213a8a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -8,10 +8,12 @@ #include "prx/libSceAgcDriver/Graphics/include/ReferenceStorage.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libkernel/Time/include/SleepWake.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/RuntimeSwitches.hpp" #include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/StagingCopyBack_spv.h" @@ -680,6 +682,22 @@ bool SnapshotOverlaps(std::uint64_t address, std::size_t bytes) { return SnapshotOverlaps(snapshot.get(), address, bytes); } +} + +bool Recorder::ReadsSettled(std::span> ranges, const WriteRanges* before) { + if (!HookSnapshotEnabled()) return false; + const bool labels = !QueuedLabelRanges().empty(); + const auto now = pendingWrites.load(std::memory_order_acquire); + for (const auto& [begin, end] : ranges) { + const auto bytes = static_cast(end - begin); + if (labels && QueuedLabelOverlaps(begin, bytes)) return false; + if (AgcDriver::Graphics::SnapshotOverlaps(before, begin, bytes) || AgcDriver::Graphics::SnapshotOverlaps(now.get(), begin, bytes)) return false; + } + return !StorageTexture::AnyPendingOverlaps(ranges) && !AnyShadowedOverlaps(ranges); +} + +namespace { + // Whether one merged range of the snapshot contains [address, end) entirely. bool SnapshotCovers(const WriteRanges* snapshot, std::uint64_t address, std::uint64_t end) { if (snapshot == nullptr || snapshot->empty()) return false; @@ -1175,7 +1193,6 @@ Recorder::Recorder(const Context& context, bool timelineSemaphores) : context(co cmdResetQueryPool = context.Function("vkCmdResetQueryPool"); cmdCopyQueryPoolResults = context.Function("vkCmdCopyQueryPoolResults"); cmdBindPipeline = context.Function("vkCmdBindPipeline"); - cmdPushConstants = context.Function("vkCmdPushConstants"); cmdDispatch = context.Function("vkCmdDispatch"); } if (!timelineSemaphores) return; @@ -1242,8 +1259,10 @@ Recorder::~Recorder() { if (labelTableOwner == this) labelTableOwner = nullptr; } if (meshArgumentPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, meshArgumentPipeline, nullptr); + if (meshArgumentHeapPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, meshArgumentHeapPipeline, nullptr); if (meshArgumentLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, meshArgumentLayout, nullptr); if (changedCopyPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, changedCopyPipeline, nullptr); + if (changedCopyHeapPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, changedCopyHeapPipeline, nullptr); if (changedCopyLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, changedCopyLayout, nullptr); if (sampleCounterState > 0) { samplesPassed.store(SamplesTotal(), std::memory_order_release); @@ -1256,6 +1275,7 @@ Recorder::~Recorder() { sampleCounter.reset(); } if (dispatchArgumentPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, dispatchArgumentPipeline, nullptr); + if (dispatchArgumentHeapPipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, dispatchArgumentHeapPipeline, nullptr); if (dispatchArgumentLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, dispatchArgumentLayout, nullptr); for (const auto fence : spare) context.Function("vkDestroyFence")(context.device, fence, nullptr); spare.clear(); @@ -1790,6 +1810,7 @@ void Recorder::ensureOpen() { } VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + NoteCommandBufferBegun(batch->commands); Check(function(beginCommandBuffer, "vkBeginCommandBuffer")(batch->commands, &begin), "vkBeginCommandBuffer recorder"); } catch (...) { release(*batch); @@ -2304,6 +2325,7 @@ bool Recorder::RecordDispatchArguments(VkCommandBuffer commands, VkDeviceAddress pipelineInfo.stage.pName = "main"; pipelineInfo.layout = dispatchArgumentLayout; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &dispatchArgumentPipeline), "vkCreateComputePipelines dispatch arguments"); + dispatchArgumentHeapPipeline = createPushOnlyHeapPipeline(pipelineInfo); context.Function("vkDestroyShaderModule")(context.device, module, nullptr); dispatchArgumentState = 1; } catch (const std::exception& error) { @@ -2313,7 +2335,7 @@ bool Recorder::RecordDispatchArguments(VkCommandBuffer commands, VkDeviceAddress } if (dispatchArgumentState < 0) return false; recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); - function(cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, dispatchArgumentPipeline); + bindPushOnlyPipeline(commands, dispatchArgumentPipeline, dispatchArgumentHeapPipeline); const auto* limit = context.limits.maxComputeWorkGroupCount; struct { VkDeviceAddress source; @@ -2323,7 +2345,7 @@ bool Recorder::RecordDispatchArguments(VkCommandBuffer commands, VkDeviceAddress } parameters{source, arguments, {limit[0], limit[1], limit[2], threadGroup != nullptr ? 1u : 0u}, {1u, 1u, 1u, 0u}}; if (threadGroup != nullptr) std::copy(threadGroup->begin(), threadGroup->end(), parameters.group.begin()); static_assert(sizeof(parameters) == PushBytes); - function(cmdPushConstants, "vkCmdPushConstants")(commands, dispatchArgumentLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, PushBytes, ¶meters); + pushOnlyParameters(commands, dispatchArgumentHeapPipeline, dispatchArgumentLayout, PushBytes, ¶meters); function(cmdDispatch, "vkCmdDispatch")(commands, 1, 1, 1); const VkPipelineStageFlags readers = VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | (threadGroup != nullptr ? VK_PIPELINE_STAGE_TRANSFER_BIT : 0u); const VkAccessFlags reads = VK_ACCESS_INDIRECT_COMMAND_READ_BIT | (threadGroup != nullptr ? VK_ACCESS_TRANSFER_READ_BIT : 0u); @@ -2636,15 +2658,14 @@ void Recorder::dispatchDumps(VkCommandBuffer commands, std::uint32_t firstDump, std::uint32_t phase; } parameters{sampleCounter->DeviceAddress(), sampleCounter->DeviceAddress() + DumpListOffset, firstDump, dumps, firstSegment, lastSegment, 0u}; static_assert(sizeof(parameters) == SampleDumpPushBytes); - const auto push = function(cmdPushConstants, "vkCmdPushConstants"); const auto dispatch = function(cmdDispatch, "vkCmdDispatch"); - push(commands, dumpLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes, ¶meters); + CmdPushConstants(context, commands, dumpLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes, ¶meters); dispatch(commands, 1, 1, 1); if (dumps == 0) return; const auto publish = [&](std::uint32_t phase) { recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT); parameters.phase = phase; - push(commands, dumpLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes, ¶meters); + CmdPushConstants(context, commands, dumpLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes, ¶meters); dispatch(commands, 1, 1, 1); }; publish(1u); @@ -2687,7 +2708,6 @@ void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { const auto commands = batch.commands; const auto copy = function(cmdCopyQueryPoolResults, "vkCmdCopyQueryPoolResults"); const auto bind = function(cmdBindPipeline, "vkCmdBindPipeline"); - const auto push = function(cmdPushConstants, "vkCmdPushConstants"); const auto dispatch = function(cmdDispatch, "vkCmdDispatch"); std::size_t done = 0; do { @@ -2709,13 +2729,13 @@ void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { std::uint32_t store; } parameters{sampleCounter->DeviceAddress(), target, count, last && target != 0 ? 1u : 0u}; static_assert(sizeof(parameters) == SampleCounterPushBytes); - push(commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); + CmdPushConstants(context, commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); dispatch(commands, 1, 1, 1); if (parameters.store == 1u) { const auto publish = [&](std::uint32_t store) { recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT); parameters.store = store; - push(commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); + CmdPushConstants(context, commands, sampleLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleCounterPushBytes, ¶meters); dispatch(commands, 1, 1, 1); }; publish(3u); @@ -2784,6 +2804,7 @@ bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress rec auto pipelineCacheLock = context.LockPipelineCacheForUse(); Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &meshArgumentPipeline), "vkCreateComputePipelines mesh arguments"); } + meshArgumentHeapPipeline = createPushOnlyHeapPipeline(pipelineInfo); context.Function("vkDestroyShaderModule")(context.device, module, nullptr); meshArgumentState = 1; } catch (const std::exception& error) { @@ -2793,20 +2814,49 @@ bool Recorder::RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress rec } if (meshArgumentState < 0) return false; recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); - function(cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, meshArgumentPipeline); + bindPushOnlyPipeline(commands, meshArgumentPipeline, meshArgumentHeapPipeline); struct { VkDeviceAddress record; VkDeviceAddress arguments; std::array rules; } parameters{record, arguments, {rules[0], rules[1], rules[2], rules[3], rules[4], rules[5], rules[6], 0u}}; static_assert(sizeof(parameters) == PushBytes); - function(cmdPushConstants, "vkCmdPushConstants")(commands, meshArgumentLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, PushBytes, ¶meters); + pushOnlyParameters(commands, meshArgumentHeapPipeline, meshArgumentLayout, PushBytes, ¶meters); function(cmdDispatch, "vkCmdDispatch")(commands, 1, 1, 1); recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT | VK_PIPELINE_STAGE_MESH_SHADER_BIT_EXT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_INDIRECT_COMMAND_READ_BIT | VK_ACCESS_SHADER_READ_BIT); CountBarriers(CommandClass::Draw, 2); return true; } +VkPipeline Recorder::createPushOnlyHeapPipeline(const VkComputePipelineCreateInfo& info) const { + if (!DescriptorHeapEnabled() || !context.descriptorHeap || !BindlessHeap::For(context).DescriptorHeapMode()) return VK_NULL_HANDLE; + auto heapFlags = DescriptorHeapFlags(); + auto heapInfo = info; + heapInfo.pNext = &heapFlags; + heapInfo.layout = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; + try { + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + if (context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &heapInfo, nullptr, &pipeline) != VK_SUCCESS) return VK_NULL_HANDLE; + } catch (const std::exception&) { + return VK_NULL_HANDLE; + } + return pipeline; +} + +void Recorder::bindPushOnlyPipeline(VkCommandBuffer commands, VkPipeline pipeline, VkPipeline heapPipeline) const { + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline != VK_NULL_HANDLE ? heapPipeline : pipeline); +} + +void Recorder::pushOnlyParameters(VkCommandBuffer commands, VkPipeline heapPipeline, VkPipelineLayout layout, std::uint32_t bytes, const void* values) const { + if (heapPipeline == VK_NULL_HANDLE) { + CmdPushConstants(context, commands, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, bytes, values); + return; + } + const VkPushDataInfoEXT data{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {values, bytes}}; + context.Function("vkCmdPushDataEXT")(commands, &data); +} + bool Recorder::RecordChangedCopy(VkCommandBuffer commands, VkDeviceAddress result, VkDeviceAddress snapshot, VkDeviceAddress target, std::uint32_t dwords, VkDeviceAddress counter) { constexpr std::uint32_t PushBytes = 40; constexpr std::uint32_t GroupSize = 64; @@ -2830,6 +2880,7 @@ bool Recorder::RecordChangedCopy(VkCommandBuffer commands, VkDeviceAddress resul pipelineInfo.stage.pName = "main"; pipelineInfo.layout = changedCopyLayout; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &changedCopyPipeline), "vkCreateComputePipelines changed copy"); + changedCopyHeapPipeline = createPushOnlyHeapPipeline(pipelineInfo); context.Function("vkDestroyShaderModule")(context.device, module, nullptr); changedCopyState = 1; } catch (const std::exception& error) { @@ -2839,7 +2890,7 @@ bool Recorder::RecordChangedCopy(VkCommandBuffer commands, VkDeviceAddress resul } if (changedCopyState < 0) return false; static const bool alwaysCopy = std::getenv("APS5_CHANGED_COPY_ALWAYS") != nullptr; - function(cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, changedCopyPipeline); + bindPushOnlyPipeline(commands, changedCopyPipeline, changedCopyHeapPipeline); const auto perDispatch = static_cast(std::min(std::uint64_t{context.limits.maxComputeWorkGroupCount[0]} * GroupSize, std::numeric_limits::max() - GroupSize)); for (std::uint32_t done = 0; done < dwords;) { const auto count = std::min(dwords - done, perDispatch); @@ -2852,7 +2903,7 @@ bool Recorder::RecordChangedCopy(VkCommandBuffer commands, VkDeviceAddress resul std::uint32_t always; } parameters{result + VkDeviceAddress{done} * 4u, snapshot + VkDeviceAddress{done} * 4u, target + VkDeviceAddress{done} * 4u, counter, count, alwaysCopy ? 1u : 0u}; static_assert(sizeof(parameters) == PushBytes); - function(cmdPushConstants, "vkCmdPushConstants")(commands, changedCopyLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, PushBytes, ¶meters); + pushOnlyParameters(commands, changedCopyHeapPipeline, changedCopyLayout, PushBytes, ¶meters); function(cmdDispatch, "vkCmdDispatch")(commands, (count + GroupSize - 1) / GroupSize, 1, 1); done += count; } @@ -3046,6 +3097,12 @@ void Recorder::KeepBytes(const void* owner, std::size_t bytes) { if (open->keptOwners.insert(owner).second) open->keptBytes += bytes; } +void Recorder::BoundDrawBatch() { + BoundKeptBytes(); + if (open == nullptr || !DescriptorHeapEnabled() || !GuestMemory::GpuMutex().HeldByThisThread()) return; + if (auto* heap = BindlessHeap::Existing(context); heap != nullptr && heap->TakeRingPressure()) Submit(); +} + void Recorder::BoundKeptBytes() { if (!GuestMemory::GpuMutex().HeldByThisThread()) return; if (open != nullptr && open->keptBytes >= KeptBytesBudget) Submit(); @@ -3970,6 +4027,7 @@ void Recorder::Submit() { VkSubmitInfo submission{VK_STRUCTURE_TYPE_SUBMIT_INFO}; submission.commandBufferCount = 1; submission.pCommandBuffers = &batch->commands; + if (auto* heap = DescriptorHeapEnabled() ? BindlessHeap::Existing(context) : nullptr; heap != nullptr) heap->Submitted(*this, submissions + 1); // The timeline reaches this batch's serial when it completes (see WaitSerial). const std::uint64_t serial = submissions + 1; std::string tokensStr = "["; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ResidentMirror.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ResidentMirror.cpp index cd0c5df69985..ce95e3c2af29 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ResidentMirror.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ResidentMirror.cpp @@ -410,8 +410,7 @@ ResidentBuffer::ResidentBuffer(const Context& context, std::uint64_t base, std:: index = last; } submit(binds); - const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; - address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + address = BufferDeviceAddress(context, buffer); if (address == 0) fail("vkGetBufferDeviceAddressKHR of a resident heap mirror", VK_ERROR_UNKNOWN); } catch (...) { release(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index f4b6a4189490..c83ee13ead79 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" @@ -50,6 +51,11 @@ VkBuffer BindExactBuffer(const Context& context, std::size_t size, VkBufferUsage return buffer; } +VkBufferUsageFlags UsageWithoutUnsupportedAddress(const Context& context, VkBufferUsageFlags requested, VkBufferUsageFlags pooled) { + if (!context.bufferDeviceAddress && (requested & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) == 0) return pooled & ~static_cast(VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + return pooled; +} + } Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : Buffer(context, size, usage, properties, false) {} @@ -59,9 +65,9 @@ std::shared_ptr Buffer::WithinDeviceLocalHeadroom(const Context& context return buffer->ready ? buffer : nullptr; } -Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties, bool headroom) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(BufferPool::Usage(usage, properties)), properties(properties) { +Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties, bool headroom) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(UsageWithoutUnsupportedAddress(context, usage, BufferPool::Usage(usage, properties))), properties(properties) { Require(size != 0, "zero-sized GPU buffer"); - const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; + const bool addressable = (this->usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; Require(!addressable || context.bufferDeviceAddress, "buffer device address is not enabled"); cache = GetBufferPool(context); if (const auto allocation = cache->Take(size, this->usage, properties, "vkAllocateMemory buffer")) { @@ -82,7 +88,14 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag deviceAddress = 0; try { buffer = BindExactBuffer(context, size, this->usage, memory, offset, allocationBytes); - initializeAddress(usage); + initializeAddress(this->usage); + } catch (...) { + release(); + throw; + } + } else if (deviceAddress == 0) { + try { + initializeAddress(this->usage); } catch (...) { release(); throw; @@ -120,7 +133,7 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag slab = true; allocationBytes = capacity; Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, offset), "vkBindBufferMemory slab"); - initializeAddress(usage); + initializeAddress(this->usage); mapping = slot.mapping; #ifdef __APPLE__ ClearPooledTail(mapping, size, capacity, properties); @@ -130,7 +143,7 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag } memory = cache->AllocateMemory(allocation, "vkAllocateMemory buffer"); Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory"); - initializeAddress(usage); + initializeAddress(this->usage); if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) Check(context.Function("vkMapMemory")(context.device, memory, 0, VK_WHOLE_SIZE, 0, &mapping), "vkMapMemory"); ready = true; } catch (...) { @@ -176,7 +189,7 @@ void Buffer::Invalidate() { Check(context.Function("vkInvalidateMappedMemoryRanges")(context.device, 1, &range), "vkInvalidateMappedMemoryRanges"); } -DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(BufferPool::Usage(usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { +DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(UsageWithoutUnsupportedAddress(context, usage, BufferPool::Usage(usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT))) { Require(size != 0, "zero-sized device buffer"); cache = GetBufferPool(context); auto pooled = cache->Take(size, this->usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, "vkAllocateMemory device buffer"); @@ -220,6 +233,8 @@ DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsa allocation.allocationSize = ClassAllocationBytes(requirements, capacity); allocationBytes = allocation.allocationSize; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, allocation.allocationSize, "device buffer"); + const VkMemoryAllocateFlagsInfo addressFlags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + if ((this->usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0) allocation.pNext = &addressFlags; const auto deviceType = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); std::uint32_t allocatedType = allocation.memoryTypeIndex; memory = cache->AllocateMemory(allocation, "vkAllocateMemory device buffer", context.RefusedDeviceMemoryFallback(requirements.memoryTypeBits, allocation.memoryTypeIndex), &allocatedType); @@ -390,6 +405,7 @@ CommandBatch::CommandBatch(const Context& context) : context(context) { fence = created; VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO}; begin.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + NoteCommandBufferBegun(commands); Check(context.Function("vkBeginCommandBuffer")(commands, &begin), "vkBeginCommandBuffer"); // Work recorded so far goes first in queue order, so this batch sees its results. if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->Recording()) recorder->Submit(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/SampleOrder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/SampleOrder.cpp index 9c6302d6730c..428af2c39e34 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/SampleOrder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/SampleOrder.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/SampleOrder.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" @@ -18,10 +19,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include @@ -44,6 +47,20 @@ struct DepthPush { std::uint32_t bit; }; +struct HeapStoragePush { + StoragePush push; + std::uint32_t image; +}; + +struct HeapDepthPush { + DepthPush push; + std::uint32_t depth; + std::uint32_t stencil; +}; + +static_assert(offsetof(HeapStoragePush, image) == 28); +static_assert(offsetof(HeapDepthPush, depth) == 20 && offsetof(HeapDepthPush, stencil) == 24); + bool exactDepth(VkFormat format) { return format == VK_FORMAT_D16_UNORM || format == VK_FORMAT_D32_SFLOAT || format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D32_SFLOAT_S8_UINT; } @@ -90,7 +107,7 @@ void transitionLocated(const Context& context, VkCommandBuffer commands, VkImage context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, sourceStages, destinationStages, 0, 0, nullptr, 0, nullptr, 1, &barrier); } -class CopyImage { +class CopyImage : public HeapSlotOwner { public: CopyImage(const Context& context, VkFormat format, std::uint32_t samples, VkExtent2D extent, bool stencil) : context(context), format(format), samples(samples), extent(extent) { try { @@ -147,6 +164,7 @@ class CopyImage { } void release() noexcept { + ReleaseHeapSlots(context); if (stencilView) context.Function("vkDestroyImageView")(context.device, stencilView, nullptr); if (depthView) context.Function("vkDestroyImageView")(context.device, depthView, nullptr); if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); @@ -165,6 +183,20 @@ class SampleOrders { void Storage(VkCommandBuffer commands, const SampleOrderTarget& target, const std::array& map, VkExtent2D region) { const auto pipeline = storagePipeline(target.format); + const StoragePush push{map, {region.width, region.height}, target.samples}; + const auto dispatch = [&] { context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, ((region.width + 1u) / 2u + 7u) / 8u, ((region.height + 1u) / 2u + 7u) / 8u, 1); }; + const auto heapPipeline = heapStoragePipelines[storageIndex(target.format)]; + if (heapPipeline != VK_NULL_HANDLE && target.owner != nullptr) { + auto& heap = BindlessHeap::For(context); + if (const auto slot = heap.ViewSlot(HeapPassStorageKind, target.owner, target.view)) { + heap.BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + const HeapStoragePush pushed{push, *slot}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + dispatch(); + return; + } + } const auto set = allocate(storageSetLayout, {{VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}}); const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, target.view, VK_IMAGE_LAYOUT_GENERAL}; VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; @@ -174,11 +206,10 @@ class SampleOrders { write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; write.pImageInfo = &imageInfo; context.Function("vkUpdateDescriptorSets")(context.device, 1, &write, 0, nullptr); - const StoragePush push{map, {region.width, region.height}, target.samples}; context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, storagePipelineLayout, 0, 1, &set, 0, nullptr); - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, storagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); - context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, ((region.width + 1u) / 2u + 7u) / 8u, ((region.height + 1u) / 2u + 7u) / 8u, 1); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, storagePipelineLayout, 0, 1, &set, SetBindSite::Detiler); + CmdPushConstants(context, commands, storagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); + dispatch(); } void Depth(VkCommandBuffer commands, const SampleOrderTarget& target, const std::array& map, VkExtent2D region, const SampleOrderPass& order, std::vector>& kept) { @@ -213,18 +244,16 @@ class SampleOrders { if (located) transitionLocated(context, commands, target.image, target.format, info, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, fragmentStages, depthAccess); RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, fragmentStages, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | depthAccess); const auto& pass = depthPass(target.format, target.samples, target.method, located, locations); - const auto set = allocate(sampledSetLayout, {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, setsPerPool * 2u}}); - std::array images{{{VK_NULL_HANDLE, copy->depthView, VK_IMAGE_LAYOUT_GENERAL}, {VK_NULL_HANDLE, stencil ? copy->stencilView : copy->depthView, VK_IMAGE_LAYOUT_GENERAL}}}; - std::array writes{}; - for (std::uint32_t binding = 0; binding < writes.size(); ++binding) { - writes[binding].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[binding].dstSet = set; - writes[binding].dstBinding = binding; - writes[binding].descriptorCount = 1; - writes[binding].descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; - writes[binding].pImageInfo = &images[binding]; + std::optional depthSlot; + std::optional stencilSlot; + if (pass.heapDepth != VK_NULL_HANDLE) { + auto& heap = BindlessHeap::For(context); + depthSlot = heap.ViewSlot(HeapPassSampledKind, copy.get(), copy->depthView); + stencilSlot = depthSlot && stencil ? heap.ViewSlot(HeapPassSampledKind, copy.get(), copy->stencilView) : depthSlot; } - context.Function("vkUpdateDescriptorSets")(context.device, stencil ? 2u : 1u, writes.data(), 0, nullptr); + const bool heapPass = depthSlot && stencilSlot; + VkDescriptorSet set = VK_NULL_HANDLE; + if (!heapPass) set = writeDepthSet(*copy, stencil); const std::array views{target.view}; auto framebuffer = std::make_shared(context, pass.renderPass, views, region); kept.push_back(framebuffer); @@ -245,18 +274,24 @@ class SampleOrders { const VkRect2D scissor{{0, 0}, region}; context.Resolved(&DeviceFunctions::cmdSetViewport, "vkCmdSetViewport")(commands, 0, 1, &viewport); context.Resolved(&DeviceFunctions::cmdSetScissor, "vkCmdSetScissor")(commands, 0, 1, &scissor); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, sampledPipelineLayout, 0, 1, &set, 0, nullptr); - const auto draw = [&](VkPipeline pipeline, std::uint32_t bit, std::uint32_t writeMask, std::uint32_t reference) { - context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + if (heapPass) BindlessHeap::For(context).BindDescriptorHeaps(commands); + else CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, sampledPipelineLayout, 0, 1, &set, SetBindSite::Detiler); + const auto draw = [&](VkPipeline pipeline, VkPipeline heapPipeline, std::uint32_t bit, std::uint32_t writeMask, std::uint32_t reference) { + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, heapPass ? heapPipeline : pipeline); context.Resolved(&DeviceFunctions::cmdSetStencilWriteMask, "vkCmdSetStencilWriteMask")(commands, VK_STENCIL_FACE_FRONT_AND_BACK, writeMask); context.Resolved(&DeviceFunctions::cmdSetStencilReference, "vkCmdSetStencilReference")(commands, VK_STENCIL_FACE_FRONT_AND_BACK, reference); const DepthPush push{map, bit}; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, sampledPipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(push), &push); + if (heapPass) { + const HeapDepthPush pushed{push, *depthSlot, *stencilSlot}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + } else { + CmdPushConstants(context, commands, sampledPipelineLayout, VK_SHADER_STAGE_FRAGMENT_BIT, 0, sizeof(push), &push); + } context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, 3, 1, 0, 0); }; - draw(pass.depth, 0u, 0xffu, 0u); + draw(pass.depth, pass.heapDepth, 0u, 0xffu, 0u); if (target.method == SampleOrderMethod::DepthStencilBits) { - for (std::uint32_t bit = 0; bit < 8u; ++bit) draw(pass.stencilBits, bit, 1u << bit, 0xffu); + for (std::uint32_t bit = 0; bit < 8u; ++bit) draw(pass.stencilBits, pass.heapStencilBits, bit, 1u << bit, 0xffu); } context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); } @@ -275,8 +310,33 @@ class SampleOrders { VkRenderPass renderPass = VK_NULL_HANDLE; VkPipeline depth = VK_NULL_HANDLE; VkPipeline stencilBits = VK_NULL_HANDLE; + VkPipeline heapDepth = VK_NULL_HANDLE; + VkPipeline heapStencilBits = VK_NULL_HANDLE; }; + VkDescriptorSet writeDepthSet(const CopyImage& copy, bool stencil) { + const auto set = allocate(sampledSetLayout, {{VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, setsPerPool * 2u}}); + std::array images{{{VK_NULL_HANDLE, copy.depthView, VK_IMAGE_LAYOUT_GENERAL}, {VK_NULL_HANDLE, stencil ? copy.stencilView : copy.depthView, VK_IMAGE_LAYOUT_GENERAL}}}; + std::array writes{}; + for (std::uint32_t binding = 0; binding < writes.size(); ++binding) { + writes[binding].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[binding].dstSet = set; + writes[binding].dstBinding = binding; + writes[binding].descriptorCount = 1; + writes[binding].descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + writes[binding].pImageInfo = &images[binding]; + } + context.Function("vkUpdateDescriptorSets")(context.device, stencil ? 2u : 1u, writes.data(), 0, nullptr); + return set; + } + + static std::size_t storageIndex(VkFormat format) { + const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R16_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32A32_UINT}; + const auto found = std::ranges::find(formats, format); + Require(found != formats.end(), "reordering the samples of a color target through a storage view of this format is not implemented"); + return static_cast(found - formats.begin()); + } + using DepthKey = std::tuple>; using CopyKey = std::tuple; @@ -317,10 +377,7 @@ class SampleOrders { } VkPipeline storagePipeline(VkFormat format) { - const std::array formats{VK_FORMAT_R8_UINT, VK_FORMAT_R16_UINT, VK_FORMAT_R32_UINT, VK_FORMAT_R32G32_UINT, VK_FORMAT_R32G32B32A32_UINT}; - const auto found = std::ranges::find(formats, format); - Require(found != formats.end(), "reordering the samples of a color target through a storage view of this format is not implemented"); - const auto index = static_cast(found - formats.begin()); + const auto index = storageIndex(format); if (storagePipelines[index] != VK_NULL_HANDLE) return storagePipelines[index]; if (storageSetLayout == VK_NULL_HANDLE) storageSetLayout = setLayout(VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1, VK_SHADER_STAGE_COMPUTE_BIT); if (storagePipelineLayout == VK_NULL_HANDLE) storagePipelineLayout = pipelineLayout(storageSetLayout, VK_SHADER_STAGE_COMPUTE_BIT, sizeof(StoragePush)); @@ -337,6 +394,10 @@ class SampleOrders { auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &storagePipelines[index]); } + if (created == VK_SUCCESS && HeapPassesActive(context)) { + const auto mapping = HeapPushIndexMapping(BindlessHeap::For(context), 0, 0, HeapPassStorageImages, offsetof(HeapStoragePush, image)); + heapStoragePipelines[index] = CreateHeapComputePipeline(context, info, std::span(&mapping, 1)); + } destroyModule(shader); Check(created, "vkCreateComputePipelines sample order"); return storagePipelines[index]; @@ -441,14 +502,27 @@ class SampleOrders { auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &pipeline); } + VkPipeline heapPipeline = VK_NULL_HANDLE; + if (created == VK_SUCCESS && HeapPassesActive(context)) { + const auto& heap = BindlessHeap::For(context); + const std::array mappings{ + HeapPushIndexMapping(heap, 0, 0, VK_SPIRV_RESOURCE_TYPE_SAMPLED_IMAGE_BIT_EXT, offsetof(HeapDepthPush, depth)), + HeapPushIndexMapping(heap, 0, 1, VK_SPIRV_RESOURCE_TYPE_SAMPLED_IMAGE_BIT_EXT, offsetof(HeapDepthPush, stencil))}; + heapPipeline = CreateHeapGraphicsPipeline(context, info, mappings); + } destroyModule(fragment); destroyModule(vertex); Check(created, "vkCreateGraphicsPipelines sample order"); - return pipeline; + return std::pair{pipeline, heapPipeline}; }; - if (method == SampleOrderMethod::DepthStencilExport) pass.depth = create(SAMPLE_ORDER_STENCILEXPORT_SPV, sizeof(SAMPLE_ORDER_STENCILEXPORT_SPV), true); - else pass.depth = create(SAMPLE_ORDER_DEPTH_SPV, sizeof(SAMPLE_ORDER_DEPTH_SPV), true); - if (method == SampleOrderMethod::DepthStencilBits) pass.stencilBits = create(SAMPLE_ORDER_STENCILBITS_SPV, sizeof(SAMPLE_ORDER_STENCILBITS_SPV), false); + if (method == SampleOrderMethod::DepthStencilExport) std::tie(pass.depth, pass.heapDepth) = create(SAMPLE_ORDER_STENCILEXPORT_SPV, sizeof(SAMPLE_ORDER_STENCILEXPORT_SPV), true); + else std::tie(pass.depth, pass.heapDepth) = create(SAMPLE_ORDER_DEPTH_SPV, sizeof(SAMPLE_ORDER_DEPTH_SPV), true); + if (method == SampleOrderMethod::DepthStencilBits) std::tie(pass.stencilBits, pass.heapStencilBits) = create(SAMPLE_ORDER_STENCILBITS_SPV, sizeof(SAMPLE_ORDER_STENCILBITS_SPV), false); + if (method == SampleOrderMethod::DepthStencilBits && (pass.heapDepth == VK_NULL_HANDLE) != (pass.heapStencilBits == VK_NULL_HANDLE)) { + if (pass.heapDepth) context.Function("vkDestroyPipeline")(context.device, pass.heapDepth, nullptr); + if (pass.heapStencilBits) context.Function("vkDestroyPipeline")(context.device, pass.heapStencilBits, nullptr); + pass.heapDepth = pass.heapStencilBits = VK_NULL_HANDLE; + } } catch (...) { destroyPass(pass); throw; @@ -512,6 +586,8 @@ class SampleOrders { } void destroyPass(const DepthPass& pass) noexcept { + if (pass.heapStencilBits) context.Function("vkDestroyPipeline")(context.device, pass.heapStencilBits, nullptr); + if (pass.heapDepth) context.Function("vkDestroyPipeline")(context.device, pass.heapDepth, nullptr); if (pass.stencilBits) context.Function("vkDestroyPipeline")(context.device, pass.stencilBits, nullptr); if (pass.depth) context.Function("vkDestroyPipeline")(context.device, pass.depth, nullptr); if (pass.renderPass) context.Function("vkDestroyRenderPass")(context.device, pass.renderPass, nullptr); @@ -521,6 +597,9 @@ class SampleOrders { copies.clear(); for (const auto& [key, pass] : depthPasses) destroyPass(pass); depthPasses.clear(); + for (const auto pipeline : heapStoragePipelines) { + if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + } for (const auto pipeline : storagePipelines) { if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); } @@ -541,6 +620,7 @@ class SampleOrders { VkPipelineLayout storagePipelineLayout = VK_NULL_HANDLE; VkPipelineLayout sampledPipelineLayout = VK_NULL_HANDLE; std::array storagePipelines{}; + std::array heapStoragePipelines{}; std::map depthPasses; std::map, std::uint64_t>> copies; std::uint64_t uses = 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp index 50d522e1dfa8..be99fcb46abf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "PipelineSpecialization.hpp" #include @@ -63,6 +64,7 @@ namespace AgcDriver::Graphics { } void Sampler::release() noexcept { + ReleaseHeapSlots(context); if (sampler) context.Function("vkDestroySampler")(context.device, sampler, nullptr); } @@ -100,6 +102,7 @@ namespace AgcDriver::Graphics { auto resource = DecodeSamplerResource(words, unnormalizedProven, forceDegammaPaired, integerBorder); resource.compareEnable = compareEnable; auto sampler = std::make_shared(context, resource); + sampler->SetCacheKey(key); // The cap keeps live samplers well below the device's limit (NVIDIA: ~4000); a set in flight // still holds the evicted sampler through its own shared_ptr. while (entries.size() >= capacity) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp index 60c40bdcd59d..d8dd6b58b0b6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp @@ -109,21 +109,6 @@ bool forEachInstruction(std::span words, const TVisit& visi return true; } -bool storesThroughAddressesCached(const ShaderRecompiler::SharedSpirv& spirv) { - static std::mutex mutex; - static std::map, bool> known; - const auto hash = spirv.ContentHash(); - const std::pair key{hash.low, hash.high}; - { - std::lock_guard lock(mutex); - if (const auto found = known.find(key); found != known.end()) return found->second; - } - const bool stores = SpirvStoresThroughAddresses(spirv.Words()); - std::lock_guard lock(mutex); - known.emplace(key, stores); - return stores; -} - bool faultReportOnlyCached(const ShaderRecompiler::SharedSpirv& spirv, std::uint32_t set, std::uint32_t binding) { static std::mutex mutex; static std::map, bool> known; @@ -141,31 +126,6 @@ bool faultReportOnlyCached(const ShaderRecompiler::SharedSpirv& spirv, std::uint } -bool ShaderStoresThroughAddresses(const ShaderRecompiler::SharedSpirv& spirv) { - return storesThroughAddressesCached(spirv); -} - -bool SpirvStoresThroughAddresses(std::span words) { - if (words.size() < 5 || words[0] != spv::MagicNumber) return true; - std::unordered_set pointerTypes; - const bool readable = forEachInstruction(words, [&](std::uint32_t opcode, std::span instruction) { - if ((opcode == spv::OpTypePointer || opcode == spv::OpTypeUntypedPointerKHR) && instruction.size() > 2 && instruction[2] == spv::StorageClassPhysicalStorageBuffer) pointerTypes.insert(instruction[1]); - }); - if (!readable) return true; - if (pointerTypes.empty()) return false; - std::unordered_set pointers; - forEachInstruction(words, [&](std::uint32_t opcode, std::span instruction) { - if (instruction.size() > 2 && !definesNoPointerValue(opcode) && pointerTypes.contains(instruction[1])) pointers.insert(instruction[2]); - }); - bool stores = false; - forEachInstruction(words, [&](std::uint32_t opcode, std::span instruction) { - const auto pointer = pointerOperandStored(opcode, instruction); - const bool extendedWithPointer = opcode == spv::OpExtInst && std::any_of(instruction.begin() + static_cast(std::min(5u, instruction.size())), instruction.end(), [&](std::uint32_t operand) { return pointers.contains(operand); }); - stores = stores || extendedWithPointer || (pointer != 0 && pointers.contains(pointer)); - }); - return stores; -} - bool SpirvFaultReportOnly(std::span words, std::uint32_t set, std::uint32_t binding) { constexpr std::uint32_t reportWords = sizeof(ShaderRecompiler::BdaAbi::Fault) / sizeof(std::uint32_t); if (words.size() < 5 || words[0] != spv::MagicNumber) return false; @@ -226,7 +186,7 @@ void ShaderResources::prepareAddressBindings(std::span sha Require(shader.program->bdaAbiVersion == (faults == 0 ? 0u : ShaderRecompiler::BdaAbi::Version), "incompatible BDA ABI version"); // Rect-list validation needs a fault buffer, but never accesses guest addresses. usesBda = usesBda || tables != 0; - if (tables != 0 && storesThroughAddressesCached(shader.program->spirv)) bdaReadOnly = false; + if (tables != 0 && shader.program->bdaStores) bdaReadOnly = false; usesFaultBuffer = usesFaultBuffer || faults != 0; const bool tableFaultsOnly = std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::ImageTableMap; }); faultChecks = faultChecks || tables != 0 || (faults != 0 && (!tableFaultsOnly || shader.program->imageTableFaults != 0 || shader.program->poisonedSrtReads != 0 || shader.program->windowLaneGuards != 0 || shader.program->descriptorFaults != 0 || loopGuarded)); @@ -241,8 +201,8 @@ void ShaderResources::prepareAddressBindings(std::span sha } } -VkDescriptorBufferInfo ShaderResources::descriptor(Allocation& allocation) { - if (allocation.guest) return wholeDwordRange(guestMemory.Descriptor(allocation.address, allocation.size, allocation.adjustment)); +VkDescriptorBufferInfo ShaderResources::descriptor(const Allocation& allocation, std::uint32_t& adjustment) const { + if (allocation.guest) return wholeDwordRange(guestMemory.Descriptor(allocation.address, allocation.size, adjustment)); if (allocation.role == ShaderRecompiler::DescriptorRole::GuestBuffers) { const auto placeholder = allocation.oversized ? context.oversizedBuffer : context.emptyBuffer; Require(placeholder != VK_NULL_HANDLE, "no placeholder buffer for a null V#"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 554e209eddde..360c4d25c738 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -10,7 +10,9 @@ #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -22,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +56,16 @@ namespace AgcDriver::Graphics { namespace { +std::uint32_t takenSlot(std::optional slot, const char* what) { + Require(slot.has_value(), what); + return *slot; +} + +bool UintStorageBinding(std::uint32_t binding) { + const auto kind = binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); + return ShaderRecompiler::RuntimeAbi::UintStorageImageKind(kind); +} + bool overlap(std::uint64_t first, std::size_t firstSize, std::uint64_t second, std::size_t secondSize) { return first < second + secondSize && second < first + firstSize; } @@ -1219,7 +1232,7 @@ struct StorageTextureCache { }; StorageTextureCache& StorageTextures() { - static StorageTextureCache cache; + static auto& cache = *new StorageTextureCache(); return cache; } @@ -1764,9 +1777,9 @@ bool StorageDedupeEnabled() { } // Whether storage element `element` of `binding` repeats the previous element's eight words. -bool SameAsPreviousStorageElement(const ShaderRecompiler::DescriptorBinding& binding, std::uint32_t element) { +bool SameAsPreviousStorageElement(std::span guest, std::uint32_t element) { if (element == 0) return false; - const auto words = binding.guestDescriptor.begin() + static_cast(element) * 8u; + const auto words = guest.begin() + static_cast(element) * 8u; return std::equal(words, words + 8, words - 8); } @@ -1840,6 +1853,7 @@ void ShaderResources::Complete() { buildPrepare(shaders, nullptr, 0, 0, 0); } buildComplete(); + if (heapStage) RefreshHeapStage(shaders); forgetDeferredInputs(); } @@ -1854,6 +1868,7 @@ void ShaderResources::forgetDeferredInputs() { void ShaderResources::build(std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes) { buildPrepare(shaders, target, colorAttachments, indexAddress, indexBytes); buildComplete(); + if (heapStage) RefreshHeapStage(shaders); } namespace { @@ -2043,16 +2058,16 @@ std::vector& ReusedTables() { return ShaderRecompiler::ThreadOwned(slot); } -bool SameTable(const ReusedTable& table, VkDevice device, const ShaderRecompiler::DescriptorBinding& binding) { - return table.device == device && table.words == binding.guestDescriptor && table.shapes == binding.imageShapes && table.depthCompare == binding.imageDepthCompare; +bool SameTable(const ReusedTable& table, VkDevice device, const ShaderRecompiler::DescriptorBinding& binding, std::span words) { + return table.device == device && std::ranges::equal(table.words, words) && table.shapes == binding.imageShapes && table.depthCompare == binding.imageDepthCompare; } -ReusedTable* FindReusedTable(VkDevice device, const ShaderRecompiler::DescriptorBinding& binding, const TableState& state) { +ReusedTable* FindReusedTable(VkDevice device, const ShaderRecompiler::DescriptorBinding& binding, std::span words, const TableState& state) { if (state.epoch == 0) return nullptr; auto& tables = ReusedTables(); std::erase_if(tables, [&](const ReusedTable& table) { return table.state != state; }); for (auto& table : tables) { - if (SameTable(table, device, binding)) return &table; + if (SameTable(table, device, binding, words)) return &table; } return nullptr; } @@ -2085,10 +2100,10 @@ bool CachedTableTextures(const ReusedTable& table, std::vector words) { auto& tables = ReusedTables(); - std::erase_if(tables, [&](const ReusedTable& entry) { return SameTable(entry, table.device, binding); }); - table.words = binding.guestDescriptor; + std::erase_if(tables, [&](const ReusedTable& entry) { return SameTable(entry, table.device, binding, words); }); + table.words.assign(words.begin(), words.end()); table.shapes = binding.imageShapes; table.depthCompare = binding.imageDepthCompare; tables.insert(tables.begin(), std::move(table)); @@ -2162,6 +2177,28 @@ void ValidateRuntimeResources(const CompiledShader& shader, std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes) { @@ -2213,28 +2250,40 @@ void ShaderResources::buildPrepare(std::span shaders, cons Require(stageStorageBuffers <= limits.maxPerStageDescriptorUpdateAfterBindStorageBuffers && stageResources <= limits.maxPerStageUpdateAfterBindResources, "shader descriptors exceed per-stage limits"); updateAfterBind = true; }; + const bool heapShader = HeapShader(shader); + if (heapShader) { + Require(DescriptorHeapEnabled(), "a GPU-mode stage needs VK_EXT_descriptor_heap (APS5_GPU_HEAP)"); + heapStage = true; + if (shader.stage == ShaderRecompiler::ShaderStage::Compute) NoteHeapComputeStage(); + } for (const auto& binding : shader.program->bindings) { - Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shader.stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "unexpected descriptor set: a shader resource must use its stage's descriptor set"); + Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || ((shader.stage == ShaderRecompiler::ShaderStage::Fragment || heapShader) && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet) || (heapShader && binding.role == ShaderRecompiler::DescriptorRole::Gds && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::GpuGdsSet), "unexpected descriptor set: a shader resource must use its stage's descriptor set"); if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) { const bool storage = binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable; Require(binding.binding == (storage ? ShaderRecompiler::RuntimeAbi::ImageStorageTableBinding : ShaderRecompiler::RuntimeAbi::ImageTableBinding) && binding.kind == (storage ? ShaderRecompiler::DescriptorKind::StorageImage : ShaderRecompiler::DescriptorKind::SampledImage) && binding.guestDescriptor.size() == static_cast(binding.count) * 8u && binding.imageShapes.size() == binding.count && binding.imageDepthCompare.size() == binding.count && binding.imageSamplers.size() == binding.count, "image table binding is malformed"); + NoteImageTableBuild(); auto& table = tableStage(shader); Require(storage ? !table.storage : !table.sampled, "a stage binds two image tables"); Require(table.images == nullptr || table.images->count == binding.count, "a stage's image tables disagree on their elements"); (storage ? table.storage : table.sampled) = true; table.images = &binding; + table.imageWords = DrawGuestWords(shader, binding); + Require(table.imageWords.size() == static_cast(binding.count) * 8u, "a draw's image table words differ in size from its binding"); table.firstSampler = firstSampler; continue; } - Require(std::find(occupied.begin(), occupied.end(), binding.binding) == occupied.end(), "duplicate shader binding"); + const bool heapStorage = heapShader && binding.role == ShaderRecompiler::DescriptorRole::GuestImages && binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet; + const bool sharedHeapArray = heapShader && (binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers || (binding.role == ShaderRecompiler::DescriptorRole::GuestImages && binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) || heapStorage); + Require(std::find(occupied.begin(), occupied.end(), binding.binding) == occupied.end() || sharedHeapArray, "duplicate shader binding"); occupied.push_back(binding.binding); if (binding.role == ShaderRecompiler::DescriptorRole::SamplerTable) { - addSamplerTable(binding, flags, tableStage(shader)); + addSamplerTable(binding, DrawGuestWords(shader, binding), flags, tableStage(shader)); continue; } if (binding.role == ShaderRecompiler::DescriptorRole::ImageTableMap) { - Require(binding.kind == ShaderRecompiler::DescriptorKind::StorageBuffer && binding.count == 1 && binding.guestDescriptor.size() >= ShaderRecompiler::ImageTableAbi::HeaderWords && !binding.readOnly && binding.guestDescriptor[ShaderRecompiler::ImageTableAbi::HeaderVersion] == ShaderRecompiler::ImageTableAbi::Version, "image table map binding is malformed"); - const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t); + const auto& mapWords = DrawGuestWords(shader, binding); + Require(binding.kind == ShaderRecompiler::DescriptorKind::StorageBuffer && binding.count == 1 && mapWords.size() >= ShaderRecompiler::ImageTableAbi::HeaderWords && !binding.readOnly && mapWords[ShaderRecompiler::ImageTableAbi::HeaderVersion] == ShaderRecompiler::ImageTableAbi::Version, "image table map binding is malformed"); + const auto size = mapWords.size() * sizeof(std::uint32_t); Require(size <= context.limits.maxStorageBufferRange, "image table map exceeds the storage buffer range limit"); stageStorageBuffers += 1; stageResources += 1; @@ -2242,19 +2291,45 @@ void ShaderResources::buildPrepare(std::span shaders, cons requireStageLimits(); auto& table = tableStage(shader); Require(table.mapAllocation == std::numeric_limits::max(), "a stage binds two image table maps"); - table.map = binding.guestDescriptor; + table.map = mapWords; allocations.push_back({0, size, false, std::make_unique(context, size, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT), ShaderRecompiler::DescriptorRole::ImageTableMap}); table.mapAllocation = allocations.size() - 1u; bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, flags, nullptr}, {allocations.size() - 1u}}); + bindings.back().heap = table.heap; continue; } + if (heapShader) { + if (heapStorage) { + Require(ShaderRecompiler::RuntimeAbi::GpuHeapStorageKind(binding.binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)), "a heap storage image binding has an atomic or unknown kind"); + addImageBinding(binding, DrawGuestWords(shader, binding), flags); + bindings.back().heap = true; + continue; + } + const auto heapClass = ClassifyHeapBinding(binding); + if (heapClass == HeapBindingClass::Unsupported) Require(false, std::string("a GPU-mode fragment stage has an unsupported ") + roleName(binding.role) + " binding"); + if (heapClass == HeapBindingClass::SharedImage) { + Require(binding.kind == ShaderRecompiler::DescriptorKind::SampledImage || binding.kind == ShaderRecompiler::DescriptorKind::Sampler, "a GPU-mode stage binds sampled images and samplers only"); + addImageBinding(binding, DrawGuestWords(shader, binding), flags); + bindings.back().heap = true; + continue; + } + if (heapClass == HeapBindingClass::ShaderData) { + Require(binding.count == 1 && !DrawGuestWords(shader, binding).empty(), "invalid GPU-mode shader data binding"); + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, DrawGuestWords(shader, binding)); + Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, flags, nullptr}, {}, {}}; + item.heap = true; + item.role = binding.role; + bindings.push_back(std::move(item)); + continue; + } + } const bool addressRole = binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer; const bool bufferRole = addressRole || binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers || binding.role == ShaderRecompiler::DescriptorRole::ShaderData || binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt || binding.role == ShaderRecompiler::DescriptorRole::Gds; const bool imageRole = binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers; if (imageRole) { if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage || binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) stageResources += binding.count; requireStageLimits(); - addImageBinding(binding, flags); + addImageBinding(binding, DrawGuestWords(shader, binding), flags); continue; } if (!bufferRole) Require(false, std::string("unsupported descriptor role ") + roleName(binding.role)); @@ -2267,7 +2342,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons requireStageLimits(); Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, binding.count, flags, nullptr}, {}}; if (binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers) { - Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4, "guest buffer descriptor must contain four DWORDs per array element"); + const auto& guestWords = DrawGuestWords(shader, binding); + Require(guestWords.size() == static_cast(binding.count) * 4, "guest buffer descriptor must contain four DWORDs per array element"); for (std::uint32_t element = 0; element < binding.count; ++element) { // An element the recompiler did not classify (a producer without the // vector) counts as written, like imageWritten below. @@ -2275,12 +2351,12 @@ void ShaderResources::buildPrepare(std::span shaders, cons const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element]; const bool read = element < binding.bufferRead.size() && binding.bufferRead[element]; const std::uint32_t extent = element < binding.bufferExtent.size() ? binding.bufferExtent[element] : 0u; - const auto index = addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read, extent); + const auto index = addGuestBuffer(std::span(guestWords).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read, extent); const bool classified = element < binding.bufferWritten.size() && element < binding.bufferRead.size() && element < binding.bufferAtomic.size(); allocations[index].writeOnly = classified && allocations[index].guest && allocations[index].written && binding.bufferWritten[element] && !read && !atomic; const auto& push = shader.program->pushConstants; const auto pushDataBytes = shader.program->stencilReferences.empty() ? push.size() : shader.program->stencilReferenceDword * sizeof(std::uint32_t); - if (pushDataBytes != 0u) { + if (pushDataBytes != 0u && !shader.program->dataAddressPush) { const auto position = shader.program->memoryOffsetDword * 4u + element; Require(position < pushDataBytes, "guest buffer offset lies outside the shader's push constants"); allocations[index].pushByte = static_cast(shader.pushConstantOffset + position); @@ -2290,27 +2366,42 @@ void ShaderResources::buildPrepare(std::span shaders, cons } item.allocations.push_back(index); } + if (heapShader && shader.program->bufferAddressCount != 0u) { + Require(shader.program->bufferAddressCount == binding.count, "a GPU-mode stage's buffer address words disagree with its buffer binding"); + item.heap = true; + item.addressed = true; + item.dataStage = static_cast(HeapStageIndex(shader.stage)); + item.addressDword = shader.program->bufferAddressDword; + addressedBuffers = true; + } } else if (addressRole) { item.allocations.push_back(allocations.size()); allocations.push_back({0, 0, false, nullptr, binding.role}); + item.heap = (heapShader && shader.program->dataAddressPush) || (heapStage && !heapShader && binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer); } else if (binding.role == ShaderRecompiler::DescriptorRole::Gds) { Require(binding.count == 1 && binding.guestDescriptor.empty(), "invalid GDS descriptor contract"); + static_assert(Pm4::GdsBytes == ShaderRecompiler::RuntimeAbi::GdsBytes); const auto gds = Pm4::GdsAddress(); guestMemory.AddWritable(gds, Pm4::GdsBytes, true); item.allocations.push_back(allocations.size()); allocations.push_back({gds, Pm4::GdsBytes, true, nullptr, ShaderRecompiler::DescriptorRole::ShaderData, true}); + if (heapShader && shader.program->dataAddressPush) { + item.heap = true; + if (gdsAllocation == std::numeric_limits::max()) gdsAllocation = item.allocations.back(); + } } else { Require(binding.count == 1, "shader data and flattened SRT descriptors must not be arrays"); - Require(!binding.guestDescriptor.empty(), "empty shader data descriptor"); + const auto& guestWords = DrawGuestWords(shader, binding); + Require(!guestWords.empty(), "empty shader data descriptor"); if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) { - ValidateRuntimeResources(shader, binding.guestDescriptor); + ValidateRuntimeResources(shader, guestWords); } - item.allocations.push_back(addDataBuffer(binding.guestDescriptor)); + item.allocations.push_back(addDataBuffer(guestWords)); if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) shaderData = static_cast(item.allocations.back()); } bindings.push_back(std::move(item)); } - for (const auto index : offsetsInData) allocations[index].dataAllocation = shaderData; + for (const auto index : offsetsInData) allocations[index].dataAllocation = heapShader ? HeapRingData(HeapStageIndex(shader.stage)) : shaderData; for (auto index = firstDeferred; index < deferredImages.size(); ++index) { deferredImages[index].firstSampler = firstSampler; deferredImages[index].samplerCount = samplers.size() - firstSampler; @@ -2331,8 +2422,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons const auto faults = map[ShaderRecompiler::ImageTableAbi::HeaderElementFaults]; const auto samplerFaults = map[ShaderRecompiler::ImageTableAbi::HeaderSamplerFaults]; Require(static_cast(faults) + elements <= map.size() && static_cast(samplerFaults) + map[ShaderRecompiler::ImageTableAbi::HeaderSamplerCount] / 4u <= map.size(), "image table map is truncated"); - if (table.images == nullptr) continue; - table.set = splitSets && table.stage == ShaderRecompiler::ShaderStage::Fragment ? 1u : 0u; + if (table.images == nullptr || table.heap) continue; + table.set = splitSets && !heapStage && table.stage == ShaderRecompiler::ShaderStage::Fragment ? 1u : 0u; table.elementBase = tableElements[table.set]; table.map[ShaderRecompiler::ImageTableAbi::HeaderElementBase] = table.elementBase; tableElements[table.set] += elements; @@ -2359,7 +2450,22 @@ void ShaderResources::buildPrepare(std::span shaders, cons // For every build, locked ones included: their stage B then takes the fast path too, and the // collects cost the same wherever they run. if (precollectImages()) phase(BuildPhase::Precollect); - if (drawBuild && !usesBda && std::all_of(allocations.begin(), allocations.end(), [&](const Allocation& allocation) { return !allocation.guest || allocation.pushByte >= 0 || (allocation.dataAllocation >= 0 && allocation.dataByte < allocations[static_cast(allocation.dataAllocation)].size); })) guestMemory.AllowAdjustedRegions(); + const auto heapDataBytes = [&](std::int64_t ring) -> std::uint64_t { + const auto stage = HeapRingStage(ring); + for (const auto& shader : shaders) { + if (HeapStageIndex(shader.stage) != stage || !HeapShader(shader)) continue; + for (const auto& binding : shader.program->bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) return binding.guestDescriptor.size() * sizeof(std::uint32_t); + } + } + return 0; + }; + const auto patchable = [&](const Allocation& allocation) { + if (!allocation.guest || allocation.pushByte >= 0) return true; + if (IsHeapRingData(allocation.dataAllocation)) return allocation.dataByte < heapDataBytes(allocation.dataAllocation); + return allocation.dataAllocation >= 0 && allocation.dataByte < allocations[static_cast(allocation.dataAllocation)].size; + }; + if (drawBuild && !usesBda && std::all_of(allocations.begin(), allocations.end(), patchable)) guestMemory.AllowAdjustedRegions(); requireGuestBuffers(); guestMemory.UploadPrepare(usesBda); timing.uploadMs = phase(BuildPhase::Upload); @@ -2367,6 +2473,7 @@ void ShaderResources::buildPrepare(std::span shaders, cons std::array, 2> descriptions; bool tablesUpdateAfterBind = false; for (const auto& binding : bindings) { + if (binding.heap) continue; const auto set = setOf(binding); descriptions[set].push_back(binding.layout); (set == 0 ? layoutBindingFlags : fragmentBindingFlags).push_back(binding.flags | (updateAfterBind && binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0)); @@ -2413,8 +2520,12 @@ void ShaderResources::buildPrepare(std::span shaders, cons owns = true; return CreateSetLayout(context, description, updateAfterBind, flags); }; + if (heapStage) { + Require(splitSets && descriptions[1].empty(), "a GPU-mode fragment stage needs split sets and no per-draw fragment bindings"); + fragmentLayoutKey = {HeapLayoutKeyTag}; + } _layout = createLayout(layoutKey, descriptions[0], layoutBindingFlags, ownsLayout); - if (splitSets) { + if (splitSets && !heapStage) { fragment.layout = createLayout(fragmentLayoutKey, descriptions[1], fragmentBindingFlags, fragment.ownsLayout); { const auto sizes = splitPoolSizes(1); @@ -2497,7 +2608,7 @@ void ShaderResources::buildPrepare(std::span shaders, cons bool SplitsSets(const Context& context, std::span shaders) { const auto fragmentSet = [](const CompiledShader& shader) { return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const auto& binding) { return binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet; }); }; const auto fragmentStage = [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Fragment; }; - return std::any_of(shaders.begin(), shaders.end(), fragmentSet) || (context.graphicsPipelineLibrary && std::any_of(shaders.begin(), shaders.end(), fragmentStage)); + return std::any_of(shaders.begin(), shaders.end(), fragmentSet) || AnyHeapShader(shaders) || (context.graphicsPipelineLibrary && std::any_of(shaders.begin(), shaders.end(), fragmentStage)); } DescribedSetLayouts DescribeSetLayouts(const Context& context, std::span shaders) { @@ -2511,21 +2622,36 @@ DescribedSetLayouts DescribeSetLayouts(const Context& context, std::spandataAddressPush; for (const auto& binding : shader.program->bindings) { - const auto set = binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet ? 1 : 0; + const auto set = !heapStage && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet ? 1 : 0; + if (heapStage && !gpuMode && binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer) continue; + if (addressedStage) { + using Role = ShaderRecompiler::DescriptorRole; + const bool tableRole = binding.role == Role::ImageTable || binding.role == Role::ImageStorageTable || binding.role == Role::SamplerTable || binding.role == Role::ImageTableMap; + const bool addressRole = binding.role == Role::BdaPagetable || binding.role == Role::FaultBuffer || (binding.role == Role::GuestBuffers && shader.program->bufferAddressCount != 0u) || (binding.role == Role::Gds && shader.program->dataAddressPush); + const bool heapStorage = binding.role == Role::GuestImages && binding.kind == ShaderRecompiler::DescriptorKind::StorageImage && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet; + if (tableRole || addressRole || heapStorage) continue; + } if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable) { - tableFlags[split && shader.stage == ShaderRecompiler::ShaderStage::Fragment ? 1 : 0] |= flags; + tableFlags[split && !heapStage && shader.stage == ShaderRecompiler::ShaderStage::Fragment ? 1 : 0] |= flags; continue; } if (binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) { storageTableFlags[split && shader.stage == ShaderRecompiler::ShaderStage::Fragment ? 1 : 0] |= flags; continue; } + if (gpuMode) { + const auto heapClass = ClassifyHeapBinding(binding); + if (heapClass == HeapBindingClass::SharedImage || heapClass == HeapBindingClass::ShaderData) continue; + } if (binding.role == ShaderRecompiler::DescriptorRole::SamplerTable) { descriptions[set].push_back({binding.binding, VK_DESCRIPTOR_TYPE_SAMPLER, context.samplerTableCapacity, flags, nullptr}); bindingFlags[set].push_back(VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT | (context.samplerTableUpdateAfterBind ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0)); @@ -2569,6 +2695,11 @@ DescribedSetLayouts DescribeSetLayouts(const Context& context, std::spanLayout(result.mainKey, descriptions[0], updateAfterBind, bindingFlags[0]); + if (heapStage) { + Require(split && descriptions[1].empty(), "a GPU-mode stage needs split sets and no per-draw fragment bindings"); + result.fragmentKey = {HeapLayoutKeyTag}; + return result; + } if (split) result.layouts.fragment = context.descriptorCache->Layout(result.fragmentKey, descriptions[1], updateAfterBind, bindingFlags[1]); return result; } @@ -2588,7 +2719,7 @@ void ShaderResources::buildComplete() { source.singleLevelImage = source.mipmappedImage = false; source.pinnedLod.reset(); } - for (const auto& deferred : deferredImages) resolveImageBinding(*deferred.binding, bindings[deferred.index], std::span>(samplers).subspan(deferred.firstSampler, deferred.samplerCount)); + for (const auto& deferred : deferredImages) resolveImageBinding(*deferred.binding, deferred.words, bindings[deferred.index], std::span>(samplers).subspan(deferred.firstSampler, deferred.samplerCount)); previousSampled = {}; deferredImages.clear(); applySamplerVariants(); @@ -2610,6 +2741,10 @@ void ShaderResources::buildComplete() { if (bda) bda->SetSrtGuards(std::move(srtGuardFaults)); finishImageTables(); phase(BuildPhase::Bda); + for (const auto& binding : bindings) { + if (!binding.addressed) continue; + for (const auto index : binding.allocations) static_cast(descriptor(allocations[index])); + } if (_set != VK_NULL_HANDLE || fragment.set != VK_NULL_HANDLE) { // One update call for the whole set: the info arrays are sized up front so every write's // pointer into them stays valid until the call. @@ -2632,6 +2767,7 @@ void ShaderResources::buildComplete() { return VkDescriptorImageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; }; for (const auto& binding : bindings) { + if (binding.heap) continue; if (binding.flags != 0) { const auto set = setOf(binding); const bool sampled = binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; @@ -2657,7 +2793,7 @@ void ShaderResources::buildComplete() { for (auto element = first; element < last; ++element) { const auto index = binding.imageAllocations[element]; if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) images.push_back({samplers[index]->Handle(), VK_NULL_HANDLE, VK_IMAGE_LAYOUT_UNDEFINED}); - else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index], ShaderRecompiler::RuntimeAbi::UintStorageImageBinding(binding.layout.binding)), VK_IMAGE_LAYOUT_GENERAL}); + else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index], UintStorageBinding(binding.layout.binding)), VK_IMAGE_LAYOUT_GENERAL}); else images.push_back({VK_NULL_HANDLE, textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(), textures[index]->Layout()}); } if (recorded) runs.emplace_back(write.dstArrayElement, write.descriptorCount); @@ -2703,8 +2839,8 @@ void ShaderResources::buildComplete() { write.pImageInfo = images.data() + images.size(); for (const auto index : binding.imageAllocations) { if (index == std::numeric_limits::max()) images.push_back(absentImage(binding.layout.binding)); - else if (storageTextures[index] == nullptr) images.push_back({VK_NULL_HANDLE, multisampleStorageViews[index], VK_IMAGE_LAYOUT_GENERAL}); - else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index], ShaderRecompiler::RuntimeAbi::UintStorageImageBinding(binding.layout.binding)), VK_IMAGE_LAYOUT_GENERAL}); + else if (storageTextures[index] == nullptr) images.push_back({VK_NULL_HANDLE, multisampleStorage[index].view, VK_IMAGE_LAYOUT_GENERAL}); + else images.push_back({VK_NULL_HANDLE, storageAtomic64[index] ? storageTextures[index]->Atomic64View(storageMips[index], storageFirstLayer[index]) : storageAtomic[index] ? storageTextures[index]->AtomicView(storageMips[index], storageFirstLayer[index]) : storageTextures[index]->StorageView(storageMips[index], storageFirstLayer[index], UintStorageBinding(binding.layout.binding)), VK_IMAGE_LAYOUT_GENERAL}); } Require(binding.imageAllocations.size() == binding.layout.descriptorCount, "descriptor allocations disagree with compact binding"); break; @@ -2740,6 +2876,10 @@ void ShaderResources::buildComplete() { pushPatches.emplace_back(static_cast(allocation.pushByte), allocation.adjustment); continue; } + if (IsHeapRingData(allocation.dataAllocation)) { + heapStages[HeapRingStage(allocation.dataAllocation)].dataPatches.emplace_back(allocation.dataByte, allocation.adjustment); + continue; + } Require(allocation.dataAllocation >= 0, "a guest buffer off the storage buffer offset alignment in a shader without shader data is not implemented"); auto& data = allocations[static_cast(allocation.dataAllocation)]; Require(data.buffer != nullptr && allocation.dataByte < data.size, "guest buffer offset lies outside the shader's data buffer"); @@ -2808,6 +2948,7 @@ void ShaderResources::noteReusable() { // Taken whether or not the object turns out reusable: the lookups' records are this build's. captureValidation(); reusable = false; + spaceReuse = false; importsReconcilable = false; spaceReuse = false; directRegions.clear(); @@ -2855,7 +2996,7 @@ std::uint64_t ShaderResources::DepthGuardRefusals() { bool ShaderResources::NeverReusable(std::span shaders) { const bool loopGuard = LoopGuardEnabled(); return std::any_of(shaders.begin(), shaders.end(), [loopGuard](const CompiledShader& shader) { - return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [&](const ShaderRecompiler::DescriptorBinding& binding) { return (binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable && (SpaceReuseDisabled() || ShaderStoresThroughAddresses(shader.program->spirv))) || (loopGuard && binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer); }); + return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [&](const ShaderRecompiler::DescriptorBinding& binding) { return (binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable && (SpaceReuseDisabled() || shader.program->bdaStores)) || (loopGuard && binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer); }); }); } @@ -2886,6 +3027,18 @@ void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableB visit(static_cast(shader.stage)); visit(static_cast(program.PipelineVariantId())); visit(static_cast(program.PipelineVariantId() >> 32u)); + if (shader.heap != nullptr && program.heapTemplateId != 0) { + visit(0u); + visit(HeapLayoutKeyTag); + visit(static_cast(program.heapTemplateId)); + visit(static_cast(program.heapTemplateId >> 32u)); + for (const auto& binding : program.bindings) { + if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages && binding.role != ShaderRecompiler::DescriptorRole::GuestSamplers) continue; + visit(binding.binding); + for (const auto word : DrawGuestWords(shader, binding)) visit(word); + } + return; + } visit(static_cast(program.bindings.size())); const auto packBits = [&](const std::vector& bits) { visit(static_cast(bits.size())); @@ -2899,17 +3052,17 @@ void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableB } }; for (const auto& binding : program.bindings) { - for (const auto word : {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(binding.guestDescriptor.size())}) visit(word); - if (movableBuffers && binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers && binding.guestDescriptor.size() == static_cast(binding.count) * 4u) { + for (const auto word : {static_cast(binding.kind), static_cast(binding.role), binding.descriptorSet, binding.binding, binding.count, binding.readOnly ? 1u : 0u, binding.imageShape.has_value() ? static_cast(*binding.imageShape) + 1u : 0u, static_cast(DrawGuestWords(shader, binding).size())}) visit(word); + if (movableBuffers && binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers && DrawGuestWords(shader, binding).size() == static_cast(binding.count) * 4u) { for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; + const auto* words = DrawGuestWords(shader, binding).data() + static_cast(element) * 4u; const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; const bool empty = words[2] == 0 || (words[0] == 0 && (words[1] & 0xffffu) == 0); if (written || empty) { for (std::uint32_t index = 0; index < 4u; ++index) visit(words[index]); } else { for (const auto word : {0u, words[1] & 0xffff0000u, 0u, words[3]}) visit(word); } } } else if ((dataWords && !movableBuffers) || !DataRole(binding.role)) { - for (const auto word : binding.guestDescriptor) visit(word); + for (const auto word : DrawGuestWords(shader, binding)) visit(word); } packBits(binding.imageWritten); packBits(binding.samplerDepthCompare); @@ -3396,7 +3549,8 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::span= storageKeys.size() || !StorageImageServesKeys(*storageTextures[i], storageKeys[i]))) return fail(FastFail::StorageKeys); if (i != 0 && storageTextures[i] == storageTextures[i - 1]) continue; @@ -3503,7 +3658,13 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::spanProvedKeys() != image->UploadedKeys()) return fail(FastFail::StorageKeys); + if (!keyProofs) return fail(FastFail::StorageKeys); + if (image->ProvedKeys() != image->UploadedKeys()) { + if (!OwnImageRefreshEnabled()) return fail(FastFail::StorageKeys); + keysMoved = true; + overlapping.push_back({i, true, false}); + continue; + } countKeysProven(true); } if (!unchanged) query(address, address + bytes, image, nullptr, {i, true, false}); @@ -3511,6 +3672,7 @@ bool ShaderResources::fastRevalidate(std::uint64_t serialBefore, std::spanGenerationFor(address, bytes)}); images.push_back(image); } + if (keysMoved) return fail(FastFail::StorageKeys); if (!pending.empty()) { if (!StorageTexture::ScanPending(pending)) return fail(FastFail::ClearedView); // A foreign image over a view under uncompressed keys is no failure while FindPending @@ -3684,7 +3846,7 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto words = std::span(DrawGuestWords(shader, binding)).subspan(static_cast(element) * 8u, 8u); if (IsNullSampledTextureResource(words)) continue; if (textureIndex >= textures.size()) return OwnRefreshFallback::Uncached; const auto index = textureIndex++; @@ -3694,7 +3856,7 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span } } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto words = std::span(DrawGuestWords(shader, binding)).subspan(static_cast(element) * 8u, 8u); if (IsNullTextureResource(words)) continue; if (storageIndex >= storageTextures.size()) return OwnRefreshFallback::Uncached; const auto index = storageIndex++; @@ -3705,7 +3867,8 @@ ShaderResources::OwnRefreshFallback ShaderResources::refreshOwnObjects(std::span } } for (const auto& table : tableStages) { - const auto* images = table.elementTextures.empty() ? nullptr : tableImages(shaders, table.stage); + std::span imageWords; + const auto* images = table.elementTextures.empty() ? nullptr : tableImages(shaders, table.stage, imageWords); for (std::size_t element = 0; element < table.elementTextures.size(); ++element) { const auto index = table.elementTextures[element]; if (index == std::numeric_limits::max()) continue; @@ -3743,11 +3906,13 @@ bool CompletePendingCpuWrites(const Context& context, std::uint64_t address, std return false; } -const ShaderRecompiler::DescriptorBinding* ShaderResources::tableImages(std::span shaders, ShaderRecompiler::ShaderStage stage) { +const ShaderRecompiler::DescriptorBinding* ShaderResources::tableImages(std::span shaders, ShaderRecompiler::ShaderStage stage, std::span& words) { for (const auto& shader : shaders) { if (shader.stage != stage || shader.program == nullptr) continue; for (const auto& binding : shader.program->bindings) { - if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable) return &binding; + if (binding.role != ShaderRecompiler::DescriptorRole::ImageTable) continue; + words = DrawGuestWords(shader, binding); + return &binding; } } return nullptr; @@ -3849,9 +4014,9 @@ bool ShaderResources::revalidateFull(std::span shaders, Pr for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestImages) continue; if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { - const auto elementWords = binding.count != 0 ? binding.guestDescriptor.size() / binding.count : 0; + const auto elementWords = binding.count != 0 ? DrawGuestWords(shader, binding).size() / binding.count : 0; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto raw = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto raw = std::span(DrawGuestWords(shader, binding)).subspan(static_cast(element) * elementWords, elementWords); std::array narrowed{}; const auto words = firstLayerWords(raw, binding.imageShape, narrowed, false); if (IsNullSampledTextureResource(words)) continue; @@ -3864,11 +4029,11 @@ bool ShaderResources::revalidateFull(std::span shaders, Pr } } else if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto words = std::span(DrawGuestWords(shader, binding)).subspan(static_cast(element) * 8u, 8u); if (IsNullTextureResource(words)) continue; if (storageIndex >= storageTextures.size()) return false; std::shared_ptr expected; - if (SameAsPreviousStorageElement(binding, element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; + if (SameAsPreviousStorageElement(DrawGuestWords(shader, binding), element) && StorageDedupeEnabled()) expected = storageTextures[storageIndex - 1]; else expected = cachedStorageTexture(context, words, DecodeTextureResource(words), storageMips[storageIndex], 0, storageWritten[storageIndex]); if (expected != storageTextures[storageIndex]) return false; ++storageIndex; @@ -3878,10 +4043,11 @@ bool ShaderResources::revalidateFull(std::span shaders, Pr } for (const auto& table : tableStages) { if (table.elementTextures.empty()) continue; - const auto* images = tableImages(shaders, table.stage); - if (images == nullptr || images->count != table.elementTextures.size()) return false; + std::span imageWords; + const auto* images = tableImages(shaders, table.stage, imageWords); + if (images == nullptr || images->count != table.elementTextures.size() || imageWords.size() != static_cast(images->count) * 8u) return false; std::vector> cached; - if (auto* reused = FindReusedTable(context.device, *images, CurrentTableState()); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { + if (auto* reused = FindReusedTable(context.device, *images, imageWords, CurrentTableState()); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { auto next = cached.begin(); bool same = true; for (std::uint32_t element = 0; same && element < images->count; ++element) { @@ -3902,7 +4068,7 @@ bool ShaderResources::revalidateFull(std::span shaders, Pr } for (std::uint32_t element = 0; element < images->count; ++element) { if (table.elementTextures[element] == std::numeric_limits::max()) continue; - const auto raw = std::span(images->guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto raw = imageWords.subspan(static_cast(element) * 8u, 8u); std::array narrowed{}; const auto words = firstLayerWords(raw, images->imageShapes.at(element), narrowed, false); const auto resource = DecodeTextureResource(words); @@ -4876,20 +5042,21 @@ std::uint64_t ShaderResources::DataWordsHash(const CompiledShader& shader) { Require(shader.program != nullptr, "missing compiled shader"); std::uint64_t hash = FnvOffset; for (const auto& binding : shader.program->bindings) { - if (DataRole(binding.role)) mixDataWords(hash, binding.guestDescriptor); + if (DataRole(binding.role)) mixDataWords(hash, DrawGuestWords(shader, binding)); } return hash; } bool ShaderResources::DataWordsDiffer(const CompiledShader& shader) const { Require(shader.program != nullptr, "missing compiled shader"); + if (heapStage) return false; const auto& program = *shader.program; for (const auto& binding : program.bindings) { if (!DataRole(binding.role)) continue; const auto held = std::ranges::find_if(bindings, [&](const Binding& candidate) { return candidate.layout.binding == binding.binding; }); if (held == bindings.end() || held->allocations.size() != 1) return true; const auto& allocation = allocations[held->allocations.front()]; - if (allocation.dataWords.size() != binding.guestDescriptor.size() || !std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) return true; + if (allocation.dataWords.size() != DrawGuestWords(shader, binding).size() || !std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), DrawGuestWords(shader, binding).begin())) return true; } return false; } @@ -4902,6 +5069,7 @@ bool ShaderResources::RefreshData(VkCommandBuffer commands, const CompiledShader bool ShaderResources::refreshData(VkCommandBuffer commands, const CompiledShader& shader, Recorder* recorder) { Require(shader.program != nullptr, "missing compiled shader"); + if (heapStage) return false; const auto& program = *shader.program; bool recorded = false; // One [gputime] range of class TemplateDataRefresh around the updates, begun at the first one @@ -4914,13 +5082,13 @@ bool ShaderResources::refreshData(VkCommandBuffer commands, const CompiledShader Require(held != bindings.end(), "template bindings disagree with the shader"); Require(held->allocations.size() == 1, "data binding without its buffer"); auto& allocation = allocations[held->allocations.front()]; - const auto size = binding.guestDescriptor.size() * sizeof(std::uint32_t); + const auto size = DrawGuestWords(shader, binding).size() * sizeof(std::uint32_t); Require(allocation.buffer != nullptr && !allocation.guest && allocation.size == size && size <= MaxRefreshBytes, "template data buffer cannot take the dispatch's words"); - if (allocation.dataWords.size() == binding.guestDescriptor.size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), binding.guestDescriptor.begin())) continue; - if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, binding.guestDescriptor); + if (allocation.dataWords.size() == DrawGuestWords(shader, binding).size() && std::equal(allocation.dataWords.begin(), allocation.dataWords.end(), DrawGuestWords(shader, binding).begin())) continue; + if (binding.role == ShaderRecompiler::DescriptorRole::ShaderData) ValidateRuntimeResources(shader, DrawGuestWords(shader, binding)); if (recorder != nullptr && timing == Recorder::NoTiming) timing = recorder->BeginGpuTiming(Recorder::CommandClass::TemplateDataRefresh); - writeDataWords(commands, held->allocations.front(), binding.guestDescriptor); - allocation.dataWords.assign(binding.guestDescriptor.begin(), binding.guestDescriptor.end()); + writeDataWords(commands, held->allocations.front(), DrawGuestWords(shader, binding)); + allocation.dataWords.assign(DrawGuestWords(shader, binding).begin(), DrawGuestWords(shader, binding).end()); refreshedBytes += size; recorded = true; } @@ -4948,6 +5116,11 @@ VkBuffer ShaderResources::shaderDataBuffer(const CompiledShader& shader) const { } void ShaderResources::forgetShaderDataWords(const CompiledShader& shader) { + if (HeapShader(shader)) { + auto& stageState = heapStages[HeapStageIndex(shader.stage)]; + stageState.data.mark.entry = 0; + stageState.dataInput.clear(); + } if (const auto allocation = shaderDataAllocation(shader)) { allocations[*allocation].dataWords.clear(); rehashDataWords(); @@ -4973,7 +5146,7 @@ void ShaderResources::PrecollectSurfaces() const { for (const auto& range : describedRanges) GuestMemory::CollectWrites(range.address, static_cast(range.bytes)); } -void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags) { +void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, VkShaderStageFlags flags) { Require(binding.count != 0, "empty descriptor binding"); const auto heap = static_cast(binding.binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); const auto capacity = binding.count; @@ -4986,20 +5159,20 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& if (binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) { // Storage images are guest textures the shader writes (looked up in stage B). Require(binding.role == ShaderRecompiler::DescriptorRole::GuestImages, "storage image binding has a non-image role"); - Require(binding.guestDescriptor.size() == static_cast(binding.count) * 8u, "guest storage image descriptors must contain 8 dwords each"); + Require(guest.size() == static_cast(binding.count) * 8u, "guest storage image descriptors must contain 8 dwords each"); Require(context.detiler != nullptr, "device texture detiler is unavailable"); Require(capacity <= context.limits.maxPerStageDescriptorStorageImages, "shader storage-image descriptors exceed per-stage limits"); bindings.push_back({{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, capacity, flags, nullptr}, {}, {}}); plannedStorageImages += binding.count; - deferredImages.push_back({&binding, bindings.size() - 1}); + deferredImages.push_back({&binding, guest, bindings.size() - 1}); return; } const bool sampledImage = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; const bool samplerKind = binding.kind == ShaderRecompiler::DescriptorKind::Sampler; if (!(sampledImage || samplerKind)) Require(false, std::string("unsupported descriptor kind ") + kindName(binding.kind) + " for role " + roleName(binding.role)); Require((sampledImage && binding.role == ShaderRecompiler::DescriptorRole::GuestImages) || (samplerKind && binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers), "guest image descriptor role disagrees with its kind"); - Require(binding.guestDescriptor.size() % binding.count == 0, "guest image descriptor size is not a multiple of the binding count"); - const auto elementWords = binding.guestDescriptor.size() / binding.count; + Require(guest.size() % binding.count == 0, "guest image descriptor size is not a multiple of the binding count"); + const auto elementWords = guest.size() / binding.count; Binding item{{binding.binding, sampledImage ? VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE : VK_DESCRIPTOR_TYPE_SAMPLER, capacity, flags, nullptr}, {}, {}}; @@ -5017,7 +5190,7 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& Require(binding.samplerDepthCompare.size() == binding.count, "guest sampler binding is missing depth comparison metadata"); Require(binding.samplerUnnormalized.empty() || binding.samplerUnnormalized.size() == binding.count, "guest sampler binding has unnormalized coordinate metadata of another size"); for (std::uint32_t element = 0; element < binding.count; ++element) { - auto guestWords = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + auto guestWords = std::span(guest).subspan(static_cast(element) * elementWords, elementWords); std::array normalizedWords{}; if (element < binding.samplerCoordinatesIgnored.size() && binding.samplerCoordinatesIgnored[element]) { std::copy(guestWords.begin(), guestWords.end(), normalizedWords.begin()); @@ -5042,7 +5215,7 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& } bindings.push_back(std::move(item)); - if (sampledImage) deferredImages.push_back({&binding, bindings.size() - 1}); + if (sampledImage) deferredImages.push_back({&binding, guest, bindings.size() - 1}); } namespace { @@ -5083,10 +5256,10 @@ bool ShaderResources::precollectImages() { auto& counters = TextureCounts(); for (const auto& deferred : deferredImages) { const auto& binding = *deferred.binding; - const auto elementWords = binding.guestDescriptor.size() / binding.count; + const auto elementWords = deferred.words.size() / binding.count; for (std::uint32_t element = 0; element < binding.count; ++element) { const bool sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; - const auto raw = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto raw = std::span(deferred.words).subspan(static_cast(element) * elementWords, elementWords); std::array narrowed{}; const auto words = firstLayerWords(raw, sampled ? binding.imageShape : std::nullopt, narrowed, false); if (sampled && !imageRecords.empty()) { @@ -5205,15 +5378,15 @@ void ShaderResources::applySamplerVariants() { } } -void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item, std::span> shaderSamplers) { +void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, Binding& item, std::span> shaderSamplers) { auto& counters = TextureCounts(); // The element's stage-A record, when the pass ran (records follow the plan order exactly). const auto nextRecord = [&]() -> ImageRecord* { return nextImageRecord < imageRecords.size() ? &imageRecords[nextImageRecord++] : nullptr; }; if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage) { Require(binding.imageSamplers.size() == binding.count, "guest sampled image binding is missing its image-sampler pairs"); - const auto elementWords = binding.guestDescriptor.size() / binding.count; + const auto elementWords = guest.size() / binding.count; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto raw = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const auto raw = std::span(guest).subspan(static_cast(element) * elementWords, elementWords); std::array narrowed{}; const auto words = firstLayerWords(raw, binding.imageShape, narrowed, true); auto* record = nextRecord(); @@ -5305,8 +5478,8 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind // storage writes). std::uint32_t mipOffset = 0; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); - const bool sameAsPrevious = SameAsPreviousStorageElement(binding, element); + const auto words = std::span(guest).subspan(static_cast(element) * 8u, 8u); + const bool sameAsPrevious = SameAsPreviousStorageElement(guest, element); if (sameAsPrevious) ++mipOffset; else mipOffset = 0; const auto* record = nextRecord(); @@ -5321,9 +5494,9 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind if (resource.dimension == TextureDimension::k2DMsaa) { Require(!(element < binding.imageAtomic.size() && binding.imageAtomic[element]) && !(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]), "storage image atomics on a multisampled texture are not implemented"); storageTextures.push_back(nullptr); - multisampleStorageViews.resize(storageTextures.size(), VK_NULL_HANDLE); - const bool uintStorage = ShaderRecompiler::RuntimeAbi::UintStorageImageBinding(binding.binding); - multisampleStorageViews.back() = MultisampleTargetStorageView(context, resource, uintStorage); + multisampleStorage.resize(storageTextures.size()); + const bool uintStorage = UintStorageBinding(binding.binding); + multisampleStorage.back() = MultisampleTargetStorageView(context, resource, uintStorage); multisampleSources.push_back(resource.baseAddress); storageMips.push_back(0); storageKeys.push_back(0); @@ -5428,6 +5601,7 @@ ShaderResources::TableStage& ShaderResources::tableStage(const CompiledShader& s table.program = shader.program; table.poison = shader.program->imageTablePoison; table.messages.assign(table.poison.size(), nullptr); + table.heap = HeapShader(shader) && shader.program->dataAddressPush; return table; } @@ -5444,21 +5618,21 @@ std::uint32_t ShaderResources::rejectTableElement(TableStage& table, std::span(table.poison.size() - 1u)); } -void ShaderResources::addSamplerTable(const ShaderRecompiler::DescriptorBinding& binding, VkShaderStageFlags flags, TableStage& table) { +void ShaderResources::addSamplerTable(const ShaderRecompiler::DescriptorBinding& binding, std::span guest, VkShaderStageFlags flags, TableStage& table) { namespace Abi = ShaderRecompiler::ImageTableAbi; - Require(binding.kind == ShaderRecompiler::DescriptorKind::Sampler && binding.guestDescriptor.size() == static_cast(binding.count) * 4u && binding.samplerDepthCompare.size() == binding.count && binding.count % Abi::SamplerEntryElements == 0u, "sampler table binding is malformed"); - Require(context.samplerTableCapacity != 0, "sampler tables need runtimeDescriptorArray and descriptorBindingPartiallyBound, which the device lacks"); + Require(binding.kind == ShaderRecompiler::DescriptorKind::Sampler && guest.size() == static_cast(binding.count) * 4u && binding.samplerDepthCompare.size() == binding.count && binding.count % Abi::SamplerEntryElements == 0u, "sampler table binding is malformed"); + Require(table.heap || context.samplerTableCapacity != 0, "sampler tables need runtimeDescriptorArray and descriptorBindingPartiallyBound, which the device lacks"); const VkDescriptorBindingFlags samplerTableFlags = VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT | (context.samplerTableUpdateAfterBind ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0); Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_SAMPLER, context.samplerTableCapacity, flags, nullptr}, {}, {}, samplerTableFlags}; static const bool noSamplerCache = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr; for (std::uint32_t entry = 0; entry < binding.count / Abi::SamplerEntryElements; ++entry) { const auto first = entry * Abi::SamplerEntryElements; - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(first) * 4u, 4u); + const auto words = guest.subspan(static_cast(first) * 4u, 4u); std::string rejected; - if (first + Abi::SamplerEntryElements > context.samplerTableCapacity) rejected = "the sampler table exceeds the device's capacity of " + std::to_string(context.samplerTableCapacity) + " samplers"; + if (!table.heap && first + Abi::SamplerEntryElements > context.samplerTableCapacity) rejected = "the sampler table exceeds the device's capacity of " + std::to_string(context.samplerTableCapacity) + " samplers"; for (std::uint32_t element = first; rejected.empty() && element < first + Abi::SamplerEntryElements; ++element) { try { - const auto elementWords = std::span(binding.guestDescriptor.data() + static_cast(element) * 4u, 4u); + const auto elementWords = std::span(guest.data() + static_cast(element) * 4u, 4u); auto resource = DecodeSamplerResource(elementWords); resource.compareEnable = binding.samplerDepthCompare.at(element); Sampler::Validate(context, resource); @@ -5477,7 +5651,7 @@ void ShaderResources::addSamplerTable(const ShaderRecompiler::DescriptorBinding& continue; } for (std::uint32_t element = first; element < first + Abi::SamplerEntryElements; ++element) { - const auto elementWords = std::span(binding.guestDescriptor).subspan(static_cast(element) * 4u, 4u); + const auto elementWords = guest.subspan(static_cast(element) * 4u, 4u); const bool compareEnable = binding.samplerDepthCompare.at(element); if (context.samplerCache != nullptr && !noSamplerCache) { samplers.push_back(context.samplerCache->Get(context, elementWords, compareEnable, false)); @@ -5492,6 +5666,10 @@ void ShaderResources::addSamplerTable(const ShaderRecompiler::DescriptorBinding& } const auto setSamplers = context.samplerTableUpdateAfterBind ? context.descriptorIndexingLimits.maxDescriptorSetUpdateAfterBindSamplers : context.limits.maxDescriptorSetSamplers; Require(samplers.size() <= setSamplers, "pipeline sampler descriptors exceed device limits"); + if (table.heap) { + table.tableSamplers = std::move(item.imageAllocations); + return; + } bindings.push_back(std::move(item)); } @@ -5579,7 +5757,7 @@ std::vector variantRepresentatives(VkDevice device, const ShaderR void ShaderResources::resolveImageTables() { const auto none = [](const std::array& set) { return set[0] == std::numeric_limits::max() && set[1] == std::numeric_limits::max(); }; - if (none(tableBindings) && none(storageTableBindings)) return; + if (none(tableBindings) && none(storageTableBindings) && std::none_of(tableStages.begin(), tableStages.end(), [](const TableStage& table) { return table.heap && table.images != nullptr; })) return; auto& counters = TextureCounts(); for (std::size_t set = 0; set < tableBindings.size(); ++set) { if (storageTableBindings[set] != std::numeric_limits::max()) bindings[storageTableBindings[set]].imageAllocations.assign(std::min(tableElements[set], StorageTableCapacity(context)), std::numeric_limits::max()); @@ -5587,11 +5765,12 @@ void ShaderResources::resolveImageTables() { } for (auto& table : tableStages) { if (table.images == nullptr) continue; - auto* item = tableBindings[table.set] != std::numeric_limits::max() ? &bindings[tableBindings[table.set]] : nullptr; + auto* item = !table.heap && tableBindings[table.set] != std::numeric_limits::max() ? &bindings[tableBindings[table.set]] : nullptr; const auto& binding = *table.images; const auto faults = table.map[ShaderRecompiler::ImageTableAbi::HeaderElementFaults]; const auto stageSamplers = std::span>(samplers).subspan(table.firstSampler, table.samplerCount); table.elementTextures.assign(binding.count, std::numeric_limits::max()); + if (table.heap) table.elementStorage.assign(binding.count, std::numeric_limits::max()); const auto capacityMessage = [&] { return "the image table exceeds the device's capacity of " + std::to_string(context.imageTableCapacity) + " images"; }; std::vector samplerKey; for (std::uint32_t sampler = 0; sampler < table.samplerCount; ++sampler) { @@ -5604,7 +5783,7 @@ void ShaderResources::resolveImageTables() { textureFirstLayer.push_back(firstLayer); describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress, true}); table.elementTextures[element] = textures.size() - 1u; - item->imageAllocations[static_cast(table.elementBase) + element] = textures.size() - 1u; + if (item != nullptr) item->imageAllocations[static_cast(table.elementBase) + element] = textures.size() - 1u; }; const auto bind = [&](std::uint32_t element, std::span words, std::shared_ptr texture, const GuestTextureResource& resource, std::uint64_t guestBytes, bool firstLayer) -> std::string { try { @@ -5632,14 +5811,14 @@ void ShaderResources::resolveImageTables() { const auto state = CurrentTableState(); std::vector> cached; const bool storageTable = std::find(binding.imageWritten.begin(), binding.imageWritten.end(), true) != binding.imageWritten.end(); - if (auto* reused = storageTable || item == nullptr ? nullptr : FindReusedTable(context.device, binding, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { + if (auto* reused = storageTable || item == nullptr ? nullptr : FindReusedTable(context.device, binding, table.imageWords, state); reused != nullptr && TableCurrent(*reused) && CachedTableTextures(*reused, cached)) { const bool replay = TableReplay() && reused->replayable && reused->imageSamplers == binding.imageSamplers && reused->samplers == samplerKey; if (!replay) reused->bound.assign(binding.count, {}); auto next = cached.begin(); for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto words = table.imageWords.subspan(static_cast(element) * 8u, 8u); const auto& entry = reused->elements[element]; - if (static_cast(table.elementBase) + element >= context.imageTableCapacity) { + if (!table.heap && static_cast(table.elementBase) + element >= context.imageTableCapacity) { if (!entry.rejected) ++next; table.map[faults + element] = rejectTableElement(table, words, capacityMessage()); } else if (entry.rejected) { @@ -5675,9 +5854,9 @@ void ShaderResources::resolveImageTables() { std::unordered_map representativeSources; std::vector substitutes; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * 8u, 8u); + const auto words = table.imageWords.subspan(static_cast(element) * 8u, 8u); const auto absolute = static_cast(table.elementBase) + element; - if (absolute >= context.imageTableCapacity) { + if (!table.heap && absolute >= context.imageTableCapacity) { table.map[faults + element] = rejectTableElement(table, words, capacityMessage()); reusable = false; continue; @@ -5709,11 +5888,11 @@ void ShaderResources::resolveImageTables() { if (element < binding.imageWritten.size() && binding.imageWritten[element]) { reusable = false; const auto storageBinding = storageTableBindings[table.set]; - if (storageBinding == std::numeric_limits::max()) { + if (!table.heap && storageBinding == std::numeric_limits::max()) { table.map[faults + element] = rejectTableElement(table, words, "set " + std::to_string(table.set) + " has no image storage table, but the shader stores to its table element " + std::to_string(absolute)); continue; } - if (absolute >= StorageTableCapacity(context)) { + if (!table.heap && absolute >= StorageTableCapacity(context)) { table.map[faults + element] = rejectTableElement(table, words, "the stored table element " + std::to_string(absolute) + " of set " + std::to_string(table.set) + " is past the image storage table's capacity of " + std::to_string(StorageTableCapacity(context)) + " images"); continue; } @@ -5737,10 +5916,11 @@ void ShaderResources::resolveImageTables() { storageAtomic.push_back(element < binding.imageAtomic.size() && binding.imageAtomic[element]); storageAtomic64.push_back(element < binding.imageAtomic64.size() && binding.imageAtomic64[element]); describedRanges.push_back({"storage", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress, true}); - bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u; + if (table.heap) table.elementStorage[element] = storageTextures.size() - 1u; + else bindings[storageBinding].imageAllocations[absolute] = storageTextures.size() - 1u; continue; } - if (item == nullptr) { + if (item == nullptr && !table.heap) { table.map[faults + element] = rejectTableElement(table, words, "a sampled image table entry has no sampled image table binding"); reusable = false; continue; @@ -5801,10 +5981,11 @@ void ShaderResources::resolveImageTables() { fresh.imageSamplers = binding.imageSamplers; fresh.samplers = std::move(samplerKey); fresh.replayable = static_cast(table.elementBase) + binding.count <= context.imageTableCapacity; - RememberTable(std::move(fresh), binding); + RememberTable(std::move(fresh), binding, table.imageWords); } } } + for (auto& table : tableStages) table.imageWords = {}; reportTextureCounters(); } @@ -5816,6 +5997,72 @@ std::uint64_t SampledLookupKeyProofs() { return sampledLookupKeyProofs; } +namespace { +std::atomic drawInputCopies{0}; +} + +std::uint64_t DrawInputCopies() { + return drawInputCopies.load(std::memory_order_relaxed); +} + +DeviceTableTexture LookupDeviceTableTexture(const Context& context, std::span descriptor, const DeviceTableView& view, std::uint32_t samplerClass) { + namespace Abi = ShaderRecompiler::DeviceImageTableAbi; + std::array words{}; + std::copy(descriptor.begin(), descriptor.end(), words.begin()); + if (view.identitySwizzle) words[3] = (words[3] & ~0xfffu) | ShaderRecompiler::ShaderImageIdentitySwizzle; + const auto resource = DecodeTextureResource(words); + const bool firstLayer = view.shape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; + if (!firstLayer && !MatchesGuestDimension(view.shape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the table entry's image shape (shape " + std::to_string(static_cast(view.shape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); + const auto guestBytes = DescribeSurface(resource).guestBytes; + if (!GuestMemory::Accessible(reinterpret_cast(resource.baseAddress), static_cast(guestBytes)) && GuestMemory::CommittedRanges(resource.baseAddress, static_cast(guestBytes), false).empty()) throw std::runtime_error("AGC graphics: the table entry's surface at 0x" + HexText(resource.baseAddress) + " (0x" + HexText(guestBytes) + " bytes) is not mapped"); + static_cast(ResolveTextureFormat(resource.format)); + CompletePendingCpuWrites(context, resource.baseAddress, static_cast(guestBytes)); + if (resource.dccAddress != 0) CompletePendingCpuWrites(context, resource.dccAddress, DccKeyBytes(guestBytes)); + DeviceTableTexture result; + result.address = resource.baseAddress; + result.guestBytes = guestBytes; + result.generation = GuestMemory::CollectWrites(resource.baseAddress, static_cast(guestBytes)); + auto texture = cachedTexture(context, words, resource, ViewComponents(resource), guestBytes, view.depthCompare); + const auto& log = lookupLog(); + const auto* looked = !log.empty() && log.back().object == texture.get() ? &log.back() : nullptr; + if (looked != nullptr && looked->depth) { + result.unservable = "the surface at 0x" + HexText(resource.baseAddress) + " is a depth surface, whose layout transitions per use the device image table does not make"; + return result; + } + if (texture->CopyKind() != Texture::PerUseCopy::None) { + result.unservable = "the texture at 0x" + HexText(resource.baseAddress) + " is a copy remade or refreshed per use, which the device image table does not make"; + return result; + } + if ((samplerClass & Abi::SamplerFilterMinmax) != 0u) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, texture->ViewFormat(), &properties); + if ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_MINMAX_BIT_EXT) == 0) throw std::runtime_error("a sampled texture whose format " + std::to_string(texture->ViewFormat()) + " does not support min/max filtering is sampled through a min or max reduction sampler with linear filtering"); + } + if ((samplerClass & Abi::SamplerOpaqueBlackBorder) != 0u && resource.bcSwizzle != 0u && resource.bcSwizzle != 4u) throw std::runtime_error("a sampled texture whose BC swizzle " + std::to_string(resource.bcSwizzle) + " moves the alpha channel is sampled through an opaque black border, which is not implemented"); + if ((samplerClass & Abi::SamplerNonSeamlessCube) != 0u && resource.dimension == TextureDimension::kCube && !context.nonSeamlessCubeMap) throw std::runtime_error("a cube texture is sampled through a sampler that disables seamless cube filtering, which needs VK_EXT_non_seamless_cube_map"); + const auto numeric = ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) != 0u ? ShaderRecompiler::IrTextureNumericClass::Float : ShaderRecompiler::SampledTextureNumericClass(static_cast(resource.format)); + const bool integerImage = numeric == ShaderRecompiler::IrTextureNumericClass::Uint || numeric == ShaderRecompiler::IrTextureNumericClass::Sint; + if (((samplerClass & Abi::SamplerFloatBorder) != 0u && integerImage) || ((samplerClass & Abi::SamplerIntegerBorder) != 0u && !integerImage)) throw std::runtime_error("an image table entry's sampler border color numeric class disagrees with its sampled image class, which needs dynamic pairing and is not implemented"); + if ((samplerClass & Abi::SamplerAnisoOverride) != 0u) throw std::runtime_error("an image table entry paired with anisotropic ANISO_OVERRIDE needs dynamic texture mip-level pairing, which is not implemented"); + result.view = firstLayer ? texture->FirstLayerView() : texture->View(); + result.layout = texture->Layout(); + result.texture = std::move(texture); + result.words = words; + result.depthCompare = view.depthCompare; + return result; +} + +bool DeviceTableTextureCurrent(const Context& context, const std::array& words, bool depthCompare, const Texture& texture) { + const auto key = MakeTextureKey(context.device, words, ViewComponents(DecodeTextureResource(words)), depthCompare); + auto& cache = Textures(); + std::lock_guard lock(cache.mutex); + const auto it = findTexture(cache, key); + if (it == cache.entries.end() || it->texture.get() != &texture || it->source != nullptr || it->keys != DccKeys::Uncompressed || it->committed.has_value()) return false; + if (Recorder::SnapshotWriteOverlaps(it->address, it->bytes.size())) return false; + const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::TextureCompare); + return GuestMemory::EqualsCommitted(it->address, it->bytes); +} + std::uint64_t SampledTexturesCleared() { return TextureCounts().cleared.load(std::memory_order_relaxed); } @@ -5864,6 +6111,10 @@ void ShaderResources::finishImageTables() { faults.push_back({IrStage(table.stage), table.program->imageTableShader, table.poison, table.messages}); ranges.insert(ranges.end(), table.program->imageTableRanges.begin(), table.program->imageTableRanges.end()); } + for (auto& table : tableStages) { + table.program = nullptr; + table.images = nullptr; + } if (bda == nullptr) return; bda->SetImageTables(std::move(faults), std::move(ranges)); } @@ -5987,6 +6238,7 @@ void ShaderResources::release() noexcept { std::uint32_t ShaderResources::setOf(const Binding& binding) const { constexpr auto count = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); if (binding.set >= 0) return static_cast(binding.set); + if (heapStage) return 0u; return splitSets && binding.layout.binding / count == static_cast(ShaderRecompiler::RuntimeAbi::Stage::Fragment) ? 1u : 0u; } @@ -6018,32 +6270,36 @@ ShaderResources::DrawBindings::~DrawBindings() { std::optional> ShaderResources::MovedReadOnlyBuffers(std::span shaders, Recorder& recorder, const ColorTarget* target, bool* patchedData) const { std::vector moved; std::size_t movedBytes = 0; - if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || (usesBda && !spaceReuse)) return moved; + if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE && !heapStage) || (usesBda && !spaceReuse)) return moved; for (const auto& shader : shaders) { if (shader.program == nullptr) return std::nullopt; + const bool heapShader = HeapShader(shader); for (const auto& binding : shader.program->bindings) { + if (heapShader && (DataRole(binding.role) || binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers)) continue; if (DataRole(binding.role)) { const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; }); if (kept == bindings.end() || kept->allocations.size() != 1) return std::nullopt; const auto index = kept->allocations.front(); const auto& item = allocations[index]; - if (item.guest || item.buffer == nullptr || item.size != binding.guestDescriptor.size() * sizeof(std::uint32_t)) return std::nullopt; + const auto& guestWords = DrawGuestWords(shader, binding); + if (item.guest || item.buffer == nullptr || item.size != guestWords.size() * sizeof(std::uint32_t)) return std::nullopt; const bool patched = std::any_of(dataPatches.begin(), dataPatches.end(), [&](const DataPatch& patch) { return patch.allocation == index; }); - const bool same = !item.dataWords.empty() ? item.dataWords == binding.guestDescriptor : !patched && std::memcmp(item.buffer->Bytes().data(), binding.guestDescriptor.data(), item.size) == 0; + const bool same = !item.dataWords.empty() ? item.dataWords == guestWords : !patched && std::memcmp(item.buffer->Bytes().data(), guestWords.data(), item.size) == 0; if (same) continue; if (item.dataWords.empty() && patched) { if (patchedData != nullptr) *patchedData = true; return std::nullopt; } if (moved.empty()) moved.reserve(allocations.size()); - moved.push_back({index, 0, item.size, binding.guestDescriptor}); + moved.push_back({index, 0, item.size, guestWords}); continue; } if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; + const auto& guestWords = DrawGuestWords(shader, binding); const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; }); - if (kept == bindings.end() || kept->allocations.size() != binding.count || binding.guestDescriptor.size() != static_cast(binding.count) * 4u) return std::nullopt; + if (kept == bindings.end() || kept->allocations.size() != binding.count || guestWords.size() != static_cast(binding.count) * 4u) return std::nullopt; for (std::uint32_t element = 0; element < binding.count; ++element) { - const auto* words = binding.guestDescriptor.data() + static_cast(element) * 4u; + const auto* words = guestWords.data() + static_cast(element) * 4u; const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; const auto address = descriptor.Base48(); const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; @@ -6059,12 +6315,19 @@ std::optional> ShaderResources::MovedR if (written || empty || item.written || size > context.limits.maxStorageBufferRange) return std::nullopt; const auto begin = address - item.adjustment; const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); - if (bytes > maxDrawSnapshotBytes - movedBytes) return std::nullopt; + const bool copied = !(usesBda && heapStage); + if (copied && bytes > maxDrawSnapshotBytes - movedBytes) return std::nullopt; if (!GuestMemory::Accessible(reinterpret_cast(begin), bytes)) return std::nullopt; if (guestMemory.WritesOverlap(begin, bytes) || recorder.PendingWriteOverlaps(begin, bytes) || PendingStorageOverlaps(begin, bytes, nullptr) || AnyShadowedOverlaps(begin, bytes)) return std::nullopt; + if (usesBda && heapStage) { + std::uint32_t adjustment = 0; + const auto inPlace = guestMemory.TryDescriptor(address, static_cast(size), adjustment); + const bool classic = std::any_of(bindings.begin(), bindings.end(), [&](const Binding& other) { return !(other.heap && other.addressed) && std::find(other.allocations.begin(), other.allocations.end(), index) != other.allocations.end(); }); + if (!inPlace.has_value() || adjustment != item.adjustment || classic) return std::nullopt; + } if (moved.empty()) moved.reserve(allocations.size()); moved.push_back({index, address, static_cast(size)}); - movedBytes += bytes; + if (copied) movedBytes += bytes; } } } @@ -6094,7 +6357,7 @@ std::optional> ShaderResources::MovedD }; std::vector moved; if (shader.program == nullptr || usesBda) return refuse(MoveRefusal::Shape); - if (_set == VK_NULL_HANDLE) return moved; + if (_set == VK_NULL_HANDLE && !heapStage) return moved; for (const auto& binding : shader.program->bindings) { if (binding.role != ShaderRecompiler::DescriptorRole::GuestBuffers) continue; const auto kept = std::find_if(bindings.begin(), bindings.end(), [&](const Binding& item) { return item.layout.binding == binding.binding; }); @@ -6145,11 +6408,16 @@ std::size_t ShaderResources::DrawUploadLimit() { std::shared_ptr ShaderResources::PrepareDrawBindings(Recorder& recorder, std::span moved, bool movedOnly, const BdaResources* faultRecord) const { requireGuestBuffers(moved); - if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE) || (usesBda && !spaceReuse)) return {}; + if ((_set == VK_NULL_HANDLE && fragment.set == VK_NULL_HANDLE && !heapStage) || (usesBda && !spaceReuse)) return {}; + const bool readsInPlace = usesBda && heapStage; + movedOnly = movedOnly || readsInPlace; + thread_local std::vector>* inPlaceSlot = nullptr; + auto& inPlace = ShaderRecompiler::ThreadOwned(inPlaceSlot); + inPlace.clear(); prepareFaultSlot(); thread_local std::vector>* readsSlot = nullptr; auto& reads = ShaderRecompiler::ThreadOwned(readsSlot); - guestMemory.InPlaceReads(reads); + if (!readsInPlace) guestMemory.InPlaceReads(reads); const auto maxSnapshotBytes = maxDrawSnapshotBytes; const auto uploadLimit = DrawUploadLimit(); std::size_t totalSnapshotBytes = 0; @@ -6166,6 +6434,12 @@ std::shared_ptr ShaderResources::PrepareDrawBindi for (std::size_t index = 0; index < allocations.size(); ++index) { const auto& item = allocations[index]; const auto override = std::find_if(moved.begin(), moved.end(), [&](const MovedBuffer& entry) { return entry.allocation == index; }); + if (override != moved.end() && override->words.empty() && readsInPlace) { + std::uint32_t adjustment = 0; + inPlace.push_back({index, wholeDwordRange(guestMemory.Descriptor(override->address, override->size, adjustment))}); + Require(adjustment == item.adjustment, "a moved buffer read in place lost its view's alignment"); + continue; + } if (override != moved.end() && !override->words.empty()) { auto slice = recorder.DrawUpload(override->size); std::memcpy(slice.bytes, override->words.data(), override->size); @@ -6225,25 +6499,28 @@ std::shared_ptr ShaderResources::PrepareDrawBindi if (GuestMemory::TrackerGeneration() != stamp[0]) stamped = false; else if (!uploaded || stamp[1] != 0) recorder.NoteUnwrittenDrawSnapshot(begin, bytes, stamp, buffer, offset); } + drawInputCopies.fetch_add(1, std::memory_order_relaxed); selected.push_back(index); keep({begin, std::move(buffer), offset, bytes}); totalSnapshotBytes += bytes; CaptureTrace::Log("draw-snapshot batch=%llu address=%llx bytes=%zu", static_cast(recorder.Submissions() + 1), static_cast(begin), bytes); } - if (selected.empty()) return {}; - writeDrawSets(*result, selected, faultRecord); + if (selected.empty() && inPlace.empty()) return {}; + Require(inPlace.empty() || addressedBuffers, "a moved buffer read in place has no addressed binding"); + writeDrawSets(*result, selected, faultRecord, inPlace); recorder.Keep(result); for (const auto& snapshot : result->snapshots) recorder.KeepBytes(snapshot.buffer.get(), snapshot.buffer->Bytes().size()); return result; } -void ShaderResources::writeDrawSets(DrawBindings& result, std::span selected, const BdaResources* faultRecord) const { +void ShaderResources::writeDrawSets(DrawBindings& result, std::span selected, const BdaResources* faultRecord, std::span> inPlace) const { Require(context.descriptorCache != nullptr, "draw snapshots require a descriptor cache"); std::array described{}; thread_local std::array, 2>* countsSlot = nullptr; auto& counts = ShaderRecompiler::ThreadOwned(countsSlot); for (auto& sizes : counts) sizes.clear(); for (const auto& binding : bindings) { + if (binding.heap) continue; const auto set = setOf(binding); described[set] = true; const auto count = (binding.flags & VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT) != 0 ? std::max(1u, static_cast(binding.imageAllocations.size())) : binding.layout.descriptorCount; @@ -6262,7 +6539,7 @@ void ShaderResources::writeDrawSets(DrawBindings& result, std::spanAllocateRewritten(fragment.layout, sizesOf(1), updateAfterBind, variableOf(1)); result.fragmentLayout = fragment.layout; Require(result.fragmentAllocation.set != VK_NULL_HANDLE, "draw snapshot descriptor allocation failed"); @@ -6282,6 +6559,7 @@ void ShaderResources::writeDrawSets(DrawBindings& result, std::spandstBinding == binding.layout.binding && nextWrite->dstSet == original) { auto& write = *nextWrite++; @@ -6298,6 +6576,24 @@ void ShaderResources::writeDrawSets(DrawBindings& result, std::span overrides(allocations.size(), VkDescriptorBufferInfo{VK_NULL_HANDLE, 0, 0}); + bool overridden = false; + for (std::size_t index = 0; index < selected.size(); ++index) { + const auto& snapshot = result.snapshots[index]; + overrides[selected[index]] = {snapshot.buffer->Handle(), snapshot.offset, snapshot.bytes}; + } + for (const auto& [index, info] : inPlace) overrides[index] = info; + for (const auto& binding : bindings) { + if (!binding.addressed) continue; + for (const auto allocation : binding.allocations) overridden = overridden || overrides[allocation].buffer != VK_NULL_HANDLE; + } + if (overridden) { + result.heapOffsets = addressedOffsets(&overrides); + result.addressOffsets = true; + result.addressOverrides = std::move(overrides); + } + } context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); } @@ -6332,8 +6628,40 @@ void ShaderResources::BindDrawUse(VkCommandBuffer commands, VkPipelineLayout lay requireGuestBuffers(); const auto set = use->bindings->allocation.set; const auto fragmentSet = use->bindings->fragmentAllocation.set; - if (set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 1, &set, 0, nullptr); - if (fragmentSet != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragmentSet, 0, nullptr); + if (set != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, 0, 1, &set, SetBindSite::Draw); + if (fragmentSet != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_GRAPHICS, layout, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragmentSet, SetBindSite::Draw); +} + +namespace { + +std::size_t DataOffsetIndex(std::size_t stage, ShaderRecompiler::RuntimeAbi::Binding binding) { + Require(binding == ShaderRecompiler::RuntimeAbi::Binding::ShaderData || binding == ShaderRecompiler::RuntimeAbi::Binding::FlattenedSrt, "only shader data and the flattened SRT have ring offsets"); + return stage * 2u + (binding == ShaderRecompiler::RuntimeAbi::Binding::FlattenedSrt ? 1u : 0u); +} + +} + +void ShaderResources::RefreshMovedAddresses(DrawBindings& rewritten) const { + if (rewritten.addressOffsets) rewritten.heapOffsets = addressedOffsets(&rewritten.addressOverrides); +} + +std::array ShaderResources::addressedOffsets(const std::vector* overrides) const { + auto offsets = heapOffsets; + auto& heap = BindlessHeap::For(context); + for (std::size_t stage = 0; stage < stageDataWords.size(); ++stage) { + if (stageDataWords[stage].empty()) continue; + auto words = stageDataWords[stage]; + writeBufferAddresses(words, static_cast(stage), overrides); + auto span = heap.Allocate(words.size()); + std::memcpy(span.words.data(), words.data(), words.size() * sizeof(std::uint32_t)); + offsets[stageDataIndex[stage]] = span.address; + } + for (std::size_t stage = 0; stage < ringDescriptorStages.size(); ++stage) { + if (!ringDescriptorStages[stage]) continue; + RingDescriptors moved; + offsets[DataOffsetIndex(stage, ShaderRecompiler::RuntimeAbi::Binding::FlattenedSrt)] = ringDescriptors(stage, overrides, moved); + } + return offsets; } void ShaderResources::prepareFaultSlot() const { @@ -6366,11 +6694,617 @@ void ShaderResources::requireGuestBuffers(std::span moved) co } } +void ShaderResources::BindDescriptorHeaps(VkCommandBuffer commands) const { + requireGuestBuffers(); + prepareFaultSlot(); + BindlessHeap::For(context).BindDescriptorHeaps(commands); +} + void ShaderResources::Bind(VkCommandBuffer commands, VkPipelineBindPoint bindPoint, VkPipelineLayout layout) const { requireGuestBuffers(); prepareFaultSlot(); - if (_set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, 0, 1, &_set, 0, nullptr); - if (fragment.set != VK_NULL_HANDLE) context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, bindPoint, layout, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragment.set, 0, nullptr); + if (_set != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, bindPoint, layout, 0, 1, &_set, bindPoint == VK_PIPELINE_BIND_POINT_COMPUTE ? SetBindSite::Dispatch : SetBindSite::Draw); + if (fragment.set != VK_NULL_HANDLE) CmdBindDescriptorSets(context, commands, bindPoint, layout, ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet, 1, &fragment.set, bindPoint == VK_PIPELINE_BIND_POINT_COMPUTE ? SetBindSite::Dispatch : SetBindSite::Draw); +} + +PipelineSetLayouts ShaderResources::SetLayouts() const { + if (!heapStage) return {_layout, fragment.layout, splitSets, layoutKey, fragmentLayoutKey}; + return {_layout, VK_NULL_HANDLE, true, layoutKey, fragmentLayoutKey}; +} + +void ShaderResources::RefreshHeapStage(std::span shaders) { + for (const auto& shader : shaders) { + if (HeapShader(shader)) writeHeapStage(shader); + } +} + +void ShaderResources::writeBufferAddresses(std::span words, std::uint32_t dataStage, const std::vector* overrides, + std::vector* ranges) const { + for (const auto& binding : bindings) { + if (!binding.addressed || binding.dataStage != dataStage) continue; + for (std::size_t element = 0; element < binding.allocations.size(); ++element) { + const auto index = binding.allocations[element]; + const auto& allocation = allocations[index]; + const bool overridden = overrides != nullptr && (*overrides)[index].buffer != VK_NULL_HANDLE; + const bool owned = !overridden && !allocation.guest && allocation.buffer != nullptr && allocation.role != ShaderRecompiler::DescriptorRole::GuestBuffers && allocation.role != ShaderRecompiler::DescriptorRole::BdaPagetable && allocation.role != ShaderRecompiler::DescriptorRole::FaultBuffer; + VkDeviceAddress address = 0; + VkDeviceSize range = 0; + if (owned) { + address = allocation.buffer->DeviceAddress(); + range = allocation.size; + } else { + auto adjustment = allocation.adjustment; + auto info = overridden ? (*overrides)[index] : descriptor(allocation, adjustment); + if (info.buffer == VK_NULL_HANDLE) info = {context.emptyBuffer, 0, EmptyBufferBytes}; + const auto base = BufferDeviceAddress(context, info.buffer); + Require(base != 0, "an addressed buffer has no device address"); + address = base + info.offset; + range = info.range; + } + if (ranges != nullptr) ranges->push_back({address, std::min(range, context.limits.maxStorageBufferRange)}); + if (words.empty()) continue; + const auto at = static_cast(binding.addressDword) + element * 4u; + Require(at + 3u < words.size(), "buffer address words lie outside the shader data"); + words[at] = static_cast(address); + words[at + 1u] = static_cast(address >> 32u); + words[at + 2u] = static_cast(std::min(range, 0xffffffffu)); + words[at + 3u] = 0u; + } + } +} + +VkDeviceAddressRangeEXT ShaderResources::srtRange(std::size_t stage) const { + const auto bytes = static_cast(heapSrtDwords[stage]) * sizeof(std::uint32_t); + if (bytes == 0u || bytes > context.limits.maxStorageBufferRange) { + const auto empty = BufferDeviceAddress(context, context.emptyBuffer); + Require(empty != 0, "the empty buffer has no device address"); + return {empty, EmptyBufferBytes}; + } + return {heapSrtOffsets[stage], bytes}; +} + +std::uint32_t ShaderResources::ringDescriptors(std::size_t stage, const std::vector* overrides, RingDescriptors& kept) const { + struct RangeStorage {}; + auto& ranges = HostThreadLocal, RangeStorage>(); + ranges.clear(); + ranges.push_back(srtRange(stage)); + writeBufferAddresses({}, static_cast(stage), overrides, &ranges); + auto& heap = BindlessHeap::For(context); + const auto equal = [](const VkDeviceAddressRangeEXT& a, const VkDeviceAddressRangeEXT& b) { return a.address == b.address && a.size == b.size; }; + if (!std::ranges::equal(ranges, kept.ranges, equal) || !heap.Held(kept.mark)) { + kept.first = heap.RingBufferDescriptors(ranges, kept.mark); + kept.ranges.assign(ranges.begin(), ranges.end()); + } + return kept.first; +} + +std::pair ShaderResources::shaderDataLocation(const CompiledShader& shader, const DrawBindings* rewritten) const { + if (!heapStage || !HeapShader(shader)) return {shaderDataBuffer(shader), 0}; + const auto& offsets = rewritten != nullptr && rewritten->addressOffsets ? rewritten->heapOffsets : heapOffsets; + return BindlessHeap::For(context).DataLocation(offsets[DataOffsetIndex(HeapStageIndex(shader.stage), ShaderRecompiler::RuntimeAbi::Binding::ShaderData)]); +} + +void ShaderResources::PatchDataAddresses(std::span bytes, std::span shaders, std::span placements, const DrawBindings* rewritten) const { + namespace Abi = ShaderRecompiler::RuntimeAbi; + if (!heapStage) return; + const auto& offsets = rewritten != nullptr && rewritten->addressOffsets ? rewritten->heapOffsets : heapOffsets; + VkDeviceAddress table = 0; + VkDeviceAddress fault = 0; + std::uint32_t tableDwords = 0; + const auto binds = [&](ShaderRecompiler::DescriptorRole role) { + return std::any_of(shaders.begin(), shaders.end(), [&](const CompiledShader& shader) { return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [&](const auto& binding) { return binding.role == role; }); }); + }; + const bool tableBound = binds(ShaderRecompiler::DescriptorRole::BdaPagetable); + const bool faultBound = binds(ShaderRecompiler::DescriptorRole::FaultBuffer); + if (bda != nullptr && (tableBound || faultBound)) { + const auto of = [&](const VkDescriptorBufferInfo& info) { + const auto base = BufferDeviceAddress(context, info.buffer); + Require(base != 0, "a BDA buffer has no device address"); + return base + info.offset; + }; + if (tableBound) { + const auto tableInfo = bda->Table(); + table = of(tableInfo); + tableDwords = static_cast(tableInfo.range / sizeof(std::uint32_t)); + } + if (faultBound) fault = of(bda->Fault()); + } + Require(placements.size() == shaders.size(), "the push layout does not describe the draw's stages"); + std::optional gds; + const auto gdsAddress = [&] { + if (gds) return *gds; + gds = VkDeviceAddress{0}; + if (gdsAllocation == std::numeric_limits::max()) return *gds; + const auto* overrides = rewritten != nullptr && rewritten->addressOffsets ? &rewritten->addressOverrides : nullptr; + auto adjustment = allocations[gdsAllocation].adjustment; + const auto info = overrides != nullptr && (*overrides)[gdsAllocation].buffer != VK_NULL_HANDLE ? (*overrides)[gdsAllocation] : descriptor(allocations[gdsAllocation], adjustment); + Require(info.buffer != VK_NULL_HANDLE, "the GDS buffer is missing"); + const auto base = BufferDeviceAddress(context, info.buffer); + Require(base != 0, "the GDS buffer has no device address"); + gds = base + info.offset; + return *gds; + }; + bool rooted = false; + for (std::size_t index = 0; index < shaders.size(); ++index) { + const auto& shader = shaders[index]; + const auto& placement = placements[index]; + const auto put = [&](Abi::RootItem item, std::uint32_t word, std::uint32_t value) { + const auto position = placement.Item(item); + if (position == Abi::NoPushPosition) return; + Require((position + word + 1u) * sizeof(std::uint32_t) <= bytes.size(), "a root item lies outside the push data"); + std::memcpy(bytes.data() + (position + word) * sizeof(std::uint32_t), &value, sizeof(value)); + }; + const auto putAddress = [&](Abi::RootItem item, VkDeviceAddress address) { + put(item, 0u, static_cast(address)); + put(item, 1u, static_cast(address >> 32u)); + }; + putAddress(Abi::RootItem::PagetableAddress, table); + putAddress(Abi::RootItem::FaultAddress, fault); + put(Abi::RootItem::PagetableDwords, 0u, tableDwords); + if (placement.Item(Abi::RootItem::GdsAddress) != Abi::NoPushPosition) putAddress(Abi::RootItem::GdsAddress, gdsAddress()); + if (shader.program == nullptr || !shader.program->dataAddressPush) continue; + const auto stage = HeapStageIndex(shader.stage); + putAddress(Abi::RootItem::ShaderData, offsets[DataOffsetIndex(stage, Abi::Binding::ShaderData)]); + put(Abi::RootItem::FlattenedSrt, 0u, static_cast(offsets[DataOffsetIndex(stage, Abi::Binding::FlattenedSrt)])); + putAddress(Abi::RootItem::SrtAddress, offsets[DataOffsetIndex(stage, Abi::Binding::FlattenedSrt)]); + put(Abi::RootItem::SrtDwords, 0u, heapSrtDwords[stage]); + putAddress(Abi::RootItem::TableMap, heapTableMapSet[stage] ? heapTableMaps[stage] : offsets[DataOffsetIndex(stage, Abi::Binding::ShaderData)]); + writePushedData(bytes, shader, placement.pushed); + rooted = true; + } + if (rooted) NoteDataAddressPush(); +} + +void ShaderResources::writePushedData(std::span bytes, const CompiledShader& shader, ShaderRecompiler::RuntimeAbi::PushedPlacement placement) const { + const auto& program = *shader.program; + if (placement.count == 0u) return; + const auto& words = heapStages[HeapStageIndex(shader.stage)].data.words; + Require(placement.count <= program.pushedData.size() && (placement.base + placement.count) * sizeof(std::uint32_t) <= bytes.size(), "a stage's pushed shader data does not fit"); + for (std::uint32_t index = 0; index < placement.count; ++index) { + const auto dword = program.pushedData[index]; + Require(dword < words.size(), "a pushed shader-data dword lies outside the stage's shader data"); + std::memcpy(bytes.data() + (placement.base + index) * sizeof(std::uint32_t), &words[dword], sizeof(std::uint32_t)); + } +} + +VkImageView ShaderResources::heapPlaceholderView(std::uint32_t kind) const { + return context.nullDescriptors ? VK_NULL_HANDLE : context.placeholderImages->View(kind); +} + +void ShaderResources::writeHeapTableMap(const CompiledShader& shader, std::size_t stageIndex) { + constexpr auto noElement = std::numeric_limits::max(); + namespace Abi = ShaderRecompiler::RuntimeAbi; + namespace TableAbi = ShaderRecompiler::ImageTableAbi; + heapTableMapSet[stageIndex] = false; + const auto table = std::find_if(tableStages.begin(), tableStages.end(), + [&](const TableStage& candidate) { return candidate.stage == shader.stage && candidate.heap; }); + if (table == tableStages.end() || table->mapAllocation == noElement) return; + const ShaderRecompiler::DescriptorBinding* images = nullptr; + for (const auto& binding : shader.program->bindings) { + if (binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable) + images = &binding; + } + auto& heap = BindlessHeap::For(context); + struct MapStorage {}; + auto& map = HostThreadLocal, MapStorage>(); + map.assign(table->map.begin(), table->map.end()); + const auto elements = images != nullptr ? static_cast(images->count) : std::size_t{0}; + TableAbi::Trailer trailer; + if (shader.program->deviceImageTables) { + auto* deviceTable = DeviceImageTable::Existing(context); + auto* recorder = Recorder::Active(); + Require(deviceTable != nullptr, "a stage reading images through the device image table needs the device's table"); + Require(recorder != nullptr, "a stage reading images through the device image table records into the device's recorder"); + const std::array programs{shader.program->Identity()}; + deviceTable->Prepare(context, *recorder, programs); + trailer = deviceTable->Trailer(*recorder); + } + const auto base = TableAbi::AppendTrailer(map, trailer); + const auto sampledAbsent = heap.AbsentSlot(Abi::FirstImageBinding, heapPlaceholderView(Abi::FirstImageBinding)); + const auto storageAbsent = heap.AbsentSlot(Abi::FirstStorageImageBinding, heapPlaceholderView(Abi::FirstStorageImageBinding)); + for (std::size_t element = 0; element < elements; ++element) { + const auto index = element < table->elementTextures.size() ? table->elementTextures[element] : noElement; + if (index == noElement) { + map.push_back(sampledAbsent); + continue; + } + const auto view = textureFirstLayer[index] ? textures[index]->FirstLayerView() : textures[index]->View(); + map.push_back(takenSlot(heap.TextureSlot(Abi::FirstImageBinding, textures[index].get(), view, textures[index]->Layout()), "the global image heap is full")); + } + const auto appendStorageSlots = [&](std::uint32_t kind, std::uint32_t absent, const char* full, const auto& wanted, const auto& viewOf) { + for (std::size_t element = 0; element < elements; ++element) { + const auto index = element < table->elementStorage.size() ? table->elementStorage[element] : noElement; + if (!wanted(element) || index == noElement || storageTextures[index] == nullptr) { + map.push_back(absent); + continue; + } + map.push_back(takenSlot(heap.StorageSlot(kind, storageTextures[index].get(), viewOf(*storageTextures[index], index)), full)); + } + }; + appendStorageSlots( + Abi::FirstStorageImageBinding, storageAbsent, "the global storage image heap is full", [](std::size_t) { return true; }, + [&](StorageTexture& image, std::size_t index) { return image.StorageView(storageMips[index], storageFirstLayer[index]); }); + const auto samplerSlotOf = [&](std::size_t index) { return takenSlot(heap.SamplerSlot(samplers[index]), "the global sampler heap is full"); }; + const auto firstSampler = std::find_if(table->tableSamplers.begin(), table->tableSamplers.end(), [&](const auto index) { return index != noElement; }); + const std::uint32_t fallbackSampler = firstSampler != table->tableSamplers.end() ? samplerSlotOf(*firstSampler) : 0u; + for (const auto index : table->tableSamplers) map.push_back(index == noElement ? fallbackSampler : samplerSlotOf(index)); + const auto atomicElement = [&](std::size_t element, bool wide) { + if (images == nullptr) return false; + const auto& flags = wide ? images->imageAtomic64 : images->imageAtomic; + return element < flags.size() && flags[element]; + }; + const auto anyAtomic = [&](bool wide) { + for (std::size_t element = 0; element < elements; ++element) { + if (atomicElement(element, wide)) return true; + } + return false; + }; + const auto samplerSpan = std::max(elements, map[TableAbi::HeaderSamplerCount]); + Require(table->tableSamplers.size() <= samplerSpan, "an image table map has more table samplers than its header counts"); + const auto extraBase = base + 2u * elements + samplerSpan; + if (anyAtomic(false) || anyAtomic(true)) { + map.resize(extraBase, fallbackSampler); + for (const bool wide : {false, true}) { + const auto kind = wide ? Abi::FirstAtomic64ImageBinding : Abi::FirstAtomicImageBinding; + appendStorageSlots( + kind, heap.AbsentSlot(kind, heapPlaceholderView(kind)), "the global atomic image heap is full", + [&](std::size_t element) { return atomicElement(element, wide); }, + [&](StorageTexture& image, std::size_t index) { + return wide ? image.Atomic64View(storageMips[index], storageFirstLayer[index]) + : image.AtomicView(storageMips[index], storageFirstLayer[index]); + }); + } + } + const bool storageTable = std::ranges::any_of(shader.program->bindings, [](const ShaderRecompiler::DescriptorBinding& binding) { + return binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable; + }); + if (storageTable) { + map.resize(extraBase + 2u * elements, fallbackSampler); + const auto kind = Abi::FirstUintStorageImageBinding; + appendStorageSlots( + kind, heap.AbsentSlot(kind, heapPlaceholderView(kind)), "the global storage image heap is full", [](std::size_t) { return true; }, + [&](StorageTexture& image, std::size_t index) { return image.UintStorageViewOrDefault(storageMips[index], storageFirstLayer[index]); }); + } + if (map.size() == base) map.push_back(sampledAbsent); + heapTableMaps[stageIndex] = heapSpan(heapStages[stageIndex].map, map); + heapTableMapSet[stageIndex] = true; +} + +std::uint64_t ShaderResources::heapSpan(HeapStageState::KeptSpan& kept, std::span words) { + auto& heap = BindlessHeap::For(context); + if (std::ranges::equal(kept.words, words) && heap.Held(kept.mark)) return kept.mark.address; + auto span = heap.Allocate(words.size(), kept.mark); + std::memcpy(span.words.data(), words.data(), words.size() * sizeof(std::uint32_t)); + kept.words.assign(words.begin(), words.end()); + return span.address; +} + +bool ShaderResources::ownsHeapBinding(const Binding& binding, VkShaderStageFlags stageBit) const { + return binding.heap && (binding.layout.stageFlags & stageBit) != 0u; +} + +VkImageView ShaderResources::heapSampledView(std::size_t index) const { + const auto& texture = textures[index]; + if (textureSrgb[index]) return texture->SrgbView(textureFirstLayer[index], textureBaseLevel[index]); + if (textureBaseLevel[index]) return texture->BaseLevelView(); + if (textureFirstLayer[index]) return texture->FirstLayerView(); + return texture->View(); +} + +VkImageView ShaderResources::heapMultisampleStorageView(std::size_t index) const { + return index < multisampleStorage.size() ? multisampleStorage[index].view : VK_NULL_HANDLE; +} + +const HeapSlotOwner* ShaderResources::heapMultisampleStorageOwner(std::size_t index) const { + return index < multisampleStorage.size() ? multisampleStorage[index].owner : nullptr; +} + +VkImageView ShaderResources::heapStorageView(std::size_t index, std::uint32_t binding) const { + const auto& texture = storageTextures[index]; + if (storageAtomic64[index]) return texture->Atomic64View(storageMips[index], storageFirstLayer[index]); + if (storageAtomic[index]) return texture->AtomicView(storageMips[index], storageFirstLayer[index]); + return texture->StorageView(storageMips[index], storageFirstLayer[index], UintStorageBinding(binding)); +} + +void ShaderResources::appendHeapSlotKey(VkShaderStageFlags stageBit, std::vector& key) const { + constexpr auto absent = std::numeric_limits::max(); + key.clear(); + for (const auto& binding : bindings) { + if (!ownsHeapBinding(binding, stageBit)) continue; + if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE) { + for (const auto index : binding.imageAllocations) { + if (index == absent) { + key.push_back(0u); + continue; + } + key.insert(key.end(), {reinterpret_cast(textures[index].get()), + reinterpret_cast(heapSampledView(index)), + static_cast(textures[index]->Layout())}); + } + } else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) { + for (const auto index : binding.imageAllocations) { + if (index != absent && storageTextures[index] == nullptr && heapMultisampleStorageView(index) != VK_NULL_HANDLE) { + key.insert(key.end(), {reinterpret_cast(heapMultisampleStorageOwner(index)), reinterpret_cast(heapMultisampleStorageView(index)), + static_cast(VK_IMAGE_LAYOUT_GENERAL)}); + continue; + } + if (index == absent || storageTextures[index] == nullptr) { + key.push_back(0u); + continue; + } + key.insert(key.end(), {reinterpret_cast(storageTextures[index].get()), + reinterpret_cast(heapStorageView(index, binding.layout.binding)), + static_cast(VK_IMAGE_LAYOUT_GENERAL)}); + } + } else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) { + for (const auto index : binding.imageAllocations) key.push_back(reinterpret_cast(samplers[index].get())); + } + } +} + +void ShaderResources::takeHeapSlots(BindlessHeap& heap, HeapStageState& state, VkShaderStageFlags stageBit, + std::span key) { + namespace Abi = ShaderRecompiler::RuntimeAbi; + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + constexpr auto absent = std::numeric_limits::max(); + for (auto& kindSlots : state.slots) kindSlots.clear(); + state.samplerSlots.clear(); + std::size_t cursor = 0; + for (const auto& binding : bindings) { + if (!ownsHeapBinding(binding, stageBit)) continue; + const auto kind = binding.layout.binding % bindingCount; + if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE) { + auto& kindSlots = state.slots[kind]; + for (const auto index : binding.imageAllocations) { + if (index == absent) { + ++cursor; + kindSlots.push_back(heap.AbsentSlot(kind, heapPlaceholderView(kind))); + continue; + } + const auto view = reinterpret_cast(key[cursor + 1]); + cursor += 3; + kindSlots.push_back(takenSlot(heap.TextureSlot(kind, textures[index].get(), view, textures[index]->Layout()), "the global image heap is full")); + } + } else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) { + auto& kindSlots = state.slots[kind]; + for (const auto index : binding.imageAllocations) { + if (index != absent && storageTextures[index] == nullptr && heapMultisampleStorageView(index) != VK_NULL_HANDLE) { + const auto view = reinterpret_cast(key[cursor + 1]); + cursor += 3; + kindSlots.push_back(takenSlot(heap.ViewSlot(kind, heapMultisampleStorageOwner(index), view), "the global storage image heap is full")); + continue; + } + if (index == absent || storageTextures[index] == nullptr) { + ++cursor; + kindSlots.push_back(heap.AbsentSlot(kind, heapPlaceholderView(kind))); + continue; + } + const auto view = reinterpret_cast(key[cursor + 1]); + cursor += 3; + kindSlots.push_back(takenSlot(heap.StorageSlot(kind, storageTextures[index].get(), view), "the global storage image heap is full")); + } + } else if (binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) { + for (const auto index : binding.imageAllocations) { + ++cursor; + state.samplerSlots.push_back(takenSlot(heap.SamplerSlot(samplers[index]), "the global sampler heap is full")); + } + } + } +} + +void ShaderResources::noteHeapSlotGenerations(const BindlessHeap& heap, HeapStageState& state) { + state.slotGenerations.clear(); + for (std::uint32_t kind = 0; kind < state.slots.size(); ++kind) { + for (const auto slot : state.slots[kind]) state.slotGenerations.push_back(heap.SlotGeneration(slot)); + } + state.samplerGenerations.clear(); + for (const auto slot : state.samplerSlots) state.samplerGenerations.push_back(heap.SamplerGeneration(slot)); +} + +void ShaderResources::verifyHeapSlots(const BindlessHeap& heap, const HeapStageState& state, VkShaderStageFlags stageBit) const { + constexpr auto bindingCount = static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count); + static std::atomic checked{0}, wrong{0}, storageChecked{0}, storageWrong{0}; + for (const auto& binding : bindings) { + if (!ownsHeapBinding(binding, stageBit)) continue; + const bool sampled = binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE; + const bool storage = binding.layout.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + if (!sampled && !storage) continue; + const auto kind = binding.layout.binding % bindingCount; + for (std::size_t element = 0; element < binding.imageAllocations.size(); ++element) { + const auto index = binding.imageAllocations[element]; + if (index == std::numeric_limits::max()) continue; + const bool bound = element < state.slots[kind].size(); + if (sampled) { + ++checked; + if (!bound || heap.SlotTexture(kind, state.slots[kind][element]) != textures[index].get()) ++wrong; + continue; + } + if (storageTextures[index] == nullptr) { + const auto view = heapMultisampleStorageView(index); + if (view == VK_NULL_HANDLE) continue; + ++storageChecked; + if (!bound || !heap.SlotHoldsStorage(kind, state.slots[kind][element], heapMultisampleStorageOwner(index), view)) ++storageWrong; + continue; + } + ++storageChecked; + if (!bound || !heap.SlotHoldsStorage(kind, state.slots[kind][element], storageTextures[index].get(), heapStorageView(index, binding.layout.binding))) ++storageWrong; + } + } + if ((checked.load() + storageChecked.load()) % 100000u < 8u) { + std::fprintf(stderr, "[heap-slots] verify: %llu image slots checked, %llu hold another texture; %llu storage slots checked, %llu hold another image or view\n", + static_cast(checked.load()), static_cast(wrong.load()), + static_cast(storageChecked.load()), static_cast(storageWrong.load())); + } +} + +namespace { + +[[noreturn]] void ThrowHeapSlotMismatch(const char* format, ...) { + char text[320]; + va_list arguments; + va_start(arguments, format); + std::vsnprintf(text, sizeof(text), format, arguments); + va_end(arguments); + throw std::runtime_error(text); +} + +} + +void ShaderResources::patchHeapSlotWords(const CompiledShader& shader, const HeapStageState& stageState, + std::vector& patched, bool current) const { + namespace Abi = ShaderRecompiler::RuntimeAbi; + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + constexpr auto metadataWords = sizeof(Abi::ResourceMetadata) / sizeof(std::uint32_t); + const auto& program = *shader.program; + const auto& slots = stageState.slots; + const auto& samplerSlots = stageState.samplerSlots; + for (std::uint32_t image = 0; image < program.runtimeImageCount; ++image) { + const auto offset = program.imageMetadataDword + image * metadataWords; + const auto kind = patched[offset + offsetof(Abi::ResourceMetadata, binding) / 4u] % bindingCount; + const bool storageKind = kind >= Abi::FirstStorageImageBinding && kind < static_cast(Abi::Binding::Samplers); + if (storageKind && (!Abi::GpuHeapStorageKind(kind) || slots[kind].empty())) continue; + auto& first = patched[offset + offsetof(Abi::ResourceMetadata, firstElement) / 4u]; + if (kind < Abi::FirstImageBinding) continue; + if (!(kind < slots.size() && first < slots[kind].size())) { + ThrowHeapSlotMismatch( + "AGC graphics: GPU-mode image metadata names an element the stage does not bind (stage %u, image %u of %u, kind %u, first %u, slots of the kind %zu, flags 0x%x, element count %u, bindings %zu, stage state reused %d)", + static_cast(shader.stage), image, program.runtimeImageCount, kind, first, + kind < slots.size() ? slots[kind].size() : std::size_t{0}, patched[offset + offsetof(Abi::ResourceMetadata, flags) / 4u], + patched[offset + offsetof(Abi::ResourceMetadata, elementCount) / 4u], bindings.size(), current ? 1 : 0); + } + first = slots[kind][first]; + } + for (const auto& word : program.heapSlotWords) { + Require(word.dword < patched.size(), "a GPU-mode slot word lies outside the shader data"); + if (word.kind == static_cast(Abi::Binding::Samplers)) { + if (word.element >= samplerSlots.size()) { + ThrowHeapSlotMismatch("AGC graphics: a GPU-mode slot word names a sampler the stage does not bind (stage %u, element %u, sampler slots %zu, bindings %zu)", + static_cast(shader.stage), word.element, samplerSlots.size(), bindings.size()); + } + patched[word.dword] = samplerSlots[word.element]; + continue; + } + if (!(word.kind < slots.size() && word.element < slots[word.kind].size())) { + ThrowHeapSlotMismatch( + "AGC graphics: a GPU-mode slot word names an image the stage does not bind (stage %u, kind %u, element %u, slots of the kind %zu, bindings %zu)", + static_cast(shader.stage), word.kind, word.element, + word.kind < slots.size() ? slots[word.kind].size() : std::size_t{0}, bindings.size()); + } + patched[word.dword] = slots[word.kind][word.element]; + } + const bool slotWords = !program.registrationModule; + Require(slotWords || program.samplerSlotDword + samplerSlots.size() <= patched.size(), "GPU-mode sampler slots exceed the shader data"); + if (!slotWords) std::copy(samplerSlots.begin(), samplerSlots.end(), patched.begin() + program.samplerSlotDword); +} + +void ShaderResources::writeHeapStage(const CompiledShader& shader) { + namespace Abi = ShaderRecompiler::RuntimeAbi; + HeapCostScope slotsCost(HeapCost::Slots); + auto& heap = BindlessHeap::For(context); + const VkShaderStageFlags stageBit = VulkanStage(shader.stage); + const auto stageIndex = HeapStageIndex(shader.stage); + auto& stageState = heapStages[stageIndex]; + struct HeapKeyStorage {}; + auto& key = HostThreadLocal, HeapKeyStorage>(); + appendHeapSlotKey(stageBit, key); + bool current = key == stageState.key; + const auto slotSerial = heap.SlotSerial(); + const bool unmoved = current && EpochProofs() && stageState.slotSerial == slotSerial; + if (!unmoved && current) + current = + heap.GenerationsCurrent(stageState.slots, stageState.slotGenerations, stageState.samplerSlots, stageState.samplerGenerations); + if (!current) { + stageState.key = key; + takeHeapSlots(heap, stageState, stageBit, key); + noteHeapSlotGenerations(heap, stageState); + } else { + stageState.touchedEpoch = heap.TouchSlots(stageState.slots, stageState.touchedEpoch); + } + stageState.slotSerial = slotSerial; + static const bool verifySlots = std::getenv("APS5_VERIFY_GPU_HEAP") != nullptr; + if (verifySlots) verifyHeapSlots(heap, stageState, stageBit); + slotsCost.Stop(); + + HeapCostScope dataCost(HeapCost::StageData); + const std::size_t dataIndex = DataOffsetIndex(stageIndex, Abi::Binding::ShaderData); + const std::size_t flatIndex = DataOffsetIndex(stageIndex, Abi::Binding::FlattenedSrt); + const auto& program = *shader.program; + if (program.dataAddressPush) writeHeapTableMap(shader, stageIndex); + const bool ringStage = context.heapBufferDescriptors && program.dataAddressPush; + ringDescriptorStages[stageIndex] = ringStage; + if (ringStage) stageDataWords[stageIndex].clear(); + bool dataKept = false; + bool inputsKept = false; + bool flattened = false; + for (const auto& binding : program.bindings) { + const bool shaderData = binding.role == ShaderRecompiler::DescriptorRole::ShaderData; + if (!shaderData && binding.role != ShaderRecompiler::DescriptorRole::FlattenedSrt) continue; + const bool heapFlattened = !shaderData && shader.heap != nullptr && !shader.heap->flattenedSrt.empty(); + const auto& words = heapFlattened ? shader.heap->flattenedSrt : DrawGuestWords(shader, binding); + const auto userData = + shader.heap != nullptr ? std::span(shader.heap->userData) : std::span{}; + const bool sameInput = + binding.role == ShaderRecompiler::DescriptorRole::ShaderData && current && + stageState.dataInput.size() == words.size() + userData.size() && + std::equal(words.begin(), words.end(), stageState.dataInput.begin()) && + std::equal(userData.begin(), userData.end(), stageState.dataInput.begin() + static_cast(words.size())); + if (sameInput && verifySlots && heap.Held(stageState.data.mark)) inputsKept = true; + if (sameInput && !verifySlots) { + if (heap.Held(stageState.data.mark)) { + heapOffsets[dataIndex] = stageState.data.mark.address; + dataKept = true; + continue; + } + stageState.data.mark.entry = 0; + } + if (!shaderData) { + (ringStage ? heapSrtOffsets[stageIndex] : heapOffsets[flatIndex]) = heapSpan(stageState.flat, words); + heapSrtDwords[stageIndex] = static_cast(words.size()); + flattened = true; + continue; + } + ValidateRuntimeResources(shader, words); + struct PatchedStorage {}; + auto& patched = HostThreadLocal, PatchedStorage>(); + patched.assign(words.begin(), words.end()); + if (shader.heap != nullptr) + std::copy_n(shader.heap->userData.begin(), std::min(shader.heap->userData.size(), patched.size()), patched.begin()); + for (const auto& [byte, adjustment] : stageState.dataPatches) { + Require(byte < patched.size() * sizeof(std::uint32_t), "a GPU-mode guest buffer offset lies outside the shader data"); + reinterpret_cast(patched.data())[byte] = static_cast(adjustment); + } + patchHeapSlotWords(shader, stageState, patched, current); + if (addressedBuffers && !ringStage) { + HeapCostScope bufferCost(HeapCost::BufferAddresses); + writeBufferAddresses(patched, static_cast(stageIndex), nullptr); + bufferCost.Stop(); + stageDataWords[stageIndex].assign(patched.begin(), patched.end()); + stageDataIndex[stageIndex] = dataIndex; + } + if (verifySlots && sameInput && !std::ranges::equal(stageState.data.words, patched)) + throw std::runtime_error("AGC graphics: a GPU-mode stage's shader data changed although its inputs and slots did not"); + heapOffsets[dataIndex] = heapSpan(stageState.data, patched); + stageState.dataInput.assign(words.begin(), words.end()); + stageState.dataInput.insert(stageState.dataInput.end(), userData.begin(), userData.end()); + } + if (!ringStage) return; + if (!flattened) heapSrtDwords[stageIndex] = 0u; + auto& kept = stageState.descriptors; + const auto srt = srtRange(stageIndex); + const bool keepable = !kept.ranges.empty() && kept.ranges.front().address == srt.address && kept.ranges.front().size == srt.size && heap.Held(kept.mark); + if (dataKept && keepable) { + heapOffsets[flatIndex] = kept.first; + return; + } + if (!(inputsKept && keepable)) { + heapOffsets[flatIndex] = ringDescriptors(stageIndex, nullptr, kept); + return; + } + const std::vector before(kept.ranges.begin(), kept.ranges.end()); + const auto first = kept.first; + heapOffsets[flatIndex] = ringDescriptors(stageIndex, nullptr, kept); + if (kept.first != first || !std::ranges::equal(before, kept.ranges, [](const auto& a, const auto& b) { return a.address == b.address && a.size == b.size; })) + throw std::runtime_error("AGC graphics: a GPU-mode stage's kept buffer descriptor run would bind other buffers although its inputs did not change"); } namespace { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index aea5f0387147..dd250983a175 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -42,6 +42,8 @@ struct Module { std::set> perPrimitiveMembers; std::map variables; std::map constants; + std::map specConstants; + std::map specIds; std::set interface; std::map inputs; std::map outputs; @@ -168,6 +170,13 @@ struct Module { } }; +bool heapImageSlotsLength(const Module& module, std::uint32_t id) { + const auto constant = module.specConstants.find(id); + const auto specId = module.specIds.find(id); + return constant != module.specConstants.end() && constant->second == ShaderRecompiler::RuntimeAbi::GpuImageHeapSlots && specId != module.specIds.end() && + specId->second == ShaderRecompiler::PipelineSpecialization::HeapImageSlots; +} + Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics, bool shaderClipDistance, bool shaderCullDistance, bool viewportIndexLayer, bool runtimeDescriptorArray, bool demoteToHelperInvocation, bool shaderTerminateInvocation, bool shaderResourceMinLod, bool fmaFloat16, std::span primitiveLocations = {}) { using Stage = ShaderRecompiler::ShaderStage; Require(compiled.program != nullptr, "missing compiled shader"); @@ -244,10 +253,11 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi capability == spv::CapabilitySignedZeroInfNanPreserve; const bool isBdaCapability = - shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && - (capability == spv::CapabilityInt64 || - capability == spv::CapabilityPhysicalStorageBufferAddresses || - capability == spv::CapabilityStorageBuffer8BitAccess); + (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && + (capability == spv::CapabilityInt64 || + capability == spv::CapabilityPhysicalStorageBufferAddresses || + capability == spv::CapabilityStorageBuffer8BitAccess)) || + ((shader.bufferAddressCount != 0u || shader.dataAddressPush) && (capability == spv::CapabilityInt64 || capability == spv::CapabilityPhysicalStorageBufferAddresses)); const bool isTessellationCapability = (control || evaluation) && @@ -360,7 +370,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi terminateExtension = true; break; } - Require(extension == "SPV_KHR_float_controls" || (fmaFloat16 && extension == "SPV_KHR_fma") || (imageInt64Atomics && extension == "SPV_EXT_shader_image_int64") || (viewportIndexLayer && extension == "SPV_EXT_shader_viewport_index_layer") || (fragment && demoteToHelperInvocation && extension == "SPV_EXT_demote_to_helper_invocation") || (mesh && (extension == "SPV_EXT_mesh_shader" || extension == "SPV_KHR_physical_storage_buffer")) || (fragment && !primitiveLocations.empty() && extension == "SPV_EXT_mesh_shader") || (geometry && state.tessellationCapture && extension == "SPV_KHR_physical_storage_buffer") || ((descriptorIndexing || runtimeDescriptorArray) && extension == "SPV_EXT_descriptor_indexing") || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (extension == "SPV_KHR_physical_storage_buffer" || extension == "SPV_KHR_8bit_storage")), "unsupported SPIR-V extension"); + Require(extension == "SPV_KHR_float_controls" || (fmaFloat16 && extension == "SPV_KHR_fma") || (imageInt64Atomics && extension == "SPV_EXT_shader_image_int64") || (viewportIndexLayer && extension == "SPV_EXT_shader_viewport_index_layer") || (fragment && demoteToHelperInvocation && extension == "SPV_EXT_demote_to_helper_invocation") || (mesh && (extension == "SPV_EXT_mesh_shader" || extension == "SPV_KHR_physical_storage_buffer")) || (fragment && !primitiveLocations.empty() && extension == "SPV_EXT_mesh_shader") || (geometry && state.tessellationCapture && extension == "SPV_KHR_physical_storage_buffer") || ((descriptorIndexing || runtimeDescriptorArray) && extension == "SPV_EXT_descriptor_indexing") || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (extension == "SPV_KHR_physical_storage_buffer" || extension == "SPV_KHR_8bit_storage")) || ((shader.bufferAddressCount != 0u || shader.dataAddressPush) && extension == "SPV_KHR_physical_storage_buffer"), "unsupported SPIR-V extension"); break; } case spv::OpTerminateInvocation: @@ -422,6 +432,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi decoration.perVertex = true; } if (kind == spv::DecorationBlock) decoration.block = true; + if (kind == spv::DecorationSpecId && count == 4) module.specIds[instruction[1]] = instruction[3]; if (kind == spv::DecorationAliased) decoration.aliased = true; break; } @@ -452,6 +463,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi if (count == 4) module.constants.emplace(instruction[2], instruction[3]); break; case spv::OpSpecConstant: + if (count == 4) module.specConstants.emplace(instruction[2], instruction[3]); break; default: break; } @@ -503,6 +515,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater || name == spv::ExecutionModePixelInterlockOrderedEXT || name == spv::ExecutionModeStencilRefReplacingEXT), "unsupported fragment execution mode"); } else Require(module.modes.empty(), "unsupported vertex execution mode"); std::set> descriptors; + std::set declared; std::map, bool> bufferViews; bool push = false; for (const auto& [id, variable] : module.variables) { @@ -565,7 +578,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } } } else if (variable.storage == spv::StorageClassPushConstant) { - Require(!push && (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl) || (geometry && state.tessellationCapture)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface"); + Require(!push && (!shader.pushConstants.empty() || shader.readsDepthRemapPush || shader.dataAddressPush || mesh || (control && !rectListControl) || (geometry && state.tessellationCapture)) && (type[0] & 0xffffu) == spv::OpTypeStruct, "invalid push constant interface"); push = true; Require(type.size() == 3 && module.decorations[typeId].block, "push constant variable must be a Block struct with exactly one member"); const auto offset = module.offsets.find({typeId, 0}); @@ -573,7 +586,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto& array = module.Type(type[2]); Require(array.size() == 4 && (array[0] & 0xffffu) == spv::OpTypeArray && module.Signature(array[2]) == "u32", "push constant member must be an array of u32"); const auto length = module.constants.find(array[3]); - Require(length != module.constants.end() && length->second == PipelinePushSlotBytes / 4, "push constant array must contain 32 elements"); + Require(length != module.constants.end() && length->second == (shader.dataAddressPush ? ShaderRecompiler::RuntimeAbi::PushDataDwords : PipelinePushSlotBytes / 4), "push constant array must contain 32 elements"); const auto stride = module.decorations[type[2]].stride; Require(stride.has_value() && *stride == 4, "push constant array must have an ArrayStride of 4"); } else if (variable.storage == spv::StorageClassWorkgroup) { @@ -582,7 +595,22 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(decoration.set && decoration.binding, "unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; }); - Require(descriptors.insert(key).second || (binding != shader.bindings.end() && (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::ImageStorageTable)), "duplicate SPIR-V resource binding"); + bool tableAlias = false; + { + namespace Abi = ShaderRecompiler::RuntimeAbi; + const bool tables = shader.dataAddressPush && std::any_of(shader.bindings.begin(), shader.bindings.end(), [](const auto& item) { return item.role == ShaderRecompiler::DescriptorRole::ImageTable || item.role == ShaderRecompiler::DescriptorRole::ImageStorageTable || item.role == ShaderRecompiler::DescriptorRole::SamplerTable; }); + const bool samplerAlias = key.second == Abi::BindingNumber(Abi::Stage::Fragment, Abi::Binding::Samplers); + const bool imageAlias = key.second == Abi::BindingNumber(Abi::Stage::Fragment, static_cast(Abi::FirstImageBinding)) || key.second == Abi::BindingNumber(Abi::Stage::Fragment, static_cast(Abi::FirstStorageImageBinding)); + tableAlias = tables && key.first == Abi::FragmentDescriptorSet && (samplerAlias || imageAlias); + if (tableAlias && binding == shader.bindings.end()) { + const auto count = type.size() == 4 && (type[0] & 0xffffu) == spv::OpTypeArray ? module.constants.find(type[3]) : module.constants.end(); + const bool array = type.size() == 4 && (type[0] & 0xffffu) == spv::OpTypeArray; + Require(array && (samplerAlias ? count != module.constants.end() && count->second == Abi::GpuSamplerHeapSlots : heapImageSlotsLength(module, type[3])), + "an image table's heap array length disagrees with the global heap"); + continue; + } + } + Require(descriptors.insert(key).second || tableAlias || (binding != shader.bindings.end() && (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::ImageStorageTable)), "duplicate SPIR-V resource binding"); Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata"); Require(binding->kind == ShaderRecompiler::DescriptorKind::Sampler || binding->kind == ShaderRecompiler::DescriptorKind::SampledImage || binding->kind == ShaderRecompiler::DescriptorKind::StorageImage, "SPIR-V descriptor type disagrees with recompiler binding metadata"); if (binding->role == ShaderRecompiler::DescriptorRole::ImageTable || binding->role == ShaderRecompiler::DescriptorRole::ImageStorageTable || binding->role == ShaderRecompiler::DescriptorRole::SamplerTable) { @@ -593,7 +621,20 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto heap = static_cast(binding->binding % static_cast(ShaderRecompiler::RuntimeAbi::Binding::Count)); const auto capacity = ShaderRecompiler::RuntimeAbi::HeapCapacity(heap); const auto count = module.constants.find(type[3]); - Require(binding->count != 0u && binding->count <= capacity && count != module.constants.end() && count->second == binding->count, "typed heap array length disagrees with compact binding"); + const bool heapArray = binding->kind != ShaderRecompiler::DescriptorKind::StorageImage || (ShaderRecompiler::RuntimeAbi::GpuHeapStorageKind(static_cast(heap)) && binding->descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet); + if (shader.gpuResolvedImages && heapArray) { + const auto slots = binding->kind == ShaderRecompiler::DescriptorKind::Sampler ? ShaderRecompiler::RuntimeAbi::GpuSamplerHeapSlots : ShaderRecompiler::RuntimeAbi::GpuImageHeapSlots; + const bool length = binding->kind == ShaderRecompiler::DescriptorKind::Sampler ? count != module.constants.end() && count->second == slots : heapImageSlotsLength(module, type[3]); + Require(binding->count != 0u && length, "GPU-mode heap array length disagrees with the global heap"); + } else { + Require(binding->count != 0u && binding->count <= capacity && count != module.constants.end() && count->second == binding->count, "typed heap array length disagrees with compact binding"); + } + } else if (variable.storage == spv::StorageClassUniform) { + namespace Abi = ShaderRecompiler::RuntimeAbi; + constexpr auto bindingCount = static_cast(Abi::Binding::Count); + Require(shader.dataAddressPush && decoration.set && decoration.binding && *decoration.set == Abi::GpuDataSet && + *decoration.binding % bindingCount == static_cast(Abi::Binding::ShaderData), + "a uniform buffer is only the GPU heap's shader data"); } else { Require(variable.storage == spv::StorageClassStorageBuffer && decoration.set && decoration.binding, "unsupported or unbound shader resource"); const auto key = std::make_pair(*decoration.set, *decoration.binding); @@ -601,15 +642,18 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(mesh && *decoration.binding == 0u, "mesh window buffer binding is wrong"); continue; } - if (*decoration.set == ShaderRecompiler::PixelFunctionLdsDescriptorSet) { + const bool gpuData = shader.gpuResolvedImages && key.first == ShaderRecompiler::RuntimeAbi::GpuDataSet && + std::any_of(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.binding == key.second && (item.role == ShaderRecompiler::DescriptorRole::ShaderData || item.role == ShaderRecompiler::DescriptorRole::FlattenedSrt); }); + if (!gpuData && *decoration.set == ShaderRecompiler::PixelFunctionLdsDescriptorSet) { Require(fragment && *decoration.binding == 0u, "pixel function LDS buffer binding is wrong"); continue; } - const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return item.descriptorSet == key.first && item.binding == key.second; }); + const auto binding = std::find_if(shader.bindings.begin(), shader.bindings.end(), [&](const auto& item) { return (gpuData || item.descriptorSet == key.first) && item.binding == key.second; }); const bool array = binding != shader.bindings.end() && binding->role == ShaderRecompiler::DescriptorRole::GuestBuffers; const auto [view, first] = bufferViews.emplace(key, decoration.aliased); Require(descriptors.insert(key).second || (!first && view->second && decoration.aliased && array), "duplicate SPIR-V resource binding"); Require(binding != shader.bindings.end(), "SPIR-V resource is absent from recompiler binding metadata"); + declared.insert(&*binding); Require(binding->kind == ShaderRecompiler::DescriptorKind::StorageBuffer, "SPIR-V descriptor type disagrees with recompiler binding metadata"); Require(!binding->readOnly, "read-only descriptor metadata is unsupported because the recompiler emits no NonWritable decoration"); Require(array || (shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version && (binding->role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding->role == ShaderRecompiler::DescriptorRole::FaultBuffer)) || binding->role == ShaderRecompiler::DescriptorRole::ShaderData || binding->role == ShaderRecompiler::DescriptorRole::FlattenedSrt || binding->role == ShaderRecompiler::DescriptorRole::ImageTableMap || binding->role == ShaderRecompiler::DescriptorRole::Gds, "SPIR-V descriptor role is unsupported"); @@ -644,8 +688,9 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi } else { Require(shader.vertexAttributes.empty(), "vertex attribute metadata is invalid for this stage"); } - Require(descriptors.size() == shader.bindings.size(), "recompiler binding metadata contains undeclared resources"); - Require(push == (!shader.pushConstants.empty() || shader.readsDepthRemapPush || mesh || (control && !rectListControl && !barycentricPatches) || (geometry && state.tessellationCapture)), "recompiler push constant metadata disagrees with SPIR-V"); + const auto addressedBindings = static_cast(std::count_if(shader.bindings.begin(), shader.bindings.end(), [&](const ShaderRecompiler::DescriptorBinding& binding) { return !declared.contains(&binding) && ((shader.bufferAddressCount != 0u && binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers) || (shader.dataAddressPush && (binding.role == ShaderRecompiler::DescriptorRole::ShaderData || binding.role == ShaderRecompiler::DescriptorRole::FlattenedSrt || binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer || binding.role == ShaderRecompiler::DescriptorRole::ImageTable || binding.role == ShaderRecompiler::DescriptorRole::ImageStorageTable || binding.role == ShaderRecompiler::DescriptorRole::SamplerTable || binding.role == ShaderRecompiler::DescriptorRole::ImageTableMap))); })); + Require(descriptors.size() + addressedBindings == shader.bindings.size(), "recompiler binding metadata contains undeclared resources"); + Require(push == (!shader.pushConstants.empty() || shader.readsDepthRemapPush || shader.dataAddressPush || mesh || (control && !rectListControl && !barycentricPatches) || (geometry && state.tessellationCapture)), "recompiler push constant metadata disagrees with SPIR-V"); for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable"); if (mesh) Require(module.primitiveIndices == state.stages.mesh->outputPrimitiveSize, "mesh shader does not export primitive indices"); if (fragment) Require(module.primitiveInputs == 0 || !primitiveLocations.empty(), "per-primitive pixel inputs need the barycentric emulation layout"); @@ -707,7 +752,7 @@ std::set ValidateShaders(std::span shaders, const auto expected = state.rectList || patches ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); - for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's"); + for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == ShaderRecompiler::RuntimeAbi::DescriptorSet || (shaders[i].program->dataAddressPush && binding.role == ShaderRecompiler::DescriptorRole::Gds && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::GpuGdsSet) || ((shaders[i].stage == ShaderRecompiler::ShaderStage::Fragment || HeapShader(shaders[i])) && binding.descriptorSet == ShaderRecompiler::RuntimeAbi::FragmentDescriptorSet), "graphics resource uses a descriptor set other than its stage's"); const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics, shaderClipDistance, shaderCullDistance, viewportIndexLayer, runtimeDescriptorArray, demoteToHelperInvocation, shaderTerminateInvocation, shaderResourceMinLod, fmaFloat16, primitiveLocations); if (i != 0) { for (const auto& [location, signature] : current.inputs) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/StagedMerge.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/StagedMerge.cpp index 9c4af04ca674..d848be7759ca 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/StagedMerge.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/StagedMerge.cpp @@ -1,9 +1,11 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/StagedMerge.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/PackedMerge_spv.h" #include +#include #include #include #include @@ -23,7 +25,15 @@ struct MergePush { std::uint32_t samples; }; -class Temporary final : public StagedTemporary { +struct HeapMergePush { + MergePush push; + std::uint32_t temporary; + std::uint32_t target; +}; + +static_assert(offsetof(HeapMergePush, temporary) == 24 && offsetof(HeapMergePush, target) == 28); + +class Temporary final : public StagedTemporary, public HeapSlotOwner { public: Temporary(const Context& context, VkExtent2D extent, std::uint32_t samples) : context(context), extent(extent), samples(samples) { try { @@ -70,6 +80,7 @@ class Temporary final : public StagedTemporary { private: void release() noexcept { + ReleaseHeapSlots(context); if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); @@ -99,8 +110,25 @@ class StagedMerges { return temporaries.back(); } - void Merge(VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area) { + void Merge(VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area, const HeapSlotOwner* targetOwner) { const auto pipeline = mergePipeline(); + const auto x = static_cast(area.offset.x); + const auto y = static_cast(area.offset.y); + const MergePush push{{x, y}, {x + area.extent.width, y + area.extent.height}, mask, samples}; + auto* owned = dynamic_cast(&temporary); + if (heapPipeline != VK_NULL_HANDLE && targetOwner != nullptr && owned != nullptr) { + auto& heap = BindlessHeap::For(context); + const auto temporarySlot = heap.ViewSlot(HeapPassStorageKind, owned, temporary.View()); + const auto targetSlot = temporarySlot ? heap.ViewSlot(HeapPassStorageKind, targetOwner, target) : std::nullopt; + if (targetSlot) { + heap.BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + const HeapMergePush pushed{push, *temporarySlot, *targetSlot}; + CmdPushData(context, commands, sizeof(pushed), &pushed); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, (area.extent.width + 7u) / 8u, (area.extent.height + 7u) / 8u, 1); + return; + } + } const auto set = allocate(); const std::array images{{{VK_NULL_HANDLE, temporary.View(), VK_IMAGE_LAYOUT_GENERAL}, {VK_NULL_HANDLE, target, VK_IMAGE_LAYOUT_GENERAL}}}; std::array writes{}; @@ -113,12 +141,9 @@ class StagedMerges { writes[binding].pImageInfo = &images[binding]; } context.Function("vkUpdateDescriptorSets")(context.device, 2, writes.data(), 0, nullptr); - const auto x = static_cast(area.offset.x); - const auto y = static_cast(area.offset.y); - const MergePush push{{x, y}, {x + area.extent.width, y + area.extent.height}, mask, samples}; context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, SetBindSite::StagedMerge); + CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), &push); context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, (area.extent.width + 7u) / 8u, (area.extent.height + 7u) / 8u, 1); } @@ -151,6 +176,13 @@ class StagedMerges { auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &info, nullptr, &pipeline); } + if (created == VK_SUCCESS && HeapPassesActive(context)) { + const auto& heap = BindlessHeap::For(context); + const std::array mappings{ + HeapPushIndexMapping(heap, 0, 0, HeapPassStorageImages, offsetof(HeapMergePush, temporary)), + HeapPushIndexMapping(heap, 0, 1, HeapPassStorageImages, offsetof(HeapMergePush, target))}; + heapPipeline = CreateHeapComputePipeline(context, info, mappings); + } context.Function("vkDestroyShaderModule")(context.device, shader, nullptr); Check(created, "vkCreateComputePipelines staged merge"); return pipeline; @@ -198,6 +230,7 @@ class StagedMerges { void release() noexcept { temporaries.clear(); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + if (heapPipeline) context.Function("vkDestroyPipeline")(context.device, heapPipeline, nullptr); for (const auto pool : pools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); if (setLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, setLayout, nullptr); @@ -207,6 +240,7 @@ class StagedMerges { VkDescriptorSetLayout setLayout = VK_NULL_HANDLE; VkPipelineLayout pipelineLayout = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline heapPipeline = VK_NULL_HANDLE; std::vector pools; std::vector serials; std::size_t current = 0; @@ -257,12 +291,13 @@ void RecordStagedClear(const Context& context, VkCommandBuffer commands, StagedT if (Recorder::Active() != nullptr) Recorder::CountBarriers(Recorder::CommandClass::Draw, 2); } -void RecordStagedMerge(const Context& context, VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area) { +void RecordStagedMerge(const Context& context, VkCommandBuffer commands, StagedTemporary& temporary, VkImageView target, std::uint32_t mask, std::uint32_t samples, VkRect2D area, + const HeapSlotOwner* targetOwner) { if (area.extent.width == 0 || area.extent.height == 0) return; RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, anyWrite, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); { std::lock_guard lock(mergesMutex()); - mergesFor(context).Merge(commands, temporary, target, mask, samples, area); + mergesFor(context).Merge(commands, temporary, target, mask, samples, area, targetOwner); } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT, anyAccess); if (Recorder::Active() != nullptr) Recorder::CountBarriers(Recorder::CommandClass::Draw, 2); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 772b923d32cc..5632f357c5df 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -13,6 +13,7 @@ #include "prx/libc/include/HostThreadSlot.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/MapImportGate.hpp" @@ -1278,6 +1279,7 @@ VkImageView Texture::BaseLevelView() const { } void Texture::release() noexcept { + ReleaseHeapSlots(context); upload.reset(); if (view) context.Function("vkDestroyImageView")(context.device, view, nullptr); view = VK_NULL_HANDLE; @@ -2154,6 +2156,13 @@ VkImageView StorageTexture::StorageView(std::uint32_t mip, bool firstLayer, bool return created; } +VkImageView StorageTexture::UintStorageViewOrDefault(std::uint32_t mip, bool firstLayer) { + const auto uintFormat = UintStorageViewFormat(storageFormat); + const bool available = PackedStorageViewFormat(storageFormat) == VK_FORMAT_UNDEFINED && uintFormat != VK_FORMAT_UNDEFINED && + StorageFormatOrUndefined(context, uintFormat) == uintFormat; + return StorageView(mip, firstLayer, available); +} + VkImageView StorageTexture::ElementView() { if (elementView != VK_NULL_HANDLE) return elementView; if (descriptor.dimension != TextureDimension::k2D || geometry.imageLayers != 1) return VK_NULL_HANDLE; @@ -2830,6 +2839,14 @@ bool StorageTexture::originalHolds(std::uint64_t address, std::size_t bytes) con return true; } +bool StorageTexture::UntrackedUnchanged() const { + if (descriptor.dccAddress != 0 || uploadedKeys != DccKeys::Uncompressed || !keptAtUnwatch.empty() || original.size() != guestBytes) return false; + constexpr std::uint64_t blockBytes = 65536; + std::vector blocks(static_cast((descriptor.baseAddress + guestBytes - 1) / blockBytes - descriptor.baseAddress / blockBytes + 1)); + if (!compareUntracked(descriptor.baseAddress, static_cast(guestBytes), blocks, true)) return false; + return std::all_of(blocks.begin(), blocks.end(), [](std::uint8_t block) { return block == GuestMemory::BlockUnchanged; }); +} + bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, std::span changed, bool memoize) const { if (!originalHolds(address, bytes)) return false; struct Exempt { @@ -3073,7 +3090,7 @@ void StorageTexture::upload(const std::vector* layers) { for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestOffset(layer, mip), elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor}); + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestOffset(layer, mip), elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor}, this); } } recordDirectUploadDone(commands); @@ -3558,10 +3575,10 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< detile.pipeBankXor = descriptor.pipeBankXor; const auto view = elementLayerView(window.level, window.layer); if (source.shadow) { - detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile); + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile, this); } else { detile.tiledBase = 0; - detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestOffset(window.layer, mip), view, mip, false, window.layer, detile); + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestOffset(window.layer, mip), view, mip, false, window.layer, detile, this); } } recordDirectUploadDone(commands); @@ -3924,9 +3941,9 @@ std::uint64_t StorageTexture::writeBackWindows(std::span se for (auto k = directRanges[i].first; k < directRanges[i].second; ++k) { auto piece = directPieces[k].window; piece.pipeBankXor = descriptor.pipeBankXor; - detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, directPieces[k].slab, directPieces[k].offset, elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, piece); + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, directPieces[k].slab, directPieces[k].offset, elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, piece, this); } - if (directRanges[i].first == directRanges[i].second) detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); + if (directRanges[i].first == directRanges[i].second) detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile, this); } singlePassMoves.fetch_add(1, std::memory_order_relaxed); } else { @@ -7172,6 +7189,7 @@ StorageTexture::~StorageTexture() { } void StorageTexture::release() noexcept { + ReleaseHeapSlots(context); for (const auto& [mip, extra] : extraViews) context.Function("vkDestroyImageView")(context.device, extra, nullptr); extraViews.clear(); for (const auto& [mip, extra] : firstLayerViews) context.Function("vkDestroyImageView")(context.device, extra, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index 9b60858076f2..fb196b069393 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" @@ -6,13 +7,18 @@ #include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/TextureRepack_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileAddressed_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/TextureRepackAddressed_spv.h" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImageWords_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileWords_spv.h" #include #include #include +#include #include #include +#include #include #include @@ -42,6 +48,67 @@ struct Push { std::uint32_t splitStride; }; +struct AddressedPush { + Push push; + std::uint64_t input; + std::uint64_t output; +}; + +struct AddressedRepackPush { + std::array push; + std::uint64_t source; + std::uint64_t destination; +}; + +struct HeapImagePush { + Push push; + std::uint64_t tiled; + std::uint32_t slot; + std::uint32_t pad; +}; + +struct HeapCmaskPush { + std::array push; + std::uint32_t pad; + std::uint64_t cmask; + std::uint64_t errors; + std::uint32_t slot; + std::uint32_t pad2; +}; + +static_assert(offsetof(HeapCmaskPush, cmask) == 32 && offsetof(HeapCmaskPush, errors) == 40 && offsetof(HeapCmaskPush, slot) == 48); + +VkDescriptorSetAndBindingMappingEXT PushAddressMapping(std::uint32_t binding, std::uint32_t offset) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = 0; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_WRITE_STORAGE_BUFFER_BIT_EXT | VK_SPIRV_RESOURCE_TYPE_READ_ONLY_STORAGE_BUFFER_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_PUSH_ADDRESS_EXT; + mapping.sourceData.pushAddressOffset = offset; + return mapping; +} + +VkDescriptorSetAndBindingMappingEXT PushIndexMapping(const BindlessHeap& heap, std::uint32_t binding, std::uint32_t offset) { + VkDescriptorSetAndBindingMappingEXT mapping{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_AND_BINDING_MAPPING_EXT}; + mapping.descriptorSet = 0; + mapping.firstBinding = binding; + mapping.bindingCount = 1; + mapping.resourceMask = VK_SPIRV_RESOURCE_TYPE_READ_WRITE_IMAGE_BIT_EXT; + mapping.source = VK_DESCRIPTOR_MAPPING_SOURCE_HEAP_WITH_PUSH_INDEX_EXT; + mapping.sourceData.pushIndex.pushOffset = offset; + mapping.sourceData.pushIndex.heapIndexStride = static_cast(heap.ImageHeapStride()); + mapping.sourceData.pushIndex.heapArrayStride = static_cast(heap.ImageHeapStride()); + return mapping; +} + +constexpr std::uint32_t DetileHeapKind = ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding + 27u; + +static_assert(offsetof(AddressedPush, input) == 80 && sizeof(AddressedPush) == 96); +static_assert(offsetof(HeapImagePush, tiled) == 80 && offsetof(HeapImagePush, slot) == 88 && sizeof(HeapImagePush) == 96); +static_assert(ShaderRecompiler::RuntimeAbi::GpuHeapStorageKind(DetileHeapKind)); +static_assert(offsetof(AddressedRepackPush, source) == 16 && sizeof(AddressedRepackPush) == 32); + std::uint32_t BlockBytesFor(TextureTileMode tileMode) { switch (tileMode) { case TextureTileMode::kLinear: return 0u; @@ -63,8 +130,8 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { - return (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image, bool addressed, bool heap) { + return (heap ? 1u << 20 : 0u) | (addressed ? 1u << 19 : 0u) | (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -97,6 +164,14 @@ TextureDetiler::TextureDetiler(const Context& context) : context(context) { Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &imageDescriptorLayout), "vkCreateDescriptorSetLayout"); pipelineLayoutInfo.pSetLayouts = &imageDescriptorLayout; Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &imagePipelineLayout), "vkCreatePipelineLayout"); + if (context.bufferDeviceAddress) { + pipelineLayoutInfo.setLayoutCount = 0; + pipelineLayoutInfo.pSetLayouts = nullptr; + pushRange.size = sizeof(AddressedPush); + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &addressedPipelineLayout), "vkCreatePipelineLayout"); + pushRange.size = sizeof(AddressedRepackPush); + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &addressedRepackLayout), "vkCreatePipelineLayout"); + } } catch (...) { release(); throw; @@ -116,12 +191,19 @@ void TextureDetiler::release() noexcept { if (readbackDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, readbackDescriptorLayout, nullptr); for (const auto pool : cmaskDescriptorSets.pools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); if (cmaskPipeline) context.Function("vkDestroyPipeline")(context.device, cmaskPipeline, nullptr); + if (cmaskHeapPipeline) context.Function("vkDestroyPipeline")(context.device, cmaskHeapPipeline, nullptr); if (cmaskPipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, cmaskPipelineLayout, nullptr); if (cmaskDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr); cmaskErrors.reset(); for (const auto pool : descriptorSets.pools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); for (const auto pool : imageDescriptorSets.pools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); if (imageModule) context.Function("vkDestroyShaderModule")(context.device, imageModule, nullptr); + if (addressedRepackPipeline) context.Function("vkDestroyPipeline")(context.device, addressedRepackPipeline, nullptr); + if (addressedRepackModule) context.Function("vkDestroyShaderModule")(context.device, addressedRepackModule, nullptr); + if (addressedModule) context.Function("vkDestroyShaderModule")(context.device, addressedModule, nullptr); + if (addressedRepackLayout) context.Function("vkDestroyPipelineLayout")(context.device, addressedRepackLayout, nullptr); + if (addressedPipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, addressedPipelineLayout, nullptr); + if (heapRepackPipeline) context.Function("vkDestroyPipeline")(context.device, heapRepackPipeline, nullptr); if (repackPipeline) context.Function("vkDestroyPipeline")(context.device, repackPipeline, nullptr); if (repackModule) context.Function("vkDestroyShaderModule")(context.device, repackModule, nullptr); if (imageWordModule) context.Function("vkDestroyShaderModule")(context.device, imageWordModule, nullptr); @@ -145,9 +227,14 @@ VkFormat TextureDetiler::ImageElementFormat(std::uint32_t elementBytes) { } } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { +VkDeviceAddress TextureDetiler::bufferAddress(VkBuffer buffer) const { + if (addressedPipelineLayout == VK_NULL_HANDLE || !BufferPool::ForceAddresses()) return 0; + return BufferDeviceAddress(context, buffer); +} + +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image, bool addressed, bool heap) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image, addressed, heap); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } @@ -180,8 +267,16 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem values[2] = 2u; std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); } - const bool words = elementBytes < 4u && (image ? retile && values[2] != 2u : !retile || values[2] != 2u); - auto& shader = image ? (words ? imageWordModule : imageModule) : (words ? wordModule : module); + const bool descriptorWords = elementBytes < 4u && (image ? retile && values[2] != 2u : !retile || values[2] != 2u); + addressed = addressed || (heap && !image && descriptorWords); + if (addressed && addressedModule == VK_NULL_HANDLE) { + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(TEXTURE_DETILE_ADDRESSED_SPV); + moduleInfo.pCode = TEXTURE_DETILE_ADDRESSED_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &addressedModule), "vkCreateShaderModule"); + } + const bool words = !addressed && descriptorWords; + auto& shader = addressed ? addressedModule : image ? (words ? imageWordModule : imageModule) : (words ? wordModule : module); if (shader == VK_NULL_HANDLE) { if (!image) shader = CreateShaderModule(context, TEXTURE_DETILE_WORDS_SPV, sizeof(TEXTURE_DETILE_WORDS_SPV), "vkCreateShaderModule detile words"); else if (words) shader = CreateShaderModule(context, TEXTURE_DETILE_IMAGE_WORDS_SPV, sizeof(TEXTURE_DETILE_IMAGE_WORDS_SPV), "vkCreateShaderModule detile image words"); @@ -201,11 +296,30 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem stage.pSpecializationInfo = &specialization; VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; createInfo.stage = stage; - createInfo.layout = image ? imagePipelineLayout : pipelineLayout; + createInfo.layout = addressed ? addressedPipelineLayout : image ? imagePipelineLayout : pipelineLayout; + std::array mappings{}; + VkShaderDescriptorSetAndBindingMappingInfoEXT mappingInfo{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}; + auto heapFlags = DescriptorHeapFlags(); + if (heap) { + if (image) { + mappings[0] = PushAddressMapping(0, offsetof(HeapImagePush, tiled)); + mappings[1] = PushIndexMapping(BindlessHeap::For(context), 1, offsetof(HeapImagePush, slot)); + } else if (!addressed) { + mappings[0] = PushAddressMapping(0, offsetof(AddressedPush, input)); + mappings[1] = PushAddressMapping(1, offsetof(AddressedPush, output)); + } + mappingInfo.mappingCount = addressed ? 0u : static_cast(mappings.size()); + mappingInfo.pMappings = mappings.data(); + createInfo.stage.pNext = &mappingInfo; + createInfo.pNext = &heapFlags; + createInfo.layout = VK_NULL_HANDLE; + } VkPipeline result = VK_NULL_HANDLE; { auto pipelineCacheLock = context.LockPipelineCacheForUse(); - Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result), "vkCreateComputePipelines"); + const auto created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result); + if (created != VK_SUCCESS && heap) result = VK_NULL_HANDLE; + else Check(created, "vkCreateComputePipelines"); } pipelines.emplace_back(key, result); return result; @@ -234,7 +348,7 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); std::uint32_t columnEnd = 0, rowEnd = 0; checkWindow(layout, window, columnEnd, rowEnd); - const auto target = pipeline(MipTileMode(tileMode, layout), elementBytes, retile, thick && !(IsGcnThick(tileMode) && layout.gcnMicroTiled)); + const bool thickPipeline = thick && !(IsGcnThick(tileMode) && layout.gcnMicroTiled); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; const auto destinationDescriptorOffset = destinationOffset - destinationOffset % alignment; @@ -248,25 +362,37 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode const auto sourceRange = (sourceBase + (retile ? linearBytes : tiledBytes) + 3) / 4 * 4; const auto destinationRange = (destinationBase + (retile ? tiledBytes : linearBytes) + 3) / 4 * 4; Require(sourceRange <= context.limits.maxStorageBufferRange && destinationRange <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); - const auto set = allocateSet(); - const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; - const VkDescriptorBufferInfo destinationInfo{destination, destinationDescriptorOffset, destinationRange}; - std::array writes{}; - writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[0].dstSet = set; - writes[0].dstBinding = 0; - writes[0].descriptorCount = 1; - writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; - writes[0].pBufferInfo = &sourceInfo; - writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[1].dstSet = set; - writes[1].dstBinding = 1; - writes[1].descriptorCount = 1; - writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; - writes[1].pBufferInfo = &destinationInfo; - context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); + const auto sourceAddress = bufferAddress(source); + const auto destinationAddress = sourceAddress != 0 ? bufferAddress(destination) : 0; + const auto heapPipeline = sourceAddress != 0 && destinationAddress != 0 && HeapPassesActive(context) ? pipeline(MipTileMode(tileMode, layout), elementBytes, retile, thickPipeline, false, false, true) : VK_NULL_HANDLE; + const bool heap = heapPipeline != VK_NULL_HANDLE; + const bool addressed = !heap && sourceAddress != 0 && destinationAddress != 0; + if (heap) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + } else if (addressed) { + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline(MipTileMode(tileMode, layout), elementBytes, retile, thickPipeline, false, true)); + } else { + const auto set = allocateSet(); + const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; + const VkDescriptorBufferInfo destinationInfo{destination, destinationDescriptorOffset, destinationRange}; + std::array writes{}; + writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[0].dstSet = set; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[0].pBufferInfo = &sourceInfo; + writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[1].dstSet = set; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[1].pBufferInfo = &destinationInfo; + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipeline(MipTileMode(tileMode, layout), elementBytes, retile, thickPipeline)); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, SetBindSite::Detiler); + } Push push{}; push.srcBase = static_cast(sourceBase); push.dstBase = static_cast(destinationBase); @@ -287,7 +413,13 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.rowBegin = window.rowBegin; push.pipeBankXor = window.pipeBankXor ^ layerBankXor(tileMode, elementBytes, layout, slice); push.splitStride = layout.gcnSplitStride; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + if (heap || addressed) { + const AddressedPush pushed{push, sourceAddress + sourceDescriptorOffset, destinationAddress + destinationDescriptorOffset}; + if (heap) CmdPushData(context, commands, sizeof(pushed), &pushed); + else CmdPushConstants(context, commands, addressedPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pushed), &pushed); + } else { + CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + } const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, groupsX, groupsY, 1); @@ -305,8 +437,12 @@ void TextureDetiler::DispatchRepack(VkCommandBuffer commands, VkBuffer source, s VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; createInfo.stage = stage; createInfo.layout = pipelineLayout; - auto pipelineCacheLock = context.LockPipelineCacheForUse(); - Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &repackPipeline), "vkCreateComputePipelines repack"); + { + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &repackPipeline), "vkCreateComputePipelines repack"); + } + const std::array mappings{PushAddressMapping(0, offsetof(AddressedRepackPush, source)), PushAddressMapping(1, offsetof(AddressedRepackPush, destination))}; + heapRepackPipeline = CreateHeapComputePipeline(context, createInfo, mappings); } const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; @@ -318,6 +454,39 @@ void TextureDetiler::DispatchRepack(VkCommandBuffer commands, VkBuffer source, s constexpr std::uint64_t rowWords = 64u * 1024u; const auto rows = (words + rowWords - 1u) / rowWords; Require(words <= UINT32_MAX && rows <= context.limits.maxComputeWorkGroupCount[1], "a texel repack exceeds the device's dispatch limits"); + const std::array repackPush{static_cast(sourceOffset - sourceDescriptorOffset), static_cast(destinationOffset - destinationDescriptorOffset), static_cast(words), static_cast(std::min(words, rowWords))}; + const auto sourceAddress = bufferAddress(source); + const auto destinationAddress = sourceAddress != 0 ? bufferAddress(destination) : 0; + if (sourceAddress != 0 && destinationAddress != 0 && heapRepackPipeline != VK_NULL_HANDLE && HeapPassesActive(context)) { + const AddressedRepackPush pushed{repackPush, sourceAddress + sourceDescriptorOffset, destinationAddress + destinationDescriptorOffset}; + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapRepackPipeline); + CmdPushData(context, commands, sizeof(pushed), &pushed); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, static_cast((std::min(words, rowWords) + 63u) / 64u), static_cast(rows), 1); + return; + } + if (sourceAddress != 0 && destinationAddress != 0) { + if (addressedRepackPipeline == VK_NULL_HANDLE) { + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(TEXTURE_REPACK_ADDRESSED_SPV); + moduleInfo.pCode = TEXTURE_REPACK_ADDRESSED_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &addressedRepackModule), "vkCreateShaderModule repack"); + VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + stage.module = addressedRepackModule; + stage.pName = "main"; + VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + createInfo.stage = stage; + createInfo.layout = addressedRepackLayout; + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &addressedRepackPipeline), "vkCreateComputePipelines repack"); + } + const AddressedRepackPush pushed{repackPush, sourceAddress + sourceDescriptorOffset, destinationAddress + destinationDescriptorOffset}; + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, addressedRepackPipeline); + CmdPushConstants(context, commands, addressedRepackLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(pushed), &pushed); + context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, static_cast((std::min(words, rowWords) + 63u) / 64u), static_cast(rows), 1); + return; + } const auto set = allocateSet(); const VkDescriptorBufferInfo sourceInfo{source, sourceDescriptorOffset, sourceRange}; const VkDescriptorBufferInfo destinationInfo{destination, destinationDescriptorOffset, destinationRange}; @@ -332,9 +501,8 @@ void TextureDetiler::DispatchRepack(VkCommandBuffer commands, VkBuffer source, s } context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, repackPipeline); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, 0, nullptr); - const std::array push{static_cast(sourceOffset - sourceDescriptorOffset), static_cast(destinationOffset - destinationDescriptorOffset), static_cast(words), static_cast(std::min(words, rowWords))}; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, pipelineLayout, 0, 1, &set, SetBindSite::Detiler); + CmdPushConstants(context, commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(repackPush), repackPush.data()); context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, static_cast((std::min(words, rowWords) + 63u) / 64u), static_cast(rows), 1); } @@ -345,7 +513,7 @@ std::uint32_t TextureDetiler::CmaskErrors() { return codes; } -void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write) { +void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write, const HeapSlotOwner* owner) { Require(commands != VK_NULL_HANDLE && cmask != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "a CMASK clear needs a command buffer, the CMASK buffer and the target view"); Require(elementBytes == 1 || elementBytes == 2 || elementBytes == 4 || elementBytes == 8, "a CMASK clear of this element size is not modeled"); Require(cmaskOffset % 4 == 0 && cmaskBytes % 4 == 0 && cmaskBytes != 0, "a CMASK clear needs a word-aligned CMASK"); @@ -380,30 +548,49 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask auto pipelineCacheLock = context.LockPipelineCacheForUse(); created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &cmaskPipeline); } + if (created == VK_SUCCESS && addressedPipelineLayout != VK_NULL_HANDLE && DescriptorHeapEnabled() && BindlessHeap::For(context).DescriptorHeapMode()) { + const auto& bindless = BindlessHeap::For(context); + const std::array mappings{PushAddressMapping(0, offsetof(HeapCmaskPush, cmask)), PushAddressMapping(1, offsetof(HeapCmaskPush, errors)), PushIndexMapping(bindless, 2, offsetof(HeapCmaskPush, slot))}; + VkShaderDescriptorSetAndBindingMappingInfoEXT mappingInfo{VK_STRUCTURE_TYPE_SHADER_DESCRIPTOR_SET_AND_BINDING_MAPPING_INFO_EXT}; + mappingInfo.mappingCount = static_cast(mappings.size()); + mappingInfo.pMappings = mappings.data(); + auto heapFlags = DescriptorHeapFlags(); + auto heapInfo = pipelineInfo; + heapInfo.pNext = &heapFlags; + heapInfo.stage.pNext = &mappingInfo; + heapInfo.layout = VK_NULL_HANDLE; + auto pipelineCacheLock = context.LockPipelineCacheForUse(); + created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &heapInfo, nullptr, &cmaskHeapPipeline); + } context.Function("vkDestroyShaderModule")(context.device, cmaskModule, nullptr); Check(created, "vkCreateComputePipelines CMASK clear"); cmaskErrors = std::make_unique(context, 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); std::memset(cmaskErrors->Bytes().data(), 0, 4); } - const auto set = allocateFrom(cmaskDescriptorSets, cmaskDescriptorLayout, {{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 64 * 2}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 64}}, "vkCreateDescriptorPool CMASK clear", "vkAllocateDescriptorSets CMASK clear", "vkResetDescriptorPool CMASK clear"); + const auto cmaskAddress = cmaskHeapPipeline != VK_NULL_HANDLE && owner != nullptr ? bufferAddress(cmask) : 0; + const auto errorsAddress = cmaskAddress != 0 ? bufferAddress(cmaskErrors->Handle()) : 0; + const auto slot = errorsAddress != 0 ? BindlessHeap::For(context).ViewSlot(DetileHeapKind, owner, view) : std::nullopt; + const auto set = slot ? VK_NULL_HANDLE : allocateFrom(cmaskDescriptorSets, cmaskDescriptorLayout, {{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 64 * 2}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 64}}, "vkCreateDescriptorPool CMASK clear", "vkAllocateDescriptorSets CMASK clear", "vkResetDescriptorPool CMASK clear"); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto descriptorOffset = cmaskOffset - cmaskOffset % alignment; const auto base = cmaskOffset - descriptorOffset; - const VkDescriptorBufferInfo cmaskInfo{cmask, descriptorOffset, base + cmaskBytes}; - const VkDescriptorBufferInfo errorInfo{cmaskErrors->Handle(), 0, 4}; - const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; - std::array writes{}; - for (std::uint32_t index = 0; index < writes.size(); ++index) { - writes[index].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[index].dstSet = set; - writes[index].dstBinding = index; - writes[index].descriptorCount = 1; - writes[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + if (!slot) { + const VkDescriptorBufferInfo cmaskInfo{cmask, descriptorOffset, base + cmaskBytes}; + const VkDescriptorBufferInfo errorInfo{cmaskErrors->Handle(), 0, 4}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + for (std::uint32_t index = 0; index < writes.size(); ++index) { + writes[index].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[index].dstSet = set; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + } + writes[0].pBufferInfo = &cmaskInfo; + writes[1].pBufferInfo = &errorInfo; + writes[2].pImageInfo = &imageInfo; + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); } - writes[0].pBufferInfo = &cmaskInfo; - writes[1].pBufferInfo = &errorInfo; - writes[2].pImageInfo = &imageInfo; - context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); const auto barrier = [&](VkPipelineStageFlags source, VkPipelineStageFlags destination, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; memory.srcAccessMask = sourceAccess; @@ -413,25 +600,32 @@ void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask constexpr VkAccessFlags anyWrite = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; constexpr VkAccessFlags anyAccess = anyWrite | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_HOST_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; barrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, anyWrite, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, 0, nullptr); const std::uint32_t mask = elementBytes >= 4 ? 0xffffffffu : (1u << (elementBytes * 8u)) - 1u; const std::array push{static_cast(base), width, height, (width + 1023u) / 1024u, clearWords[0] & mask, elementBytes == 8 ? clearWords[1] : 0u, write ? 1u : 0u}; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + if (slot) { + BindlessHeap::For(context).BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskHeapPipeline); + const HeapCmaskPush pushed{push, 0u, cmaskAddress + descriptorOffset, errorsAddress, *slot, 0u}; + const VkPushDataInfoEXT data{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {&pushed, sizeof(pushed)}}; + context.Function("vkCmdPushDataEXT")(commands, &data); + } else { + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, SetBindSite::Detiler); + CmdPushConstants(context, commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + } context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, anyAccess); context.Resolved(&DeviceFunctions::cmdFillBuffer, "vkCmdFillBuffer")(commands, cmask, cmaskOffset, cmaskBytes, 0xffffffffu); barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); } -void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window) { +void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window, const HeapSlotOwner* owner) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); Require(tiled != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "texture detiling requires a tiled buffer and an image view"); Require(ImageElementFormat(elementBytes) != VK_FORMAT_UNDEFINED, "unsupported element size for texture detiling through an image"); std::uint32_t columnEnd = 0, rowEnd = 0; checkWindow(layout, window, columnEnd, rowEnd); - const auto target = pipeline(MipTileMode(tileMode, layout), elementBytes, retile, false, true); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto descriptorOffset = tiledOffset - tiledOffset % alignment; const auto base = tiledOffset - descriptorOffset; @@ -439,25 +633,6 @@ void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode til const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min(layout.tiledSize, window.rangeEnd) - window.tiledBase; const auto range = (base + tiledBytes + 3) / 4 * 4; Require(range <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); - const auto set = allocateSet(true); - const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range}; - const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; - std::array writes{}; - writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[0].dstSet = set; - writes[0].dstBinding = 0; - writes[0].descriptorCount = 1; - writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; - writes[0].pBufferInfo = &bufferInfo; - writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; - writes[1].dstSet = set; - writes[1].dstBinding = 1; - writes[1].descriptorCount = 1; - writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; - writes[1].pImageInfo = &imageInfo; - context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); - context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); - context.Resolved(&DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); Push push{}; push.srcBase = static_cast(retile ? 0 : base); push.dstBase = static_cast(retile ? base : 0); @@ -478,7 +653,40 @@ void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode til push.rowBegin = window.rowBegin; push.pipeBankXor = window.pipeBankXor ^ layerBankXor(tileMode, elementBytes, layout, slice); push.splitStride = layout.gcnSplitStride; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + std::optional slot; + const auto tiledAddress = owner != nullptr && DescriptorHeapEnabled() ? bufferAddress(tiled) : 0; + const auto heapPipeline = tiledAddress != 0 ? pipeline(MipTileMode(tileMode, layout), elementBytes, retile, false, true, false, true) : VK_NULL_HANDLE; + if (heapPipeline != VK_NULL_HANDLE) slot = BindlessHeap::For(context).ViewSlot(DetileHeapKind, owner, view); + if (slot) { + auto& bindless = BindlessHeap::For(context); + bindless.BindDescriptorHeaps(commands); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, heapPipeline); + const HeapImagePush pushed{push, tiledAddress + descriptorOffset, *slot, 0u}; + const VkPushDataInfoEXT data{VK_STRUCTURE_TYPE_PUSH_DATA_INFO_EXT, nullptr, 0, {&pushed, sizeof(pushed)}}; + context.Function("vkCmdPushDataEXT")(commands, &data); + } else { + const auto target = pipeline(MipTileMode(tileMode, layout), elementBytes, retile, false, true); + const auto set = allocateSet(true); + const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[0].dstSet = set; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[0].pBufferInfo = &bufferInfo; + writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[1].dstSet = set; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[1].pImageInfo = &imageInfo; + context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + CmdBindDescriptorSets(context, commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, SetBindSite::Detiler); + CmdPushConstants(context, commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + } const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; context.Resolved(&DeviceFunctions::cmdDispatch, "vkCmdDispatch")(commands, groupsX, groupsY, 1); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index c9a8e2e6a614..fdeba894d802 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Graphics/include/MemoryAccounting.hpp" #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/SlimMutex.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -194,12 +195,16 @@ std::shared_ptr makeSlab(const Context& context, std::uint64_t first VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; info.size = static_cast(units) * UnitBytes; info.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + const bool addressable = BufferPool::ForceAddresses() && context.bufferDeviceAddress; + if (addressable) info.usage |= VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; VkBuffer buffer = VK_NULL_HANDLE; if (context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer) != VK_SUCCESS) return nullptr; VkMemoryRequirements requirements{}; context.Function("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + const VkMemoryAllocateFlagsInfo addressFlags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + if (addressable) allocation.pNext = &addressFlags; allocation.allocationSize = requirements.size; VkDeviceMemory memory = VK_NULL_HANDLE; try { diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaContracts.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaContracts.cpp index c284b9aca44e..58a317311dc7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaContracts.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaContracts.cpp @@ -2,14 +2,12 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "SpirvBackend/SpirvBda.hpp" -#include #include #include #include #include #include #include -#include namespace { @@ -28,46 +26,6 @@ struct ScanWords { } }; -enum ScanId : std::uint32_t { - IdVoid = 1, IdVoidFunction, IdUint, IdUlong, IdUintPointer, IdAddress, IdOne, IdDeviceScope, IdNoSemantics, IdPointerPointer, IdPointerFunction, IdBool, IdTrue, IdFloat, IdFloatPointer, IdGlsl, IdFunctionUintPointer, IdUntypedPointer, IdFloatOne, IdMain, IdMainLabel, IdPointer, IdHelper, IdHelperParameter, IdHelperLabel, IdBody, -}; - -bool ScanStores(const std::function& main, const std::function& after = {}) { - ScanWords module{{spv::MagicNumber, 0x00010500u, 0u, 200u, 0u}}; - module.Op(spv::OpCapability, {spv::CapabilityShader}); - module.Op(spv::OpCapability, {spv::CapabilityInt64}); - module.Op(spv::OpCapability, {spv::CapabilityPhysicalStorageBufferAddresses}); - module.OpString(spv::OpExtInstImport, {IdGlsl}, "GLSL.std.450"); - module.Op(spv::OpMemoryModel, {spv::AddressingModelPhysicalStorageBuffer64, spv::MemoryModelGLSL450}); - module.OpString(spv::OpEntryPoint, {spv::ExecutionModelGLCompute, IdMain}, "main"); - module.Op(spv::OpTypeVoid, {IdVoid}); - module.Op(spv::OpTypeFunction, {IdVoidFunction, IdVoid}); - module.Op(spv::OpTypeInt, {IdUint, 32u, 0u}); - module.Op(spv::OpTypeInt, {IdUlong, 64u, 0u}); - module.Op(spv::OpTypeFloat, {IdFloat, 32u}); - module.Op(spv::OpTypeBool, {IdBool}); - module.Op(spv::OpTypePointer, {IdUintPointer, spv::StorageClassPhysicalStorageBuffer, IdUint}); - module.Op(spv::OpTypePointer, {IdPointerPointer, spv::StorageClassPhysicalStorageBuffer, IdUintPointer}); - module.Op(spv::OpTypePointer, {IdFloatPointer, spv::StorageClassPhysicalStorageBuffer, IdFloat}); - module.Op(spv::OpTypePointer, {IdPointerFunction, spv::StorageClassFunction, IdUint}); - module.Op(spv::OpTypeUntypedPointerKHR, {IdUntypedPointer, spv::StorageClassPhysicalStorageBuffer}); - module.Op(spv::OpTypeFunction, {IdFunctionUintPointer, IdVoid, IdUintPointer}); - module.Op(spv::OpConstant, {IdUlong, IdAddress, 0x1000u, 0u}); - module.Op(spv::OpConstant, {IdUint, IdOne, 1u}); - module.Op(spv::OpConstant, {IdUint, IdDeviceScope, spv::ScopeDevice}); - module.Op(spv::OpConstant, {IdUint, IdNoSemantics, 0u}); - module.Op(spv::OpConstant, {IdFloat, IdFloatOne, 0x3f800000u}); - module.Op(spv::OpConstantTrue, {IdBool, IdTrue}); - module.Op(spv::OpFunction, {IdVoid, IdMain, spv::FunctionControlMaskNone, IdVoidFunction}); - module.Op(spv::OpLabel, {IdMainLabel}); - module.Op(spv::OpConvertUToPtr, {IdUintPointer, IdPointer, IdAddress}); - main(module); - module.Op(spv::OpReturn, {}); - module.Op(spv::OpFunctionEnd, {}); - if (after) after(module); - return AgcDriver::Graphics::SpirvStoresThroughAddresses(module.words); -} - enum FaultScanId : std::uint32_t { FaultVoid = 1, FaultVoidFunction, FaultUint, FaultWords, FaultBlock, FaultBlockPointer, FaultWordPointer, FaultHelperFunction, FaultVariable, FaultZero, FaultReportLast, FaultOverflowWord, FaultScope, FaultSemantics, FaultMain, FaultMainLabel, FaultHelper, FaultHelperParameter, FaultHelperLabel, FaultBody, }; @@ -137,53 +95,6 @@ void RunFaultReportScanTests() { }), "fault scan: the fault buffer handed to a function was not followed"); } -void RunAddressStoreScanTests() { - using AgcDriver::Graphics::Require; - Require(!ScanStores([](ScanWords& m) { m.Op(spv::OpLoad, {IdUint, IdBody, IdPointer}); }), "store scan: a load through a physical pointer was taken for a store"); - Require(!ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicLoad, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics}); }), "store scan: an atomic load was taken for a store"); - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpStore, {IdPointer, IdOne}); }), "store scan: a store was missed"); - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicStore, {IdPointer, IdDeviceScope, IdNoSemantics, IdOne}); }), "store scan: an atomic store was missed"); - for (const auto opcode : {spv::OpAtomicExchange, spv::OpAtomicIAdd, spv::OpAtomicISub, spv::OpAtomicSMin, spv::OpAtomicUMin, spv::OpAtomicSMax, spv::OpAtomicUMax, spv::OpAtomicAnd, spv::OpAtomicOr, spv::OpAtomicXor}) { - Require(ScanStores([opcode](ScanWords& m) { m.Op(opcode, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics, IdOne}); }), "store scan: atomic opcode " + std::to_string(static_cast(opcode)) + " was missed"); - } - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicIIncrement, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics}); }), "store scan: an atomic increment was missed"); - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpAtomicCompareExchange, {IdUint, IdBody, IdPointer, IdDeviceScope, IdNoSemantics, IdNoSemantics, IdOne, IdOne}); }), "store scan: an atomic compare-exchange was missed"); - Require(ScanStores([](ScanWords& m) { - m.Op(spv::OpConvertUToPtr, {IdPointerPointer, IdBody, IdAddress}); - m.Op(spv::OpLoad, {IdUintPointer, IdBody + 1u, IdBody}); - m.Op(spv::OpStore, {IdBody + 1u, IdOne}); - }), "store scan: a store through a pointer loaded from guest memory was missed"); - Require(ScanStores([](ScanWords& m) { - m.Op(spv::OpSelect, {IdUintPointer, IdBody, IdTrue, IdPointer, IdPointer}); - m.Op(spv::OpStore, {IdBody, IdOne}); - }), "store scan: a store through a selected pointer was missed"); - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpFunctionCall, {IdVoid, IdBody, IdHelper, IdPointer}); }, [](ScanWords& m) { - m.Op(spv::OpFunction, {IdVoid, IdHelper, spv::FunctionControlMaskNone, IdFunctionUintPointer}); - m.Op(spv::OpFunctionParameter, {IdUintPointer, IdHelperParameter}); - m.Op(spv::OpLabel, {IdHelperLabel}); - m.Op(spv::OpStore, {IdHelperParameter, IdOne}); - m.Op(spv::OpReturn, {}); - m.Op(spv::OpFunctionEnd, {}); - }), "store scan: a store through a pointer parameter of a called function was missed"); - Require(!ScanStores([](ScanWords& m) { - m.Op(spv::OpVariable, {IdPointerFunction, IdBody, spv::StorageClassFunction}); - m.Op(spv::OpCopyMemory, {IdBody, IdPointer}); - }), "store scan: a copy out of guest memory was taken for a store"); - Require(ScanStores([](ScanWords& m) { - m.Op(spv::OpVariable, {IdPointerFunction, IdBody, spv::StorageClassFunction}); - m.Op(spv::OpCopyMemory, {IdPointer, IdBody}); - }), "store scan: a copy into guest memory was missed"); - Require(ScanStores([](ScanWords& m) { - m.Op(spv::OpConvertUToPtr, {IdUntypedPointer, IdBody, IdAddress}); - m.Op(spv::OpStore, {IdBody, IdOne}); - }), "store scan: a store through an untyped physical pointer was missed"); - Require(ScanStores([](ScanWords& m) { - m.Op(spv::OpConvertUToPtr, {IdFloatPointer, IdBody, IdAddress}); - m.Op(spv::OpExtInst, {IdFloat, IdBody + 1u, IdGlsl, GLSLstd450Modf, IdFloatOne, IdBody}); - }), "store scan: an extended instruction writing through a physical pointer was missed"); - Require(ScanStores([](ScanWords& m) { m.Op(spv::OpCooperativeMatrixStoreKHR, {IdPointer, IdOne, IdNoSemantics}); }), "store scan: a cooperative matrix store was missed"); -} - template void reject(TAction action, const char* expected) { @@ -210,22 +121,11 @@ std::uint32_t CallSites(const std::vector& spirv, const char* nam void RunBdaContractTests() { using namespace ShaderRecompiler; - using AgcDriver::Graphics::SpirvStoresThroughAddresses; - RunAddressStoreScanTests(); RunFaultReportScanTests(); - AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(MakeBdaTestShader(0x1000, 32)), "a BDA read was taken for a store through an address"); - AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(MakeBdaDwordReadTestShader(0x1000, 4, false, false)), "BDA dword reads were taken for a store through an address"); - AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaStoreTestShader(0x1000, 7, 32, false)), "a BDA store was not seen"); - AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaWriteTestShader(0x1000, 7, 32, false)), "a BDA write was not seen"); - const std::array stored{7u, 9u}; - AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(MakeBdaDwordWriteTestShader(0x1000, static_cast(stored.size()), stored.data())), "BDA dword writes were not seen"); const std::array headerOnly{spv::MagicNumber, 0x00010300u, 0u, 1u, 0u}; - AgcDriver::Graphics::Require(!SpirvStoresThroughAddresses(headerOnly), "a module without physical pointers was taken for one storing through addresses"); const std::array truncated{spv::MagicNumber, 0u}; - AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(truncated), "an unreadable module was taken for one without stores"); auto unparsed = MakeBdaStoreTestShader(0x1000, 7, 32, false); unparsed[5] |= 0xffff0000u; - AgcDriver::Graphics::Require(SpirvStoresThroughAddresses(unparsed), "a module with an instruction past its end was taken for one without stores"); using AgcDriver::Graphics::SpirvFaultReportOnly; AgcDriver::Graphics::Require(SpirvFaultReportOnly(MakeBdaTestShader(0x1000, 32), 0, 1) && SpirvFaultReportOnly(MakeBdaDwordReadTestShader(0x1000, 4, false, false), 0, 1) && SpirvFaultReportOnly(MakeBdaFaultSequenceTestShader({BdaAbi::FaultReason::Unmapped, BdaAbi::FaultReason::Unaligned}), 0, 1), "a BDA read program, which writes only the fault report, was given a full fault buffer"); AgcDriver::Graphics::Require(!SpirvFaultReportOnly(MakeBdaStoreTestShader(0x1000, 7, 32, false), 0, 1) && !SpirvFaultReportOnly(MakeBdaWriteTestShader(0x1000, 7, 32, false), 0, 1), "a BDA store program, which records written pages, was given a report-only fault buffer"); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index 42f03743e85c..5408c6cecdb4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -613,6 +613,9 @@ void importJoinTests(const Context& context) { Require(prefix != nullptr && prefix->base == heapBase && prefix->bytes == 3 * piece && prefixBuffer == prefix->buffer && prefixCopied == 0, "a buffer bound from the first of three pieces into the last was not imported over the pieces it crosses"); const auto* kept = HostImportFor(importing, first, 16); Require(kept != nullptr && kept->buffer == joinedBuffer && kept->bytes == 2 * large, "a registry change elsewhere re-imported the joined ranges"); + const auto [startedBuffer, startedCopied] = bind(heapBase, 2 * piece + 32); + const auto* started = HostImportFor(importing, heapBase, 2 * piece + 32); + Require(started != nullptr && started->base == heapBase && started->bytes == 3 * piece && startedBuffer == started->buffer && startedCopied == 0, "a buffer bound from the start of one piece into the next was not imported as one run"); change(guest + 2 * large, large, true); Require(HostImportFor(importing, first, 16)->buffer == joinedBuffer, "a range registered next to the joined ranges re-imported them"); const auto [grownBuffer, grownCopied] = bind(third - 16); @@ -4616,6 +4619,7 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { const std::array snapshots{{{0x7fff12340000ULL, source}}}; ShaderRecompiler::RecompileResult shader; shader.bindings = {binding(Role::BdaPagetable, 4), binding(Role::FaultBuffer, 5)}; + shader.bdaStores = true; CompiledShader compiled{ShaderRecompiler::ShaderStage::Compute, &shader, 0}; reject([&] { ShaderResources resources(context, compiled, snapshots); }, "ABI version"); shader.bdaAbiVersion = ShaderRecompiler::BdaAbi::Version; diff --git a/core/libs/prx/libSceAgcDriver/tests/BindlessHeapSlots.cpp b/core/libs/prx/libSceAgcDriver/tests/BindlessHeapSlots.cpp new file mode 100644 index 000000000000..4665c44bc05b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/BindlessHeapSlots.cpp @@ -0,0 +1,643 @@ +#include "execution/VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DescriptorHeapObjects.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; +namespace Abi = ShaderRecompiler::RuntimeAbi; + +void require(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error(what); +} + +BindlessHeap::DataSpan ringAllocate(BindlessHeap& heap, std::size_t words) { + BindlessHeap::SpanMark mark; + return heap.RingAllocate(words, mark); +} + +struct FakeOwner : HeapSlotOwner { + FakeOwner() = default; +}; + +alignas(Texture) std::array, 12> fakeTextures{}; + +std::shared_ptr fakeTexture(std::size_t index) { + return std::shared_ptr(reinterpret_cast(fakeTextures[index].data()), [](Texture*) {}); +} + +void textureSlots(const Context& context, VkImageView view) { + BindlessHeap heap(context); + const auto kind = Abi::FirstImageBinding; + auto first = fakeTexture(0); + const auto slot = heap.TextureSlot(kind, first.get(), view, VK_IMAGE_LAYOUT_GENERAL); + require(slot.has_value(), "the first texture takes a slot"); + require(heap.SlotTexture(kind, *slot) == first.get(), "the slot names the first texture"); + require(first.use_count() == 1, "a heap slot holds a reference to its texture"); + const auto generation = heap.SlotGeneration(*slot); + + auto second = fakeTexture(1); + require(!heap.TextureSlot(kind, second.get(), view, VK_IMAGE_LAYOUT_GENERAL).has_value(), "the slot of a live texture is not reused"); + heap.ReleaseSlots(first.get()); + const auto reused = heap.TextureSlot(kind, second.get(), view, VK_IMAGE_LAYOUT_GENERAL); + require(reused.has_value() && *reused == *slot, "the released texture's slot is reused"); + require(heap.SlotGeneration(*slot) != generation, "a freed slot moves its generation"); + require(heap.SlotTexture(kind, *slot) == second.get(), "the reused slot names the second texture"); + const auto counts = heap.Statistics(); + require(counts.slotsFreed == 1u && counts.slotsReused == 1u, "one slot freed and reused"); + + auto shared = fakeTexture(2); + const auto otherKind = kind + 1u; + require(!heap.TextureSlot(otherKind, shared.get(), view, VK_IMAGE_LAYOUT_GENERAL).has_value(), + "another kind finds the capped heap's only slot taken"); + heap.ReleaseSlots(second.get()); + const auto sharedSlot = heap.TextureSlot(otherKind, shared.get(), view, VK_IMAGE_LAYOUT_GENERAL); + require(sharedSlot.has_value() && *sharedSlot == *slot, "another kind takes the slot the first kind's texture left"); + require(heap.SlotTexture(kind, *slot) == nullptr && heap.SlotTexture(otherKind, *slot) == shared.get(), + "the slot names the other kind's texture"); + const auto freedBefore = heap.Statistics().slotsFreed; + heap.ReleaseSlots(shared.get()); + heap.ReleaseSlots(shared.get()); + require(heap.Statistics().slotsFreed == freedBefore + 1u, "a texture's release frees its slot once"); +} + +void samplerSlots(const Context& context) { + auto& heap = BindlessHeap::For(context); + auto cache = std::make_unique(); + const std::array firstWords{0u, 0u, 0u, 0u}; + const std::array secondWords{0u, 0u, 0x00100000u, 0u}; + auto first = cache->Get(context, firstWords, false); + const auto uses = first.use_count(); + const auto slot = heap.SamplerSlot(first); + require(slot.has_value(), "the first sampler takes a slot"); + require(first.use_count() == uses, "a heap slot holds a reference to its sampler"); + const auto generation = heap.SamplerGeneration(*slot); + require(first->CacheKey() != nullptr, "a cached sampler carries its key"); + first.reset(); + cache.reset(); + require(heap.SamplerGeneration(*slot) != generation, "a destroyed sampler did not release its slot"); + cache = std::make_unique(); + auto second = cache->Get(context, secondWords, false); + const auto reused = heap.SamplerSlot(second); + require(reused.has_value() && *reused == *slot, "the destroyed sampler's slot is reused"); +} + +void viewSlots(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + const auto kind = Abi::FirstStorageImageBinding + 27u; + FakeOwner first, second; + const auto slot = heap.ViewSlot(kind, &first, view); + require(slot.has_value() && heap.ViewSlot(kind, &first, view) == slot, "a view keeps its slot"); + require(!heap.ViewSlot(kind, &second, view).has_value(), "a full heap refuses another owner's view"); + const auto generation = heap.SlotGeneration(*slot); + heap.ReleaseSlots(&first); + require(heap.SlotGeneration(*slot) != generation, "a released view slot moves its generation"); + require(heap.ViewSlot(kind, &second, view) == slot, "a released view slot is reused"); + heap.ReleaseSlots(&second); +} + +void encodedViews(const Context& context) { + BindlessHeap heap(context); + const auto base = heap.Statistics(); + const auto encodes = [&] { return heap.Statistics().encodes - base.encodes; }; + const auto copies = [&] { return heap.Statistics().encodedCopies - base.encodedCopies; }; + VkImageView destroyed = VK_NULL_HANDLE; + const auto storageKey = static_cast(VK_IMAGE_LAYOUT_GENERAL) | 0x80000000u; + std::vector bytes(static_cast(heap.ImageHeapStride())); + { + const PlaceholderImages placeholders(context); + const auto view = placeholders.View(Abi::FirstStorageImageBinding); + heap.AbsentSlot(Abi::FirstStorageImageBinding + 2u, view); + require(encodes() == 1u && copies() == 0u, "a view's first heap slot encodes its descriptor"); + heap.AbsentSlot(Abi::FirstStorageImageBinding + 3u, view); + require(encodes() == 1u && copies() == 1u, "another slot for the same view copies the encoded descriptor"); + require(CopyEncodedImageView(view, storageKey, bytes), "a live view keeps its encoded descriptor"); + destroyed = view; + } + require(!CopyEncodedImageView(destroyed, storageKey, bytes), "a destroyed view drops its encoded descriptor"); + const PlaceholderImages again(context); + heap.AbsentSlot(Abi::FirstStorageImageBinding + 4u, again.View(Abi::FirstStorageImageBinding)); + require(encodes() == 2u && copies() == 1u, "a destroyed view's encoded descriptor is not copied to a new view"); +} + +void ownerSlots(const Context& context, VkImageView view) { + BindlessHeap heap(context); + FakeOwner image, other; + const auto slot = heap.StorageSlot(Abi::FirstStorageImageBinding, &image, view); + require(slot.has_value(), "a storage owner takes the capped heap's only slot"); + require(!heap.StorageSlot(Abi::FirstStorageImageBinding + 1u, &image, view).has_value(), "the owner's second view finds the heap full"); + heap.ReleaseSlots(&other); + require(!heap.StorageSlot(Abi::FirstStorageImageBinding, &other, view).has_value(), "another owner's release freed this owner's slot"); + heap.ReleaseSlots(&image); + require(heap.StorageSlot(Abi::FirstStorageImageBinding, &other, view) == slot, "a storage owner's release frees its slot"); + heap.ReleaseSlots(&other); +} + +void ownerViewsRelease(const Context& context, VkImageView view) { + BindlessHeap heap(context); + FakeOwner owner; + const auto firstKind = Abi::FirstStorageImageBinding, secondKind = Abi::FirstStorageImageBinding + 1u; + const auto first = heap.StorageSlot(firstKind, &owner, view); + const auto second = heap.ViewSlot(secondKind, &owner, view); + require(first.has_value() && second.has_value() && *first != *second, "one owner takes a slot for each of its two views"); + const auto before = heap.Statistics(); + const auto serial = heap.SlotSerial(); + require(heap.StorageSlot(firstKind, &owner, view) == first && heap.ViewSlot(secondKind, &owner, view) == second, "a lookup hit returns the owner's existing slots"); + const auto hit = heap.Statistics(); + require(hit.slotsFreed == before.slotsFreed && hit.slotsReused == before.slotsReused && hit.encodes == before.encodes && hit.encodedCopies == before.encodedCopies, + "a lookup hit changed the counts"); + require(heap.SlotSerial() == serial, "a lookup hit moved the slot serial"); + const auto firstGeneration = heap.SlotGeneration(*first), secondGeneration = heap.SlotGeneration(*second); + heap.ReleaseSlots(&owner); + const auto released = heap.Statistics(); + require(released.slotsFreed == before.slotsFreed + 2u, "one release freed " + std::to_string(released.slotsFreed - before.slotsFreed) + " of the owner's two slots"); + require(heap.SlotGeneration(*first) != firstGeneration && heap.SlotGeneration(*second) != secondGeneration, "the release moved both slots' generations"); + require(heap.SlotSerial() != serial, "a release moved the slot serial"); + const auto releasedSerial = heap.SlotSerial(); + const auto releasedGenerations = std::array{heap.SlotGeneration(*first), heap.SlotGeneration(*second)}; + heap.ReleaseSlots(&owner); + require(heap.Statistics().slotsFreed == released.slotsFreed, "a second release of the same owner freed slots"); + require(heap.SlotSerial() == releasedSerial, "a second release moved the slot serial"); + require(heap.SlotGeneration(*first) == releasedGenerations[0] && heap.SlotGeneration(*second) == releasedGenerations[1], "a second release moved a generation"); +} + +struct SharedOwner : std::enable_shared_from_this, HeapSlotOwner { + explicit SharedOwner(const Context& context) : context(context) {} + ~SharedOwner() { ReleaseHeapSlots(context); } + Context context; +}; + +void sharedOwnerRelease(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + auto owner = std::make_shared(context); + const HeapSlotOwner* base = owner.get(); + require(static_cast(base) != static_cast(owner.get()), "the slot owner base does not sit behind another base"); + const auto slot = heap.StorageSlot(Abi::FirstStorageImageBinding, owner.get(), view); + require(slot.has_value(), "an owner behind another base takes a slot"); + const auto generation = heap.SlotGeneration(*slot); + const auto freed = heap.Statistics().slotsFreed; + owner.reset(); + require(heap.SlotGeneration(*slot) != generation && heap.Statistics().slotsFreed == freed + 1u, + "destroying an owner behind another base did not free its slot"); +} + +void ringSpans(const Context& context) { + BindlessHeap heap(context); + BindlessHeap::SpanMark unwritten; + require(!heap.Held(unwritten), "a span never written is not held"); + BindlessHeap::SpanMark first; + const auto span = heap.RingAllocate(16u, first); + require(first.offset == span.offset && first.words == 16u && heap.Held(first), "a span written in the open chunk is held"); + constexpr std::size_t chunkWords = AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t); + BindlessHeap::SpanMark filler; + static_cast(heap.RingAllocate(chunkWords - 1024u, filler)); + BindlessHeap::SpanMark crossing; + static_cast(heap.RingAllocate(4096u, crossing)); + require(heap.Held(crossing), "a span over two chunks is kept"); + require(heap.Held(first), "a span stays held while its batch keeps the chunk open"); + const auto counts = heap.Statistics(); + require(counts.spansWritten == 3u && counts.spansKept == 3u, "three spans written and kept"); + BindlessHeap::SpanMark third; + static_cast(heap.RingAllocate(chunkWords, third)); + Recorder recorder(context); + heap.Submitted(recorder, 0u); + require(heap.Held(first), "a span stays held after its batch is submitted while the ring has not come back to its chunk"); + BindlessHeap::SpanMark fill; + const auto untilWrap = [&] { + const auto wraps = heap.Statistics().ringWraps; + BindlessHeap::DataSpan last{}; + while (heap.Statistics().ringWraps == wraps) last = heap.RingAllocate(chunkWords, fill); + return last.offset; + }; + require(untilWrap() == AgcDriver::Graphics::BindlessHeap::RingChunkBytes, "the ring skips a chunk a later batch still reads kept spans from"); + require(heap.Held(first), "the skipped chunk's kept span is still held"); + require(heap.Statistics().keptChunksSkipped == 1u, "one kept chunk skipped"); + heap.Submitted(recorder, 0u); + require(untilWrap() == 0u && !heap.Held(first) && heap.Statistics().spansExpired == 1u, "a span the ring wrote over has expired"); +} + +bool ringLargeSpans(const Context& context) { + BindlessHeap heap(context); + if (heap.RingChunkCount() < 5u) return false; + constexpr std::size_t chunkWords = AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t); + constexpr std::size_t largeWords = chunkWords + 1024u; + BindlessHeap::SpanMark pin; + static_cast(heap.RingAllocate(16u, pin)); + BindlessHeap::SpanMark large; + const auto span = heap.RingAllocate(largeWords, large); + require(span.words.size() == largeWords && heap.Held(large), "a span over two chunks is written and held"); + Recorder recorder(context); + BindlessHeap::SpanMark fill; + const auto untilWrap = [&](const BindlessHeap::SpanMark& kept) { + const auto wraps = heap.Statistics().ringWraps; + BindlessHeap::DataSpan last{}; + while (heap.Statistics().ringWraps == wraps) { + heap.Submitted(recorder, 0u); + require(heap.Held(kept), "a kept span stays held while later batches fill the ring"); + last = heap.RingAllocate(chunkWords, fill); + } + return last.offset; + }; + require(untilWrap(large) == 2u * AgcDriver::Graphics::BindlessHeap::RingChunkBytes, "the ring skips every chunk a kept large span covers"); + require(untilWrap(pin) == AgcDriver::Graphics::BindlessHeap::RingChunkBytes, "the ring skips only the pinned first chunk"); + require(!heap.Held(large), "a large span expires once the ring writes over one of its later chunks"); + require(heap.Held(pin), "the pinned span outlives the large span"); + bool refused = false; + try { + static_cast(ringAllocate(heap, heap.RingChunkCount() * AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t) / 4u + 1u)); + } catch (const std::runtime_error&) { + refused = true; + } + require(refused, "a span over a quarter of the ring is refused"); + return true; +} + +bool ringPressure(const Context& context) { + BindlessHeap heap(context); + if (heap.RingChunkCount() < 4u) return false; + constexpr std::size_t chunkWords = AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t); + Recorder recorder(context); + BindlessHeap::SpanMark fill; + require(!heap.TakeRingPressure(), "an empty ring is under no pressure"); + const auto half = heap.RingChunkCount() / 2u; + for (std::size_t chunk = 1; chunk < half; ++chunk) static_cast(heap.RingAllocate(chunkWords, fill)); + require(!heap.TakeRingPressure(), "a batch below half the ring's chunks is under no pressure"); + static_cast(heap.RingAllocate(chunkWords, fill)); + require(heap.TakeRingPressure() && heap.Statistics().ringPressureSubmits == 1u, "a batch holding half the ring's chunks asks for a submit"); + heap.Submitted(recorder, 0u); + require(!heap.TakeRingPressure(), "a submit relieves the pressure"); + BindlessHeap::SpanMark kept; + static_cast(heap.RingAllocate(16u, kept)); + for (std::size_t chunk = 1; chunk < half; ++chunk) { + heap.Submitted(recorder, 0u); + static_cast(heap.RingAllocate(chunkWords, fill)); + } + heap.Submitted(recorder, 0u); + require(heap.Held(kept) && !heap.TakeRingPressure(), "chunks of earlier batches do not count"); + return true; +} + +void ringCurrentChunk(const Context& context) { + BindlessHeap heap(context); + constexpr std::size_t chunkWords = AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t); + Recorder recorder(context); + std::vector pins(heap.RingChunkCount() - 1u); + for (auto& pin : pins) { + static_cast(heap.RingAllocate(chunkWords, pin)); + heap.Submitted(recorder, 0u); + for (const auto& kept : pins) { + if (kept.entry != 0u) require(heap.Held(kept), "every pinned chunk stays held"); + } + } + BindlessHeap::SpanMark live; + static_cast(heap.RingAllocate(16u, live)); + bool refused = false; + try { + static_cast(ringAllocate(heap, chunkWords)); + } catch (const std::runtime_error&) { + refused = true; + } + require(refused, "a span never wraps onto the current chunk behind data this batch already wrote"); + require(heap.Held(live), "the current chunk's span is still held"); +} + +bool ringBusyChunks(const Context& context) { + BindlessHeap heap(context); + if (heap.RingChunkCount() < 3u) return false; + Recorder recorder(context, true); + if (!recorder.HasTimeline()) return false; + constexpr std::size_t chunkWords = AgcDriver::Graphics::BindlessHeap::RingChunkBytes / sizeof(std::uint32_t); + const auto unreached = recorder.Submissions() + (1ull << 40u); + BindlessHeap::SpanMark busy; + static_cast(heap.RingAllocate(16u, busy)); + static_cast(ringAllocate(heap, chunkWords)); + heap.Submitted(recorder, unreached); + heap.Submitted(recorder, 0u); + BindlessHeap::SpanMark fill; + const auto wraps = heap.Statistics().ringWraps; + BindlessHeap::DataSpan last{}; + while (heap.Statistics().ringWraps == wraps) { + last = heap.RingAllocate(chunkWords, fill); + heap.Submitted(recorder, 0u); + } + require(last.offset >= AgcDriver::Graphics::BindlessHeap::RingChunkBytes, "the ring skips the chunk a batch still in flight reads"); + const auto counts = heap.Statistics(); + require(counts.busyChunksSkipped >= 1u && counts.ringWaits == 0u, "the busy chunk is skipped without a wait"); + require(heap.Held(busy), "a span in the skipped busy chunk is still held"); + return true; +} + +bool dataPool(const Context& context) { + Recorder recorder(context, true); + if (!recorder.HasTimeline()) return false; + constexpr std::size_t blockWords = AgcDriver::Graphics::BindlessHeap::PoolBlockBytes / sizeof(std::uint32_t); + { + BindlessHeap heap(context); + std::vector marks(5); + for (auto& mark : marks) static_cast(heap.Allocate(blockWords / 2u, mark)); + require(heap.Statistics().poolBlocksMade == 3u, "one batch's shader data grows into new blocks instead of waiting"); + for (const auto& mark : marks) require(heap.Held(mark), "every span of the batch is held"); + BindlessHeap::SpanMark big; + const auto span = heap.Allocate(blockWords * 3u / 2u, big); + require(span.words.size() == blockWords * 3u / 2u && heap.Statistics().poolBlocksMade == 4u, "a span larger than a block gets a block of its own"); + const auto [buffer, offset] = heap.DataLocation(span.address + 64u); + require(buffer != VK_NULL_HANDLE && offset == span.offset + 64u, "a pooled address maps back to its buffer and offset"); + heap.Submitted(recorder, recorder.Submissions() + (1ull << 40u)); + const auto made = heap.Statistics().poolBlocksMade; + BindlessHeap::SpanMark fill; + for (int index = 0; index < 4; ++index) static_cast(heap.Allocate(blockWords / 2u + 1u, fill)); + const auto busy = heap.Statistics(); + require(busy.poolBlocksRecycled == 0u && busy.poolBlocksMade > made, "blocks a batch in flight still reads are not reused"); + } + { + BindlessHeap heap(context); + BindlessHeap::SpanMark kept; + BindlessHeap::SpanMark first; + static_cast(heap.Allocate(blockWords - 64u, kept)); + static_cast(heap.Allocate(blockWords - 64u, first)); + static_cast(heap.Allocate(blockWords - 64u)); + heap.Submitted(recorder, 0u); + require(heap.Held(kept), "a span a later draw reuses stays held"); + const auto made = heap.Statistics().poolBlocksMade; + BindlessHeap::SpanMark fill; + static_cast(heap.Allocate(blockWords - 64u, fill)); + const auto counts = heap.Statistics(); + require(counts.poolBlocksRecycled == 1u && counts.poolBlocksMade == made, "a completed block is reused instead of making a new one"); + require(heap.Held(kept), "the block a kept span reopened is not reused"); + require(!heap.Held(first), "a span in a reused block has expired"); + } + { + BindlessHeap heap(context); + constexpr std::size_t blocks = 2u * AgcDriver::Graphics::BindlessHeap::PoolIdleBytes / AgcDriver::Graphics::BindlessHeap::PoolBlockBytes; + for (std::size_t index = 0; index < blocks; ++index) static_cast(heap.Allocate(blockWords - 64u)); + heap.Submitted(recorder, 0u); + const auto counts = heap.Statistics(); + require(counts.poolBlocksFreed != 0u && counts.poolBlockBytes <= AgcDriver::Graphics::BindlessHeap::PoolIdleBytes + AgcDriver::Graphics::BindlessHeap::PoolBlockBytes, + "idle blocks beyond the pool's budget are freed"); + } + return true; +} + +void heapComesBack(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + const auto kind = Abi::FirstImageBinding; + std::vector> held; + for (std::size_t index = 0; index < 7u; ++index) { + held.push_back(fakeTexture(index)); + require(heap.TextureSlot(kind, held.back().get(), view, VK_IMAGE_LAYOUT_GENERAL).has_value(), + "a texture takes one of the eight capped slots"); + } + require(BindlessHeap::NearlyFull(), "one free slot of eight refuses the heap"); + for (std::size_t index = 0; index < 2u; ++index) heap.ReleaseSlots(held[index].get()); + require(BindlessHeap::NearlyFull(), "three free slots of eight leave the heap refused"); + heap.ReleaseSlots(held[2].get()); + require(!BindlessHeap::NearlyFull(), "a release that leaves twice the margin free takes the heap again"); + for (std::size_t index = 3; index < held.size(); ++index) heap.ReleaseSlots(held[index].get()); + BindlessHeap::Release(context.device); +} + +bool idleSlotsEvicted(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + const auto kind = Abi::FirstImageBinding; + Recorder recorder(context, true); + if (!recorder.HasTimeline()) return false; + recorder.Activate(); + const auto submit = [&] { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + static_cast(recorder.Commands()); + recorder.Submit(); + }; + std::vector> held; + std::vector slots; + const auto take = [&](std::size_t index) { + if (held.size() <= index) held.resize(index + 1u); + if (held[index] == nullptr) held[index] = fakeTexture(index); + return heap.TextureSlot(kind, held[index].get(), view, VK_IMAGE_LAYOUT_GENERAL); + }; + for (std::size_t index = 0; index < 6u; ++index) { + const auto slot = take(index); + require(slot.has_value(), "a texture takes one of the eight capped slots"); + slots.push_back(*slot); + } + const auto idleGeneration = heap.SlotGeneration(slots[0]); + submit(); + submit(); + require(take(5) == slots[5], "a texture a draw uses again keeps its slot"); + require(take(6).has_value(), "the seventh texture takes a slot"); + require(heap.Statistics().slotsEvicted == 5u, "the five slots idle for two submissions are evicted"); + require(heap.SlotTexture(kind, slots[0]) == nullptr && heap.SlotGeneration(slots[0]) != idleGeneration, + "an evicted slot names no texture and moves its generation"); + require(heap.SlotTexture(kind, slots[5]) == held[5].get(), "the slot used again stays with its texture"); + require(BindlessHeap::NearlyFull(), "evicted slots no submission covers yet do not count as free"); + require(take(7).has_value(), "the eighth texture takes the last never-used slot"); + require(!take(8).has_value(), "an evicted slot is not reused before a submission covers the draws that used it"); + submit(); + require(!BindlessHeap::NearlyFull(), "evicted slots a submission covers take the heap again"); + require(recorder.WaitSerialFor(recorder.Submissions(), ~std::uint64_t{0}), "the submission completes"); + const auto reused = take(8); + require(reused.has_value() && std::find(slots.begin(), slots.begin() + 5, *reused) != slots.begin() + 5, + "an evicted slot is reused once the submission after its eviction completes"); + const auto back = take(0); + require(back.has_value() && heap.SlotTexture(kind, *back) == held[0].get(), "an evicted texture takes a slot again when a draw uses it"); + for (const auto& texture : held) heap.ReleaseSlots(texture.get()); + BindlessHeap::Release(context.device); + return true; +} + +void storageComesBack(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + std::array owners{}; + for (auto& owner : owners) { + require(heap.StorageSlot(Abi::FirstStorageImageBinding, &owner, view).has_value(), + "a storage owner takes one of the eight capped slots"); + } + require(BindlessHeap::NearlyFull(), "storage owners that leave one free slot of eight refuse the heap"); + for (auto& owner : owners) heap.ReleaseSlots(&owner); + require(!BindlessHeap::NearlyFull(), "releasing the storage owners takes the heap again"); + BindlessHeap::Release(context.device); +} + +void storageViewsComeBack(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + std::array owners{}; + for (std::size_t index = 0; index < 3u; ++index) { + for (std::uint32_t kind = 0; kind < 2u; ++kind) { + require(heap.StorageSlot(Abi::FirstStorageImageBinding + kind, &owners[index], view).has_value(), + "a storage owner takes one of the eight capped slots for each of its two views"); + } + } + require(heap.StorageSlot(Abi::FirstStorageImageBinding, &owners[3], view).has_value(), "a single-view storage owner takes the seventh slot"); + require(BindlessHeap::NearlyFull(), "storage owners that leave one free slot of eight refuse the heap"); + const auto freed = heap.Statistics().slotsFreed; + heap.ReleaseSlots(&owners[0]); + require(heap.Statistics().slotsFreed == freed + 2u, "a storage owner's release freed " + std::to_string(heap.Statistics().slotsFreed - freed) + " of its two slots"); + require(BindlessHeap::NearlyFull(), "three free slots of eight leave the heap refused"); + heap.ReleaseSlots(&owners[1]); + require(!BindlessHeap::NearlyFull(), "releasing a second two-view owner takes the heap again"); + heap.ReleaseSlots(&owners[2]); + heap.ReleaseSlots(&owners[3]); + BindlessHeap::Release(context.device); +} + +void mixedComesBack(const Context& context, VkImageView sampledView, VkImageView storageView) { + auto& heap = BindlessHeap::For(context); + std::vector> textures; + for (std::size_t index = 0; index < 4u; ++index) { + textures.push_back(fakeTexture(index)); + require(heap.TextureSlot(Abi::FirstImageBinding, textures.back().get(), sampledView, VK_IMAGE_LAYOUT_GENERAL).has_value(), + "a texture takes one of the eight capped slots"); + } + require(!BindlessHeap::NearlyFull(), "four textures in eight capped slots leave the heap taken"); + std::array owners{}; + for (auto& owner : owners) { + require(heap.StorageSlot(Abi::FirstStorageImageBinding, &owner, storageView).has_value(), + "a storage owner takes one of the slots the textures left"); + } + require(BindlessHeap::NearlyFull(), "storage owners that take the slots the textures left refuse the heap"); + for (auto& owner : owners) heap.ReleaseSlots(&owner); + require(!BindlessHeap::NearlyFull(), "releasing the storage owners takes the heap again while the textures stay"); + for (const auto& texture : textures) heap.ReleaseSlots(texture.get()); + BindlessHeap::Release(context.device); +} + +void viewsComeBack(const Context& context, VkImageView view) { + auto& heap = BindlessHeap::For(context); + const auto kind = Abi::FirstStorageImageBinding + 27u; + std::array owners{}; + for (auto& owner : owners) require(heap.ViewSlot(kind, &owner, view).has_value(), "a view takes one of the eight capped slots"); + require(BindlessHeap::NearlyFull(), "views that leave one free slot of eight refuse the heap"); + for (auto& owner : owners) heap.ReleaseSlots(&owner); + require(!BindlessHeap::NearlyFull(), "releasing the views takes the heap again"); + BindlessHeap::Release(context.device); +} + +void twoHeapsComeBack(const Context& context, VkImageView view) { + auto first = std::make_unique(context); + auto second = std::make_unique(context); + std::array firstOwners{}, secondOwners{}; + for (auto& owner : firstOwners) require(first->ViewSlot(Abi::FirstStorageImageBinding, &owner, view).has_value(), "a view takes one of the first heap's eight capped slots"); + for (auto& owner : secondOwners) require(second->ViewSlot(Abi::FirstStorageImageBinding, &owner, view).has_value(), "a view takes one of the second heap's eight capped slots"); + require(BindlessHeap::NearlyFull(), "two nearly full heaps refuse the heap"); + for (auto& owner : secondOwners) second->ReleaseSlots(&owner); + require(BindlessHeap::NearlyFull(), "room in the second heap took the heap back while the first stays nearly full"); + second.reset(); + require(BindlessHeap::NearlyFull(), "destroying the second heap took the heap back while the first stays nearly full"); + for (auto& owner : firstOwners) first->ReleaseSlots(&owner); + require(!BindlessHeap::NearlyFull(), "room in the last refused heap did not take the heap back"); + first.reset(); +} + +} + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : ""; + const bool largeSpans = mode == "ring-large-spans"; + const bool busyChunks = mode == "ring-busy-chunks"; + const bool comesBack = !mode.empty() && !largeSpans && !busyChunks; + try { + SetTestEnvironment("APS5_GPU_HEAP", "1"); + SetTestEnvironment("APS5_HEAP_SLOT_CAP", comesBack ? "8" : "1"); + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->graphicsContext().descriptorHeap) { + std::printf("skipped, the device has no VK_EXT_descriptor_heap\n"); + return VulkanTestSkipped; + } + if (busyChunks) { + bool ran = false; + { + const auto context = device->graphicsContext(); + ran = ringBusyChunks(context); + device->WaitIdle(); + } + device.reset(); + if (!ran) { + std::printf("skipped, the heap ring has fewer than three chunks or the recorder no timeline\n"); + return VulkanTestSkipped; + } + std::cout << "bindless heap ring busy chunk tests passed\n"; + return 0; + } + if (largeSpans) { + bool ran = false; + { + const auto context = device->graphicsContext(); + ran = ringLargeSpans(context); + device->WaitIdle(); + } + device.reset(); + if (!ran) { + std::printf("skipped, the heap ring has fewer than five chunks\n"); + return VulkanTestSkipped; + } + std::cout << "bindless heap ring large span tests passed\n"; + return 0; + } + if (comesBack) { + { + const auto context = device->graphicsContext(); + const PlaceholderImages placeholders(context); + const auto sampled = placeholders.View(Abi::FirstImageBinding); + const auto storage = placeholders.View(Abi::FirstStorageImageBinding); + if (mode == "comes-back") heapComesBack(context, sampled); + else if (mode == "storage-comes-back") storageComesBack(context, storage); + else if (mode == "storage-views-come-back") storageViewsComeBack(context, storage); + else if (mode == "mixed-comes-back") mixedComesBack(context, sampled, storage); + else if (mode == "views-come-back") viewsComeBack(context, storage); + else if (mode == "two-heaps-come-back") twoHeapsComeBack(context, storage); + else if (mode == "owner-views-release") ownerViewsRelease(context, storage); + else if (mode == "shared-owner-release") sharedOwnerRelease(context, storage); + else if (mode == "idle-slots-evicted") { + if (!idleSlotsEvicted(context, sampled)) { + std::printf("skipped, the recorder has no timeline\n"); + return VulkanTestSkipped; + } + } + else throw std::runtime_error("unknown mode " + mode); + device->WaitIdle(); + } + device.reset(); + std::cout << "the refused heap comes back once released slots leave room (" << mode << ")\n"; + return 0; + } + { + const auto context = device->graphicsContext(); + const PlaceholderImages placeholders(context); + textureSlots(context, placeholders.View(Abi::FirstImageBinding)); + samplerSlots(context); + if (!ringPressure(context)) std::printf("the ring pressure test needs at least four chunks\n"); + if (!dataPool(context)) std::printf("the data pool tests need the recorder timeline\n"); + if (BindlessHeap(context).RingChunkCount() != 0u) { + ringSpans(context); + ringCurrentChunk(context); + } else { + std::printf("the device's resource heap holds no descriptor ring\n"); + } + viewSlots(context, placeholders.View(Abi::FirstStorageImageBinding)); + encodedViews(context); + ownerSlots(context, placeholders.View(Abi::FirstStorageImageBinding)); + device->WaitIdle(); + } + device.reset(); + std::cout << "bindless heap slot reuse tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "bindless heap slot reuse tests failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceImageKeys.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceImageKeys.cpp new file mode 100644 index 000000000000..c48c7003799d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceImageKeys.cpp @@ -0,0 +1,234 @@ +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageKeys.hpp" +#include +#include +#include +#include +#include + +using namespace AgcDriver::Graphics; +namespace Abi = ShaderRecompiler::DeviceImageTableAbi; + +namespace { + +int failures = 0; + +void Expect(bool condition, const char* what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; +} + +DeviceImageKeys::Key MakeKey(std::uint32_t seed) { + DeviceImageKeys::Key key{}; + for (std::uint32_t word = 0; word < key.size(); ++word) key[word] = seed * 0x9e3779b9u + word * 0x7f4a7c15u + (seed >> 3u); + return key; +} + +std::uint32_t Home(const DeviceImageKeys::Key& key, std::uint32_t buckets) { + return Abi::Hash(key) & (buckets - 1u); +} + +std::vector SameHome(std::uint32_t buckets, std::size_t count) { + std::vector keys; + const auto home = Home(MakeKey(1u), buckets); + for (std::uint32_t seed = 1u; keys.size() < count; ++seed) { + const auto key = MakeKey(seed); + if (Home(key, buckets) == home) keys.push_back(key); + } + return keys; +} + +void testHash() { + Expect(Abi::Hash(DeviceImageKeys::Key{}) == 0xfa15124cu, "the hash of the zero key is MurmurHash3_x86_32 of 60 zero bytes with seed 0x9747b28c"); + Expect(Abi::Hash(DeviceImageKeys::Key{1u, 2u, 3u, 4u, 5u, 6u, 7u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, 15u}) == 0x1c73aadcu, "the hash of 1..15 matches the reference"); + Expect(Abi::Hash(DeviceImageKeys::Key{0x00123400u, 0x01600000u, 0x8000c03fu, 0x9000fac0u, 0u, 0u, 0u, 0u, 0x89abcdefu, 0x01234567u, 5u, 0u, 0u, 0u, 0u}) == 0x4dbc4501u, "the hash of a T# key matches the reference"); + Expect(Abi::Hash(DeviceImageKeys::Key{0x00123400u, 0x01600000u, 0x8000c03fu, 0x9000fac0u, 0u, 0u, 0u, 0u, 0x89abcdefu, 0x01234567u, Abi::RuntimeSampler, 0x92u, 0x400000u, 0x500000u, 0u}) == 0xd28701c6u, "the hash of a T# and S# key matches the reference"); + Expect(Abi::Hash(DeviceImageKeys::Key{0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu}) == 0x1f3adfbdu, "the hash of all ones matches the reference"); + auto changed = MakeKey(7u); + const auto original = Abi::Hash(changed); + for (std::uint32_t word = 0; word < changed.size(); ++word) { + auto other = changed; + other[word] ^= 1u; + Expect(Abi::Hash(other) != original, "a key differing in one bit of any word hashes differently"); + } +} + +void testInsertFind() { + DeviceImageKeys keys(64u, 64u); + Expect(keys.Buckets() == 64u && keys.Entries() == 0u && keys.Words().size() == 64u * Abi::BucketWords, "a new table has its buckets and no entries"); + Expect(!keys.Find(MakeKey(1u)).has_value(), "an empty table finds nothing"); + for (std::uint32_t seed = 1; seed <= 20; ++seed) Expect(keys.Insert(MakeKey(seed), {seed, seed % 3u, 0u}) == DeviceImageKeys::Insertion::Inserted, "a new key is inserted"); + Expect(keys.Entries() == 20u, "the table counts its entries"); + for (std::uint32_t seed = 1; seed <= 20; ++seed) { + const auto found = keys.Find(MakeKey(seed)); + Expect(found.has_value() && found->slot == seed && found->mode == seed % 3u && found->fault == 0u, "every inserted key is found with its value"); + } + Expect(!keys.Find(MakeKey(21u)).has_value(), "a key never inserted is not found"); + Expect(keys.Insert(MakeKey(5u), {99u, 7u, 3u, 42u}) == DeviceImageKeys::Insertion::Updated && keys.Entries() == 20u, "inserting a present key rewrites its value without a new entry"); + Expect(keys.Find(MakeKey(5u)) == DeviceImageValue{99u, 7u, 3u, 42u}, "the rewritten value, sampler slot included, is found"); + std::size_t live = 0; + keys.ForEach([&](const DeviceImageKeys::Key& key, const DeviceImageValue& value) { + ++live; + Expect(keys.Find(key) == value, "every visited entry is the one Find returns"); + }); + Expect(live == 20u, "ForEach visits every entry once"); +} + +void testLayout() { + DeviceImageKeys keys(16u, 16u); + const auto key = MakeKey(3u); + Expect(keys.Insert(key, {11u, 2u, 5u, 9u}) == DeviceImageKeys::Insertion::Inserted, "the layout key is inserted"); + const auto bucket = Home(key, 16u); + const auto* at = keys.Words().data() + bucket * Abi::BucketWords; + Expect(std::equal(key.begin(), key.end(), at), "the key words lie at the start of the home bucket"); + Expect(at[Abi::StateWord] == Abi::LiveState && at[Abi::SlotWord] == 11u && at[Abi::ModeWord] == 2u && at[Abi::FaultWord] == 5u && at[Abi::SamplerSlotWord] == 9u, "the bucket holds the state, slot, mode, fault and sampler slot words"); + std::size_t nonzero = 0; + for (std::size_t word = 0; word < keys.Words().size(); ++word) nonzero += (word / Abi::BucketWords != bucket && keys.Words()[word] != 0u) ? 1u : 0u; + Expect(nonzero == 0u, "the other buckets stay zero"); +} + +void testCollisions() { + DeviceImageKeys keys(64u, 64u); + const auto colliding = SameHome(64u, 3u); + for (std::uint32_t index = 0; index < colliding.size(); ++index) Expect(keys.Insert(colliding[index], {index + 1u, 0u, 0u}) == DeviceImageKeys::Insertion::Inserted, "keys of one home bucket are all inserted"); + const auto home = Home(colliding[0], 64u); + for (std::uint32_t index = 0; index < colliding.size(); ++index) { + const auto* at = keys.Words().data() + ((home + index) & 63u) * Abi::BucketWords; + Expect(std::equal(colliding[index].begin(), colliding[index].end(), at), "colliding keys take the buckets after their home in insertion order"); + Expect(keys.Find(colliding[index]) == DeviceImageValue{index + 1u, 0u, 0u}, "every colliding key is found"); + } +} + +void testProbeLimit() { + DeviceImageKeys bounded(64u, 64u); + const auto colliding = SameHome(64u, Abi::ProbeSteps + 1u); + for (std::uint32_t index = 0; index < Abi::ProbeSteps; ++index) Expect(bounded.Insert(colliding[index], {index, 0u, 0u}) == DeviceImageKeys::Insertion::Inserted, "keys within the probe window of their home are inserted"); + Expect(bounded.Insert(colliding[Abi::ProbeSteps], {99u, 0u, 0u}) == DeviceImageKeys::Insertion::Full, "a key with no free bucket within the probe window of a table at its maximum is refused"); + Expect(!bounded.Find(colliding[Abi::ProbeSteps]).has_value(), "the refused key is not found"); + for (std::uint32_t index = 0; index < Abi::ProbeSteps; ++index) Expect(bounded.Find(colliding[index]) == DeviceImageValue{index, 0u, 0u}, "a refused insertion leaves the entries in place"); + + DeviceImageKeys growing(64u, 1024u); + for (std::uint32_t index = 0; index <= Abi::ProbeSteps; ++index) Expect(growing.Insert(colliding[index], {index, 0u, 0u}) == DeviceImageKeys::Insertion::Inserted, "a key past the probe window grows the table"); + Expect(growing.Buckets() > 64u && growing.Rebuilt(), "the table grew and reports the rebuild"); + for (std::uint32_t index = 0; index <= Abi::ProbeSteps; ++index) Expect(growing.Find(colliding[index]) == DeviceImageValue{index, 0u, 0u}, "every key is found after the growth"); +} + +void testLoadAndFull() { + DeviceImageKeys keys(16u, 64u); + std::uint32_t inserted = 0; + for (std::uint32_t seed = 1; seed <= 200 && keys.Insert(MakeKey(seed), {seed, 0u, 0u}) != DeviceImageKeys::Insertion::Full; ++seed) ++inserted; + Expect(keys.Buckets() == 64u, "the table grows up to its maximum"); + Expect(inserted <= 32u && keys.Entries() == inserted, "the table holds at most half its maximum buckets"); + for (std::uint32_t seed = 1; seed <= inserted; ++seed) Expect(keys.Find(MakeKey(seed)) == DeviceImageValue{seed, 0u, 0u}, "every entry of a full table is found"); + keys.Clear(); + Expect(keys.Entries() == 0u && keys.Rebuilt() && !keys.Find(MakeKey(1u)).has_value(), "a cleared table is empty and reports the rebuild"); + Expect(std::all_of(keys.Words().begin(), keys.Words().end(), [](std::uint32_t word) { return word == 0u; }), "a cleared table is all zeros"); + Expect(keys.Insert(MakeKey(1u), {1u, 0u, 0u}) == DeviceImageKeys::Insertion::Inserted, "a cleared table takes keys again"); +} + +void testChanged() { + DeviceImageKeys keys(64u, 64u); + static_cast(keys.TakeChanged()); + Expect(!keys.Rebuilt(), "taking the changes clears the rebuild flag"); + const auto first = MakeKey(1u); + const auto second = MakeKey(2u); + static_cast(keys.Insert(first, {1u, 0u, 0u})); + static_cast(keys.Insert(second, {2u, 0u, 0u})); + static_cast(keys.Insert(first, {3u, 0u, 0u})); + auto changed = keys.TakeChanged(); + std::vector expected{Home(first, 64u), Home(second, 64u)}; + if (expected[0] == expected[1]) expected[1] = (expected[1] + 1u) & 63u; + std::ranges::sort(expected); + Expect(changed == expected, "the changed buckets are reported once each, sorted"); + Expect(keys.TakeChanged().empty(), "the changes are taken once"); +} + +void testRing() { + constexpr std::uint32_t capacity = 4u; + std::vector ring(Abi::RingRecordsWord + capacity * Abi::RecordWords, 0u); + const auto write = [&](std::uint32_t record, const DeviceImageKeys::Key& key, std::uint32_t pc) { + auto* at = ring.data() + Abi::RingRecordsWord + record * Abi::RecordWords; + std::copy(key.begin(), key.end(), at); + at[Abi::RecordPcWord] = pc; + }; + write(0, MakeKey(1u), 0x40u); + write(1, MakeKey(2u), 0x44u); + write(2, MakeKey(1u), 0x48u); + ring[Abi::RingCountWord] = 3u; + const auto decoded = DecodeDeviceImageRing(ring, capacity); + Expect(decoded.written == 3u && decoded.dropped == 0u && decoded.misses.size() == 3u, "a ring within its capacity decodes every record"); + Expect(decoded.misses[1].key == MakeKey(2u) && decoded.misses[1].pc == 0x44u, "a record carries its key and program counter"); + const auto distinct = DistinctDeviceImageMisses(decoded.misses); + Expect(distinct.size() == 2u && distinct[0].key == MakeKey(1u) && distinct[0].pc == 0x40u && distinct[1].key == MakeKey(2u), "the drain keeps the first record of every key in ring order"); + write(3, MakeKey(3u), 0x4cu); + ring[Abi::RingCountWord] = 9u; + const auto overflowed = DecodeDeviceImageRing(ring, capacity); + Expect(overflowed.written == 9u && overflowed.dropped == 5u && overflowed.misses.size() == capacity, "a count past the capacity reports the dropped records"); + bool refused = false; + try { + static_cast(DecodeDeviceImageRing(std::span(ring).first(ring.size() - 1u), capacity)); + } catch (const std::exception&) { + refused = true; + } + Expect(refused, "a ring buffer smaller than its capacity is refused"); +} + +void testConstruction() { + bool refused = false; + try { + DeviceImageKeys keys(24u, 64u); + } catch (const std::exception&) { + refused = true; + } + Expect(refused, "a bucket count that is not a power of two is refused"); + refused = false; + try { + DeviceImageKeys keys(8u, 64u); + } catch (const std::exception&) { + refused = true; + } + Expect(refused, "a table smaller than the probe window is refused"); +} + +} + +void testRetain() { + DeviceImageKeys keys(64u, 64u); + const auto colliding = SameHome(64u, 6u); + for (std::uint32_t index = 0; index < colliding.size(); ++index) Expect(keys.Insert(colliding[index], {index + 1u, 0u, 0u}) == DeviceImageKeys::Insertion::Inserted, "a colliding key is inserted"); + for (std::uint32_t seed = 5000; seed < 5010; ++seed) Expect(keys.Insert(MakeKey(seed), {seed, 1u, 0u}) == DeviceImageKeys::Insertion::Inserted, "a spread key is inserted"); + static_cast(keys.TakeChanged()); + const auto dropped = keys.Retain([&](const DeviceImageKeys::Key& key) { return key != colliding[0] && key != colliding[2] && key != MakeKey(5004u); }); + Expect(dropped == 3u && keys.Entries() == 13u, "Retain drops the keys it is not asked to keep and counts the rest"); + Expect(!keys.Find(colliding[0]).has_value() && !keys.Find(colliding[2]).has_value() && !keys.Find(MakeKey(5004u)).has_value(), "a dropped key is no longer found"); + for (std::uint32_t index : {1u, 3u, 4u, 5u}) { + const auto found = keys.Find(colliding[index]); + Expect(found.has_value() && found->slot == index + 1u, "a kept key behind a dropped one in its probe sequence is still found with its value"); + } + for (std::uint32_t seed = 5000; seed < 5010; ++seed) { + if (seed == 5004u) continue; + const auto found = keys.Find(MakeKey(seed)); + Expect(found.has_value() && found->slot == seed, "a kept spread key is found with its value"); + } + Expect(keys.Rebuilt() && keys.TakeChanged().empty(), "Retain rebuilds the table, which uploads it whole"); + Expect(keys.Retain([](const DeviceImageKeys::Key&) { return true; }) == 0u && keys.Entries() == 13u, "keeping every key drops none"); +} + +int main() { + testHash(); + testRetain(); + testInsertFind(); + testLayout(); + testCollisions(); + testProbeLimit(); + testLoadAndFull(); + testChanged(); + testRing(); + testConstruction(); + if (failures != 0) { + std::fprintf(stderr, "%d device image key checks failed\n", failures); + return 1; + } + std::printf("device image key tests passed\n"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index ca9e414a98ba..d444782f2cb3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -6268,6 +6268,46 @@ void copyInputKeptBytesTests() { Require(mock.live == 0, "copy input accounting tests leaked Vulkan objects"); } +void pooledBufferAddressTests() { + using namespace AgcDriver::Graphics; + constexpr std::size_t bytes = 4096; + constexpr auto properties = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + const bool forced = BufferPool::ForceAddresses(); + mock = MockVulkan{}; + { + auto context = mockContext(); + context.device = makeHandle(); + context.bufferDeviceAddress = true; + context.memory.memoryTypes[0].propertyFlags |= properties; + context.memory.memoryHeapCount = 1; + context.memory.memoryHeaps[0] = {std::uint64_t{4} << 30u, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + context.limits.maxMemoryAllocationCount = 4096; + context.bufferPool = std::make_shared(context); + BufferPool::SetForceAddresses(true); + VkBuffer pooled = VK_NULL_HANDLE; + { + DeviceBuffer texture(context, bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + pooled = texture.Handle(); + } + VkDeviceAddress address = 0; + VkDeviceAddress expected = 0; + { + Buffer staging(context, bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, properties); + Require(staging.Handle() == pooled, "a device-local buffer of the pooled device buffer's size and usage did not reuse its allocation"); + const VkBufferDeviceAddressInfo info{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, pooled}; + expected = mockGetBufferDeviceAddress(context.device, &info); + Require(expected != 0, "the mock device address of the pooled buffer is zero"); + try { + address = staging.DeviceAddress(); + } catch (const std::exception&) { + } + } + BufferPool::SetForceAddresses(forced); + Require(address == expected, "a buffer that reuses a pooled device buffer's allocation has no device address"); + } + Require(mock.live == 0, "pooled buffer address tests leaked Vulkan objects"); +} + void depthCopyKeptBytesTests() { using namespace AgcDriver::Graphics; constexpr std::size_t bytes = (8 * 8 + 4 * 4) * sizeof(float); @@ -13362,6 +13402,11 @@ int main(int argc, char** argv) { std::cout << "GPU-copy input snapshot accounting and retention passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--pooled-buffer-address") { + pooledBufferAddressTests(); + std::cout << "a reused pooled allocation keeps a device address\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--depth-copy-kept-bytes") { depthCopyKeptBytesTests(); std::cout << "Depth-copy linear buffer accounting and retention passed\n"; diff --git a/core/libs/prx/libSceAgcDriver/tests/HeapWalkMemo.cpp b/core/libs/prx/libSceAgcDriver/tests/HeapWalkMemo.cpp new file mode 100644 index 000000000000..a1cd55a36bee --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/HeapWalkMemo.cpp @@ -0,0 +1,162 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapWalkMemo.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::DriverDetail::HeapWalkEntry; +using AgcDriver::DriverDetail::HeapWalkEntryBytes; +using AgcDriver::DriverDetail::HeapWalkMemo; +using AgcDriver::DriverDetail::HeapWalkMemoKeys; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +HeapWalkEntry sizedEntry(std::size_t words, std::shared_ptr result = nullptr) { + HeapWalkEntry entry; + entry.values.resize(words); + entry.values.shrink_to_fit(); + entry.result = std::move(result); + return entry; +} + +std::uint64_t heldBytes(const HeapWalkMemo& memo) { + std::uint64_t bytes = 0; + std::unordered_set pinned; + for (const auto* generation : {&memo.current, &memo.previous}) { + for (const auto& [key, ways] : *generation) { + for (const auto& entry : ways) { + bytes += AgcDriver::DriverDetail::HeapWalkEntryOwnBytes(entry); + if (entry.capture != nullptr && pinned.insert(entry.capture.get()).second) bytes += AgcDriver::DriverDetail::HeapWalkCaptureBytes(*entry.capture); + if (entry.result != nullptr && pinned.insert(entry.result.get()).second) bytes += AgcDriver::DriverDetail::HeapWalkResultBytes(*entry.result); + if (entry.words != nullptr && pinned.insert(entry.words.get()).second) bytes += AgcDriver::DriverDetail::HeapWalkWordsBytes(*entry.words); + } + } + } + require(pinned.size() == memo.pins.size(), "the heap walk memo pins " + std::to_string(memo.pins.size()) + " objects for the " + std::to_string(pinned.size()) + " its entries hold"); + return bytes; +} + +void requireWithinBudget(const HeapWalkMemo& memo, const std::string& where) { + const auto held = heldBytes(memo); + require(held == memo.heldBytes, where + ": the heap walk memo charged " + std::to_string(memo.heldBytes) + " bytes for the " + std::to_string(held) + " it holds"); + require(held <= memo.budget, where + ": the heap walk memo holds " + std::to_string(held) + " bytes, over its budget of " + std::to_string(memo.budget)); +} + +void oldestEntriesDroppedTests() { + constexpr std::size_t words = 4096; + require(HeapWalkEntryBytes(sizedEntry(words)) == sizeof(HeapWalkEntry) + words * sizeof(std::uint32_t), "a heap walk entry is not charged its struct and the capacity of its vectors"); + const auto entryBytes = HeapWalkEntryBytes(sizedEntry(words, std::make_shared())); + HeapWalkMemo memo; + memo.budget = 16 * entryBytes; + constexpr std::uint64_t keys = 64; + std::vector> results; + for (std::uint64_t key = 1; key <= keys; ++key) { + auto result = std::make_shared(); + results.push_back(result); + memo.remember(key, sizedEntry(words, std::move(result))); + requireWithinBudget(memo, "after key " + std::to_string(key)); + } + require(memo.find(1) == nullptr && memo.find(keys / 2) == nullptr, "the oldest heap walk entries stayed past the budget"); + require(results.front().expired() && results[keys / 2 - 1].expired(), "a heap walk entry the budget dropped still holds its result"); + const auto* newest = memo.find(keys); + require(newest != nullptr && newest->size() == 1 && newest->front().values.size() == words, "the newest heap walk entry is not findable under the budget"); + require(!results.back().expired(), "the newest heap walk entry lost its result"); + std::cout << "the heap walk memo keeps " << memo.current.size() << " of " << keys << " entries within its budget of " << memo.budget << " bytes\n"; +} + +void previousGenerationDroppedFirstTests() { + const auto tinyBytes = HeapWalkEntryBytes(sizedEntry(1)); + constexpr std::size_t largeWords = std::size_t{2} << 20u; + const auto largeBytes = HeapWalkEntryBytes(sizedEntry(largeWords)); + HeapWalkMemo memo; + memo.budget = largeBytes + 64 * tinyBytes; + require((HeapWalkMemoKeys + 1) * tinyBytes < memo.budget, "the generation fixture does not fit its budget"); + for (std::uint64_t key = 1; key <= HeapWalkMemoKeys + 1; ++key) memo.remember(key, sizedEntry(1)); + require(memo.previous.size() == HeapWalkMemoKeys && memo.current.size() == 1, "the heap walk memo did not start a generation at its key cap"); + requireWithinBudget(memo, "after the generation swap"); + memo.remember(HeapWalkMemoKeys + 2, sizedEntry(largeWords)); + requireWithinBudget(memo, "after the large entry"); + require(memo.previous.empty(), "the heap walk memo kept its previous generation over the budget"); + require(memo.find(HeapWalkMemoKeys + 1) != nullptr && memo.find(HeapWalkMemoKeys + 2) != nullptr, "the heap walk memo dropped current entries while its previous generation could go"); + std::cout << "the heap walk memo drops its previous generation before its current entries\n"; +} + +void hitEntriesKeptTests() { + constexpr std::size_t words = 4096; + const auto entryBytes = HeapWalkEntryBytes(sizedEntry(words)); + HeapWalkMemo memo; + memo.budget = 4 * entryBytes; + for (std::uint64_t key = 1; key <= 4; ++key) memo.remember(key, sizedEntry(words)); + auto* hit = memo.find(1); + require(hit != nullptr && hit->size() == 1, "the first heap walk entry is not findable under the budget"); + memo.touch(hit->front()); + memo.remember(5, sizedEntry(words)); + requireWithinBudget(memo, "after the trim"); + require(memo.find(1) != nullptr, "the heap walk memo dropped an entry that was hit after the entries it kept"); + require(memo.find(2) == nullptr, "the heap walk memo kept its least recently used entry past the budget"); + std::cout << "the heap walk memo keeps its recently hit entries\n"; +} + +void promotedHitKeptTests() { + const auto tinyBytes = HeapWalkEntryBytes(sizedEntry(1)); + constexpr std::size_t largeWords = std::size_t{2} << 20u; + const auto largeBytes = HeapWalkEntryBytes(sizedEntry(largeWords)); + HeapWalkMemo memo; + memo.budget = largeBytes + 64 * tinyBytes; + for (std::uint64_t key = 1; key <= HeapWalkMemoKeys + 1; ++key) memo.remember(key, sizedEntry(1)); + auto* promoted = memo.find(1); + require(promoted != nullptr && memo.current.contains(1), "a hit did not move the heap walk entry into the current generation"); + memo.touch(promoted->front()); + memo.remember(HeapWalkMemoKeys + 2, sizedEntry(largeWords)); + requireWithinBudget(memo, "after the large entry"); + require(memo.find(1) != nullptr, "the heap walk memo dropped an entry promoted by a hit with its previous generation"); + std::cout << "the heap walk memo keeps an entry a hit promoted from its previous generation\n"; +} + +void sharedResultsChargedOnceTests() { + constexpr std::size_t resultWords = std::size_t{1} << 16u; + auto result = std::make_shared(); + result->heapUserWords.resize(resultWords); + result->heapUserWords.shrink_to_fit(); + const std::shared_ptr shared = std::move(result); + const auto ownBytes = AgcDriver::DriverDetail::HeapWalkEntryOwnBytes(sizedEntry(1)); + const auto sharedBytes = AgcDriver::DriverDetail::HeapWalkResultBytes(*shared); + HeapWalkMemo memo; + memo.budget = 2 * HeapWalkEntryBytes(sizedEntry(1, shared)); + constexpr std::uint64_t keys = 16; + for (std::uint64_t key = 1; key <= keys; ++key) memo.remember(key, sizedEntry(1, shared)); + require(memo.heldBytes == keys * ownBytes + sharedBytes, "the heap walk memo charged " + std::to_string(memo.heldBytes) + " bytes for " + std::to_string(keys) + " entries that share one result, not " + std::to_string(keys * ownBytes + sharedBytes)); + for (std::uint64_t key = 1; key <= keys; ++key) require(memo.find(key) != nullptr, "the heap walk memo dropped an entry whose shared result it already charged"); + for (std::uint64_t key = keys + 1; key <= keys + 64; ++key) { + memo.remember(key, sizedEntry(1, std::make_shared())); + require(memo.heldBytes <= memo.budget, "after key " + std::to_string(key) + ": the heap walk memo holds " + std::to_string(memo.heldBytes) + " bytes, over its budget of " + std::to_string(memo.budget)); + } + requireWithinBudget(memo, "after the shared result's entries aged out"); + std::cout << "the heap walk memo charges a result its entries share once\n"; +} + +} + +int main() { + try { + oldestEntriesDroppedTests(); + previousGenerationDroppedFirstTests(); + hitEntriesKeptTests(); + promotedHitKeptTests(); + sharedResultsChargedOnceTests(); + std::cout << "heap walk memo budget tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "heap walk memo budget tests failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp index 53e1f7541059..c02a15035066 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "Recompiler.hpp" #include "MeshGeometryFixture.hpp" @@ -173,7 +174,8 @@ bool ReadsNumWorkgroups(std::span words) { } void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { - const auto target = device.Target(); + auto target = device.Target(); + target.gpuResolvedImages = AgcDriver::Graphics::DescriptorHeapEnabled(); std::vector userData(12, 0u); const auto index = AgcDriver::Graphics::MeshIndexBufferDescriptor(setup.draw); std::copy(index.begin(), index.end(), userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord); @@ -190,7 +192,7 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { const auto meshResult = ShaderRecompiler::Recompile(geometry); Require(ReadsNumWorkgroups(meshResult.spirv), "the mesh program does not read NumWorkgroups, so RADV can keep a previous vertex pipeline's user SGPR count for its dispatch"); const auto meshPush = static_cast(meshResult.pushConstants.size()); - Require((meshPush == 0) == (setup.geometryPushBytes == 0), "the geometry program's push data does not follow its layout"); + Require(meshResult.gpuResolvedImages || (meshPush == 0) == (setup.geometryPushBytes == 0), "the geometry program's push data does not follow its layout"); ShaderRecompiler::ShaderPixelStageInfo pixel{}; pixel.interpolatorCount = 1; @@ -206,6 +208,7 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { std::nullopt }; const auto pixelResult = ShaderRecompiler::Recompile(fragment); + Require(meshResult.gpuResolvedImages == target.gpuResolvedImages && pixelResult.gpuResolvedImages == target.gpuResolvedImages, "the mesh draw's stages did not take GPU mode"); const std::array shaders{{ {ShaderStage::Mesh, &meshResult, 0}, {ShaderStage::Fragment, &pixelResult, PixelPushOffset(meshPush, target)} diff --git a/core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp b/core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp index 4449d50b9a15..d50b80305a02 100644 --- a/core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/PresentationPass.cpp @@ -1,3 +1,5 @@ +#include "execution/VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/AspectFit.hpp" #include "prx/libSceAgcDriver/Execution/include/DisplayBuffer.hpp" #include "prx/libSceAgcDriver/Execution/include/DisplayFormat.hpp" @@ -26,7 +28,6 @@ using AgcDriver::DisplayBuffer; using AgcDriver::PresentationPass; using AgcDriver::PresentationScaler; -constexpr int VulkanTestSkipped = 77; constexpr std::uint64_t Bgra8 = 0x8000000000000000ull; constexpr std::uint64_t Rgba8 = 0x8000000022000000ull; constexpr std::uint64_t TenBit = 0x0100000000000000ull; @@ -789,7 +790,7 @@ void sourceImageOutsideDeviceMemoryTests(const Context& base) { base.Function("vkGetImageMemoryRequirements")(base.device, probe, &requirements); base.Function("vkDestroyImage")(base.device, probe, nullptr); if (!base.HostMemoryType(requirements.memoryTypeBits)) { - std::printf("presentation source in host memory: skipped, the device offers no host memory type for the image\n"); + std::printf("presentation source in host memory: skipped, the device has no host memory type for the image\n"); return; } auto context = base; @@ -826,6 +827,14 @@ int main() { residentPresentTests(device->GetContext()); overlayPlaneTests(device->GetContext()); sourceImageOutsideDeviceMemoryTests(device->GetContext()); + if (GpuHeapEnabled()) { + const auto heapDevice = OpenVulkanTestDevice(); + if (!heapDevice) return VulkanTestSkipped; + if (SkipWithoutDescriptorHeap()) return VulkanTestSkipped; + const auto context = heapDevice->graphicsContext(); + Require(BindlessHeap::For(context).DescriptorHeapMode(), "the test device did not open the descriptor heap"); + residentPresentTests(context); + } std::cout << "resident present format, presentation pass, overlay plane and letterbox tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/PushDataPlacements.cpp b/core/libs/prx/libSceAgcDriver/tests/PushDataPlacements.cpp new file mode 100644 index 000000000000..945800920555 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/PushDataPlacements.cpp @@ -0,0 +1,154 @@ +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "PushDataLayout.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::CompiledShader; +using AgcDriver::Graphics::PushDataPlacements; +using ShaderRecompiler::ShaderStage; +using ShaderRecompiler::StagePushPlacement; +namespace Abi = ShaderRecompiler::RuntimeAbi; +using Item = Abi::RootItem; + +void require(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error(what); +} + +struct Stage { + ShaderStage stage; + std::uint32_t items; + std::size_t pushed; +}; + +std::uint32_t bits(std::initializer_list items) { + std::uint32_t mask = 0; + for (const auto item : items) mask |= Abi::RootItemBit(item); + return mask; +} + +std::vector place(const std::vector& stages) { + static std::vector programs; + programs.assign(stages.size(), {}); + std::vector shaders; + for (std::size_t index = 0; index < stages.size(); ++index) { + programs[index].dataAddressPush = true; + programs[index].gpuResolvedImages = true; + programs[index].pushItems = stages[index].items; + programs[index].pushedData.resize(stages[index].pushed); + for (std::size_t dword = 0; dword < stages[index].pushed; ++dword) programs[index].pushedData[dword] = static_cast(dword); + shaders.push_back({stages[index].stage, &programs[index], 0u}); + } + return PushDataPlacements(shaders); +} + +void disjoint(const std::vector& stages, const std::vector& placements, std::uint64_t occupied, const char* what) { + std::uint64_t used = occupied; + std::uint64_t globals = 0; + for (std::size_t index = 0; index < placements.size(); ++index) { + const auto claim = [&](std::uint32_t first, std::uint32_t width, bool shared) { + for (auto dword = first; dword < first + width; ++dword) { + require(dword < Abi::PushDataDwords, std::string("a push entry leaves the push data: ") + what); + const auto bit = 1ull << dword; + if (shared && (globals & bit) != 0u) continue; + require((used & bit) == 0u, std::string("two push entries share a dword: ") + what); + used |= bit; + if (shared) globals |= bit; + } + }; + for (std::uint32_t item = 0; item < Abi::RootItemCount; ++item) { + const bool wanted = (stages[index].items & (1u << item)) != 0u; + const auto position = placements[index].items[item]; + require(wanted == (position != Abi::NoPushPosition), std::string("a stage's root items and its placement disagree: ") + what); + if (!wanted) continue; + const auto kind = static_cast(item); + if (Abi::RootAddressItem(kind)) require(position % 2u == 0u, std::string("a root address is not 8-byte aligned: ") + what); + claim(position, Abi::RootAddressItem(kind) ? 2u : 1u, static_cast(kind) < static_cast(Abi::RootItem::ShaderData)); + } + claim(placements[index].pushed.base, placements[index].pushed.count, false); + require(placements[index].pushed.count <= stages[index].pushed, std::string("a stage pushes more than it asked for: ") + what); + } +} + +} + +int main() { + try { + const auto vertexItems = bits({Item::PagetableAddress, Item::ShaderData, Item::FlattenedSrt}); + const auto pixelItems = bits({Item::PagetableAddress, Item::ShaderData, Item::TableMap}); + { + const std::vector stages{{ShaderStage::Vertex, vertexItems, 6}, {ShaderStage::Fragment, pixelItems, 30}}; + const auto placements = place(stages); + disjoint(stages, placements, 0, "vertex and pixel"); + require(placements[0].pushed.count == 6 && placements[1].pushed.count == 30, "stages that fit push everything they want"); + require(place({stages[1]})[0] == placements[1], "a pixel stage that fits is placed as it is alone, so its fragment library is shared"); + require(placements[0].Item(Item::PagetableAddress) != placements[1].Item(Item::PagetableAddress), "the pre-rasterization and fragment parts hold their own globals"); + } + { + const std::vector stages{{ShaderStage::Vertex, vertexItems, 40}, {ShaderStage::Fragment, pixelItems, 40}}; + const auto placements = place(stages); + disjoint(stages, placements, 0, "stages that want too much"); + require(placements[0].pushed.count == 27 && placements[1].pushed.count == 26, "stages that want too much share the room evenly"); + } + { + const std::vector stages{{ShaderStage::Local, vertexItems, 4}, {ShaderStage::TessellationControl, bits({Item::PagetableAddress, Item::ShaderData}), 3}, {ShaderStage::TessellationEvaluation, vertexItems, 10}, {ShaderStage::Fragment, pixelItems, 48}}; + const auto placements = place(stages); + disjoint(stages, placements, ShaderRecompiler::PushFeatureDwords(false, true, false), "a tessellation pipeline"); + require(placements[0].Item(Item::PagetableAddress) == placements[1].Item(Item::PagetableAddress), "pre-rasterization stages share their part's globals"); + } + { + const std::vector stages{{ShaderStage::Mesh, vertexItems, 8}, {ShaderStage::Fragment, pixelItems, 20}}; + disjoint(stages, place(stages), ShaderRecompiler::PushFeatureDwords(true, false, false), "a mesh pipeline"); + } + { + const std::vector stages{{ShaderStage::Compute, bits({Item::PagetableAddress, Item::GdsAddress, Item::ShaderData}), 60}}; + const auto placements = place(stages); + disjoint(stages, placements, ShaderRecompiler::StagePushFeatures(ShaderStage::Compute, 0u), "a compute stage"); + require(placements[0] == ShaderRecompiler::StandalonePushPlacement(ShaderStage::Compute, stages[0].items, stages[0].pushed), "the compiler's standalone compute placement is the driver's"); + } + { + std::vector programs(2); + programs[0].dataAddressPush = true; + programs[0].gpuResolvedImages = true; + programs[0].pushItems = vertexItems; + programs[0].pushedData.resize(48); + programs[1].pushConstants.resize(12); + programs[1].stencilReferences.push_back(1u); + programs[1].stencilReferenceDword = 2u; + const std::array shaders{CompiledShader{ShaderStage::Vertex, &programs[0], 0u}, CompiledShader{ShaderStage::Fragment, &programs[1], 128u}}; + const auto placements = PushDataPlacements(shaders); + const auto stencil = 128u / 4u + programs[1].stencilReferenceDword; + const auto pushed = placements[0].pushed; + require(stencil < pushed.base || stencil >= pushed.base + pushed.count, "a pushed run covers a per-draw stage's stencil reference"); + } + { + std::vector programs(3); + for (const std::size_t index : {0u, 2u}) { + programs[index].dataAddressPush = true; + programs[index].gpuResolvedImages = true; + } + programs[0].pushItems = vertexItems; + programs[0].pushedData.resize(6); + programs[1].pushConstants.resize(8); + programs[2].pushItems = pixelItems; + programs[2].pushedData.resize(20); + const std::array shaders{CompiledShader{ShaderStage::Vertex, &programs[0], 0u}, CompiledShader{ShaderStage::TessellationControl, &programs[1], 200u}, CompiledShader{ShaderStage::Fragment, &programs[2], 0u}}; + const auto placements = PushDataPlacements(shaders); + const std::vector stages{{ShaderStage::Vertex, vertexItems, 6}, {ShaderStage::TessellationControl, 0u, 0}, {ShaderStage::Fragment, pixelItems, 20}}; + disjoint(stages, placements, ShaderRecompiler::PushBytesMask(200u, 8u) | ShaderRecompiler::PushFeatureDwords(false, true, false), "a pixel stage wider than the top free run"); + require(placements[2].pushed.count == 20, "a pixel stage wider than the top free run takes a lower run instead of being cut"); + } + std::puts("push data placements passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index 5adecaa4a582..4fc9eb32d9ee 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -15,6 +15,7 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/MapImportGate.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PlaceholderImages.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" @@ -9474,6 +9475,46 @@ void refreshProofTests(const Device& device, Recorder& recorder) { recorder.Sync(); } recorder.Sync(); + { + std::memset(keys, 0xff, keyCount); + const auto meta = resource.dccAddress >> 8u; + const std::vector words{ + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (56u << 20u) | (((side - 1u) & 3u) << 30u), + ((side - 1u) >> 2u) | ((side - 1u) << 14u), + 0xfacu | (0x1bu << 20u) | (9u << 28u), + 0u, + 0u, + (1u << 19u) | (1u << 20u) | (1u << 21u) | (static_cast(meta & 0xffu) << 24u), + static_cast(meta >> 8u), + }; + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding images{ShaderRecompiler::DescriptorKind::StorageImage, ShaderRecompiler::DescriptorRole::GuestImages, 0, ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding, 1, words}; + images.imageShape = ShaderRecompiler::DescriptorImageShape::Image2D; + images.imageWritten = {true}; + program.bindings.push_back(images); + const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + const PlaceholderImages placeholders(context); + auto imageContext = context; + imageContext.placeholderImages = &placeholders; + std::lock_guard gpu(GpuMutex()); + ShaderResources resources(imageContext, compute); + Require(resources.Reusable(), "a set writing a keyed storage image is not kept for reuse"); + ShaderResources::ProofReport report; + Require(resources.Revalidate(std::span(&compute, 1), &report), "a set over an unchanged keyed storage image did not revalidate"); + Require(resources.Revalidate(std::span(&compute, 1), &report) && report.path == ShaderResources::ProofPath::Fast, "a set over an unchanged keyed storage image left the fast proof"); + const auto image = StorageTexture::FindLive(address, surfaceBytes); + Require(image != nullptr && image->UploadedKeys() == DccKeys::Uncompressed, "the storage image was not uploaded under uncompressed keys"); + std::memset(keys, 0x00, keyCount); + Require(resources.Revalidate(std::span(&compute, 1), &report), "a set over a fast-cleared keyed storage image did not revalidate"); + Require(report.path == ShaderResources::ProofPath::OwnRefreshed, "a fast clear of a storage image's keys took the full walk instead of the image's own refresh"); + Require(image->UploadedKeys() != DccKeys::Uncompressed, "the own refresh did not apply the fast clear"); + Require(resources.Revalidate(std::span(&compute, 1), &report) && report.path == ShaderResources::ProofPath::Fast, "the set did not return to the fast proof after the fast clear"); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + ClearCachedTextures(context.device); + } } bool pendingKeyRefreshTests(const Device& device, Recorder& recorder) { @@ -15051,6 +15092,11 @@ int main(int argc, char** argv) { unchangedCpuStampTests(device, recorder); return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--refresh-proofs-only") { + refreshProofTests(device, recorder); + std::cout << "refresh proof tests passed\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--rewritten-blocks-only") return rewrittenBlockTests(device, recorder) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--rewritten-stale-units-only") return rewrittenStaleUnitTests(device, recorder) ? 0 : 77; if (argc == 2 && std::string_view(argv[1]) == "--idle-writeback-race-only") { diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index cae1e73c7378..b75664128a4c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -164,6 +164,7 @@ void requireSameArtifact(const CompiledShaderArtifact& left, const CompiledShade require(left.bdaAbiVersion == right.bdaAbiVersion, prefix + "BDA ABI version differs"); require(left.descriptorFaults == right.descriptorFaults, prefix + "descriptor fault count differs"); require(left.windowLaneGuards == right.windowLaneGuards, prefix + "window lane guards differ"); + require(left.deviceImageTables == right.deviceImageTables, prefix + "device image table flag differs"); require(left.runtimeAbiVersion == right.runtimeAbiVersion, prefix + "runtime ABI version differs"); require(left.memoryOffsetDword == right.memoryOffsetDword, prefix + "memory offset differs"); require(left.shaderDataDwords == right.shaderDataDwords && left.imageMetadataDword == right.imageMetadataDword && left.runtimeImageCount == right.runtimeImageCount && left.runtimeImageResources == right.runtimeImageResources, prefix + "compact data layout differs"); @@ -200,6 +201,7 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right require(left.imageTableRanges == right.imageTableRanges && left.imageTableShader == right.imageTableShader && left.imageTableFaults == right.imageTableFaults && left.imageTablePoison.size() == right.imageTablePoison.size(), prefix + "image table invocation data differs"); require(left.imageTableReadRanges == right.imageTableReadRanges, prefix + "image table read ranges differ"); require(left.userDataRegisterReads == right.userDataRegisterReads, prefix + "user data register read mask differs"); + require(left.deviceImageInterfaces == right.deviceImageInterfaces && left.deviceSamplerClasses == right.deviceSamplerClasses, prefix + "device image interfaces differ"); for (std::size_t i = 0; i < left.imageTablePoison.size(); ++i) { const auto& a = left.imageTablePoison[i]; const auto& b = right.imageTablePoison[i]; @@ -271,6 +273,9 @@ RecompileResult sampleResult() { result.bdaAbiVersion = 3; result.descriptorFaults = 2; result.windowLaneGuards = 7u; + result.deviceImageTables = true; + result.deviceImageInterfaces = {0x1234567890abcdefull, 0u, 42u}; + result.deviceSamplerClasses = {3u, 0u}; result.memoryOffsetDword = 7; result.hostSubgroupSize = 32; result.fullSubgroups = true; @@ -365,6 +370,7 @@ CompiledVariant sampleVariant() { image.layerRows = true; image.maxArrayLayers = 0x01020304u; image.optionalDescriptor = true; + image.deviceTable = true; image.byElements = 4; image.byComponents = 1; image.emulatedCompare = 0x1a07u; @@ -400,6 +406,7 @@ CompiledVariant sampleVariant() { info.bindings.memoryOffsetDword = 1; info.bindings.memoryOffsetCount = 5; info.bindings.stencilReferenceDword = 23u; + info.bindings.dataAddress = true; info.bindings.userDataRegisters = {4, 5, 9}; info.bindings.descriptors = {{DescriptorBindingKind::Buffers, {0, 1}}, {DescriptorBindingKind::FlattenedSrt, {}}}; variant.bindings.layout = info.bindings; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 44b4ee7a069a..65da12c076e0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -31,6 +31,7 @@ #include "CacheKey.hpp" #include "CompiledVariant.hpp" #include "BdaAbi.hpp" +#include "DeviceImageTableAbi.hpp" #include "ImageTableAbi.hpp" #include "ShaderDiskCache.hpp" #include @@ -44,6 +45,7 @@ #include #include #include +#include #include #include #include @@ -55,6 +57,7 @@ #include #include #include +#include #include #include #include @@ -5625,6 +5628,261 @@ void verifyScalarAddressRule() { } } +ShaderRecompiler::RecompileRequest DeviceTableRequest(const std::vector& code, std::span userData, bool pixel, bool deviceTables, bool gpuMode) { + using namespace ShaderRecompiler; + static constexpr std::array capabilities{29u, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + static constexpr std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + std::uint64_t hash = 0xcbf29ce484222325ull; + for (const auto word : code) hash = (hash ^ word) * 0x100000001b3ull; + RecompileRequest request{}; + request.shader = {pixel ? ShaderStage::Fragment : ShaderStage::Compute, 0x200000u + (hash & 0xffffu) * 0x100u, code, 0, {}}; + request.context.waveSize = pixel ? 32u : 64u; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + if (pixel) { + request.context.pixel = ShaderPixelStageInfo{}; + request.context.pixel->inputAddr = PixelInputBit(PixelInput::PositionX); + request.context.pixel->posX = true; + request.context.pixel->targetOutputMode[0] = 9u; + request.context.pixel->targetExportMapping.fill(0xe4u); + } else { + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + } + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32u; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = BdaAbi::Version; + request.target.fragmentShaderBarycentricEnabled = false; + request.target.gpuResolvedImages = gpuMode; + request.target.descriptorMemory = gpuMode; + request.target.deviceImageTables = deviceTables; + request.layout.pushConstantSizeBytes = 128; + return request; +} + +struct DeviceTableSpirvScan { + bool lookup = false; + std::set lookupSubtrahends; + std::set sets; + std::vector> tableArrays; + std::vector> lookupKeys; +}; + +DeviceTableSpirvScan ScanDeviceTableSpirv(std::span words) { + DeviceTableSpirvScan result; + std::map sets; + std::map bindings; + std::map names; + std::map constants; + std::map> composites; + std::vector> calls; + std::vector subtracted; + std::uint32_t lookup = 0; + std::uint32_t function = 0; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + require(count != 0 && count <= words.size() - cursor, "device image tables: truncated SPIR-V instruction"); + const auto op = words[cursor] & 0xffffu; + if (op == spv::OpName && count > 2u) { + const auto* text = reinterpret_cast(words.data() + cursor + 2u); + names[words[cursor + 1]] = std::string(text, std::min(std::strlen(text), (count - 2u) * 4u)); + if (names[words[cursor + 1]] == "device_image_lookup") lookup = words[cursor + 1]; + } + if (op == spv::OpDecorate && count == 4u && words[cursor + 2] == spv::DecorationDescriptorSet) sets[words[cursor + 1]] = words[cursor + 3]; + if (op == spv::OpDecorate && count == 4u && words[cursor + 2] == spv::DecorationBinding) bindings[words[cursor + 1]] = words[cursor + 3]; + if (op == spv::OpConstant && count == 4u) constants[words[cursor + 2]] = words[cursor + 3]; + if ((op == spv::OpCompositeConstruct || op == spv::OpConstantComposite) && count > 3u) composites[words[cursor + 2]] = std::vector(words.begin() + cursor + 3, words.begin() + cursor + count); + if (op == spv::OpFunctionCall && count >= 7u) calls.emplace_back(words[cursor + 3], words[cursor + 6]); + if (op == spv::OpFunction && count >= 5u) function = words[cursor + 2]; + if (op == spv::OpFunctionEnd) function = 0; + if (op == spv::OpISub && count == 5u && function != 0u && function == lookup) subtracted.push_back(words[cursor + 4]); + cursor += count; + } + result.lookup = lookup != 0u; + for (const auto operand : subtracted) { + if (const auto constant = constants.find(operand); constant != constants.end()) result.lookupSubtrahends.insert(constant->second); + } + for (const auto& [callee, extra] : calls) { + const auto composite = composites.find(extra); + if (callee != lookup || composite == composites.end() || composite->second.size() != 4u) continue; + std::array values{}; + bool known = true; + for (std::size_t component = 0; component < 4u; ++component) { + const auto constant = constants.find(composite->second[component]); + known = known && constant != constants.end(); + if (known) values[component] = constant->second; + } + if (known) result.lookupKeys.push_back(values); + } + for (const auto& [id, set] : sets) { + result.sets.insert(set); + const auto binding = bindings.find(id); + const auto name = names.find(id); + if (binding != bindings.end() && name != names.end() && name->second.rfind("table_images_", 0) == 0) result.tableArrays.emplace_back(set, binding->second); + } + return result; +} + +const std::vector DeviceTableComputeCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000030u, 0xbe900302u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; +const std::vector DeviceTablePixelCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0x7e020300u, 0xbe900302u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xf800180fu, 0x03020100u, 0xbf810000u}; + +void verifyDeviceImageTables() { + using namespace ShaderRecompiler; + namespace Abi = DeviceImageTableAbi; + namespace TableAbi = ImageTableAbi; + constexpr std::uint32_t Entries = 2000; + static TableTexture textures[2]; + alignas(4096) static std::array, Entries> heap{}; + alignas(4096) static std::array srt{}; + static std::array output{}; + for (std::uint32_t entry = 0; entry < Entries; ++entry) heap[entry] = TableT(textures[entry % 2u].bytes.data(), TableFormat8888UNorm, TableType2D); + const auto heapV = TableV(heap.data(), 32u, Entries); + const auto outputV = TableV(output.data(), 0u, 16u); + std::copy(heapV.begin(), heapV.end(), srt.begin()); + srt[4] = 6u; + srt[5] = 0u; + srt[6] = 0u; + srt[7] = 1u << 30u; + std::copy(outputV.begin(), outputV.end(), srt.begin() + 12); + const auto address = static_cast(reinterpret_cast(srt.data())); + const std::array userData{static_cast(address), static_cast(address >> 32u), 7u}; + constexpr std::uint32_t samplerClass = Abi::SamplerOpaqueBlackBorder | Abi::SamplerFloatBorder; + const std::set trailerWords{TableAbi::TrailerWords - TableAbi::TrailerBucketsLow, TableAbi::TrailerWords - TableAbi::TrailerBucketsHigh, TableAbi::TrailerWords - TableAbi::TrailerRingLow, TableAbi::TrailerWords - TableAbi::TrailerRingHigh}; + for (const bool pixel : {false, true}) { + const auto& code = pixel ? DeviceTablePixelCode : DeviceTableComputeCode; + const auto request = DeviceTableRequest(code, userData, pixel, true, true); + const auto plan = GetResourcePlan(request); + require(plan->info.images.size() == 1u && plan->info.images[0].table != NoTable && plan->info.images[0].deviceTable, "device image tables: the GPU-mode stage's V# table column is not read at run time"); + require(!plan->requiresSpecializationMemory, "device image tables: a program whose only table is a device table still needs specialization memory"); + const auto compiled = CompileTable(request); + require(!TableTouched(compiled.regions, heap.data(), sizeof(heap)), "device image tables: the capture read the table records"); + const auto& result = *compiled.result; + require(result.deviceImageTables && result.gpuResolvedImages && result.dataAddressPush, "device image tables: the result is not a GPU-mode stage reading the device image table"); + const auto* map = TableBinding(result, DescriptorRole::ImageTableMap); + require(map != nullptr && map->guestDescriptor.size() >= TableAbi::HeaderWords && map->guestDescriptor[TableAbi::HeaderVersion] == TableAbi::Version, "device image tables: the GPU-mode stage has no table map to carry the trailer"); + const auto& mapWords = map->guestDescriptor; + require(mapWords[TableAbi::HeaderElementCount] == 0u && mapWords[TableAbi::TableHeader(0) + TableAbi::TableKeyCount] == 0u, "device image tables: the device column's records went into the CPU map"); + require(TableBinding(result, DescriptorRole::SamplerTable) == nullptr, "device image tables: a device-table program bound a sampler table"); + const auto scan = ScanDeviceTableSpirv(result.spirv.Words()); + require(scan.lookup, "device image tables: the module has no device image lookup"); + require(std::ranges::includes(scan.lookupSubtrahends, trailerWords), "device image tables: the lookup does not read the bucket and ring addresses from the four trailer words before the element base"); + require(scan.sets.size() <= 2u && !scan.sets.contains(2u) && !scan.sets.contains(3u), "device image tables: the module declares descriptor sets 2 or 3"); + require(!scan.tableArrays.empty(), "device image tables: the module has no table image array for the device slots"); + for (const auto& [set, binding] : scan.tableArrays) require(set == RuntimeAbi::FragmentDescriptorSet && binding == NativeBinding(IrShaderStage::Pixel, static_cast(RuntimeAbi::FirstImageBinding)), "device image tables: a table image array does not alias the GPU heap's first sampled binding"); + const auto& modes = plan->info.runtimeImageModes.at(0); + require(!scan.lookupKeys.empty(), "device image tables: the lookup is not keyed by constants"); + for (const auto& key : scan.lookupKeys) { + require(key[2] == samplerClass, "device image tables: the lookup's sampler class is not the S#'s"); + const auto registered = ResourceMaterializer::FindDeviceImageInterface(key[0] | (static_cast(key[1]) << 32u)); + require(registered != nullptr && registered->modes.size() == modes.size() && registered->image.deviceTable, "device image tables: the module's image interface was not registered with its modes"); + } +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(result.spirv, request.target.vulkanVersion, request.target.spirvVersion)); +#endif + require(plan->info.deviceImageUses.size() == 1u && plan->info.deviceImageUses[0].image == 0u && plan->info.deviceImageUses[0].sampler == 0u, "device image tables: the plan does not record the device column's sampled use"); + SrtRuntime runtime; + runtime.userData = userData; + runtime.shaderBase = request.shader.codeAddress; + bool complete = false; + auto expected = ResourceMaterializer::ExpectedDeviceImages(*plan, runtime, complete); + require(complete && expected.size() == 1u && !expected[0].candidate && std::equal(heap[7].begin(), heap[7].end(), expected[0].key.begin()), "device image tables: the pre-pass did not expect the T# at key 7 of the 2,000-entry table"); + const auto sample = static_cast(std::distance(code.begin(), std::find(code.begin(), code.end(), 0xf09c0f08u)) * 4u); + require(expected[0].pc == sample && expected[0].sampler == 0u, "device image tables: the expected use does not name its first pc and its S#"); + require(result.deviceSamplerClasses.size() == 1u && result.deviceSamplerClasses[expected[0].sampler] == samplerClass, "device image tables: the result does not carry the S#'s device sampler class"); + require(result.deviceImageInterfaces.size() == 1u && result.deviceImageInterfaces[0] != 0u, "device image tables: the result does not carry its interface ids"); + Abi::NameInterfaces(expected, result.deviceImageInterfaces); + Abi::NameSamplerClasses(expected, result.deviceSamplerClasses); + require(std::ranges::any_of(scan.lookupKeys, [&](const std::array& key) { return key[0] == expected[0].key[Abi::InterfaceWord] && key[1] == expected[0].key[Abi::InterfaceWord + 1u] && key[2] == expected[0].key[Abi::SamplerClassWord]; }), "device image tables: the expected key names another interface or sampler class than the module's lookup"); + auto readLane = code; + *std::find(readLane.begin(), readLane.end(), 0xbe900302u) = 0x7e200500u; + const auto lanePlan = GetResourcePlan(DeviceTableRequest(readLane, userData, pixel, true, true)); + require(lanePlan->info.images.size() == 1u && lanePlan->info.images[0].deviceTable, "device image tables: a key read from a lane is not a device column"); + static_cast(ResourceMaterializer::ExpectedDeviceImages(*lanePlan, runtime, complete)); + require(!complete, "device image tables: the pre-pass called a key read from a lane complete"); + + const auto offRequest = DeviceTableRequest(code, userData, pixel, false, true); + const auto offPlan = GetResourcePlan(offRequest); + require(offPlan->info.images.size() == 1u && offPlan->info.images[0].table != NoTable && !offPlan->info.images[0].deviceTable && offPlan->requiresSpecializationMemory, "device image tables: the table column left the CPU map with the option off"); + const auto off = CompileTable(offRequest); + require(!off.result->deviceImageTables && !ScanDeviceTableSpirv(off.result->spirv.Words()).lookup, "device image tables: the option off still emitted the device lookup"); + + const auto perDraw = DeviceTableRequest(code, userData, pixel, true, false); + const auto perDrawPlan = GetResourcePlan(perDraw); + require(perDrawPlan->info.images.size() == 1u && perDrawPlan->info.images[0].table != NoTable && !perDrawPlan->info.images[0].deviceTable, "device image tables: a stage on the per-draw path read a column through the device image table"); + } + + alignas(4096) static TableGuest guest; + std::fill(guest.heap.begin(), guest.heap.end(), std::array{}); + guest.heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + FillTableSrt(guest, guest.heap.data(), 32u, 4u); + const auto guestData = TableUserData(guest); + const std::vector storeCode{0xf4080100u, 0xfa000000u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf0200f08u, 0x00050200u, 0xbf810000u}; + const std::vector compareCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf0fc0108u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + const std::vector samplerTableCode{0xf4080100u, 0xfa000000u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0xf4000400u, 0xfa000040u, 0x7e240500u, 0x8f118412u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf4280206u, 0x22000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + const std::vector unnormalizedCode{0xf4080100u, 0xfa000000u, 0xbe8803ffu, 0x00008000u, 0xbe890380u, 0xbe8a0380u, 0xbe8b0380u, 0xf4080700u, 0xfa000030u, 0xf4000400u, 0xfa000040u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; + for (const auto* code : {&storeCode, &compareCode, &samplerTableCode, &unnormalizedCode}) { + auto request = TableRequest(*code, guestData); + request.target.gpuResolvedImages = true; + request.target.descriptorMemory = true; + request.target.deviceImageTables = true; + const auto plan = GetResourcePlan(request); + require(!plan->info.images.empty() && plan->info.images[0].table != NoTable && !plan->info.images[0].deviceTable, "device image tables: a column with a storage write, a depth compare, a sampler table or an unnormalized sampler left the CPU map"); + require(code != &samplerTableCode || (plan->info.samplers.size() == 1u && plan->info.samplers[0].table != NoTable), "device image tables: the sampler table program has no sampler table"); + } +} + +void verifyTableTrailer() { + using namespace ShaderRecompiler; + namespace Abi = ImageTableAbi; + static TableTexture textures[2]; + alignas(4096) static TableGuest guest; + std::fill(guest.heap.begin(), guest.heap.end(), std::array{}); + guest.heap[0] = TableT(textures[0].bytes.data(), TableFormat8888UNorm, TableType2D); + guest.heap[1] = TableT(textures[1].bytes.data(), TableFormat32UInt, TableType2D); + guest.heap[3] = guest.heap[0]; + guest.materials = {{{0u, 1u, 0u, 0u}, {0u, 0u, 0u, 0u}, {0u, 3u, 0u, 0u}}}; + FillTableSrt(guest, guest.heap.data(), 32u, 4u); + const auto compiled = CompileTable(TableRequest(TableMaterialCode, TableUserData(guest))); + const auto& recompiled = TableMap(*compiled.result); + require(Abi::Version == 3u && recompiled[Abi::HeaderVersion] == Abi::Version, "table trailer: the recompiled map does not carry ABI version 3"); + const auto tables = recompiled[Abi::HeaderTableCount]; + require(tables != 0u, "table trailer: the program has no image table"); + std::size_t recordsEnd = Abi::TableHeader(tables); + for (std::uint32_t table = 0; table < tables; ++table) { + const auto header = Abi::TableHeader(table); + recordsEnd = std::max(recordsEnd, static_cast(recompiled[header + Abi::TableMapStart]) + recompiled[header + Abi::TableKeyCount]); + } + recordsEnd = std::max(recordsEnd, static_cast(recompiled[Abi::HeaderElementFaults]) + recompiled[Abi::HeaderElementCount]); + recordsEnd = std::max(recordsEnd, static_cast(recompiled[Abi::HeaderSamplerFaults]) + recompiled[Abi::HeaderSamplerCount] / Abi::SamplerEntryElements); + require(recordsEnd <= recompiled.size(), "table trailer: the recompiled map is shorter than its records"); + + const auto check = [&](const Abi::Trailer& trailer, const char* what) { + auto map = recompiled; + const auto base = Abi::AppendTrailer(map, trailer); + require(base == recompiled.size() + Abi::TrailerWords && map.size() == base && map[Abi::HeaderElementBase] == base, (std::string("table trailer: ") + what + ": the element base does not follow the four trailer words").c_str()); + for (std::size_t word = 0; word < recompiled.size(); ++word) { + if (word != Abi::HeaderElementBase) require(map[word] == recompiled[word], (std::string("table trailer: ") + what + ": a header or record word moved").c_str()); + } + require(Abi::TrailerWord(base, 0u) == recompiled.size() && Abi::TrailerWord(base, 0u) >= recordsEnd && Abi::TrailerWord(base, Abi::TrailerWords) == base, (std::string("table trailer: ") + what + ": the trailer does not sit between the records and the element base").c_str()); + require(map[Abi::TrailerWord(base, Abi::TrailerBucketsLow)] == static_cast(trailer.buckets) && map[Abi::TrailerWord(base, Abi::TrailerBucketsHigh)] == static_cast(trailer.buckets >> 32u), (std::string("table trailer: ") + what + ": the bucket address words are wrong").c_str()); + require(map[Abi::TrailerWord(base, Abi::TrailerRingLow)] == static_cast(trailer.ring) && map[Abi::TrailerWord(base, Abi::TrailerRingHigh)] == static_cast(trailer.ring >> 32u), (std::string("table trailer: ") + what + ": the ring address words are wrong").c_str()); + require(Abi::ReadTrailer(map) == trailer, (std::string("table trailer: ") + what + ": reading the trailer back gives other addresses").c_str()); + map.resize(base + 2u * recompiled[Abi::HeaderElementCount], 0x5a5a5a5au); + require(Abi::ReadTrailer(map) == trailer, (std::string("table trailer: ") + what + ": slot parts after the element base changed the trailer").c_str()); + return map; + }; + const auto absent = check({}, "without a device table"); + for (std::uint32_t field = 0; field < Abi::TrailerWords; ++field) require(absent[Abi::TrailerWord(static_cast(recompiled.size() + Abi::TrailerWords), field)] == 0u, "table trailer: without a device table a trailer word is not zero"); + static_cast(check({0x0000123456789ab0ull, 0xfedcba9876543200ull}, "with a device table")); + auto truncated = recompiled; + truncated[Abi::HeaderElementBase] = Abi::HeaderWords + Abi::TrailerWords - 1u; + require(!Abi::ReadTrailer(truncated).has_value(), "table trailer: an element base inside the header read a trailer"); + truncated[Abi::HeaderElementBase] = static_cast(truncated.size() + 1u); + require(!Abi::ReadTrailer(truncated).has_value(), "table trailer: an element base past the map read a trailer"); +} + void verifyPointerTableUnwrittenReuse() { using namespace ShaderRecompiler; constexpr std::uint32_t Palette = 0x40; @@ -7508,6 +7766,28 @@ void verifySingleLaneVertexWave() { require(withoutKey != withKey, "single-lane vertex wave: vertex subgroup support did not change the cache key"); } +void verifyExtensionKey() { + using namespace ShaderRecompiler; + RecompileRequest request{}; + request.shader.stage = ShaderStage::Compute; + const std::array first{"VK_KHR_a", "VK_KHR_bb"}; + const std::string copied[2] = {"VK_KHR_a", "VK_KHR_bb"}; + const std::array same{copied[0], copied[1]}; + const std::array second{"VK_KHR_a", "VK_KHR_bc"}; + std::vector firstKey, againKey, sameKey, secondKey, backKey; + request.target.supportedExtensions = first; + RecompileCacheKey::BuildInterface(request, firstKey); + RecompileCacheKey::BuildInterface(request, againKey); + request.target.supportedExtensions = same; + RecompileCacheKey::BuildInterface(request, sameKey); + request.target.supportedExtensions = second; + RecompileCacheKey::BuildInterface(request, secondKey); + request.target.supportedExtensions = first; + RecompileCacheKey::BuildInterface(request, backKey); + require(firstKey == againKey && firstKey == sameKey && firstKey == backKey, "extension key: the same extension names built different keys"); + require(firstKey != secondKey, "extension key: a changed extension name kept the key"); +} + void verifySingleLaneElectedAtomicMax() { using namespace ShaderRecompiler; static constexpr std::array capabilities{spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle, spv::CapabilityGroupNonUniformArithmetic, spv::CapabilityGroupNonUniformShuffleRelative, spv::CapabilityClipDistance, spv::CapabilityInt64}; @@ -8177,7 +8457,7 @@ void verifyPixelRequestSerialization() { minimal.context.pixel->layerExported = true; minimal.context.pixel->ditheredAlphaToCoverage = true; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQTUAAAA=", "new requests did not use serialization version 53"); + require(requestPrefix(encoded, 8u) == "NVNQQTcAAAA=", "new requests did not use serialization version 55"); auto withBarycentric = minimal; withBarycentric.target.fragmentBarycentricBuiltins = true; require(serializer.Deserialize(serializer.Serialize(withBarycentric)).request.target.fragmentBarycentricBuiltins && !serializer.Deserialize(encoded).request.target.fragmentBarycentricBuiltins, "barycentric built-ins were lost in serialization"); @@ -8298,7 +8578,7 @@ void verifyPixelRequestSerialization() { for (std::size_t bytes = 0; bytes < 45u; ++bytes) { expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing, dual-source, alpha-to-coverage, stencil reference export flag, primitive class, layer export, dither or front-face flag, quad pixel mask or quad sample mask was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQTIAAAA=", "NVNQQTYAAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQTIAAAA=", "NVNQQTgAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } static constexpr std::string_view version18Request{ @@ -8352,10 +8632,15 @@ std::vector decodeBase64(std::string_view text) { bytes.push_back(static_cast((chunk >> bits) & 0xffu)); } } + if (bytes.size() > 5u && bytes[4] == 55u) { + require(bytes.back() == 0u, "a version 55 request in these checks set the GPU heap buffer descriptors"); + bytes.pop_back(); + } return bytes; } -std::string encodeBase64(const std::vector& bytes) { +std::string encodeBase64(std::vector bytes) { + if (bytes.size() > 4u && bytes[4] == 55u) bytes.push_back(0u); constexpr std::string_view alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; std::string text; for (std::size_t index = 0; index < bytes.size(); index += 3u) { @@ -8417,14 +8702,15 @@ void verifyDepthExportClamp() { const auto both = RequestSerializer{}.Deserialize(RequestSerializer{}.Serialize(tail)); require(both.request.target.largePixelFunctionLdsBuffer && both.request.target.pixelFunctionStorageDwords == 2048u && both.request.context.pixel->depthExportMin == 0.25f && both.request.context.pixel->depthExportMax == 0.75f, - "depth export: a version 53 request lost the bounds or the version 45 target tail"); + "depth export: a version 55 request lost the bounds or the version 45 target tail"); auto bytes = decodeBase64(RequestSerializer{}.Serialize(tail)); - require(bytes.size() > 8u && bytes[4] == 53u, "depth export: a depth exporting request did not use serialization version 53"); + require(bytes.size() > 8u && bytes[4] == 55u, "depth export: a depth exporting request did not use serialization version 55"); static constexpr std::array boundBytes{0x00u, 0x00u, 0x80u, 0x3eu, 0x00u, 0x00u, 0x40u, 0x3fu}; const auto found = std::search(bytes.begin(), bytes.end(), boundBytes.begin(), boundBytes.end()); require(found != bytes.end() && std::search(found + 1, bytes.end(), boundBytes.begin(), boundBytes.end()) == bytes.end(), "depth export: the clamp bounds were not written exactly once"); bytes.erase(found, found + boundBytes.size()); + bytes.resize(bytes.size() - 12u); bytes.pop_back(); bytes.pop_back(); bytes.pop_back(); @@ -8447,12 +8733,13 @@ void verifyDepthExportClamp() { ditheredPixel.ditheredAlphaSamples = 2u; dithered.context.pixel = ditheredPixel; auto ditheredBytes = decodeBase64(RequestSerializer{}.Serialize(dithered)); - require(ditheredBytes.size() > 8u && ditheredBytes[4] == 53u, "a two-sample dithered request did not use serialization version 53"); + require(ditheredBytes.size() > 8u && ditheredBytes[4] == 55u, "a two-sample dithered request did not use serialization version 55"); if (depthExport) { const auto bounds = std::search(ditheredBytes.begin(), ditheredBytes.end(), boundBytes.begin(), boundBytes.end()); require(bounds != ditheredBytes.end(), "a two-sample dithered depth exporting request lacks the clamp bounds"); ditheredBytes.erase(bounds, bounds + boundBytes.size()); } + ditheredBytes.resize(ditheredBytes.size() - 12u); ditheredBytes.pop_back(); ditheredBytes.pop_back(); ditheredBytes.pop_back(); @@ -8616,7 +8903,7 @@ void verifyVertexInfoVersion19() { const auto version43Bytes = [&](std::vector bytes) { require(bytes.end() - bytes.begin() > mergeOffset + static_cast(mergeBytes) && std::equal(plainPixel.begin() + mergeOffset, plainPixel.begin() + mergeOffset + static_cast(mergeBytes), bytes.begin() + mergeOffset), "a replay fixture does not hold the default color target merge after the sample order"); bytes.erase(bytes.begin() + mergeOffset, bytes.begin() + mergeOffset + static_cast(mergeBytes)); - bytes.resize(bytes.size() - sizeof(std::uint32_t) - 6u); + bytes.resize(bytes.size() - sizeof(std::uint32_t) - 6u - 12u); return bytes; }; auto version42 = version43Bytes(plainPixel); @@ -8778,11 +9065,10 @@ void verifyDepthRemapRequests() { const auto plainBytes = decodeBase64(plain); const auto remappedBytes = decodeBase64(remapped); constexpr std::size_t pointTailBytes = 13u; - require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 17u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 17 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - 10 - pointTailBytes, plainBytes.end(), remappedBytes.end() - 10 - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags, the point tail, the fused arithmetic flag, the pixel function LDS flag, the pixel function storage size, the barycentric built-in flag, the tessellation evaluation subgroup flag, the host interpolation first flag and the shader image minimum LOD flag of a vertex request"); + require(plainBytes.size() == remappedBytes.size() && plainBytes.size() > 29u + pointTailBytes && std::equal(plainBytes.begin(), plainBytes.end() - 29 - pointTailBytes, remappedBytes.begin()) && std::equal(plainBytes.end() - 22 - pointTailBytes, plainBytes.end(), remappedBytes.end() - 22 - pointTailBytes), "the depth remap flag and mode are not the 6 bytes before the f32 flush and denormal preserve flags, the point tail, the fused arithmetic flag, the pixel function LDS flag, the pixel function storage size, the barycentric built-in flag, the tessellation evaluation subgroup flag, the host interpolation first flag, the heap profile and the other keyed target flags of a vertex request"); for (const std::uint8_t legacyVersion : {32u, 34u}) { auto legacyBytes = remappedBytes; - legacyBytes.pop_back(); - legacyBytes.pop_back(); + legacyBytes.resize(legacyBytes.size() - 14u); legacyBytes.resize(legacyBytes.size() - sizeof(std::uint32_t)); legacyBytes.pop_back(); legacyBytes.pop_back(); @@ -8796,8 +9082,7 @@ void verifyDepthRemapRequests() { require(serializer.Serialize(legacy.request) == plain, ("a version " + std::to_string(legacyVersion) + " vertex request misread the fields before the depth remap").c_str()); } auto version35 = remappedBytes; - version35.pop_back(); - version35.pop_back(); + version35.resize(version35.size() - 14u); version35.resize(version35.size() - sizeof(std::uint32_t)); version35.pop_back(); version35.pop_back(); @@ -8851,8 +9136,8 @@ void verifyMergedHullConfiguration() { require(!serializer.Deserialize(separate).request.graphics->tessellation->mergedHull, "a separate hull configuration was read as merged"); constexpr std::size_t pointTailBytes = 13u; auto legacy = decodeBase64(merged); - require(legacy.size() > 58u + pointTailBytes && legacy[4] == 53u, "a merged hull request did not use serialization version 53"); - legacy.resize(legacy.size() - sizeof(std::uint32_t) - 5u - pointTailBytes); + require(legacy.size() > 58u + pointTailBytes && legacy[4] == 55u, "a merged hull request did not use serialization version 55"); + legacy.resize(legacy.size() - sizeof(std::uint32_t) - 5u - pointTailBytes - 12u); legacy.pop_back(); legacy.pop_back(); require(legacy[legacy.size() - 52u] == 1u, "the merged hull flag is not the last byte of the tessellation configuration"); @@ -9532,7 +9817,7 @@ void verifyCoherentGds() { require(std::ranges::find(decorations[gds], static_cast(spv::DecorationCoherent)) != decorations[gds].end(), "coherent GDS: the gds buffer is not Coherent"); } -void verifyCoherentBufferAccesses() { +auto recompileCoherent(const std::vector& code, std::uint32_t address, std::span capabilities, bool addressed) { using namespace ShaderRecompiler; alignas(256) static std::array input{}; alignas(256) static std::array output{}; @@ -9542,6 +9827,28 @@ void verifyCoherentBufferAccesses() { static_cast(inputAddress), static_cast((inputAddress >> 32u) & 0xffffu), 256u, 0x31016facu, static_cast(outputAddress), static_cast((outputAddress >> 32u) & 0xffffu), 256u, 0x31016facu}; const std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, address, code, 0, {}}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.bdaAbiVersion = BdaAbi::Version; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.fragmentShaderBarycentricEnabled = false; + request.target.gpuResolvedImages = addressed; + request.target.descriptorMemory = addressed; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return Recompile(request).spirv; +} + +void verifyCoherentBufferAccesses() { + using namespace ShaderRecompiler; const std::array capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; struct Module { std::map names; @@ -9553,22 +9860,7 @@ void verifyCoherentBufferAccesses() { std::vector code{0x34020082u}; if (invalidate) code.insert(code.end(), {0xe1c40000u, 0u}); code.insert(code.end(), {0xe0301000u | (glcLoad ? 0x4000u : 0u), 0x80000201u, 0xbf8c3f70u, 0xe0701000u | (glcStore ? 0x4000u : 0u), 0x80010201u, 0xbf810000u}); - RecompileRequest request{}; - request.shader = {ShaderStage::Compute, 0x5a000u, code, 0, {}}; - request.context.waveSize = 32; - request.context.userDataBaseRegister = 0; - request.context.userData = userData; - request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; - request.target.vulkanVersion = 0x00401000u; - request.target.spirvVersion = 0x00010300u; - request.target.subgroupSize = 32; - request.target.bdaAbiVersion = BdaAbi::Version; - request.target.supportedCapabilities = capabilities; - request.target.supportedExtensions = extensions; - request.target.fragmentShaderBarycentricEnabled = false; - request.layout.pushConstantSizeBytes = 128; - request.useCache = false; - const auto words = Recompile(request).spirv; + const auto words = recompileCoherent(code, 0x5a000u, capabilities, false); Module module; std::map bases; const auto base = [&](std::uint32_t pointer) { @@ -9628,6 +9920,51 @@ void verifyCoherentBufferAccesses() { require(only(invalidated.loads, "buffers") && only(invalidated.stores, "buffers"), "coherent buffers: a program that invalidates the vector caches does not use the coherent buffer array"); } +void verifyAddressedCoherentBufferAccesses() { + using namespace ShaderRecompiler; + const std::array capabilities{spv::CapabilityInt64, spv::CapabilityInt64Atomics, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}; + struct Counts { + std::uint32_t loads = 0, stores = 0, wideLoads = 0; + }; + const auto compile = [&](bool glcLoad, bool glcStore, bool pair, bool invalidate) { + const std::uint32_t loadOp = pair ? 0xe0341000u : 0xe0301000u; + const std::uint32_t storeOp = pair ? 0xe0741000u : 0xe0701000u; + std::vector code{0x34020082u}; + if (invalidate) code.insert(code.end(), {0xe1c40000u, 0u}); + code.insert(code.end(), {loadOp | (glcLoad ? 0x4000u : 0u), 0x80000201u, 0xbf8c3f70u, storeOp | (glcStore ? 0x4000u : 0u), 0x80010201u, 0xbf810000u}); + const auto words = recompileCoherent(code, 0x5b000u, capabilities, true); + std::set wideTypes; + Counts counts; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto opcode = words[cursor] & 0xffffu; + require(count != 0 && count <= words.size() - cursor, "addressed coherent buffers: truncated SPIR-V instruction"); + if (opcode == spv::OpTypeInt && words[cursor + 2] == 64u) wideTypes.insert(words[cursor + 1]); + if (opcode == spv::OpAtomicLoad) ++(wideTypes.contains(words[cursor + 1]) ? counts.wideLoads : counts.loads); + if (opcode == spv::OpAtomicStore) ++counts.stores; + cursor += count; + } + return counts; + }; + + const auto plain = compile(false, false, false, false); + require(plain.loads == 0u && plain.stores == 0u && plain.wideLoads == 0u, "addressed coherent buffers: accesses without GLC became atomics"); + + const auto single = compile(true, true, false, false); + require(single.loads != 0u, "addressed coherent buffers: a GLC load through a buffer address is not an atomic load"); + require(single.stores != 0u, "addressed coherent buffers: a GLC store through a buffer address is not an atomic store"); + + const auto wide = compile(true, true, true, false); + require(wide.wideLoads != 0u, "addressed coherent buffers: a GLC two-dword load through a buffer address is not one 64-bit atomic load"); + require(wide.stores != 0u, "addressed coherent buffers: a GLC two-dword store through a buffer address is not an atomic store"); + + const auto invalidated = compile(false, true, false, true); + require(invalidated.loads != 0u, "addressed coherent buffers: a load without GLC in a program that invalidates the vector caches is not an atomic load"); + + const auto invalidatedPlain = compile(false, false, false, true); + require(invalidatedPlain.loads == 0u && invalidatedPlain.stores == 0u, "addressed coherent buffers: a program that invalidates the vector caches without GLC or DLC accesses made its accesses atomic"); +} + void verifyUnnormalizedSamplers() { using namespace ShaderRecompiler; constexpr std::uint32_t Format8888UNorm = 56; @@ -11693,6 +12030,16 @@ int main(int argc, char** argv) { std::cout << "Image table keys, maps, records and registration passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--device-image-tables") { + verifyDeviceImageTables(); + std::cout << "GPU-mode stages read V# table columns through the device image table, whose addresses they take from the map trailer\n"; + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--table-trailer") { + verifyTableTrailer(); + std::cout << "The heap map's device table trailer sits between the records and the element base\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--registered-view-churn") { verifyRegisteredViewsUnderUnrelatedChurn(); std::cout << "Registered image views stay current while unrelated allocations change\n"; @@ -11802,6 +12149,7 @@ int main(int argc, char** argv) { verifyFusedExactArithmetic(); verifyNarrowVertexWave(); verifySingleLaneVertexWave(); + verifyExtensionKey(); verifySingleLaneElectedAtomicMax(); verifySingleLaneDomainWave(); verifyMeshConfiguration(); @@ -11823,6 +12171,7 @@ int main(int argc, char** argv) { verifyInt64AtomicCapabilities(); verifySpecializedBufferAtomicBindings(); verifyCoherentBufferAccesses(); + verifyAddressedCoherentBufferAccesses(); verifyUnnormalizedSamplers(); verifyUnusedUnnormalizedSampler(); verifyNullVolumeUnnormalizedSampler(); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderReplayRequests.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderReplayRequests.cpp index e34aeeb6faf9..3293ff101a72 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderReplayRequests.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderReplayRequests.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include int main(int argc, char** argv) try { @@ -75,6 +76,27 @@ int main(int argc, char** argv) try { file << RequestSerializer{}.Serialize(lds); if (!file) throw std::runtime_error("cannot write synthetic replay request"); } + for (const auto& [name, dwords] : {std::pair{"heap-addressed.req", 0u}, std::pair{"heap-4096.req", 4096u}}) { + auto heap = lds; + heap.target.gpuResolvedImages = true; + heap.target.descriptorMemory = true; + heap.target.heapDataUniformDwords = dwords; + std::ofstream file(directory / name, std::ios::binary); + file << RequestSerializer{}.Serialize(heap); + if (!file) throw std::runtime_error("cannot write synthetic replay request"); + } + srt[20] = 0u; + const std::array dataCapabilities{spv::CapabilityImageGatherExtended, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilitySampledImageArrayDynamicIndexing, spv::CapabilityStorageImageArrayDynamicIndexing}; + for (const auto& [name, dwords] : {std::pair{"heap-data-addressed.req", 0u}, std::pair{"heap-data-4096.req", 4096u}}) { + auto heap = request; + heap.target.supportedCapabilities = dataCapabilities; + heap.target.gpuResolvedImages = true; + heap.target.descriptorMemory = true; + heap.target.heapDataUniformDwords = dwords; + std::ofstream file(directory / name, std::ios::binary); + file << RequestSerializer{}.Serialize(heap); + if (!file) throw std::runtime_error("cannot write synthetic replay request"); + } std::puts("synthetic guarded replay requests written"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderReplayTool.cmake b/core/libs/prx/libSceAgcDriver/tests/ShaderReplayTool.cmake index 528f42d8e28f..5316a1f81fc2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderReplayTool.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderReplayTool.cmake @@ -42,3 +42,70 @@ foreach(mode IN ITEMS --prepare --materialize) message(FATAL_ERROR "${mode} ignored --wave-lds-memory-barriers\n${baseline}") endif() endforeach() + +function(expect_heap_profile description profile) + execute_process(COMMAND "${TOOL}" ${ARGN} RESULT_VARIABLE code OUTPUT_VARIABLE output ERROR_VARIABLE errors) + if(NOT code STREQUAL "0" OR NOT output MATCHES "heap profile ${profile}\n") + message(FATAL_ERROR "${description}: expected heap profile ${profile}: ${code}\n${output}${errors}") + endif() +endfunction() + +expect_heap_profile("captured heap off" "\\(captured\\): no heap, gpuResolvedImages 0, descriptorMemory 0, data uniform dwords 0" "${CAPTURE}/lds.req") +expect_heap_profile("captured heap with limit 0" "\\(captured\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 0" "${CAPTURE}/heap-addressed.req") +expect_heap_profile("captured heap with limit 4096" "\\(captured\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 4096" "${CAPTURE}/heap-4096.req") +expect_heap_profile("--gpu-heap on a heap-off capture" "\\(override\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 16384" --gpu-heap "${CAPTURE}/lds.req") +expect_heap_profile("--gpu-heap on a limit 0 capture" "\\(override\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 16384" --gpu-heap "${CAPTURE}/heap-addressed.req") +expect_heap_profile("--gpu-heap with a limit" "\\(override\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 8192" --gpu-heap --heap-data-uniform-dwords 8192 "${CAPTURE}/lds.req") +expect_heap_profile("--no-gpu-heap on a heap capture" "\\(override\\): no heap, gpuResolvedImages 0, descriptorMemory 0, data uniform dwords 0" --no-gpu-heap "${CAPTURE}/heap-4096.req") +expect_heap_profile("--heap-data-uniform-dwords alone keeps the captured heap" "\\(override\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 0" --heap-data-uniform-dwords 0 "${CAPTURE}/heap-4096.req") + +execute_process(COMMAND "${TOOL}" --heap-data-uniform-dwords x "${CAPTURE}/lds.req" RESULT_VARIABLE code OUTPUT_VARIABLE output ERROR_VARIABLE errors) +if(NOT code STREQUAL "2") + message(FATAL_ERROR "a malformed --heap-data-uniform-dwords value was accepted: ${code}\n${output}${errors}") +endif() + +function(expect_usage_error description message) + execute_process(COMMAND "${TOOL}" ${ARGN} RESULT_VARIABLE code OUTPUT_VARIABLE output ERROR_VARIABLE errors) + if(NOT code STREQUAL "2" OR NOT errors MATCHES "${message}") + message(FATAL_ERROR "${description}: expected exit 2 with '${message}': ${code}\n${output}${errors}") + endif() +endfunction() + +expect_usage_error("--gpu-heap then --no-gpu-heap" "--gpu-heap and --no-gpu-heap conflict" --gpu-heap --no-gpu-heap "${CAPTURE}/lds.req") +expect_usage_error("--no-gpu-heap then --gpu-heap" "--gpu-heap and --no-gpu-heap conflict" --no-gpu-heap --gpu-heap "${CAPTURE}/lds.req") +expect_usage_error("--no-gpu-heap with a limit" "--no-gpu-heap conflicts with --heap-data-uniform-dwords 4096" --no-gpu-heap --heap-data-uniform-dwords 4096 "${CAPTURE}/lds.req") +expect_usage_error("a limit then --no-gpu-heap" "--no-gpu-heap conflicts with --heap-data-uniform-dwords 4096" --heap-data-uniform-dwords 4096 --no-gpu-heap "${CAPTURE}/lds.req") +expect_usage_error("a missing limit" "needs an unsigned 32-bit number" --heap-data-uniform-dwords) +expect_usage_error("an overflowing limit" "needs an unsigned 32-bit number" --heap-data-uniform-dwords 99999999999 "${CAPTURE}/lds.req") +expect_usage_error("a negative limit" "needs an unsigned 32-bit number" --heap-data-uniform-dwords -1 "${CAPTURE}/lds.req") +expect_heap_profile("--no-gpu-heap with limit 0" "\\(override\\): no heap, gpuResolvedImages 0, descriptorMemory 0, data uniform dwords 0" --no-gpu-heap --heap-data-uniform-dwords 0 "${CAPTURE}/heap-4096.req") +expect_heap_profile("--gpu-heap replaces a captured limit" "\\(override\\): heap, gpuResolvedImages 1, descriptorMemory 1, data uniform dwords 16384" --gpu-heap "${CAPTURE}/heap-4096.req") + +execute_process(COMMAND "${TOOL}" --robust2 --lds-lock-fences "${CAPTURE}/lds.req" RESULT_VARIABLE code OUTPUT_VARIABLE output ERROR_VARIABLE errors) +if(NOT code STREQUAL "0" OR NOT output MATCHES "target flags forced on: robust2, lds-lock-fences\n" OR NOT output MATCHES "heap profile \\(captured\\)") + message(FATAL_ERROR "forced target flags were not reported next to the captured heap profile: ${code}\n${output}${errors}") +endif() + +function(replay_spv result) + execute_process(COMMAND "${TOOL}" --spv ${ARGN} RESULT_VARIABLE code OUTPUT_VARIABLE output ERROR_VARIABLE errors) + if(NOT code STREQUAL "0" OR NOT output MATCHES "recompiled: [0-9]+ SPIR-V words") + message(FATAL_ERROR "--spv ${ARGN} failed: ${code}\n${output}${errors}") + endif() + string(FIND "${output}" "\n" firstLine) + math(EXPR firstLine "${firstLine} + 1") + string(SUBSTRING "${output}" ${firstLine} -1 output) + string(REGEX REPLACE "[^\n]*heap profile[^\n]*\n" "" output "${output}") + set(${result} "${output}" PARENT_SCOPE) +endfunction() + +replay_spv(addressed "${CAPTURE}/heap-data-addressed.req") +replay_spv(uniform "${CAPTURE}/heap-data-4096.req") +replay_spv(addressedAgain --heap-data-uniform-dwords 0 "${CAPTURE}/heap-data-addressed.req") +replay_spv(uniformOverride --heap-data-uniform-dwords 4096 "${CAPTURE}/heap-data-addressed.req") +replay_spv(addressedOverride --heap-data-uniform-dwords 0 "${CAPTURE}/heap-data-4096.req") +if(addressed STREQUAL uniform) + message(FATAL_ERROR "the captured limits 0 and 4096 replayed to the same SPIR-V\n${addressed}") +endif() +if(NOT addressed STREQUAL addressedAgain OR NOT uniform STREQUAL uniformOverride OR NOT addressed STREQUAL addressedOverride) + message(FATAL_ERROR "--heap-data-uniform-dwords did not reproduce the captured limit's SPIR-V\n${addressed}\n${addressedAgain}\n${uniform}\n${uniformOverride}\n${addressedOverride}") +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_device_image_keys_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_device_image_keys_tests.cmake new file mode 100644 index 000000000000..df7d3014bed8 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_device_image_keys_tests.cmake @@ -0,0 +1,9 @@ +add_test_executable(agc_device_image_keys_tests EXCLUDE_FROM_ALL tests/DeviceImageKeys.cpp Graphics/src/DeviceImageKeys.cpp) +target_include_directories(agc_device_image_keys_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler) +target_link_libraries(agc_device_image_keys_tests PRIVATE libc) +set_target_properties(agc_device_image_keys_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_device_image_keys_tests) + +if(BUILD_TESTING) + agc_add_default_test(agc_device_image_keys_tests) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_draw_space_reuse_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_draw_space_reuse_tests.cmake index bcf7c0bf2ed6..fcfb73e7b6a2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_draw_space_reuse_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_draw_space_reuse_tests.cmake @@ -23,4 +23,10 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_address_draw_space_reuse_held PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + agc_add_env_test(agc_driver_address_draw_space_reuse_heap agc_driver_address_draw_space_reuse_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 TIMEOUT 60 NO_WORKING_DIRECTORY) + agc_add_env_test(agc_driver_address_draw_space_reuse_heap_stages agc_driver_address_draw_space_reuse_tests + ARGS --heap-stages ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 TIMEOUT 60 NO_WORKING_DIRECTORY) + agc_add_env_test(agc_driver_address_draw_space_reuse_heap_transitions agc_driver_address_draw_space_reuse_tests + ARGS --transitions ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_DRAW_TRANSITIONS=1 TIMEOUT 60 NO_WORKING_DIRECTORY) endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_passes_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_passes_tests.cmake index 9208eaf2d256..c8e637395a5f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_passes_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_address_passes_tests.cmake @@ -12,3 +12,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_address_passes) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bda_span_reads_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bda_span_reads_tests.cmake index a04e3566680a..76823a0a095d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bda_span_reads_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bda_span_reads_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_bda_span_reads) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_bda_span_reads) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bindless_images_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bindless_images_tests.cmake new file mode 100644 index 000000000000..3892cefc3235 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_bindless_images_tests.cmake @@ -0,0 +1,7 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_bindless_images) +endif() +if(BUILD_TESTING AND TARGET agc_driver_bindless_images_tests) + agc_add_env_test(agc_driver_bindless_images_heap agc_driver_bindless_images_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 NO_WORKING_DIRECTORY) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_atomics_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_atomics_tests.cmake new file mode 100644 index 000000000000..bd2fde5c5e7d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_atomics_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_buffer_atomics) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_format_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_format_tests.cmake new file mode 100644 index 000000000000..d73e107870ad --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_format_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_buffer_format) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_load_store_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_load_store_tests.cmake index 6657a149501c..59e092a61eb2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_load_store_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_buffer_load_store_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_buffer_load_store) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_buffer_load_store) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_compute_table_calls_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_compute_table_calls_tests.cmake new file mode 100644 index 000000000000..3a5f7014a351 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_compute_table_calls_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_compute_table_calls) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_copied_writer_copy_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_copied_writer_copy_tests.cmake index 86b69033ad36..fac8a17c07e5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_copied_writer_copy_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_copied_writer_copy_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_copied_writer_copy) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_copied_writer_copy) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_dispatch_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_dispatch_tests.cmake new file mode 100644 index 000000000000..85967a92e666 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_dispatch_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_depth_storage_dispatch) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_reuse_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_reuse_tests.cmake index ae4b99b882dc..06ed98cdd187 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_reuse_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_depth_storage_reuse_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_depth_storage_reuse) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_depth_storage_reuse) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_path_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_path_tests.cmake new file mode 100644 index 000000000000..76980dd8b583 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_path_tests.cmake @@ -0,0 +1,9 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_device_image_path_gpu_path COMMAND agc_driver_device_image_path_tests) + set_tests_properties(agc_driver_device_image_path_gpu_path PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_HEAP=1;APS5_GPU_PATH=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + add_test(NAME agc_driver_device_image_path_without_heap COMMAND agc_driver_device_image_path_tests) + set_tests_properties(agc_driver_device_image_path_without_heap PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_PATH=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + if(WIN32) + set_tests_properties(agc_driver_device_image_path_gpu_path agc_driver_device_image_path_without_heap PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_table_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_table_tests.cmake new file mode 100644 index 000000000000..9bbd9ba1eafd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_device_image_table_tests.cmake @@ -0,0 +1,7 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_device_image_table_heap COMMAND agc_driver_device_image_table_tests --heap) + set_tests_properties(agc_driver_device_image_table_heap PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_HEAP=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + if(WIN32) + set_tests_properties(agc_driver_device_image_table_heap PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_dispatch_move_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_dispatch_move_tests.cmake new file mode 100644 index 000000000000..401ae774dfbe --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_dispatch_move_tests.cmake @@ -0,0 +1,4 @@ +if(BUILD_TESTING AND TARGET agc_driver_dispatch_move_tests) + agc_add_env_test(agc_driver_dispatch_move_gpu_heap agc_driver_dispatch_move_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_relocation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_relocation_tests.cmake index 510aeafff229..d146ab39b559 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_relocation_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_relocation_tests.cmake @@ -14,3 +14,7 @@ if(BUILD_TESTING) endif() agc_add_draw_pipeline_test(agc_driver_draw_cache_relocation) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_draw_cache_relocation) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_reuse_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_reuse_tests.cmake index 835d1aa0cf25..c82d3ec00171 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_reuse_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_cache_reuse_tests.cmake @@ -26,3 +26,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_draw_cache_reuse) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_data_hit_user_words_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_data_hit_user_words_tests.cmake new file mode 100644 index 000000000000..959f90b10a4a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_data_hit_user_words_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_draw_data_hit_user_words) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_offset_indices_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_offset_indices_tests.cmake index da26ba76dbc0..554bfae39456 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_offset_indices_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_offset_indices_tests.cmake @@ -16,4 +16,12 @@ if(BUILD_TESTING) endif() endforeach() endforeach() + add_test(NAME agc_driver_draw_heap_attributes_compact_restart COMMAND agc_driver_draw_offset_indices_tests heap-attributes) + set_tests_properties(agc_driver_draw_heap_attributes_compact_restart PROPERTIES + ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" TIMEOUT 45 SKIP_RETURN_CODE 77 + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_draw_heap_attributes_compact_restart PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_relocate_user_values_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_relocate_user_values_tests.cmake new file mode 100644 index 000000000000..a7861b87515a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_relocate_user_values_tests.cmake @@ -0,0 +1,4 @@ +if(BUILD_TESTING) + agc_add_env_test(agc_driver_draw_relocate_user_values_gpu_heap agc_driver_draw_relocate_user_values_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_VERIFY_DRAW_ENTRIES=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_unsupported_buffer_type_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_unsupported_buffer_type_tests.cmake index 807d44f0c0f1..a641a28029e0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_unsupported_buffer_type_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_unsupported_buffer_type_tests.cmake @@ -8,3 +8,7 @@ if(BUILD_TESTING) endif() agc_add_draw_pipeline_test(agc_driver_draw_unsupported_buffer_type) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_draw_unsupported_buffer_type) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_write_ranges_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_write_ranges_tests.cmake index 230d88b1669c..5eb2f7eb1d70 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_write_ranges_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_draw_write_ranges_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_draw_write_ranges) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_draw_write_ranges) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_finisher_label_order_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_finisher_label_order_tests.cmake index 889a5ae09ccc..f0bcb3e2d923 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_finisher_label_order_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_finisher_label_order_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_finisher_label_order) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_finisher_label_order) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_fragment_user_data_tail_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_fragment_user_data_tail_tests.cmake new file mode 100644 index 000000000000..6b01e6f504e5 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_fragment_user_data_tail_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_fragment_user_data_tail) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gated_dispatch_capture_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gated_dispatch_capture_tests.cmake index d933f3070ddd..2a50895389de 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gated_dispatch_capture_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gated_dispatch_capture_tests.cmake @@ -9,3 +9,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_gated_dispatch_capture) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_misaligned_writer_ahead_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_misaligned_writer_ahead_tests.cmake new file mode 100644 index 000000000000..79bc85e76449 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_misaligned_writer_ahead_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_gpu_indirect_arguments_misaligned_writer_ahead) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_written_ahead_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_written_ahead_tests.cmake new file mode 100644 index 000000000000..ab633bf75e25 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gpu_indirect_arguments_written_ahead_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_gpu_indirect_arguments_written_ahead) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake index d6285bd4bf3b..99ea10d4061f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_graphics_tests.cmake @@ -94,7 +94,8 @@ if(BUILD_TESTING) add_test(NAME agc_sparse_buffer_holes COMMAND agc_driver_graphics_tests --sparse-buffer-holes) add_test(NAME agc_copy_input_kept_bytes COMMAND agc_driver_graphics_tests --copy-input-kept-bytes) add_test(NAME agc_depth_copy_kept_bytes COMMAND agc_driver_graphics_tests --depth-copy-kept-bytes) - set_tests_properties(agc_copy_input_kept_bytes agc_depth_copy_kept_bytes PROPERTIES + add_test(NAME agc_pooled_buffer_address COMMAND agc_driver_graphics_tests --pooled-buffer-address) + set_tests_properties(agc_copy_input_kept_bytes agc_depth_copy_kept_bytes agc_pooled_buffer_address PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" ENVIRONMENT_MODIFICATION "APS5_CPU_COPIES=unset:;APS5_GPU_COPY_MAX_KIB=unset:;APS5_CAPTURE_GPU_COPIES=unset:;APS5_BUFFER_POOL_SLOTS=unset:;APS5_BUFFER_POOL_SHARED=unset:;APS5_NO_BUFFER_CLASSES=unset:;APS5_STAGING_POOL_MIB=unset:") if(WIN32) @@ -102,6 +103,8 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") set_property(TEST agc_depth_copy_kept_bytes APPEND PROPERTY ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_property(TEST agc_pooled_buffer_address APPEND PROPERTY ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") endif() set_tests_properties(agc_sparse_buffer_holes PROPERTIES TIMEOUT 30) add_test(NAME agc_pipeline_journal COMMAND agc_driver_graphics_tests --pipeline-journal) diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_compute_sample_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_compute_sample_tests.cmake new file mode 100644 index 000000000000..a9d4ef8d3cbb --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_compute_sample_tests.cmake @@ -0,0 +1,8 @@ +if(BUILD_TESTING AND TARGET agc_driver_heap_compute_sample_tests) + agc_add_env_test(agc_driver_heap_compute_sample_heap_pipelined agc_driver_heap_compute_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_PIPELINE_DISPATCH=1 APS5_PIPELINED_DRAWS=32) + agc_add_env_test(agc_driver_heap_compute_sample_heap agc_driver_heap_compute_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1) + agc_add_env_test(agc_driver_heap_compute_sample_full_heap agc_driver_heap_compute_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_HEAP_SLOT_CAP=4) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_draw_sample_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_draw_sample_tests.cmake new file mode 100644 index 000000000000..c468b0a1252b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_draw_sample_tests.cmake @@ -0,0 +1,12 @@ +if(BUILD_TESTING AND TARGET agc_driver_heap_draw_sample_tests) + agc_add_env_test(agc_driver_heap_draw_sample_heap agc_driver_heap_draw_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1) + agc_add_env_test(agc_driver_heap_draw_sample_full_heap agc_driver_heap_draw_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_HEAP_SLOT_CAP=1) + agc_add_env_test(agc_driver_heap_draw_sample_full_heap_pipelined agc_driver_heap_draw_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_HEAP_SLOT_CAP=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1) + agc_add_env_test(agc_driver_heap_draw_sample_nogpl agc_driver_heap_draw_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_NO_GPL=1) + agc_add_env_test(agc_driver_heap_draw_sample_nogpl_heap agc_driver_heap_draw_sample_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_NO_GPL=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_slots_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_slots_tests.cmake new file mode 100644 index 000000000000..8b9e7cfee37b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_slots_tests.cmake @@ -0,0 +1,32 @@ +add_test_executable(agc_driver_heap_slots_tests EXCLUDE_FROM_ALL + tests/BindlessHeapSlots.cpp +) + +target_include_directories(agc_driver_heap_slots_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR} +) + +target_link_libraries(agc_driver_heap_slots_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_heap_slots_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_heap_slots_tests) + +if(BUILD_TESTING) + agc_add_default_test(agc_driver_heap_slots_tests) + agc_add_env_test(agc_driver_heap_slots_ring_large_spans agc_driver_heap_slots_tests + ARGS ring-large-spans ENV ANYPS5_NO_SHADER_CACHE=1) + agc_add_env_test(agc_driver_heap_slots_ring_busy_chunks agc_driver_heap_slots_tests + ARGS ring-busy-chunks ENV ANYPS5_NO_SHADER_CACHE=1 TIMEOUT 60) + foreach(agcHeapSlotsMode IN ITEMS comes-back storage-comes-back storage-views-come-back mixed-comes-back views-come-back two-heaps-come-back owner-views-release shared-owner-release idle-slots-evicted) + string(REPLACE "-" "_" agcHeapSlotsName "${agcHeapSlotsMode}") + agc_add_env_test(agc_driver_heap_slots_${agcHeapSlotsName} agc_driver_heap_slots_tests + ARGS ${agcHeapSlotsMode} ENV ANYPS5_NO_SHADER_CACHE=1) + endforeach() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_walk_memo_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_walk_memo_tests.cmake new file mode 100644 index 000000000000..56ccd917a0c6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_heap_walk_memo_tests.cmake @@ -0,0 +1,24 @@ +add_test_executable(agc_driver_heap_walk_memo_tests EXCLUDE_FROM_ALL + tests/HeapWalkMemo.cpp +) + +target_include_directories(agc_driver_heap_walk_memo_tests PRIVATE + ${LIBS_INCLUDE_DIR} + $ + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR} +) + +target_link_libraries(agc_driver_heap_walk_memo_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_heap_walk_memo_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_heap_walk_memo_tests) + +if(BUILD_TESTING) + agc_add_default_test(agc_driver_heap_walk_memo_tests) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics64_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics64_tests.cmake new file mode 100644 index 000000000000..1906060b5d14 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics64_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_atomics64) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_bitwise_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_bitwise_tests.cmake new file mode 100644 index 000000000000..ffa07becb676 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_bitwise_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_atomics_bitwise) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_float_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_float_tests.cmake new file mode 100644 index 000000000000..7e2d6ef5b4fd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_float_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_atomics_float) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_tests.cmake new file mode 100644 index 000000000000..4d1590c46853 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_atomics_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_atomics) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_cache_bits_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_cache_bits_tests.cmake new file mode 100644 index 000000000000..e1ad2f4b0be2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_cache_bits_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_cache_bits) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_converted_unorm_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_converted_unorm_tests.cmake index fa9a73bb7c25..e214c6fabc3b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_converted_unorm_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_converted_unorm_tests.cmake @@ -10,3 +10,7 @@ if(BUILD_TESTING) endif() endforeach() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_converted_unorm) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_gather_one_dimensional_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_gather_one_dimensional_tests.cmake new file mode 100644 index 000000000000..70324af9eab4 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_gather_one_dimensional_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_gather_one_dimensional) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_load_packed_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_load_packed_tests.cmake new file mode 100644 index 000000000000..1a1d9b83a008 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_load_packed_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_load_packed) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_border_color_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_border_color_tests.cmake new file mode 100644 index 000000000000..6adc925a198f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_border_color_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_sample_border_color) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_derivatives_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_derivatives_tests.cmake new file mode 100644 index 000000000000..6e94552f53e9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_derivatives_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_sample_derivatives) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_lod_clamp_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_lod_clamp_tests.cmake index 4cf0a2069597..9a6913007160 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_lod_clamp_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_lod_clamp_tests.cmake @@ -1,4 +1,5 @@ if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_sample_lod_clamp) add_test(NAME agc_driver_image_sample_lod_clamp_shader_lane COMMAND agc_driver_image_sample_lod_clamp_tests --shader-lane) set_tests_properties(agc_driver_image_sample_lod_clamp_shader_lane PROPERTIES ENVIRONMENT "APS5_NO_IMAGE_VIEW_MIN_LOD=1;APS5_NO_GPL=1;ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 60) if(WIN32) diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_shared_sampler_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_shared_sampler_tests.cmake new file mode 100644 index 000000000000..15964a4cfc42 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sample_shared_sampler_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_sample_shared_sampler) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sint8_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sint8_tests.cmake new file mode 100644 index 000000000000..c6412e790609 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_sint8_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_sint8) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_srgb_load_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_srgb_load_tests.cmake new file mode 100644 index 000000000000..6e2bbe454302 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_srgb_load_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_srgb_load) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_array_through_volume_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_array_through_volume_tests.cmake new file mode 100644 index 000000000000..8d54dc08f592 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_array_through_volume_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_array_through_volume) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_face_on_cube_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_face_on_cube_tests.cmake new file mode 100644 index 000000000000..0ab9fe54fdbc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_face_on_cube_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_face_on_cube) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_float_rounding_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_float_rounding_tests.cmake new file mode 100644 index 000000000000..3d16cb85557c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_float_rounding_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_float_rounding) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_array_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_array_tests.cmake new file mode 100644 index 000000000000..342b19006c96 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_array_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_line_through_plane_array) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_tests.cmake new file mode 100644 index 000000000000..3fab0a37b350 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_line_through_plane_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_line_through_plane) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_packed_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_packed_tests.cmake new file mode 100644 index 000000000000..942cc404cb0b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_packed_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_packed) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_sint_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_sint_tests.cmake new file mode 100644 index 000000000000..6179aa7e5991 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_sint_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_sint) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_unorm_rounding_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_unorm_rounding_tests.cmake new file mode 100644 index 000000000000..a72eed6ae44f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_unorm_rounding_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_unorm_rounding) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_volume_through_array_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_volume_through_array_tests.cmake new file mode 100644 index 000000000000..f45dde563ae9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_store_volume_through_array_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_store_volume_through_array) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_cache_bounds_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_cache_bounds_tests.cmake new file mode 100644 index 000000000000..a3bf9bfb8051 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_cache_bounds_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_table_cache_bounds) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_reuse_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_reuse_tests.cmake index 725d651511cc..e8dbefae5b50 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_reuse_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_table_reuse_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_image_table_reuse) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_table_reuse) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_texel_status_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_texel_status_tests.cmake new file mode 100644 index 000000000000..dd91b04aa2b4 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_image_texel_status_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_image_texel_status) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_arguments_written_ahead_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_arguments_written_ahead_tests.cmake new file mode 100644 index 000000000000..8c80cbfe825e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_arguments_written_ahead_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_indirect_arguments_written_ahead) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_patched_stage_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_patched_stage_tests.cmake new file mode 100644 index 000000000000..fdc74f343c1f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_patched_stage_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_indirect_patched_stage) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_thread_dimensions_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_thread_dimensions_tests.cmake index e1e265cfd939..9173a6930ae8 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_thread_dimensions_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_indirect_thread_dimensions_tests.cmake @@ -6,3 +6,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_indirect_thread_dimensions) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_lds_beyond_device_limit_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_lds_beyond_device_limit_tests.cmake new file mode 100644 index 000000000000..441dfc8499c6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_lds_beyond_device_limit_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_lds_beyond_device_limit) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_draw_relocation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_draw_relocation_tests.cmake new file mode 100644 index 000000000000..1168df7bb5ae --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_draw_relocation_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_mesh_draw_relocation) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_index_cache_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_index_cache_tests.cmake new file mode 100644 index 000000000000..5db5e725cd88 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_index_cache_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_mesh_index_cache) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_provoking_vertex_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_provoking_vertex_tests.cmake new file mode 100644 index 000000000000..dc8ecfc0f1a3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_provoking_vertex_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_mesh_provoking_vertex) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_tests.cmake index 2f26ed8c15dd..d26f10a7ef5a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_mesh_tests.cmake @@ -12,3 +12,7 @@ configure_windows_unwind(agc_driver_mesh_tests) if(BUILD_TESTING) agc_add_default_test(agc_driver_mesh_tests) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_mesh) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipeline_journal_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipeline_journal_tests.cmake index d44ec5a784f4..b4adc276eb74 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipeline_journal_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipeline_journal_tests.cmake @@ -1,2 +1,6 @@ target_include_directories(agc_driver_pipeline_journal_tests PRIVATE $) target_link_libraries(agc_driver_pipeline_journal_tests PRIVATE ${CMAKE_DL_LIBS}) +if(BUILD_TESTING AND TARGET agc_driver_pipeline_journal_tests) + agc_add_env_test(agc_driver_pipeline_journal_gpu_heap agc_driver_pipeline_journal_tests + ARGS --gpu-heap ENV APS5_GPU_HEAP=1 TIMEOUT 30) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_dma_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_dma_tests.cmake index 63a078aa117a..6b60fe164558 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_dma_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_dma_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_pipelined_dma) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pipelined_dma) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_fill_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_fill_tests.cmake index 38a274318427..42f881e669ff 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_fill_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_fill_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_pipelined_fill) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pipelined_fill) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_metadata_pass_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_metadata_pass_tests.cmake index b4df42c2ced8..716726765017 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_metadata_pass_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_metadata_pass_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_pipelined_metadata_pass) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pipelined_metadata_pass) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predicated_label_order_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predicated_label_order_tests.cmake index f80d0d69b445..74273173b184 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predicated_label_order_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predicated_label_order_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_pipelined_predicated_label_order) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pipelined_predicated_label_order) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predication_read_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predication_read_tests.cmake index 59f85e5bf0b1..2b8b222f1bbe 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predication_read_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pipelined_predication_read_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_pipelined_predication_read) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pipelined_predication_read) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pixel_barycentric_emulation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pixel_barycentric_emulation_tests.cmake new file mode 100644 index 000000000000..4912dbdf62ba --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_pixel_barycentric_emulation_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_pixel_barycentric_emulation) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_prepared_graphics_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_prepared_graphics_tests.cmake index 47e76a68790b..1ae6f19fa991 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_prepared_graphics_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_prepared_graphics_tests.cmake @@ -17,3 +17,7 @@ if(BUILD_TESTING) set_tests_properties(${prebuildPolicyTest} PROPERTIES ENVIRONMENT_MODIFICATION "${prebuildPolicyEnvironment}") endforeach() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_prepared_graphics) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_presentation_pass_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_presentation_pass_tests.cmake index 3ce36f91fbe3..bf65e0d9043f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_presentation_pass_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_presentation_pass_tests.cmake @@ -6,6 +6,7 @@ target_include_directories(agc_driver_presentation_pass_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR} ) target_link_libraries(agc_driver_presentation_pass_tests PRIVATE @@ -21,8 +22,10 @@ if(BUILD_TESTING) set_target_properties(agc_driver_presentation_pass_tests PROPERTIES EXCLUDE_FROM_ALL OFF) add_test(NAME agc_driver_presentation_pass COMMAND agc_driver_presentation_pass_tests) set_tests_properties(agc_driver_presentation_pass PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_driver_presentation_pass_gpu_heap COMMAND agc_driver_presentation_pass_tests) + set_tests_properties(agc_driver_presentation_pass_gpu_heap PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_GPU_HEAP=1;APS5_VERIFY_GPU_HEAP=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_driver_presentation_pass PROPERTIES ENVIRONMENT_MODIFICATION + set_tests_properties(agc_driver_presentation_pass agc_driver_presentation_pass_gpu_heap PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_provoking_vertex_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_provoking_vertex_tests.cmake index 023e8c55df76..c3e9092bdafb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_provoking_vertex_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_provoking_vertex_tests.cmake @@ -19,3 +19,7 @@ target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestSh if(BUILD_TESTING) set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_provoking_vertex) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_data_placements_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_data_placements_tests.cmake new file mode 100644 index 000000000000..d21b0671b563 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_data_placements_tests.cmake @@ -0,0 +1,23 @@ +add_test_executable(agc_driver_push_data_placements_tests EXCLUDE_FROM_ALL + tests/PushDataPlacements.cpp +) + +target_include_directories(agc_driver_push_data_placements_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR} + ${ANYPS5_SPIRV_HEADERS_INCLUDE_DIR} +) + +target_link_libraries(agc_driver_push_data_placements_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_push_data_placements_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_push_data_placements_tests) + +if(BUILD_TESTING) + agc_add_default_test(agc_driver_push_data_placements_tests) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_only_heap_passes_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_only_heap_passes_tests.cmake new file mode 100644 index 000000000000..df3f856331fe --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_push_only_heap_passes_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_push_only_heap_passes) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorded_draw_staging_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorded_draw_staging_tests.cmake new file mode 100644 index 000000000000..8ae69dded719 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorded_draw_staging_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_recorded_draw_staging) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake index 29321dcfbe7e..9c4196c745ad 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_recorder_tests.cmake @@ -182,6 +182,11 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() add_test(NAME agc_storage_kept_unit_settle COMMAND agc_driver_recorder_tests --kept-unit-settle-only) + add_test(NAME agc_storage_refresh_proofs COMMAND agc_driver_recorder_tests --refresh-proofs-only) + set_tests_properties(agc_storage_refresh_proofs PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60) + if(WIN32) + set_tests_properties(agc_storage_refresh_proofs PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() add_test(NAME agc_storage_idle_writeback_race COMMAND agc_driver_recorder_tests --idle-writeback-race-only) set_tests_properties(agc_storage_idle_writeback_race PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 60) set_tests_properties(agc_storage_kept_unit_settle PROPERTIES SKIP_RETURN_CODE 77 TIMEOUT 30) diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_rect_list_indirect_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_rect_list_indirect_tests.cmake new file mode 100644 index 000000000000..1867b03a37e7 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_rect_list_indirect_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_rect_list_indirect) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_same_queue_label_wait_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_same_queue_label_wait_tests.cmake index 2740681a9589..249fa785b3a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_same_queue_label_wait_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_same_queue_label_wait_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_same_queue_label_wait) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_same_queue_label_wait) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sample_multisample_target_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sample_multisample_target_tests.cmake index bf0f20fa31b2..d602d1e40d2c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sample_multisample_target_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sample_multisample_target_tests.cmake @@ -14,3 +14,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_sample_multisample_target) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sampled_cache_budget_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sampled_cache_budget_tests.cmake index b5027b671fc0..70823848a447 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sampled_cache_budget_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_sampled_cache_budget_tests.cmake @@ -1,6 +1,8 @@ if(BUILD_TESTING) add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory) set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) + agc_add_env_test(agc_driver_sampled_cache_budget_heap_view agc_driver_sampled_cache_budget_tests + ARGS heap-view ENV ANYPS5_NO_SHADER_CACHE=1 APS5_NO_IMAGE_SLABS=1 APS5_RELEASE_UNDER_LOCK=1) add_test(NAME agc_driver_sampled_cache_budget_without_storage_image_multisample COMMAND agc_driver_sampled_cache_budget_tests without-storage-image-multisample) set_tests_properties(agc_driver_sampled_cache_budget_without_storage_image_multisample PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 TIMEOUT 30) add_test(NAME agc_sampled_cache_device_selection COMMAND agc_driver_sampled_cache_budget_tests device-selection-cpu) @@ -40,3 +42,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_sampled_cache_budget) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_scratch_pool_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_scratch_pool_tests.cmake new file mode 100644 index 000000000000..73356a9c5815 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_scratch_pool_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_scratch_pool) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_staged_clear_sampled_read_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_staged_clear_sampled_read_tests.cmake index f982191576a9..f37fcc32f203 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_staged_clear_sampled_read_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_staged_clear_sampled_read_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_staged_clear_sampled_read) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_staged_clear_sampled_read) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_cache_budget_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_cache_budget_tests.cmake index 16f71c2e6d0c..a8eec88bbfdf 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_cache_budget_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_cache_budget_tests.cmake @@ -14,4 +14,8 @@ if(BUILD_TESTING) set_tests_properties(agc_driver_storage_cache_retained_batch PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + agc_add_env_test(agc_driver_storage_cache_budget_heap_slots agc_driver_storage_cache_budget_tests + ARGS --heap-slots ENV ANYPS5_NO_SHADER_CACHE=1 APS5_NO_IMAGE_SLABS=1 TIMEOUT 30) + agc_add_env_test(agc_driver_storage_cache_retained_batch_heap_slots agc_driver_storage_cache_budget_tests + ARGS --retained-batch-heap-slots ENV ANYPS5_NO_SHADER_CACHE=1 TIMEOUT 30) endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_flush_ordering_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_flush_ordering_tests.cmake new file mode 100644 index 000000000000..744af26a1091 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_flush_ordering_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_storage_flush_ordering) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_one_slice_alias_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_one_slice_alias_tests.cmake new file mode 100644 index 000000000000..ae27a3bdfeef --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_one_slice_alias_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_storage_one_slice_alias) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_write_back_overhang_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_write_back_overhang_tests.cmake index 688fd4820eab..862745ecb13b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_write_back_overhang_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_storage_write_back_overhang_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_storage_write_back_overhang) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_storage_write_back_overhang) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_submit_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_submit_tests.cmake new file mode 100644 index 000000000000..7c5e93e4d111 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_submit_tests.cmake @@ -0,0 +1,6 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_tessellation_submit) + agc_add_env_test(agc_driver_tessellation_submit_geometry agc_driver_tessellation_submit_tests ARGS --geometry) + agc_add_env_test(agc_driver_tessellation_submit_geometry_gpu_heap agc_driver_tessellation_submit_tests ARGS --geometry + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 APS5_DRAW_PIPELINE=1 APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_tests.cmake index f616a8f244b2..8a7f8e593836 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_tessellation_tests.cmake @@ -7,3 +7,7 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_tessellation) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_unimported_label_order_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_unimported_label_order_tests.cmake index fe6fd86b0953..409dbf893501 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_unimported_label_order_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_unimported_label_order_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_unimported_label_order) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_unimported_label_order) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_view_past_last_mip_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_view_past_last_mip_tests.cmake new file mode 100644 index 000000000000..8a1e081e24de --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_view_past_last_mip_tests.cmake @@ -0,0 +1,3 @@ +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_view_past_last_mip) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_write_data_written_ahead_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_write_data_written_ahead_tests.cmake index 46785e7b7461..2eb60cce0d6d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_write_data_written_ahead_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_write_data_written_ahead_tests.cmake @@ -1,3 +1,7 @@ if(BUILD_TESTING) agc_add_draw_pipeline_test(agc_driver_write_data_written_ahead) endif() + +if(BUILD_TESTING) + agc_add_gpu_heap_test(agc_driver_write_data_written_ahead) +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake index c8d961b5e8c6..c411e6aa74fa 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_pipeline_cache_allocation_tests.cmake @@ -1,5 +1,5 @@ add_test_executable(agc_pipeline_cache_allocation_tests EXCLUDE_FROM_ALL tests/PipelineCacheFailures.cpp - Graphics/src/PipelineCache.cpp tests/AllocationFailure.cpp) + Graphics/src/PipelineCache.cpp Graphics/src/DescriptorHeapObjects.cpp tests/AllocationFailure.cpp) target_include_directories(agc_pipeline_cache_allocation_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) target_link_libraries(agc_pipeline_cache_allocation_tests PRIVATE shader_recompiler_agc_driver libSceAgcDriver libc) set_target_properties(agc_pipeline_cache_allocation_tests PROPERTIES CXX_EXTENSIONS OFF) diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake index 1e66e009a013..f92621a2ebf7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_shader_memory_tests.cmake @@ -49,12 +49,24 @@ if(BUILD_TESTING) set_tests_properties(agc_shader_table_snapshot_reuse PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_shader_table_snapshot_reuse_unmasked COMMAND agc_shader_memory_tests --table-snapshot-reuse) set_tests_properties(agc_shader_table_snapshot_reuse_unmasked PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_MASKED_RESULT_MEMO=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_table_trailer COMMAND agc_shader_memory_tests --table-trailer) + add_test(NAME agc_shader_device_image_tables COMMAND agc_shader_memory_tests --device-image-tables) + set_tests_properties(agc_shader_device_image_tables PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_shader_table_trailer PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") add_test(NAME agc_driver_image_table_vouched_lookups COMMAND agc_driver_image_table_cache_bounds_tests --vouched-lookups) set_tests_properties(agc_driver_image_table_vouched_lookups PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_WRITE_WATCH_IMPORTS=watch;APS5_NO_SPANNED_WAVE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + agc_add_env_test(agc_driver_image_table_atomics agc_driver_image_table_cache_bounds_tests + ARGS --atomic-table ENV ANYPS5_NO_SHADER_CACHE=1 TIMEOUT 60) + agc_add_env_test(agc_driver_image_table_atomics_gpu_heap agc_driver_image_table_cache_bounds_tests + ARGS --atomic-table ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 TIMEOUT 60) + agc_add_env_test(agc_driver_image_table_store_gpu_heap agc_driver_image_table_cache_bounds_tests + ARGS --store-table ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_VERIFY_GPU_HEAP=1 TIMEOUT 60) add_test(NAME agc_driver_image_table_vouch_queued_draws COMMAND agc_driver_image_table_cache_bounds_tests --queued-draws) set_tests_properties(agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 60) + agc_add_env_test(agc_driver_image_table_cache_bounds_heap_small agc_driver_image_table_cache_bounds_tests + ENV ANYPS5_NO_SHADER_CACHE=1 APS5_GPU_HEAP=1 APS5_HEAP_SLOT_CAP=1 TIMEOUT 60) if(WIN32) - set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_table_snapshot_reuse agc_shader_table_snapshot_reuse_unmasked agc_shader_table_trailer agc_shader_device_image_tables agc_driver_image_table_vouched_lookups agc_driver_image_table_vouch_queued_draws PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() add_test(NAME agc_shader_type_zero_image_materialization COMMAND agc_shader_memory_tests --type-zero-image-materialization) set_tests_properties(agc_shader_type_zero_image_materialization PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/AddressDrawSpaceReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/AddressDrawSpaceReuse.cpp index cc06c649cd96..0ec36bed065e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/AddressDrawSpaceReuse.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/AddressDrawSpaceReuse.cpp @@ -2,10 +2,13 @@ #include "prx/libSceAgcDriver/Execution/include/Recipe.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/BdaResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" @@ -36,6 +39,7 @@ #include #include #include +#include #include namespace AgcDriver { @@ -61,6 +65,7 @@ constexpr std::size_t PixelCodeOffset = 256; constexpr std::size_t IndirectCodeOffset = 512; constexpr std::size_t PointerSlots = 1024; constexpr std::size_t MovedVerticesOffset = 32768; +constexpr std::size_t ShiftedVerticesOffset = MovedVerticesOffset + 4100; constexpr std::size_t RingSlotBytes = 256; constexpr std::size_t RingSlots = 8; constexpr std::uint32_t PinWaitMs = 4000; @@ -295,6 +300,8 @@ class Outcomes { AgcDriver::Graphics::AddressDrawReuseCounts before = AgcDriver::Graphics::AddressDrawReuseCounters(); }; +bool HeapStages = false; + std::uint64_t Builds() { const auto counters = AgcDriver::Graphics::BdaResources::TableCacheCounters(); return counters.hits + counters.misses - AgcDriver::Graphics::AddressDrawReuseVerifications(); @@ -324,6 +331,7 @@ struct Scene { pixels.Register(true); std::memcpy(vertices.Data(), Triangle.data(), sizeof(Triangle)); std::memcpy(vertices.Data() + MovedVerticesOffset, Triangle.data(), sizeof(Triangle)); + std::memcpy(vertices.Data() + ShiftedVerticesOffset, Triangle.data(), sizeof(Triangle)); vertices.Register(false); std::memcpy(code.Data() + VertexCodeOffset, VertexCode.data(), sizeof(VertexCode)); std::memcpy(code.Data() + PixelCodeOffset, PixelCode.data(), sizeof(PixelCode)); @@ -362,7 +370,7 @@ struct Scene { ShaderRecompiler::RecompileRequest vertex{ {ShaderStage::Vertex, code.Address(VertexCodeOffset), VertexCode, 0, {}}, {64, 0, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, - device.Target(), + Target(), {0, 0, 0, 64} }; vertex.useCache = cachedVertex; @@ -372,8 +380,14 @@ struct Scene { shaders[0] = {ShaderStage::Vertex, &vertexResults.back(), 0}; } + ShaderRecompiler::SpirvTarget Target() const { + auto target = device.Target(); + target.gpuResolvedImages = HeapStages; + return target; + } + void Point(std::uint64_t slot, std::span program = PixelCode, std::size_t offset = PixelCodeOffset) { - const auto target = device.Target(); + const auto target = Target(); ShaderRecompiler::ShaderPixelStageInfo pixel{}; pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); pixel.posX = true; @@ -464,8 +478,13 @@ void CpuWrite(Scene& scene) { Require(scene.DrawnColor() == 120, "address draw space reuse: the draw before the CPU write did not read the guest constant"); scene.Store(scene.constants, 0, 40); const auto before = Builds(); + const auto copies = AgcDriver::Graphics::DrawInputCopies(); const auto color = scene.DrawnColor(); Require(color == 40, "address draw space reuse: a draw after a CPU write into its constant read " + std::to_string(color) + " instead of the written 40"); + if (HeapStages) { + const auto taken = AgcDriver::Graphics::DrawInputCopies() - copies; + Require(taken == 0, "address draw space reuse: heap stages of an address draw read " + std::to_string(taken) + " per-draw copies instead of guest memory in place"); + } Require(Builds() == before, "address draw space reuse: a CPU write into memory the draw reads in place made it build its resources again (" + std::to_string(Builds() - before) + " builds)"); } @@ -746,10 +765,24 @@ void RingRefusals(Scene& scene) { { const Outcomes outcomes; scene.Fetch(MovedVerticesOffset); + const auto copies = AgcDriver::Graphics::DrawInputCopies(); const auto color = scene.DrawnColor(); Require(color == 170, "address draw space reuse: the draw after its read-only vertex buffer moved read " + std::to_string(color) + " instead of 170"); + if (HeapStages) { + const auto taken = AgcDriver::Graphics::DrawInputCopies() - copies; + Require(taken == 0, "address draw space reuse: heap stages read " + std::to_string(taken) + " copies of a vertex buffer that moved inside the space instead of the buffer in place"); + } Require(outcomes[AddressDrawReuse::Reused] == 1 && outcomes.Total() == 1, "address draw space reuse: a draw whose read-only vertex buffer moved inside the space counted " + outcomes.Text() + ", expected one reuse with the buffer bound anew"); } + if (HeapStages) { + const Outcomes outcomes; + scene.Fetch(ShiftedVerticesOffset); + const auto copies = AgcDriver::Graphics::DrawInputCopies(); + const auto color = scene.DrawnColor(); + Require(color == 170, "address draw space reuse: the draw after its vertex buffer moved off its view's alignment read " + std::to_string(color) + " instead of 170"); + Require(outcomes[AddressDrawReuse::MovedBuffer] == 1 && outcomes.Total() == 1, "address draw space reuse: a heap-stage draw whose vertex buffer moved off its view's alignment counted " + outcomes.Text() + ", expected one refusal for a moved GuestBuffers element"); + Require(AgcDriver::Graphics::DrawInputCopies() == copies, "address draw space reuse: heap stages read a copy of a vertex buffer that moved off its view's alignment"); + } { std::memcpy(scene.spare.Data(), Triangle.data(), sizeof(Triangle)); constexpr std::size_t Filled = 64; @@ -1006,20 +1039,27 @@ constexpr std::array TransitionCases{{ int main(int argc, char** argv) { try { - const std::string mode = argc > 1 ? argv[1] : ""; - const std::string only = mode == "--case" && argc > 2 ? argv[2] : ""; - Require(mode.empty() || mode == "--off" || mode == "--transitions" || mode == "--verify" || mode == "--held" || !only.empty(), "usage: [--off | --transitions | --verify | --held | --case ]"); + HeapStages = argc > 1 && std::string_view(argv[1]) == "--heap-stages"; + const int first = HeapStages ? 2 : 1; + const std::string mode = argc > first ? argv[first] : HeapStages ? "--heap-stages" : ""; + const std::string only = mode == "--case" && argc > first + 1 ? argv[first + 1] : ""; + Require(mode.empty() || mode == "--off" || mode == "--transitions" || mode == "--verify" || mode == "--held" || mode == "--heap-stages" || !only.empty(), "usage: [--off | --transitions | --verify | --held | --heap-stages [--case ] | --case ]"); Require(mode != "--verify" || std::getenv("APS5_VERIFY_ADDRESS_DRAW_REUSE") != nullptr, "address draw space reuse: --verify needs APS5_VERIFY_ADDRESS_DRAW_REUSE=1"); Require(mode != "--held" || std::getenv("APS5_RELEASE_ON_UNLOCK") != nullptr, "address draw space reuse: --held needs APS5_RELEASE_ON_UNLOCK=1"); LimitPinWait(); const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (HeapStages && !AgcDriver::Graphics::DescriptorHeapEnabled()) { + std::puts("skipped, the device has no VK_EXT_descriptor_heap, which --heap-stages needs"); + return VulkanTestSkipped; + } { Scene scene(*device); if (!Imported(scene)) { std::puts("skipped, host imports are unavailable: the cached address space needs one"); return VulkanTestSkipped; } + if (HeapStages) Require(std::all_of(scene.shaders.begin(), scene.shaders.end(), [](const AgcDriver::Graphics::CompiledShader& shader) { return AgcDriver::Graphics::HeapShader(shader); }), "address draw space reuse: --heap-stages built a stage that is not a heap stage"); const Outcomes outcomes; const auto verifiedBefore = AgcDriver::Graphics::AddressDrawReuseVerifications(); if (mode == "--off") { @@ -1029,6 +1069,7 @@ int main(int argc, char** argv) { bool ran = false; for (const auto& entry : cases) { if (!only.empty() && only != entry.name) continue; + if (HeapStages && std::string_view(entry.name) == "fault-per-use") continue; entry.run(scene); ran = true; } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp index bc4e3ffab8e9..5c9abaf2908f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/AddressPasses.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -213,9 +214,12 @@ std::string Counts(const AgcDriver::Graphics::DrawPassCounts& before, const AgcD void ReadersShareAPass(const GuestBlock& vertices, const GuestBlock& data, const GuestBlock& target) { const auto before = AddressDraws(); + const auto heapBinds = AgcDriver::Graphics::DescriptorHeapBinds(); Submit(vertices, target, {{PixelRead.data(), data.At(0)}, {PixelRead.data(), data.At(0)}}); const auto after = AddressDraws(); - std::printf("two draws reading through the table: %s\n", Counts(before, after).c_str()); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds() - heapBinds; + std::printf("two draws reading through the table: %s, %llu heap binds\n", Counts(before, after).c_str(), static_cast(binds)); + Expect(!AgcDriver::Graphics::DescriptorHeapEnabled() || binds == 1, "two heap draws in one command buffer bound the descriptor heaps " + std::to_string(binds) + " times"); Expect(after.draws - before.draws == 2 && after.passesContinued - before.passesContinued == 1, "a draw reading through the table after another did not continue its pass: " + Counts(before, after)); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp index 3268ff39d811..cfeefcbba3ba 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp @@ -1,9 +1,11 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" #include "ImageTableAbi.hpp" #include "VulkanTestDevice.hpp" #include #include +#include #include #include @@ -56,8 +58,12 @@ void Run(AgcDriver::VulkanDevice& device) { const auto output = Buffer(data.output.data(), 0u, 256u); std::copy(output.begin(), output.end(), data.srt.begin() + 12u); const ShaderComputeStageInfo compute{{threads, 1u, 1u}, 0u, {false, false, false}, false, 1u}; - RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, regions}, device.Target(), {0u, 0u, 0u, 128u}}; + const bool heap = AgcDriver::Graphics::DescriptorHeapEnabled(); + auto target = device.Target(); + target.gpuResolvedImages = heap; + RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, {32u, 0u, userData, compute, std::nullopt, std::nullopt, regions}, target, {0u, 0u, 0u, 128u}}; const auto shader = Recompile(request); + Require(shader.gpuResolvedImages == heap, heap ? "the compute heap did not take the table stage" : "a table stage took the heap without it"); if (artifact.variantId == 0u) artifact = shader; else Require(shader.cacheHit && shader.variantId == artifact.variantId, "bindless table changed the compiled artifact"); if (iteration == 1u) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp index 550cbd7f0bea..88c1230b46b7 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomics.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" @@ -77,11 +78,12 @@ void Run(AgcDriver::VulkanDevice& device) { ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(32), {0, 0, 0, 128} }; request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + Require(result.gpuResolvedImages == AgcDriver::Graphics::DescriptorHeapEnabled(), "the compute heap did not take the buffer stage"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferFormat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferFormat.cpp index 7394269e39b1..3067384aad32 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferFormat.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" @@ -155,11 +156,12 @@ void Run(AgcDriver::VulkanDevice& device) { ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(32), {0, 0, 0, 128} }; request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + Require(result.gpuResolvedImages == AgcDriver::Graphics::DescriptorHeapEnabled(), "the compute heap did not take the buffer stage"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp index da69c6d9d0b6..0932f742a89f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferLoadStore.cpp @@ -304,7 +304,7 @@ void CheckRuntimeDescriptors(AgcDriver::VulkanDevice& device, std::uint32_t wave const auto output = BufferDescriptor(Output.data(), threads * 4u); std::copy(output.begin(), output.end(), userData.begin() + 4); const ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1u, 1u}, 0u, {false, false, false}, false, 1u}; - ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {waveSize, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.Target(), {0u, 0u, 0u, 128u}}; + ShaderRecompiler::RecompileRequest request{{ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {waveSize, 0u, userData, compute, std::nullopt, std::nullopt, {}}, device.ComputeTarget(waveSize), {0u, 0u, 0u, 128u}}; ShaderRecompiler::RecompileResult first; for (const auto& value : cases) { auto input = BufferDescriptor(Input.data(), value.records); @@ -345,9 +345,9 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device, Wave32Code, 32, device->Target()); + Run(*device, Wave32Code, 32, device->ComputeTarget(32)); Check(Expected32, Names, "wave32"); - Run(*device, Wave64Code, 64, device->Target()); + Run(*device, Wave64Code, 64, device->ComputeTarget(64)); Check(Expected64, Names, "wave64"); Run(*device, Wave64Code, 64, device->ComputeTarget(32)); Check(Expected64, Names, "wave64 split"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageDispatch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageDispatch.cpp index 4c2e019b69da..e72c76e7006e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageDispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageDispatch.cpp @@ -141,7 +141,7 @@ ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std:: ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageReuse.cpp index dbd774f671a8..1f44d1771099 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageReuse.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthStorageReuse.cpp @@ -45,7 +45,7 @@ void Read(AgcDriver::VulkanDevice& device, std::uint32_t format, std::uint32_t w const ShaderRecompiler::ShaderComputeStageInfo compute{{32, 1, 1}, 0u, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, - {32, 0, data, compute, std::nullopt, std::nullopt, memory}, device.Target(), {0, 0, 0, 128}, + {32, 0, data, compute, std::nullopt, std::nullopt, memory}, device.ComputeTarget(64), {0, 0, 0, 128}, }; request.useCache = false; const auto shader = ShaderRecompiler::Recompile(request); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DeviceFunctionCache.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceFunctionCache.cpp new file mode 100644 index 000000000000..4df226b5bf5c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceFunctionCache.cpp @@ -0,0 +1,51 @@ +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" + +#include +#include + +namespace { + +PFN_vkGetDeviceProcAddr realProc = nullptr; +int countedCalls = 0; + +PFN_vkVoidFunction VKAPI_CALL countingProc(VkDevice device, const char* name) { + ++countedCalls; + return realProc(device, name); +} + +void require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr char DispatchName[] = "vkCmdDispatch"; + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + auto context = device->graphicsContext(); + require(context.functions != nullptr && context.functionCache != nullptr, "the device context carries its function cache"); + const auto first = context.Function(DispatchName); + const auto lookups = AgcDriver::Graphics::DeviceProcLookups(); + require(context.Function(DispatchName) == first && AgcDriver::Graphics::DeviceProcLookups() == lookups, + "a repeated lookup is served from the device's cache"); + realProc = context.deviceProc; + context.deviceProc = countingProc; + require(context.Function(DispatchName) == first && countedCalls == 1, "another resolver is not served from the cache"); + context.deviceProc = realProc; + require(context.Function(DispatchName) == first && AgcDriver::Graphics::DeviceProcLookups() == lookups + 1u, + "the device's resolver is still served from the cache"); + context.deviceProc = countingProc; + context.functions = nullptr; + static_cast(context.Function(DispatchName)); + require(countedCalls == 2, "a context without the device's table looks up every time"); + std::printf("device function cache tests passed\n"); + return 0; + } catch (const std::exception& error) { + std::printf("device function cache tests failed: %s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImagePath.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImagePath.cpp new file mode 100644 index 000000000000..2e5055613d8f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImagePath.cpp @@ -0,0 +1,372 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" +#include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::DeviceImageTable; + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr std::uint32_t Entries = 2000; +constexpr std::uint32_t RecordBytes = 32; +constexpr std::uint32_t Side = 4; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Textures = 16; +constexpr std::size_t BlockBytes = std::size_t{2} << 20u; +constexpr std::size_t TextureStride = 0x1000; +constexpr std::size_t TableOffset = 0x100000; +constexpr std::size_t SrtOffset = 0x180000; +constexpr std::array Formats{1u, 13u, 14u, 22u, 56u}; +constexpr std::array FormatNames{"8 unorm", "16 float", "8_8 unorm", "32 float", "8_8_8_8 unorm"}; +constexpr std::array HalfValues{0x3266u, 0x38cdu, 0x3666u, 0x3a66u}; +constexpr std::array FloatValues{0.2f, 0.6f, 0.4f, 0.8f}; + +alignas(256) constexpr std::array ComputeCode{ + 0x7e0002f0u, 0x7e0202f0u, 0xf4080200u, 0xfa000000u, 0xf4080300u, 0xfa000010u, 0xbf8cc07fu, 0x8f108502u, 0xf42c0504u, 0x20000000u, + 0xbf8cc07fu, 0xf09c0f08u, 0x00650000u, 0xbf8c0000u, 0xe0780000u, 0x80010000u, 0xbf810000u, 0xbf810000u, +}; + +alignas(4096) std::array, 64> Outputs{}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Texture(std::uint32_t index) const { return block + static_cast(index) * TextureStride; } + std::uint32_t* Table() const { return reinterpret_cast(block + TableOffset); } + std::uint32_t* Srt() const { return reinterpret_cast(block + SrtOffset); } + +private: + std::uint8_t* block = nullptr; +}; + +float HalfToFloat(std::uint16_t half) { + const auto exponent = static_cast((half >> 10u) & 0x1fu); + const auto mantissa = static_cast(half & 0x3ffu); + return std::ldexp(1.0f + mantissa / 1024.0f, exponent - 15); +} + +struct Texel { + std::uint32_t format = 0; + std::vector bytes; + std::array value{}; +}; + +Texel TexelOf(std::uint32_t texture) { + const auto kind = texture % static_cast(Formats.size()); + const auto variant = texture / static_cast(Formats.size()); + Texel texel; + texel.format = Formats[kind]; + const auto unorm = [](std::uint8_t byte) { return static_cast(byte) / 255.0f; }; + switch (kind) { + case 0: { + const auto red = static_cast(0x40u + 0x30u * variant); + texel.bytes = {red}; + texel.value = {unorm(red), 0.0f, 0.0f, 1.0f}; + break; + } + case 1: { + const auto half = HalfValues[variant]; + texel.bytes = {static_cast(half), static_cast(half >> 8u)}; + texel.value = {HalfToFloat(half), 0.0f, 0.0f, 1.0f}; + break; + } + case 2: { + const auto red = static_cast(0x11u + 0x22u * variant); + const auto green = static_cast(0x99u - 0x11u * variant); + texel.bytes = {red, green}; + texel.value = {unorm(red), unorm(green), 0.0f, 1.0f}; + break; + } + case 3: { + const auto bits = std::bit_cast(FloatValues[variant]); + texel.bytes = {static_cast(bits), static_cast(bits >> 8u), static_cast(bits >> 16u), static_cast(bits >> 24u)}; + texel.value = {FloatValues[variant], 0.0f, 0.0f, 1.0f}; + break; + } + default: { + const std::array rgba{static_cast(0x10u + 0x30u * variant), static_cast(0x80u - 0x20u * variant), static_cast(0xc0u - 0x30u * variant), static_cast(0xffu - 0x40u * variant)}; + texel.bytes.assign(rgba.begin(), rgba.end()); + texel.value = {unorm(rgba[0]), unorm(rgba[1]), unorm(rgba[2]), unorm(rgba[3])}; + break; + } + } + return texel; +} + +void FillTexture(std::uint8_t* memory, std::span bytes) { + for (std::size_t offset = 0; offset + bytes.size() <= TextureStride; offset += bytes.size()) std::memcpy(memory + offset, bytes.data(), bytes.size()); +} + +std::array TextureDescriptor(const void* data, std::uint32_t format) { + const auto address = static_cast(reinterpret_cast(data)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = static_cast(reinterpret_cast(data)); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::uint32_t TextureOf(std::uint32_t entry) { + return entry % Textures; +} + +void FillGuest(const GuestBlock& block) { + for (std::uint32_t texture = 0; texture < Textures; ++texture) FillTexture(block.Texture(texture), TexelOf(texture).bytes); + auto* table = block.Table(); + for (std::uint32_t entry = 0; entry < Entries; ++entry) { + const auto texture = TextureOf(entry); + const auto descriptor = TextureDescriptor(block.Texture(texture), TexelOf(texture).format); + std::copy(descriptor.begin(), descriptor.end(), table + static_cast(entry) * (RecordBytes / 4u)); + } + auto* srt = block.Srt(); + const auto tableWords = BufferDescriptor(table, Entries * RecordBytes); + std::copy(tableWords.begin(), tableWords.end(), srt); + std::fill_n(srt + 4, 4, 0u); +} + +struct Header { + Shader shader{}; + ShaderUserData userData{}; +}; + +alignas(8) Header ComputeHeader{}; + +void Registered(Header& header, const std::uint32_t* code, std::size_t bytes) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(Header); + header.shader.shader_size = static_cast(bytes); + header.shader.code = code; + header.shader.type = 0; + header.shader.user_data = &header.userData; + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + +void Append(std::vector& commands, std::uint32_t opcode, std::initializer_list payload) { + commands.push_back(0xc0000000u | (static_cast(payload.size() - 1) << 16u) | (opcode << 8u)); + commands.insert(commands.end(), payload); +} + +void Dispatch(std::vector& words, const GuestBlock& block, std::uint32_t key, std::array& output) { + const auto address = static_cast(reinterpret_cast(ComputeCode.data())); + const auto srt = static_cast(reinterpret_cast(block.Srt())); + const auto set = [&](std::uint32_t offset, std::uint32_t value) { Append(words, 0x76, {offset, value}); }; + set(0x20c, static_cast(address >> 8u)); + set(0x20d, static_cast(address >> 40u) & 0xffu); + for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) set(offset, 1); + set(0x212, 0); + set(0x213, 16u << 1u); + std::array userData{}; + userData[0] = static_cast(srt); + userData[1] = static_cast(srt >> 32u); + userData[2] = key; + const auto outputWords = BufferDescriptor(output.data(), sizeof(output)); + std::copy(outputWords.begin(), outputWords.end(), userData.begin() + 4); + for (std::uint32_t index = 0; index < userData.size(); ++index) set(0x240u + index, userData[index]); + Append(words, 0x15, {1, 1, 1, 0x8041u}); +} + +void SubmitCompute(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(0x20, &packet) == 0, "the compute submission failed"); + AgcDriverWaitIdle_nid_postfix(); + std::vector stored(sizeof(Outputs)); + AgcDriver::GuestMemory::Read(static_cast(reinterpret_cast(Outputs.data())), stored, 4); + std::memcpy(Outputs.data(), stored.data(), stored.size()); +} + +struct Counters { + std::uint64_t heapComputeStages = 0; + std::uint64_t tables = 0; + DeviceImageTable::Counts device; +}; + +Counters Read() { + Counters counters; + counters.heapComputeStages = AgcDriver::Graphics::HeapComputeStages(); + const auto totals = DeviceImageTable::DeviceTotals(); + counters.tables = totals.size(); + for (const auto& table : totals) { + counters.device.prepares += table.prepares; + counters.device.drained += table.drained; + counters.device.inserted += table.inserted; + counters.device.expected += table.expected; + counters.device.expectedInserted += table.expectedInserted; + counters.device.faults += table.faults; + counters.device.rejected += table.rejected; + counters.device.fallbacks += table.fallbacks; + } + return counters; +} + +std::string Describe(const Counters& before, const Counters& after) { + return " (GPU-mode compute stages " + std::to_string(after.heapComputeStages - before.heapComputeStages) + ", device table prepares " + std::to_string(after.device.prepares - before.device.prepares) + ", expected " + std::to_string(after.device.expected - before.device.expected) + ", inserted " + std::to_string(after.device.inserted - before.device.inserted) + ", miss records " + std::to_string(after.device.drained - before.device.drained) + ", fallbacks " + std::to_string(after.device.fallbacks - before.device.fallbacks) + ")"; +} + +void RequireTexel(const std::array& output, const std::array& expected, const std::string& what) { + for (std::size_t channel = 0; channel < 4; ++channel) { + Require(std::fabs(output[channel] - expected[channel]) < 1e-6f, what + ": channel " + std::to_string(channel) + " is " + std::to_string(output[channel]) + ", expected " + std::to_string(expected[channel])); + } +} + +constexpr std::array ComputeKeys{3u, 1764u, 37u, 54u, 1495u, 72u, 1657u, 1600u, 33u, 1810u}; + +void RunComputeKeys(const GuestBlock& block) { + for (std::size_t index = 0; index < ComputeKeys.size(); ++index) Outputs[index].fill(-1.0f); + std::vector words; + for (std::size_t index = 0; index < ComputeKeys.size(); ++index) Dispatch(words, block, ComputeKeys[index], Outputs[index]); + SubmitCompute(words); +} + +void RequireComputeTexels(const std::string& what) { + for (std::size_t index = 0; index < ComputeKeys.size(); ++index) { + const auto texture = TextureOf(ComputeKeys[index]); + RequireTexel(Outputs[index], TexelOf(texture).value, what + ", key " + std::to_string(ComputeKeys[index]) + " (" + FormatNames[texture % Formats.size()] + ")"); + } +} + +void ComputeTable(const GuestBlock& block, bool device) { + auto before = Read(); + RunComputeKeys(block); + auto after = Read(); + RequireComputeTexels("first batch"); + if (device) { + Require(after.tables == 1u, "the device keeps one device image table" + Describe(before, after)); + Require(after.heapComputeStages > before.heapComputeStages, "first batch: the sampling compute stage is not GPU mode" + Describe(before, after)); + Require(after.device.prepares > before.device.prepares, "first batch: the GPU-mode stage did not read its table through the device image table" + Describe(before, after)); + Require(after.device.expectedInserted == before.device.expectedInserted + ComputeKeys.size(), "first batch: the driver did not insert the ten expected keys before the batch" + Describe(before, after)); + Require(after.device.drained == before.device.drained, "first batch: the device path recorded misses for keys it expected" + Describe(before, after)); + Require(after.device.fallbacks == before.device.fallbacks, "first batch: the program fell back to the CPU table map" + Describe(before, after)); + } else { + Require(after.tables == 0u, "without APS5_GPU_PATH the device keeps no device image table"); + } + std::printf("first batch of %zu dispatches: %s%s\n", ComputeKeys.size(), device ? "texels through the device image table, keys inserted before the batch" : "texels through the CPU table map", Describe(before, after).c_str()); + before = after; + RunComputeKeys(block); + after = Read(); + RequireComputeTexels("second batch"); + if (device) { + Require(after.device.inserted == before.device.inserted, "second batch: the device path inserted keys it already held" + Describe(before, after)); + Require(after.device.drained == before.device.drained, "second batch: the device path recorded misses for keys it held" + Describe(before, after)); + } + std::printf("second batch: the same texels%s\n", Describe(before, after).c_str()); +} + +} + +int main() { + const bool requested = AgcDriver::Graphics::GpuPathEnabled(); +#ifdef _WIN32 + _putenv_s("APS5_EAGER_WRITEBACK", "1"); +#else + setenv("APS5_EAGER_WRITEBACK", "1", 1); +#endif + try { + bool served = false; + bool heapUsable = false; + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + served = device->DeviceImagePath(); + heapUsable = AgcDriver::Graphics::DescriptorHeapEnabled(); + Require(device->Target().deviceImageTables == served && device->ComputeTarget(64u).deviceImageTables == served, "the recompile targets ask for device image tables exactly when the device serves the path"); + Require((DeviceImageTable::Existing(device->graphicsContext()) != nullptr) == served, "the device keeps a device image table exactly when it serves the path"); + } + const bool gpuHeap = AgcDriver::Graphics::GpuHeapEnabled(); + Require(served == (requested && gpuHeap && heapUsable), "the device serves the device image path exactly when APS5_GPU_PATH and APS5_GPU_HEAP are set on a device whose descriptor heap the GPU heap uses"); + if (requested && gpuHeap && !served) { + std::printf("skipped, the device lacks VK_EXT_descriptor_heap, which the GPU heap the device image path reads its tables through needs\n"); + return VulkanTestSkipped; + } + GuestBlock block; + FillGuest(block); + Registered(ComputeHeader, ComputeCode.data(), sizeof(ComputeCode)); + std::printf("device image path %s\n", served ? "on" : "off"); + ComputeTable(block, served); + AgcDriverShutdown_nid_postfix(); + std::cout << "device image path tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "device image path tests failed: " << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception& shutdownError) { + std::cerr << shutdownError.what() << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImageTable.cpp new file mode 100644 index 000000000000..e3f46a6d80a6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DeviceImageTable.cpp @@ -0,0 +1,922 @@ +#include "prx/libSceAgcDriver/Graphics/include/DeviceImageTable.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Optimization/ResourceMaterializer.hpp" +#include "IntermediateRepresentation/IrProgram.hpp" +#include "DeviceImageTableAbi.hpp" +#include "ImageTableAbi.hpp" +#include "RuntimeAbi.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; +namespace Abi = ShaderRecompiler::DeviceImageTableAbi; +namespace Runtime = ShaderRecompiler::RuntimeAbi; +using Key = DeviceImageTable::Key; + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr std::uint32_t Side = 4; +constexpr std::uint32_t Type2D = 9; +constexpr std::array Formats{1u, 13u, 14u, 22u, 56u}; +constexpr std::uint64_t InterfaceId = 0x1234'5678'9abc'def0ull; +constexpr ShaderRecompiler::PipelineIdentity Program{0x5151'0000'0000'0001ull, 0x5151'0000'0000'0002ull}; +constexpr std::uint64_t ProgramAddress = 0x7700'0000ull; + +alignas(256) std::array, Formats.size()> Textures{}; + +std::array TextureDescriptor(const void* data, std::uint32_t format) { + const auto address = static_cast(reinterpret_cast(data)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +std::shared_ptr SampledInterface() { + static const auto sampled = [] { + auto made = std::make_shared(); + made->id = InterfaceId; + made->image.resourceClass = ShaderRecompiler::ImageResourceClass::Sampled; + made->image.numericClass = ShaderRecompiler::IrTextureNumericClass::Float; + made->image.dimension = ShaderRecompiler::RdnaImageDimension::Dim2D; + made->image.read = true; + made->image.table = 0; + made->modes = ShaderRecompiler::ResourceMaterializer::RuntimeImageModes(made->image); + return made; + }(); + return sampled; +} + +DeviceImageTable::InterfaceLookup Lookup() { + return [](std::uint64_t id) { return id == InterfaceId ? SampledInterface() : ShaderRecompiler::ResourceMaterializer::FindDeviceImageInterface(id); }; +} + +struct FakeHeap { + std::uint32_t cap = Runtime::GpuImageHeapSlots; + bool nearlyFull = false; + std::uint64_t serial = 1; + std::map, std::uint32_t> slots; + std::map, std::weak_ptr> holders; + std::map, std::uint32_t> generations; + std::map next; + std::uint32_t epoch = 1; + std::uint32_t touches = 0; + std::map uses; + + void Sweep(std::uint32_t kind, std::uint32_t slot) { + for (auto found = slots.begin(); found != slots.end();) { + if (std::get<0>(found->first) == kind && found->second == slot) found = slots.erase(found); + else ++found; + } + holders.erase({kind, slot}); + ++generations[{kind, slot}]; + ++serial; + } +}; + +DeviceImageTable::Slots FakeSlots(const std::shared_ptr& heap) { + DeviceImageTable::Slots slots; + slots.take = [heap](std::uint32_t kind, const std::shared_ptr& texture, VkImageView view, VkImageLayout) -> std::optional { + const auto key = std::make_tuple(kind, static_cast(texture.get()), view); + if (const auto found = heap->slots.find(key); found != heap->slots.end()) return found->second; + auto& next = heap->next[kind]; + if (next >= heap->cap) return std::nullopt; + const auto slot = next++; + heap->slots.emplace(key, slot); + heap->holders[{kind, slot}] = texture; + return slot; + }; + slots.generation = [heap](std::uint32_t kind, std::uint32_t slot) { return heap->generations[{kind, slot}]; }; + slots.serial = [heap] { return heap->serial; }; + slots.nearlyFull = [heap] { return heap->nearlyFull; }; + slots.touch = [heap](std::span> held, std::uint32_t touched) { + if (touched == heap->epoch) return heap->epoch; + ++heap->touches; + for (const auto& kind : held) { + for (const auto slot : kind) heap->uses[slot] = heap->epoch; + } + return heap->epoch; + }; + return slots; +} + +Key KeyFor(const std::array& descriptor, std::uint32_t samplerClass = 0u) { + Key key{}; + std::copy(descriptor.begin(), descriptor.end(), key.begin()); + key[Abi::InterfaceWord] = static_cast(InterfaceId); + key[Abi::InterfaceWord + 1u] = static_cast(InterfaceId >> 32u); + key[Abi::SamplerClassWord] = samplerClass; + return key; +} + +void WriteRecords(std::span ring, std::span keys, std::uint32_t count, std::uint32_t pcFlags = 0u) { + ring[Abi::RingCountWord] = count; + for (std::uint32_t record = 0; record < std::min(count, DeviceImageTable::RingCapacity); ++record) { + auto* at = ring.data() + Abi::RingRecordsWord + static_cast(record) * Abi::RecordWords; + const auto& key = keys[record % keys.size()]; + std::copy(key.begin(), key.end(), at); + at[Abi::RecordPcWord] = (0x100u + record * 4u) | pcFlags; + } +} + +std::string HexWords(std::span words) { + std::ostringstream text; + text << std::hex << std::setfill('0'); + for (std::size_t index = 0; index < words.size(); ++index) text << (index == 0 ? "" : " ") << std::setw(8) << words[index]; + return text.str(); +} + +std::vector ReadTable(const Context& context, Recorder& recorder, const DeviceImageTable& table, std::size_t words) { + Buffer readback(context, words * sizeof(std::uint32_t), VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + const VkBufferCopy region{0, 0, words * sizeof(std::uint32_t)}; + context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, table.TableBuffer(), readback.Handle(), 1, ®ion); + recorder.Sync(); + std::vector result(words); + std::memcpy(result.data(), readback.Bytes().data(), words * sizeof(std::uint32_t)); + return result; +} + +std::vector FormatKeys() { + std::vector keys; + for (std::size_t index = 0; index < Formats.size(); ++index) keys.push_back(KeyFor(TextureDescriptor(Textures[index].data(), Formats[index]))); + return keys; +} + +ShaderRecompiler::RuntimeImageModeMatch ExpectedMode(const Key& key) { + ShaderRecompiler::DescriptorValue descriptor; + descriptor.dwordCount = Abi::DescriptorWords; + descriptor.availableDwordMask = 0xffu; + std::copy_n(key.begin(), Abi::DescriptorWords, descriptor.dwords.begin()); + const auto sampled = SampledInterface(); + return ShaderRecompiler::ResourceMaterializer::TryRuntimeImageMode(sampled->image, descriptor, sampled->modes); +} + +std::uint32_t KindOf(const DeviceImageValue&) { + return Runtime::FirstImageBinding; +} + +void HeaderAtCreation(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "the device has an active recorder"); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + const auto words = ReadTable(context, *recorder, table, DeviceImageTable::TableWords); + const auto header = std::span(words).first(Abi::HeaderWords); + Require(words[Abi::HeaderVersion] == Abi::Version && words[Abi::HeaderMask] == table.Totals().buckets - 1u && words[Abi::HeaderRingCapacity] == DeviceImageTable::RingCapacity, "a new table's header names the version, the bucket mask and the ring capacity before its first prepare (" + HexWords(header) + ")"); + Require(std::all_of(words.begin() + Abi::HeaderRingCapacity + 1u, words.end(), [](std::uint32_t word) { return word == 0u; }), "a new table's other header words and all its buckets are zero"); + table.Prepare(context, *recorder); + Require(table.Totals().uploadBytes == 0u, "a first prepare with nothing resolved writes no header again (" + std::to_string(table.Totals().uploadBytes) + " bytes)"); + recorder->Sync(); +} + +void ResolvesAndUploads(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "the device has an active recorder"); + const auto context = device.graphicsContext(); + const auto heap = std::make_shared(); + DeviceImageTable table(context, Lookup(), FakeSlots(heap)); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, 8u); + recorder->Sync(); + auto counts = table.Totals(); + Require(counts.drained == 8u && counts.dropped == 0u, "the batch's eight ring records are drained when it completes"); + Require(counts.inserted == 0u, "nothing is inserted before the next prepare"); + table.Prepare(context, *recorder); + counts = table.Totals(); + Require(counts.distinct == keys.size() && counts.inserted == keys.size() && counts.entries == keys.size(), "the five distinct records are resolved and inserted (distinct " + std::to_string(counts.distinct) + ", inserted " + std::to_string(counts.inserted) + ")"); + std::set> slots; + for (std::size_t index = 0; index < keys.size(); ++index) { + const auto value = table.Find(keys[index]); + Require(value.has_value() && value->fault == 0u && value->samplerSlot == 0u, "format " + std::to_string(Formats[index]) + " is inserted without a fault"); + const auto expected = ExpectedMode(keys[index]); + Require(expected.mode.has_value() && value->mode == *expected.mode, "format " + std::to_string(Formats[index]) + " names the mode the CPU table path picks"); + const auto kind = KindOf(*value); + const auto holder = heap->holders.find({kind, value->slot}); + Require(holder != heap->holders.end() && holder->second.lock() == table.EntryTexture(keys[index]), "format " + std::to_string(Formats[index]) + " holds its entry's texture in its heap slot"); + Require(slots.insert({kind, value->slot}).second, "every format has its own heap slot"); + } + const auto words = table.KeyWords(); + const auto uploaded = ReadTable(context, *recorder, table, Abi::HeaderWords + words.size()); + Require(uploaded[Abi::HeaderVersion] == Abi::Version && uploaded[Abi::HeaderMask] == counts.buckets - 1u && uploaded[Abi::HeaderRingCapacity] == DeviceImageTable::RingCapacity, "the header names the version, the bucket mask and the ring capacity"); + Require(std::all_of(uploaded.begin() + Abi::HeaderRingCapacity + 1u, uploaded.begin() + Abi::HeaderWords, [](std::uint32_t word) { return word == 0u; }), "the header's other words stay zero"); + Require(std::equal(words.begin(), words.end(), uploaded.begin() + Abi::HeaderWords), "the device's buckets equal the host key table"); + const auto uploads = table.Totals().uploads; + table.Prepare(context, *recorder); + Require(table.Totals().uploads == uploads, "a prepare with nothing changed uploads nothing"); + recorder->Sync(); + std::printf("resolved %zu formats: %llu uploads, %llu bytes\n", keys.size(), static_cast(table.Totals().uploads), static_cast(table.Totals().uploadBytes)); +} + +void TrailerAddresses(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + static_cast(recorder->Commands()); + const auto first = table.Trailer(*recorder); + Require(first.buckets != 0u && first.ring != 0u, "the trailer names a bucket address and a ring chunk address"); + Require(table.Trailer(*recorder) == first, "one batch reads one ring chunk"); + recorder->Submit(); + static_cast(recorder->Commands()); + const auto second = table.Trailer(*recorder); + Require(second.buckets == first.buckets, "every batch reads the same buckets"); + const auto stride = second.ring > first.ring ? second.ring - first.ring : first.ring - second.ring; + Require(stride >= static_cast(DeviceImageTable::RingWords) * sizeof(std::uint32_t) && stride % context.limits.minStorageBufferOffsetAlignment == 0u, "a batch still in flight keeps its chunk, so the next batch reads another one, a whole aligned chunk away"); + recorder->Sync(); + std::printf("trailer: buckets 0x%llx, ring chunks 0x%llx and 0x%llx\n", static_cast(first.buckets), static_cast(first.ring), static_cast(second.ring)); +} + +void SweptSlotsLeave(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + const auto heap = std::make_shared(); + DeviceImageTable table(context, Lookup(), FakeSlots(heap)); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder); + Require(table.Totals().entries == keys.size(), "every format is inserted"); + ++heap->serial; + table.Prepare(context, *recorder); + Require(table.Totals().entries == keys.size() && table.Totals().swept == 0u, "a heap whose slots moved elsewhere leaves the table whole"); + const auto swept = table.Find(keys[2]); + Require(swept.has_value(), "the swept key was inserted"); + const auto texture = table.EntryTexture(keys[2]); + heap->Sweep(KindOf(*swept), swept->slot); + const auto uploads = table.Totals().uploads; + table.Prepare(context, *recorder); + const auto counts = table.Totals(); + Require(!table.Find(keys[2]).has_value() && counts.swept == 1u && counts.entries == keys.size() - 1u, "a key whose heap slot was swept leaves the table (" + std::to_string(counts.swept) + " swept, " + std::to_string(counts.entries) + " left)"); + Require(counts.uploads == uploads + 1u, "the table without the swept key is uploaded before the batch"); + for (std::size_t index = 0; index < keys.size(); ++index) { + if (index != 2u) Require(table.Find(keys[index]).has_value(), "a key whose slot was not swept stays"); + } + Require(table.EntryTexture(keys[2]) == nullptr && texture != nullptr && texture.use_count() >= 2, "the swept entry lets go of its texture, which the open batch keeps"); + WriteRecords(table.RingChunk(*recorder), std::span(keys).subspan(2, 1), 1u); + recorder->Sync(); + table.Prepare(context, *recorder); + const auto again = table.Find(keys[2]); + Require(again.has_value() && again->fault == 0u && table.Totals().inserted == keys.size() + 1u, "the swept key's next miss inserts it again with a fresh slot"); + recorder->Sync(); +} + +void SweptWhileWaitingForAChunk(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + const auto heap = std::make_shared(); + DeviceImageTable table(context, Lookup(), FakeSlots(heap)); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder); + Require(table.Totals().entries == keys.size(), "every format is inserted"); + const auto held = table.Find(keys[2]); + Require(held.has_value(), "the key whose slot moves is inserted"); + const auto kind = KindOf(*held); + recorder->Sync(); + for (std::uint32_t chunk = 0; chunk < DeviceImageTable::RingChunks; ++chunk) { + static_cast(recorder->Commands()); + static_cast(table.Trailer(*recorder)); + if (chunk == 0u) recorder->OnComplete([heap, kind, slot = held->slot] { heap->Sweep(kind, slot); }); + recorder->Submit(); + } + static_cast(recorder->Commands()); + const auto waits = table.Totals().chunkWaits; + table.Prepare(context, *recorder); + Require(table.Totals().chunkWaits == waits + 1u, "the prepare waited for a miss-ring chunk, every one held by a batch in flight"); + Require(heap->generations[{kind, held->slot}] == 1u, "the batch that finished during the wait moved the key's heap slot"); + Require(!table.Find(keys[2]).has_value(), "a prepare serves no key whose heap slot moved while it waited for a miss-ring chunk (slot " + std::to_string(held->slot) + ", " + std::to_string(table.Totals().swept) + " swept)"); + for (std::size_t index = 0; index < keys.size(); ++index) { + if (index != 2u) Require(table.Find(keys[index]).has_value(), "a key whose slot did not move stays"); + } + const auto words = table.KeyWords(); + const auto uploaded = ReadTable(context, *recorder, table, Abi::HeaderWords + words.size()); + Require(std::equal(words.begin(), words.end(), uploaded.begin() + Abi::HeaderWords), "the batch after the wait reads buckets without the moved key"); + recorder->Sync(); +} + +void LiveSlotsTouched(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + const auto heap = std::make_shared(); + DeviceImageTable table(context, Lookup(), FakeSlots(heap)); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + Require(heap->touches == 0u, "a table without entries touches no slot"); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder); + const auto allUsed = [&] { + for (const auto& key : keys) { + const auto value = table.Find(key); + if (value.has_value() && heap->uses[value->slot] != heap->epoch) return false; + } + return true; + }; + Require(heap->touches == 1u && allUsed(), "the prepare that inserts the keys marks every live slot used in the heap's epoch (" + std::to_string(heap->touches) + " touches)"); + table.Prepare(context, *recorder); + Require(heap->touches == 1u, "a second prepare in the same epoch touches nothing again"); + ++heap->epoch; + table.Prepare(context, *recorder); + Require(heap->touches == 2u && allUsed(), "a prepare in a later epoch marks every live slot used again"); + const auto swept = table.Find(keys[1]); + Require(swept.has_value(), "the key to sweep is inserted"); + heap->Sweep(KindOf(*swept), swept->slot); + ++heap->epoch; + table.Prepare(context, *recorder); + Require(!table.Find(keys[1]).has_value() && heap->uses[swept->slot] == heap->epoch - 1u && heap->touches == 3u && allUsed(), "a swept key's slot is no longer marked used, and the others are"); + recorder->Sync(); +} + +void MarginRefuses(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + const auto heap = std::make_shared(); + heap->nearlyFull = true; + DeviceImageTable table(context, Lookup(), FakeSlots(heap)); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + const auto keys = FormatKeys(); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder, programs); + const auto counts = table.Totals(); + Require(counts.inserted == 0u && counts.heapFull == keys.size() && heap->slots.empty(), "a heap inside its margin gives the table no slot (" + std::to_string(counts.heapFull) + " refused)"); + Require(!table.ServesProgram(ProgramAddress) && counts.fallbacks == 1u, "the program whose keys found no slot falls back to the CPU path"); + Require(table.LastFallback().find("outside its margin") != std::string::npos, "the fallback names the margin: " + table.LastFallback()); + recorder->Sync(); +} + +void RejectsAndFaults(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + auto invalid = TextureDescriptor(Textures[0].data(), Formats[0]); + invalid[1] &= ~(0x1ffu << 20u); + auto unmapped = TextureDescriptor(reinterpret_cast(std::uintptr_t{0x10000}), Formats[4]); + const std::array keys{KeyFor(invalid), KeyFor(unmapped)}; + const auto checked = ExpectedMode(keys[0]); + Require(!checked.mode.has_value(), "the CPU table path rejects a descriptor of format 0"); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, 2u); + recorder->Sync(); + table.Prepare(context, *recorder); + const auto first = table.Find(keys[0]); + Require(first.has_value() && first->fault == 1u + static_cast(checked.reason), "a descriptor the mode check rejects is inserted with 1 + its reason"); + const auto second = table.Find(keys[1]); + Require(second.has_value() && second->fault == 1u + static_cast(ShaderRecompiler::ImageTableAbi::PoisonReason::DriverRejected), "a descriptor over unmapped memory is inserted as rejected by the driver"); + Require(table.Totals().rejected == 2u && table.Totals().inserted == 0u, "both are rejections, neither holds a texture"); + WriteRecords(table.RingChunk(*recorder), std::span(keys).first(1), 1u, Abi::FaultFlag); + recorder->Sync(); + const auto fault = table.LastFault(); + const auto name = ShaderRecompiler::ImageTableAbi::PoisonReasonName(checked.reason); + Require(table.Totals().faults == 1u, "the fault record is counted"); + Require(fault.find("image table: pc 0x100 selected the descriptor (words " + HexWords(std::span(invalid)) + ")") != std::string::npos && fault.find(std::string(name)) != std::string::npos, "the fault names the pc, the descriptor's words and the reason: " + fault); + std::printf("fault: %s\n", fault.c_str()); +} + +constexpr std::size_t BlockBytes = std::size_t{4} << 20u; +constexpr std::size_t TextureStride = 1024u; +constexpr std::uint32_t WatchedTextures = 2000u; + +class GuestBlock { +public: + explicit GuestBlock(bool watched = true) : watched(watched) { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(BlockBytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, BlockBytes, PAGE_READWRITE, BlockBytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, BlockBytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + BlockBytes != begin + BlockBytes + alignment) munmap(reinterpret_cast(aligned + BlockBytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + if (watched) GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, BlockBytes); + } +#endif + Require(block != nullptr, "cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, BlockBytes); + GuestArena::GuestArenaRelease_nid_postfix(block, BlockBytes); +#else + munmap(block, BlockBytes); + if (watched) GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, BlockBytes); +#endif + } + + std::uint8_t* Texture(std::uint32_t index) const { return block + static_cast(index) * TextureStride; } + +private: + bool watched = true; + std::uint8_t* block = nullptr; +}; + +double Microseconds(std::chrono::steady_clock::time_point start) { + return std::chrono::duration(std::chrono::steady_clock::now() - start).count(); +} + +void RefreshesWrittenEntries(AgcDriver::VulkanDevice& device) { + GuestBlock block; + for (std::uint32_t index = 0; index < WatchedTextures; ++index) std::memset(block.Texture(index), static_cast(index & 0xffu), Side * Side * 4u); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::GuestMemory::BumpCollectEpoch(); + if (!AgcDriver::GuestMemory::WriteWatched()) { + std::puts("guest memory has no write watch: device image entry refreshes are not tested"); + return; + } + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + std::vector keys; + for (std::uint32_t index = 0; index < WatchedTextures; ++index) keys.push_back(KeyFor(TextureDescriptor(block.Texture(index), Formats[4]))); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, WatchedTextures); + recorder->Sync(); + auto start = std::chrono::steady_clock::now(); + table.Prepare(context, *recorder); + const auto insertUs = Microseconds(start); + auto counts = table.Totals(); + Require(counts.inserted == WatchedTextures && counts.entries == WatchedTextures, "the 2000 entries are inserted (" + std::to_string(counts.inserted) + ")"); + const auto insertBytes = counts.uploadBytes; + recorder->Sync(); + table.Prepare(context, *recorder); + start = std::chrono::steady_clock::now(); + constexpr int Prepares = 100; + for (int round = 0; round < Prepares; ++round) table.Prepare(context, *recorder); + const auto sameEpochUs = Microseconds(start) / Prepares; + Require(table.Totals().refreshed == 0u, "nothing is refreshed while nothing was written"); + double newEpochUs = 0; + for (int round = 0; round < Prepares; ++round) { + AgcDriver::GuestMemory::BumpCollectEpoch(); + start = std::chrono::steady_clock::now(); + table.Prepare(context, *recorder); + newEpochUs += Microseconds(start); + } + newEpochUs /= Prepares; + counts = table.Totals(); + Require(counts.refreshed == 0u, "a new collect epoch with nothing written refreshes nothing (" + std::to_string(counts.refreshed) + ")"); + const auto uploads = counts.uploads; + const auto written = block.Texture(777u); + std::memset(written, 0x5a, Side * Side * 4u); + AgcDriver::GuestMemory::BumpCollectEpoch(); + start = std::chrono::steady_clock::now(); + table.Prepare(context, *recorder); + const auto refreshUs = Microseconds(start); + counts = table.Totals(); + Require(counts.refreshed == 1u, "a CPU write to one texture refreshes exactly its entry (" + std::to_string(counts.refreshed) + " refreshed)"); + const auto value = table.Find(keys[777]); + Require(value.has_value() && value->fault == 0u, "the refreshed entry stays in the table"); + recorder->Sync(); + std::printf("2000 entries: insert %.1f us per key (%.1f bytes uploaded per insert); prepare with nothing changed %.2f us in the same epoch, %.1f us in a new epoch (%llu scans); one written texture refreshed in %.1f us (%llu entries of its 64 KiB blocks checked), %llu uploads after it\n", + insertUs / WatchedTextures, static_cast(insertBytes) / WatchedTextures, sameEpochUs, newEpochUs, static_cast(counts.scans), refreshUs, static_cast(counts.checked), static_cast(counts.uploads - uploads)); +} + +void RefreshesOnOwnStores(AgcDriver::VulkanDevice& device) { + GuestBlock block; + constexpr std::uint32_t Count = 16u; + for (std::uint32_t index = 0; index < Count; ++index) std::memset(block.Texture(index), static_cast(index + 1u), Side * Side * 4u); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::GuestMemory::BumpCollectEpoch(); + if (!AgcDriver::GuestMemory::WriteWatched()) { + std::puts("guest memory has no write watch: refreshes after the driver's own stores are not tested"); + return; + } + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + std::vector keys; + for (std::uint32_t index = 0; index < Count; ++index) keys.push_back(KeyFor(TextureDescriptor(block.Texture(index), Formats[4]))); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, Count); + recorder->Sync(); + table.Prepare(context, *recorder); + table.Prepare(context, *recorder); + Require(table.Totals().entries == Count && table.Totals().refreshed == 0u, "the entries are inserted and nothing is refreshed"); + const auto before = table.EntryTexture(keys[5]); + const auto address = reinterpret_cast(block.Texture(5)); + std::memset(block.Texture(5), 0x77, Side * Side * 4u); + static_cast(AgcDriver::GuestMemory::MarkWritten(address, Side * Side * 4u)); + table.Prepare(context, *recorder); + const auto counts = table.Totals(); + Require(counts.refreshed == 1u && table.EntryTexture(keys[5]) != before, "a store the driver made in the same collect epoch refreshes the entry it covers (" + std::to_string(counts.refreshed) + " refreshed)"); + recorder->Sync(); +} + +void KeepsUnwatchedEntries(AgcDriver::VulkanDevice& device) { + GuestBlock block(false); + constexpr std::uint32_t Count = 16u; + for (std::uint32_t index = 0; index < Count; ++index) std::memset(block.Texture(index), static_cast(0x40u + index), Side * Side * 4u); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + std::vector keys; + for (std::uint32_t index = 0; index < Count; ++index) keys.push_back(KeyFor(TextureDescriptor(block.Texture(index), Formats[4]))); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, Count); + recorder->Sync(); + table.Prepare(context, *recorder); + Require(table.Totals().entries == Count, "the unwatched entries are inserted"); + AgcDriver::GuestMemory::BumpCollectEpoch(); + table.Prepare(context, *recorder); + const auto settled = table.Totals(); + for (int round = 0; round < 4; ++round) { + AgcDriver::GuestMemory::BumpCollectEpoch(); + table.Prepare(context, *recorder); + } + auto counts = table.Totals(); + Require(counts.checked == settled.checked && counts.refreshed == 0u, "a new collect epoch over unwatched entries whose bytes still match looks none of them up again (" + std::to_string(counts.checked - settled.checked) + " checked)"); + const auto before = table.EntryTexture(keys[9]); + std::memset(block.Texture(9), 0x99, Side * Side * 4u); + AgcDriver::GuestMemory::BumpCollectEpoch(); + table.Prepare(context, *recorder); + counts = table.Totals(); + Require(counts.refreshed == 1u && table.EntryTexture(keys[9]) != before, "a CPU write to one unwatched texture refreshes its entry at the next collect epoch (" + std::to_string(counts.refreshed) + " refreshed)"); + std::printf("unwatched entries: %llu checked over 5 epochs with nothing written, 1 refreshed after a CPU write\n", static_cast(settled.checked)); + recorder->Sync(); +} + +void EvictsLeastRecentlyUsed(AgcDriver::VulkanDevice& device) { + std::unique_lock gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared()), DeviceImageTable::Budget{4u, std::uint64_t{1} << 40u, 0u, 0u}); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), std::span(keys).first(4), 4u); + recorder->Sync(); + table.Prepare(context, *recorder); + Require(table.Totals().entries == 4u && table.Totals().evicted == 0u, "four entries fit the budget"); + auto oldest = table.EntryTexture(keys[0]); + Require(oldest != nullptr, "the oldest entry holds its texture"); + const auto held = oldest.use_count(); + WriteRecords(table.RingChunk(*recorder), std::span(keys).subspan(4), 1u); + recorder->Sync(); + table.Prepare(context, *recorder); + const auto counts = table.Totals(); + Require(counts.entries == 3u && counts.evicted == 2u, "a fifth entry over a budget of four evicts down to three (" + std::to_string(counts.entries) + " left, " + std::to_string(counts.evicted) + " evicted)"); + Require(!table.Find(keys[0]).has_value() && !table.Find(keys[1]).has_value(), "the two least recently used entries are evicted"); + Require(table.Find(keys[2]).has_value() && table.Find(keys[3]).has_value() && table.Find(keys[4]).has_value(), "the recently used entries stay"); + Require(oldest.use_count() == held, "an evicted entry's texture is kept by the open batch"); + recorder->Sync(); + recorder->FlushDeferredReleases(); + gpu.unlock(); + for (int wait = 0; wait < 5000 && oldest.use_count() >= held; ++wait) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(oldest.use_count() == held - 1, "the evicted texture is released once its batch completed (" + std::to_string(oldest.use_count()) + " of " + std::to_string(held) + ")"); +} + +void CountsOverflow(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, DeviceImageTable::RingCapacity + 100u); + recorder->Sync(); + const auto counts = table.Totals(); + Require(counts.drained == DeviceImageTable::RingCapacity && counts.dropped == 100u, "records past the ring's capacity are counted as dropped (drained " + std::to_string(counts.drained) + ", dropped " + std::to_string(counts.dropped) + ")"); + table.Prepare(context, *recorder); + Require(table.Totals().inserted == keys.size(), "the records that fit are still resolved"); + recorder->Sync(); +} + +void OverflowFallsBack(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + const auto keys = FormatKeys(); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + for (std::uint32_t batch = 1; batch <= DeviceImageTable::OverflowBatches; ++batch) { + Require(table.ServesProgram(ProgramAddress), "the program keeps the device path before " + std::to_string(DeviceImageTable::OverflowBatches) + " overflowing batches"); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, DeviceImageTable::RingCapacity + 1u); + recorder->Sync(); + } + Require(!table.ServesProgram(ProgramAddress) && !table.ServesIdentity(programs[0]) && table.Totals().fallbacks == 1u, "a program whose misses overflow the ring in three batches in a row after the table resolved its misses falls back to the CPU path"); + Require(table.ServesProgram(ProgramAddress + 0x100u), "only that program falls back"); + DeviceImageTable other(context, Lookup(), FakeSlots(std::make_shared())); + other.NoteProgram(Program, ProgramAddress); + constexpr std::uint32_t Overflow = DeviceImageTable::RingCapacity + 1u; + for (const auto count : {1u, Overflow, Overflow, 1u, Overflow, Overflow}) { + other.Prepare(context, *recorder, programs); + WriteRecords(other.RingChunk(*recorder), keys, count); + recorder->Sync(); + } + Require(other.ServesProgram(ProgramAddress) && other.OverflowStreak(programs[0]) == 2u, "a batch without overflow between them resets the count (streak " + std::to_string(other.OverflowStreak(programs[0])) + ")"); +} + +void OverflowsBeforeAResolveCountOnce(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + const auto keys = FormatKeys(); + for (std::uint32_t batch = 0; batch < DeviceImageTable::OverflowBatches + 1u; ++batch) { + static_cast(recorder->Commands()); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, DeviceImageTable::RingCapacity + 1u); + recorder->Submit(); + } + Require(table.Totals().drained == 0u, "the overflowing batches are still in flight"); + recorder->Sync(); + const auto drained = table.Totals(); + Require(drained.dropped == DeviceImageTable::OverflowBatches + 1u, "every batch overflowed the ring (dropped " + std::to_string(drained.dropped) + ")"); + Require(table.ServesProgram(ProgramAddress) && drained.fallbacks == 0u, "batches that overflow before the table resolved the misses of the first count as one overflow"); + table.Prepare(context, *recorder, programs); + Require(table.Totals().inserted == keys.size(), "the records that fit are resolved"); + recorder->Sync(); +} + +void OverflowsBeforeTheirInsertsDoNotCount(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + const auto keys = FormatKeys(); + std::vector serials; + for (std::uint32_t batch = 0; batch < DeviceImageTable::OverflowBatches + 2u; ++batch) { + static_cast(recorder->Commands()); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, DeviceImageTable::RingCapacity + 1u); + serials.push_back(recorder->SubmitAndEpoch()); + } + Require(table.Totals().drained == 0u, "the overflowing batches are still in flight"); + for (const auto serial : serials) { + recorder->FinishUpTo(serial); + static_cast(recorder->Commands()); + table.Prepare(context, *recorder, programs); + } + const auto drained = table.Totals(); + Require(drained.dropped == serials.size() && drained.inserted == keys.size(), "every batch overflowed the ring and the first batch's records were resolved before the others drained (dropped " + std::to_string(drained.dropped) + ", inserted " + std::to_string(drained.inserted) + ")"); + const auto streak = table.OverflowStreak(programs[0]); + Require(table.ServesProgram(ProgramAddress) && drained.fallbacks == 0u && streak == 0u, "batches taken before the inserts that resolved their misses neither count toward the overflow streak nor make the program fall back (streak " + std::to_string(streak) + ", " + std::to_string(drained.fallbacks) + " fallbacks)"); + recorder->Sync(); +} + +void WarmOverflowFallsBack(AgcDriver::VulkanDevice& device) { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared())); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + const auto warm = FormatKeys(); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), warm, static_cast(warm.size())); + recorder->Sync(); + std::vector keys(DeviceImageTable::RingCapacity + 1u, warm[0]); + for (std::uint32_t batch = 0; batch < DeviceImageTable::OverflowBatches; ++batch) { + Require(table.ServesProgram(ProgramAddress), "the program keeps the device path before " + std::to_string(DeviceImageTable::OverflowBatches) + " overflowing batches"); + for (std::uint32_t index = 0; index < keys.size(); ++index) keys[index][Abi::SamplerWord] = batch * static_cast(keys.size()) + index + 1u; + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + } + const auto counts = table.Totals(); + const auto resolved = warm.size() + (DeviceImageTable::OverflowBatches - 1u) * DeviceImageTable::RingCapacity; + Require(counts.inserted == resolved && counts.dropped == DeviceImageTable::OverflowBatches, "the table resolved the warm keys and each batch's recorded misses before the next batch was taken (" + std::to_string(counts.inserted) + " inserted, " + std::to_string(counts.dropped) + " dropped)"); + Require(!table.ServesProgram(ProgramAddress) && counts.fallbacks == 1u, "a program that misses more distinct keys than the ring holds in three batches in a row, each taken after the previous batch's misses were inserted, falls back to the CPU path"); + const auto reason = "its misses overflowed the ring of " + std::to_string(DeviceImageTable::RingCapacity) + " records in " + std::to_string(DeviceImageTable::OverflowBatches) + " batches in a row"; + Require(table.LastFallback() == reason, "the fallback gives the overflow reason: " + table.LastFallback()); +} + +void FullTableFallsBack(AgcDriver::VulkanDevice& device) { + GuestBlock block; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + const auto context = device.graphicsContext(); + DeviceImageTable table(context, Lookup(), FakeSlots(std::make_shared()), DeviceImageTable::Budget{1u << 20u, std::uint64_t{1} << 40u, 0u, 0u, 16u}); + table.NoteProgram(Program, ProgramAddress); + const std::array programs{Program}; + std::vector keys; + for (std::uint32_t index = 0; index < 12u; ++index) keys.push_back(KeyFor(TextureDescriptor(block.Texture(index), Formats[4]))); + table.Prepare(context, *recorder, programs); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder, programs); + const auto counts = table.Totals(); + Require(counts.buckets == 16u && counts.entries < keys.size() && counts.tableFull != 0u, "a table of at most 16 buckets fills (" + std::to_string(counts.entries) + " entries)"); + Require(!table.ServesProgram(ProgramAddress) && counts.fallbacks == 1u, "a program whose key finds the table full falls back to the CPU path"); + recorder->Sync(); +} + +alignas(4096) std::array CubeTexels{}; + +void NonSeamlessCube(AgcDriver::VulkanDevice& device) { + constexpr std::uint32_t TypeCube = 11; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + const auto context = device.graphicsContext(); + auto cube = TextureDescriptor(CubeTexels.data(), Formats[4]); + cube[3] = 0xfacu | (TypeCube << 28u); + cube[4] = 5u; + const DeviceTableView view{ShaderRecompiler::DescriptorImageShape::Image2DArray, false, false}; + const auto seamless = LookupDeviceTableTexture(context, cube, view, 0u); + Require(seamless.texture != nullptr && seamless.unservable.empty(), "a cube T# resolves through the texture cache"); + try { + const auto resolved = LookupDeviceTableTexture(context, cube, view, Abi::SamplerNonSeamlessCube); + Require(context.nonSeamlessCubeMap && resolved.texture != nullptr, "a non-seamless cube sampler is served only by a device with VK_EXT_non_seamless_cube_map"); + } catch (const std::runtime_error& error) { + Require(!context.nonSeamlessCubeMap && std::string(error.what()).find("needs VK_EXT_non_seamless_cube_map") != std::string::npos, std::string("a non-seamless cube sampler is rejected only without VK_EXT_non_seamless_cube_map: ") + error.what()); + } + std::printf("non-seamless cube sampler: %s (VK_EXT_non_seamless_cube_map %s)\n", context.nonSeamlessCubeMap ? "served" : "rejected", context.nonSeamlessCubeMap ? "on" : "off"); +} + +void RegistersInterfaces() { + using ShaderRecompiler::ResourceMaterializer; + auto image = SampledInterface()->image; + image.source = 7u; + image.firstUsePc = 0x40u; + image.table = 3u; + const auto modes = ResourceMaterializer::RuntimeImageModes(image); + const auto id = ResourceMaterializer::DeviceImageInterfaceId(image, modes); + Require(ResourceMaterializer::FindDeviceImageInterface(id) == nullptr, "an interface no program registered is not found"); + Require(ResourceMaterializer::RegisterDeviceImageInterface(image, modes) == id, "registering an interface gives its id"); + const auto found = ResourceMaterializer::FindDeviceImageInterface(id); + Require(found != nullptr && found->id == id && found->modes.size() == modes.size() && found->image.source == 0u && found->image.table == 0u, "the registry holds the interface without the program's source, pc or table number"); + auto elsewhere = image; + elsewhere.source = 9u; + elsewhere.firstUsePc = 0x80u; + elsewhere.table = 1u; + Require(ResourceMaterializer::DeviceImageInterfaceId(elsewhere, modes) == id && ResourceMaterializer::RegisterDeviceImageInterface(elsewhere, modes) == id, "the same image read from another source, pc or table shares the interface"); + auto integer = image; + integer.numericClass = ShaderRecompiler::IrTextureNumericClass::Uint; + Require(ResourceMaterializer::DeviceImageInterfaceId(integer, ResourceMaterializer::RuntimeImageModes(integer)) != id, "an image of another numeric class has another interface"); + const auto differingIn = [&](const std::string& field, bool hashed, auto change) { + auto other = image; + change(other); + const auto otherId = ResourceMaterializer::DeviceImageInterfaceId(other, modes); + Require(ResourceMaterializer::RegisterDeviceImageInterface(other, modes) == otherId, "an image differing only in " + field + " registers"); + Require((otherId != id) == hashed, "an image differing only in " + field + (hashed ? " has another interface" : " shares the interface")); + }; + differingIn("adjustedSample", false, [](ShaderRecompiler::ImageResource& other) { other.adjustedSample = !other.adjustedSample; }); + differingIn("dimensionQueryOnly", true, [](ShaderRecompiler::ImageResource& other) { other.dimensionQueryOnly = !other.dimensionQueryOnly; }); + differingIn("sampledOnly", true, [](ShaderRecompiler::ImageResource& other) { other.sampledOnly = !other.sampledOnly; }); + differingIn("optionalDescriptor", false, [](ShaderRecompiler::ImageResource& other) { other.optionalDescriptor = !other.optionalDescriptor; }); + differingIn("targetMerge", true, [](ShaderRecompiler::ImageResource& other) { other.targetMerge = !other.targetMerge; }); + auto otherModes = modes; + otherModes.front().sampledOnly = !otherModes.front().sampledOnly; + const auto modeId = ResourceMaterializer::DeviceImageInterfaceId(image, otherModes); + Require(ResourceMaterializer::RegisterDeviceImageInterface(image, otherModes) == modeId && modeId != id, "a mode differing only in sampledOnly registers under another interface"); +} + +void DevicePathRefusals() { + Require(!DeviceImagePathRefusal({true, true}).has_value(), "the GPU heap on a device with VK_EXT_descriptor_heap serves the device image path"); + const auto withoutHeap = DeviceImagePathRefusal({false, true}); + Require(withoutHeap.has_value() && withoutHeap->find("APS5_GPU_HEAP") != std::string::npos, "the device image path stays off without APS5_GPU_HEAP and says so"); + const auto withoutExtension = DeviceImagePathRefusal({true, false}); + Require(withoutExtension.has_value() && withoutExtension->find("VK_EXT_descriptor_heap") != std::string::npos, "the device image path stays off without VK_EXT_descriptor_heap and says so"); + Require(DeviceImagePathRefusal({false, false}).has_value(), "the device image path stays off with neither"); +} + +int HeapRetention() { + auto device = OpenVulkanTestDevice(); + if (device == nullptr) return VulkanTestSkipped; + if (!device->graphicsContext().descriptorHeap) { + std::printf("skipped, the device lacks VK_EXT_descriptor_heap, which the GPU heap the device image table takes its slots from needs\n"); + return VulkanTestSkipped; + } + { + const auto context = device->graphicsContext(); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + DeviceImageTable table(context, Lookup(), DeviceImageTable::HeapSlots(context)); + const auto keys = FormatKeys(); + table.Prepare(context, *recorder); + WriteRecords(table.RingChunk(*recorder), keys, static_cast(keys.size())); + recorder->Sync(); + table.Prepare(context, *recorder); + Require(table.Totals().entries == keys.size(), "the heap gives every format a slot"); + auto& heap = BindlessHeap::For(context); + std::vector> generations; + for (const auto& key : keys) { + const auto value = table.Find(key); + Require(value.has_value(), "every format is inserted"); + Require(heap.SlotTexture(KindOf(*value), value->slot) == table.EntryTexture(key).get(), "the heap slot holds the entry's texture"); + generations.emplace_back(KindOf(*value), heap.SlotGeneration(value->slot)); + } + table.Prepare(context, *recorder); + for (std::size_t index = 0; index < keys.size(); ++index) { + const auto value = table.Find(keys[index]); + Require(value.has_value() && heap.SlotGeneration(value->slot) == generations[index].second, "a texture the table holds keeps its heap slot"); + } + Require(table.Totals().swept == 0u, "nothing left the table"); + recorder->Sync(); + } + device->WaitIdle(); + device.reset(); + std::cout << "device image table heap retention tests passed\n"; + return 0; +} + +} + +int main(int argc, char** argv) { + try { + const std::string mode = argc > 1 ? argv[1] : ""; + for (std::size_t index = 0; index < Textures.size(); ++index) { + for (std::size_t texel = 0; texel < Textures[index].size(); ++texel) Textures[index][texel] = static_cast(0x01020304u * (index + 1u) + texel); + } + if (mode == "--heap") return HeapRetention(); + RegistersInterfaces(); + DevicePathRefusals(); + auto device = OpenVulkanTestDevice(); + if (device == nullptr) return VulkanTestSkipped; + HeaderAtCreation(*device); + ResolvesAndUploads(*device); + TrailerAddresses(*device); + SweptSlotsLeave(*device); + SweptWhileWaitingForAChunk(*device); + LiveSlotsTouched(*device); + MarginRefuses(*device); + RejectsAndFaults(*device); + CountsOverflow(*device); + RefreshesWrittenEntries(*device); + RefreshesOnOwnStores(*device); + KeepsUnwatchedEntries(*device); + EvictsLeastRecentlyUsed(*device); + FullTableFallsBack(*device); + OverflowFallsBack(*device); + OverflowsBeforeAResolveCountOnce(*device); + OverflowsBeforeTheirInsertsDoNotCount(*device); + WarmOverflowFallsBack(*device); + NonSeamlessCube(*device); + device->WaitIdle(); + device.reset(); + std::cout << "device image table tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << "device image table tests failed: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp index 34895d5545ec..f8fbeea2a51c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp @@ -383,7 +383,7 @@ class Fixture { return state; } - void Draw(const ShaderRecompiler::RecompileResult& vertex, const AgcDriver::Pm4::DrawParameters& draw, bool compactRestart = false) { + void Draw(const ShaderRecompiler::RecompileResult& vertex, const AgcDriver::Pm4::DrawParameters& draw, bool compactRestart = false, const ShaderRecompiler::HeapStageWords* heap = nullptr) { const auto vertexPush = static_cast(vertex.pushConstants.size()); ShaderRecompiler::ShaderPixelStageInfo pixelInfo{}; pixelInfo.wave32 = true; @@ -392,7 +392,7 @@ class Fixture { ShaderRecompiler::RecompileRequest pixelRequest{{ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {32u, 0u, {}, std::nullopt, pixelInfo, std::nullopt, {}}, device->Target(), PixelPushLayout(vertexPush, device->Target())}; pixelRequest.useCache = false; const auto pixel = ShaderRecompiler::Recompile(pixelRequest); - const std::array shaders{{{ShaderStage::Vertex, &vertex, 0u}, {ShaderStage::Fragment, &pixel, PixelPushOffset(vertexPush, device->Target())}}}; + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0u, nullptr, heap}, {ShaderStage::Fragment, &pixel, PixelPushOffset(vertexPush, device->Target())}}}; Output.fill(Unwritten); if (compactRestart) { auto context = device->graphicsContext(); @@ -458,8 +458,8 @@ std::string PathName(const std::optional& path) { return path ? "the CPU path (" + std::string(AgcDriver::Graphics::IndirectDrawPathName(*path)) + ")" : std::string("the GPU"); } -std::string Classified(Fixture& fixture, const std::string& what, const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters& draw, std::optional expectedPath, Rule expectedRule, bool rectList = false) { - const auto path = AgcDriver::DriverDetail::Driver::ClassifyIndirectDraw(vertex, fixture.State(rectList), Program(userData), fixture.device, draw, false); +std::string Classified(Fixture& fixture, const std::string& what, const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters& draw, std::optional expectedPath, Rule expectedRule, bool rectList = false, const ShaderRecompiler::HeapStageWords* heap = nullptr) { + const auto path = AgcDriver::DriverDetail::Driver::ClassifyIndirectDraw(vertex, fixture.State(rectList), Program(userData), fixture.device, draw, false, heap); const auto& indirect = *draw.indirect; const bool rules = expectedPath || (indirect.vertexRule == expectedRule && indirect.instanceRule == expectedRule); std::cout << what << ": classified to " << PathName(path) << '\n'; @@ -519,6 +519,28 @@ std::string RuntimeFetch(Fixture& fixture, std::uint32_t index, bool compactRest return std::equal(expected.begin(), expected.end(), Output.begin()) ? "" : "runtime-fetched VertexIndex was changed\n"; } +std::string HeapAttributes(Fixture& fixture) { + const auto userData = fixture.UserData(0, 0); + const auto perDraw = fixture.Vertex(VertexCode, userData); + const auto records = BufferTable[2]; + BufferTable[2] = 1u; + const auto shared = fixture.Vertex(VertexCode, userData); + BufferTable[2] = records; + Require(!perDraw.vertexAttributes.empty() && shared.vertexAttributes.size() == perDraw.vertexAttributes.size(), "the vertex buffers did not become fixed vertex attributes"); + ShaderRecompiler::HeapStageWords words; + words.vertexAttributes = perDraw.vertexAttributes; + const auto index = static_cast(VertexSlots.size() - 1u); + const std::array fetched{index}; + fixture.Place(0, fetched, 0); + alignas(256) static std::array indices{}; + indices = {0xffffffffu, index, index, index, 0xffffffffu}; + const AgcDriver::Pm4::DrawParameters draw{reinterpret_cast(indices.data()), 5, 4, 1, 0, true, 0, 0}; + fixture.Draw(shared, draw, true, &words); + std::cout << "heap vertex attributes, index " << index << ", compact restart: readback " << Output[0] << ' ' << Output[1] << ' ' << Output[2] << ' ' << Output[3] << '\n'; + const std::array expected{index, 0, index, 0}; + return std::equal(expected.begin(), expected.end(), Output.begin()) ? "" : "a compacted index inside the draw's own vertex buffer was clamped by the shared program's buffer size\n"; +} + std::string Run(Fixture& fixture) { std::string failures = Analysis(fixture); const std::array autoIndices{0, 1, 2}; @@ -592,6 +614,13 @@ std::string Run(Fixture& fixture) { AgcDriver::Pm4::DrawParameters draw{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; draw.indirect = indexedIndirect; failures += Classified(fixture, what, vertex, staleUserData, draw, std::nullopt, Rule::InPlace); + if (code == &PostAddCode) { + ShaderRecompiler::HeapStageWords words; + words.vertexAttributes = vertex.vertexAttributes; + for (auto& attribute : words.vertexAttributes) attribute.resource.fields[2] = 0x2000000u; + auto heapDraw = draw; + failures += Classified(fixture, what + ", with a 128 MiB vertex buffer in its heap words", vertex, staleUserData, heapDraw, Path::VertexRange, Rule::InPlace, false, &words); + } fixture.Place(indexedRecord[3], bufferIndices, indexedRecord[4]); fixture.Draw(vertex, draw); failures += fixture.Check(what + ", drawn in place on the GPU", bufferIndices, indexedRecord[3], indexedRecord[4], false); @@ -633,6 +662,7 @@ int main(int argc, char** argv) { Fixture fixture(device); std::string failures; if (argc == 1) failures = Run(fixture); + else if (argc == 2 && std::string(argv[1]) == "heap-attributes") failures = HeapAttributes(fixture); else { Require(argc == 3 || argc == 4, "expected runtime-fetch INDEX [compact-restart]"); Require(std::string(argv[1]) == "runtime-fetch", "unknown case"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawRelocateUserValues.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawRelocateUserValues.cpp index 1c2407762fac..e89a689ee1a6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DrawRelocateUserValues.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawRelocateUserValues.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" @@ -9,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -163,6 +165,30 @@ void BufferValueWordsArePatchedInTheirBinding() { Require(sharedCount->valueSlots.size() == 1u && sharedCount->valueSlots[0] == std::array{4u, 0u, 0u}, "a V# whose record count builds a second descriptor recorded a record-count patch"); const auto ownCount = shared(descriptor[2] + 1u); Require((ownCount->valueWords >> 6u & 1u) != 0u, "a record count used by one descriptor only was not tagged next to an independent second descriptor"); + + const auto tagged = [&](bool forwardedAsPushData, bool shaderDataCopy, bool heapMode) { + auto compiled = result(false); + if (!shaderDataCopy) { + compiled->pushConstants.resize(sizeof(std::uint32_t)); + std::memcpy(compiled->pushConstants.data(), &descriptor[0], sizeof(std::uint32_t)); + } + if (forwardedAsPushData) compiled->shaderDataUserWords = {4u}; + const std::vector userWords(program.userData.begin(), program.userData.end()); + if (shaderDataCopy) compiled->bindings.push_back(ShaderRecompiler::DescriptorBinding{ShaderRecompiler::DescriptorKind::UniformBuffer, ShaderRecompiler::DescriptorRole::ShaderData, 1, 0, 1, heapMode ? std::vector(userWords.size(), 0u) : userWords}); + const auto variant = Variant(compiled); + ShaderRecompiler::HeapStageWords words; + words.userData = userWords; + words.bindings = {descriptor, {}}; + Driver::tagUserValueWords(*variant, program, heapMode ? &words : nullptr); + return variant->valueWords; + }; + for (const auto forwardedAsPushData : {false, true}) { + const auto expected = forwardedAsPushData ? std::uint64_t {(1u << 4u) | (1u << 6u)} : std::uint64_t {0}; + Require(tagged(forwardedAsPushData, false, false) == expected, std::string("default: a V# base ") + (forwardedAsPushData ? "forwarded as push data from its own user word" : "also present in push data but not forwarded from it") + " was tagged as " + std::to_string(tagged(forwardedAsPushData, false, false))); + Require(tagged(forwardedAsPushData, false, true) == tagged(forwardedAsPushData, false, false), std::string("heap and default disagree on a V# base ") + (forwardedAsPushData ? "forwarded as push data" : "that is not forwarded") + " (heap " + std::to_string(tagged(forwardedAsPushData, false, true)) + ", default " + std::to_string(tagged(forwardedAsPushData, false, false)) + ")"); + } + Require(tagged(false, true, false) == 0u, "default: a V# base copied into a shader data binding was tagged"); + Require(tagged(false, true, true) == ((1u << 4u) | (1u << 6u)), "heap: the user-word copy in shader data hid a V# from tagging (mask " + std::to_string(tagged(false, true, true)) + ")"); } alignas(256) constexpr std::array VertexCode{ @@ -177,6 +203,10 @@ alignas(256) constexpr std::array OwnCountPixelCode{ 0xbe880304u, 0xbe890305u, 0xbe8a0306u, 0xbe8b0307u, 0xf4200300u, 0xfa000000u, 0xf4200344u, 0xfa000004u, 0xbf8cc07fu, 0x7e00020cu, 0x7e02020du, 0x7e040280u, 0x7e0602f2u, 0xf800180fu, 0x03020100u, 0xbf810000u, }; +alignas(256) constexpr std::array ForwardedBasePixelCode{ + 0xbe880304u, 0xbe890305u, 0xbe8a0306u, 0xbe8b0307u, 0xf4200300u, 0xfa000000u, 0xf4200344u, 0xfa000004u, + 0xbf8cc07fu, 0x7e00020cu, 0x7e02020du, 0x7e040280u, 0x7e062200u, 0xf800180fu, 0x03020100u, 0xbf810000u, +}; alignas(256) std::array ColorMemory{}; struct Header { @@ -187,6 +217,7 @@ struct Header { alignas(8) Header VertexHeader{}; alignas(8) Header SharedCountHeader{}; alignas(8) Header OwnCountHeader{}; +alignas(8) Header ForwardedBaseHeader{}; void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) { header.shader.file_header = 0x34333231u; @@ -266,12 +297,21 @@ void MovedRecordCountsReachEveryDescriptor(bool sharedCount) { return std::array{Low(first[i].data()), High(first[i].data()), sharedCount ? records : 12u, 0x20077facu, Low(second[i].data()), High(second[i].data()), sharedCount ? 4u : records, 0x20077facu}; }; const auto start = Driver::Get().DrawCacheCounters(); + const auto heapStart = AgcDriver::DriverDetail::HeapStagesTaken(); + const auto ringStart = AgcDriver::DriverDetail::HeapRingHitsVerified(); for (std::size_t i = 0; i < 5; ++i) { const auto pixel = DrawWithUserData(pixelCode, userData(i, 4u)); Require(pixel == std::array{255, 0, 0, 255}, mode + ": a draw whose second descriptor holds one dword drew " + std::to_string(pixel[0]) + "," + std::to_string(pixel[1]) + "," + std::to_string(pixel[2]) + "," + std::to_string(pixel[3])); + first[i] = {0.0f, 0.0f, 1.0f, 0.0f}; + second[i] = {1.0f, 0.0f, 1.0f, 0.0f}; } const auto taught = Driver::Get().DrawCacheCounters(); Require(taught.hits > start.hits, mode + ": draws whose descriptors moved to equal contents never reused the cached draw (" + std::to_string(taught.lookups - start.lookups) + " lookups)"); + if (std::getenv("APS5_VERIFY_GPU_HEAP") != nullptr) { + Require(AgcDriver::DriverDetail::HeapStagesTaken() > heapStart, mode + ": the draws never reached the GPU heap"); + Require(AgcDriver::DriverDetail::HeapRingHitsVerified() > ringStart, mode + ": no relocated ring hit was compared with a fresh capture"); + Require(AgcDriver::DriverDetail::HeapRingHitsDiffering() == 0, mode + ": a relocated ring hit's rebuilt words differ from a fresh capture"); + } const auto pixel = DrawWithUserData(pixelCode, userData(5, 8u)); Require(pixel == std::array{255, 255, 0, 255}, mode + ": after the record count grew to two dwords, the second descriptor's second dword drew " + std::to_string(pixel[1]) + " (a reused draw kept the earlier count)"); if (!sharedCount) { @@ -280,6 +320,38 @@ void MovedRecordCountsReachEveryDescriptor(bool sharedCount) { } } +struct alignas(256) Block { + std::array, 16> entries{}; +}; + +void ForwardedBaseWordsFollowRelocatedDraws() { + Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2); + Registered(ForwardedBaseHeader, ForwardedBasePixelCode.data(), sizeof(ForwardedBasePixelCode), 1); + static std::array blocks{}; + alignas(16) static std::array, 8> other{}; + for (auto& block : blocks) { + for (auto& entry : block.entries) entry = {1.0f, 0.0f, 0.0f, 0.0f}; + } + for (auto& entry : other) entry = {0.0f, 1.0f, 0.0f, 0.0f}; + const auto ringStart = AgcDriver::DriverDetail::HeapRingHitsVerified(); + const auto start = Driver::Get().DrawCacheCounters(); + for (std::size_t i = 0; i < blocks.size(); ++i) { + const bool lowByteSet = i % 2 == 1; + const auto* first = blocks[i].entries[lowByteSet ? 1 : 0].data(); + const auto* second = other[i % other.size()].data(); + const std::array words{Low(first), High(first), 4u, 0x20077facu, Low(second), High(second), 4u, 0x20077facu}; + const auto pixel = DrawWithUserData(ForwardedBasePixelCode.data(), words); + const std::array expected{255, 0, 0, static_cast(lowByteSet ? 255 : 0)}; + Require(pixel == expected, "draw " + std::to_string(i) + " read a stale forwarded V# base word: drew " + std::to_string(pixel[0]) + "," + std::to_string(pixel[1]) + "," + std::to_string(pixel[2]) + "," + std::to_string(pixel[3]) + ", expected alpha " + std::to_string(expected[3])); + } + const auto after = Driver::Get().DrawCacheCounters(); + Require(after.hits > start.hits, "draws that moved only their first V# never reused the cached draw, so the forwarded base word was never rebuilt for a hit"); + if (std::getenv("APS5_VERIFY_GPU_HEAP") != nullptr) { + Require(AgcDriver::DriverDetail::HeapRingHitsVerified() > ringStart, "no relocated ring hit of the forwarded-base draws was compared with a fresh capture"); + Require(AgcDriver::DriverDetail::HeapRingHitsDiffering() == 0, "a relocated ring hit of the forwarded-base draws differs from a fresh capture"); + } +} + void BufferValueWordsKeyByNothing() { const RelocatableUserWord word{4u, {}, true, 0u}; bool readable = true; @@ -301,6 +373,7 @@ int main() { } MovedRecordCountsReachEveryDescriptor(true); MovedRecordCountsReachEveryDescriptor(false); + ForwardedBaseWordsFollowRelocatedDraws(); AgcDriverShutdown_nid_postfix(); std::cout << "User-data V# values and fetch subroutine pointers relocate\n"; return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HeapComputeSample.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HeapComputeSample.cpp new file mode 100644 index 000000000000..d9e3e2aa3178 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HeapComputeSample.cpp @@ -0,0 +1,175 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "Recompiler.hpp" +#include "HeapSampleKernel.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr std::array OtherTexel{0xf0u, 0x20u, 0x00u, 0x40u}; + +alignas(256) constexpr std::array LdsSampleCode{ + 0x7e0002f0u, 0x7e0202f0u, 0xf09c0f08u, 0x00400000u, 0x7e080280u, 0xbf8c0000u, 0xd8340000u, 0x00000004u, + 0xbf8c0000u, 0xd8d80000u, 0x00000004u, 0xbf8c0000u, 0xe0780000u, 0x80030000u, 0xbf810000u, 0xbf810000u}; + +alignas(256) std::array TextureMemory{}; +alignas(256) std::array OtherTextureMemory{}; +alignas(4096) std::array, 4> Outputs{}; +constexpr std::size_t FullHeapTextures = 16; +alignas(256) std::array, FullHeapTextures> ManyTextures{}; + +struct Header { + Shader shader{}; + ShaderUserData userData{}; +}; + +alignas(8) Header SampleHeader{}; + +void Registered(Header& header, const std::uint32_t* code, std::size_t bytes) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(Header); + header.shader.shader_size = static_cast(bytes); + header.shader.code = code; + header.shader.type = 0; + header.shader.user_data = &header.userData; + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + +void SetShaderRegister(std::vector& words, std::uint32_t offset, std::uint32_t value) { + words.insert(words.end(), {0xc0017600u, offset, value}); +} + +void Sample(std::vector& words, const void* image, float* output) { + const auto address = static_cast(reinterpret_cast(SampleCode.data())); + SetShaderRegister(words, 0x20c, static_cast(address >> 8u)); + SetShaderRegister(words, 0x20d, static_cast(address >> 40u) & 0xffu); + for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) SetShaderRegister(words, offset, 1); + SetShaderRegister(words, 0x212, 0); + SetShaderRegister(words, 0x213, 16u << 1u); + const auto userData = SampleUserData(image, output); + std::uint32_t offset = 0x240; + for (const auto value : userData) SetShaderRegister(words, offset++, value); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void Submit(std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(0x20, &packet) == 0, "the compute submission failed"); + AgcDriverWaitIdle_nid_postfix(); +} + +void DriverSample(bool heap) { + Outputs[0].fill(-1.0f); + const auto before = AgcDriver::Graphics::HeapComputeStages(); + std::vector words; + Sample(words, TextureMemory.data(), Outputs[0].data()); + Sample(words, TextureMemory.data(), Outputs[0].data()); + Submit(words); + RequireSampled(Outputs[0].data(), SampleTexel, "the sampled texture", heap); + if (heap) Require(AgcDriver::Graphics::HeapComputeStages() > before, "the heap did not take the sampling compute stage"); +} + +void FullHeapSample() { + Outputs[0].fill(-1.0f); + std::vector words; + for (std::size_t index = 0; index < FullHeapTextures; ++index) Sample(words, ManyTextures[index].data(), Outputs[0].data() + 4u * index); + Submit(words); + for (std::size_t index = 0; index < FullHeapTextures; ++index) + RequireSampled(Outputs[0].data() + 4u * index, SampleTexel, "texture " + std::to_string(index) + " of a heap with too few image slots", true); + Require(AgcDriver::Graphics::BindlessHeap::NearlyFull(), "the capped heap never ran short of image slots"); +} + +void DeviceSample(AgcDriver::VulkanDevice& device, bool heap) { + for (std::size_t index = 1; index < Outputs.size(); ++index) Outputs[index].fill(-1.0f); + const auto first = CompileSampleKernel(device, OtherTextureMemory.data(), Outputs[1].data(), false); + const auto second = CompileSampleKernel(device, TextureMemory.data(), Outputs[2].data(), false); + Require(first.gpuResolvedImages == heap && second.gpuResolvedImages == heap, + heap ? "a sampling compute stage is not GPU mode" : "a compute stage took the heap without it"); + const auto before = AgcDriver::Graphics::HeapComputeStages(); + const auto pushesBefore = AgcDriver::Graphics::DataAddressPushes(); + device.Dispatch(first, 1u, 1u, 1u, {}, reinterpret_cast(SampleCode.data())); + device.Dispatch(second, 1u, 1u, 1u, {}, reinterpret_cast(SampleCode.data())); + device.WaitIdle(); + RequireSampled(Outputs[1].data(), OtherTexel, "the first of two textures in one batch", heap); + RequireSampled(Outputs[2].data(), SampleTexel, "the second of two textures in one batch", heap); + if (heap) { + Require(AgcDriver::Graphics::HeapComputeStages() >= before + 2u, "the heap did not take both sampling compute stages"); + Require(AgcDriver::Graphics::DataAddressPushes() >= pushesBefore + 2u, + "a heap dispatch did not read its shader data through pushed addresses"); + } + const auto again = CompileSampleKernel(device, TextureMemory.data(), Outputs[3].data(), false); + device.Dispatch(again, 1u, 1u, 1u, {}, reinterpret_cast(SampleCode.data())); + device.WaitIdle(); + RequireSampled(Outputs[3].data(), SampleTexel, "the first texture again in a later batch", heap); + Outputs[1].fill(-1.0f); + const auto lds = CompileSampleKernel(device, TextureMemory.data(), Outputs[1].data(), false, LdsSampleCode, + (device.ComputeTarget(32u).maxWorkgroupSharedMemoryBytes + 4096u) / 4u); + Require(lds.workgroupMemoryDwords != 0u && lds.gpuResolvedImages == heap, + "the LDS sampling stage did not get workgroup memory in the expected mode"); + const auto heapPipelines = device.DescriptorHeapComputePipelines(); + device.Dispatch(lds, 1u, 1u, 1u, {}, reinterpret_cast(LdsSampleCode.data())); + device.WaitIdle(); + RequireSampled(Outputs[1].data(), SampleTexel, "the texture sampled through workgroup memory", heap); + if (heap) + Require(device.DescriptorHeapComputePipelines() > heapPipelines, "the LDS sampling stage did not take the descriptor heap"); +} + +} + +int main() { + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + const bool heap = std::getenv("APS5_GPU_HEAP") != nullptr; + if (heap && SkipWithoutDescriptorHeap()) return VulkanTestSkipped; + FillTexels(TextureMemory, SampleTexel); + FillTexels(OtherTextureMemory, OtherTexel); + for (auto& texture : ManyTextures) FillTexels(texture, SampleTexel); + Registered(SampleHeader, SampleCode.data(), sizeof(SampleCode)); + if (heap && std::getenv("APS5_HEAP_SLOT_CAP") != nullptr) { + FullHeapSample(); + AgcDriverShutdown_nid_postfix(); + std::cout << "compute stages sampled every texture after the heap ran short of image slots\n"; + return 0; + } + DriverSample(heap); + AgcDriverShutdown_nid_postfix(); + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + DeviceSample(*device, heap); + } + std::cout << "compute stages sampled their textures" << (heap ? " through the heap" : "") << '\n'; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception& shutdownError) { + std::cerr << shutdownError.what() << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HeapDrawSample.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HeapDrawSample.cpp new file mode 100644 index 000000000000..28faa160a936 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HeapDrawSample.cpp @@ -0,0 +1,470 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::byte Kept{0x40}; +constexpr std::array Texel{0x10u, 0x80u, 0xc0u, 0xffu}; + +alignas(256) constexpr std::array VertexCode{ + 0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u, + 0x7e060280u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, +}; +alignas(256) constexpr std::array PixelCode{0x7e0002f0u, 0x7e0202f0u, 0xf09c0f08u, 0x00400000u, + 0xbf8c0000u, 0xf800180fu, 0x03020100u, 0xbf810000u, + 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array BufferPixelCode{ + 0x7e0002f0u, 0x7e0202f0u, 0xf4200506u, 0xfa000000u, 0xf09c0f08u, 0x00400000u, 0xbf8c0000u, 0x7e060214u, + 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array GdsPixelCode{ + 0xbefc03ffu, 0x0000ffffu, 0x7e0002f0u, 0x7e0202f0u, 0xd8fa0010u, 0x04000000u, 0xf09c0f08u, 0x00400000u, + 0xbf8c0000u, 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array StoragePixelCode{ + 0x7e0002f0u, 0x7e0202f0u, 0xf09c0f08u, 0x00400000u, 0xbf8c0000u, 0x7e100280u, 0x7e120280u, 0xf0201f08u, + 0x00040008u, 0xbf8c0000u, 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array LdsPixelCode{ + 0x7e0002f0u, 0x7e0202f0u, 0xf09c0f08u, 0x00400000u, 0xbf8c0000u, 0x7e080280u, 0xd8344400u, 0x00000004u, + 0xbf8cc07fu, 0xd8d84400u, 0x00000004u, 0xbf8cc07fu, 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u}; +alignas(4096) std::array StorageMemory{}; +alignas(256) constexpr std::array SamplingVertexCode{ + 0x7e1402f0u, 0x7e1602f0u, 0xf09c0f08u, 0x00820c0au, 0xbf8c0000u, 0xf800020fu, 0x0f0e0d0cu, 0x7e000d05u, + 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, 0x10040500u, 0x060404f3u, 0x7e060280u, + 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u, +}; +alignas(256) constexpr std::array InterpolantPixelCode{0xc8020002u, 0xc8060102u, 0xc80a0202u, 0xc80e0302u, + 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array BdaPixelCode{ + 0x7e0002f0u, 0x7e0202f0u, 0x7e08020cu, 0x7e0a020du, 0xdc308000u, 0x067d0004u, 0xf09c0f08u, 0x00400000u, + 0xbf8c0000u, 0x7e060306u, 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array TablePixelCode{ + 0xf4200404u, 0xfa000000u, 0xbf8cc07fu, 0x8f108510u, 0xf42c0500u, 0x20000000u, 0xbf8cc07fu, 0x7e0002f0u, + 0x7e0202f0u, 0xf09c0f08u, 0x00250000u, 0xbf8c0000u, 0xf800180fu, 0x03020100u, 0xbf810000u, 0xbf810000u}; +alignas(256) constexpr std::array TableVertexCode{ + 0xf4200508u, 0xfa000000u, 0xbf8cc07fu, 0x8f148514u, 0xf42c0604u, 0x28000000u, 0xbf8cc07fu, 0x7e1402f0u, 0x7e1602f0u, 0xf09c0f08u, + 0x00660c0au, 0xbf8c0000u, 0xf800020fu, 0x0f0e0d0cu, 0x7e000d05u, 0x080200f4u, 0x10020300u, 0x100202f4u, 0x060202f3u, 0x060400f3u, + 0x10040500u, 0x060404f3u, 0x7e060280u, 0x7e0802f2u, 0xf80008cfu, 0x04030201u, 0xbf810000u, 0xbf810000u, +}; +alignas(256) std::array TableMemory{}; +alignas(256) std::array MixedVertexCode{}; +alignas(256) std::array MixedPixelCode{}; +alignas(256) std::array MaterialMemory{}; +alignas(256) std::array OtherTextureMemory{}; +constexpr float BufferAlpha = 0.2f; +constexpr std::uint8_t BufferAlphaByte = 51u; +constexpr float SecondAlpha = 0.6f; +constexpr std::uint8_t SecondAlphaByte = 153u; +alignas(4096) std::array BufferMemory{}; +alignas(256) std::array ColorMemory{}; +alignas(256) std::array TextureMemory{}; + +struct Header { + Shader shader{}; + ShaderUserData userData{}; +}; + +alignas(8) Header VertexHeader{}; +alignas(8) Header PixelHeader{}; + +void Registered(Header& header, const std::uint32_t* code, std::size_t bytes, std::uint8_t type) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(Header); + header.shader.shader_size = static_cast(bytes); + header.shader.code = code; + header.shader.type = type; + header.shader.user_data = &header.userData; + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + +void Append(std::vector& commands, std::uint32_t opcode, std::initializer_list payload) { + commands.push_back(0xc0000000u | (static_cast(payload.size() - 1) << 16u) | (opcode << 8u)); + commands.insert(commands.end(), payload); +} + +enum class Mode { Texture, Buffer, Storage, Gds, Vertex, Bda, Table, VertexTable, MovedBuffer, MovedTexture, PixelLds }; + +const std::uint32_t* VertexProgram(Mode mode) { + if (mode == Mode::VertexTable) return TableVertexCode.data(); + if (mode == Mode::Vertex) return SamplingVertexCode.data(); + return VertexCode.data(); +} + +std::uint32_t PixelUserWords(Mode mode, bool vertexSampling) { + switch (mode) { + case Mode::Storage: + return 24u; + case Mode::Buffer: + case Mode::MovedBuffer: + return 16u; + case Mode::Bda: + return 14u; + default: + return vertexSampling ? 0u : 12u; + } +} + +unsigned ExpectedChannel(Mode mode, bool buffer, std::size_t channel) { + if (mode == Mode::MovedTexture) return 0x33u; + if ((mode == Mode::Bda || mode == Mode::MovedBuffer) && channel == 3) return SecondAlphaByte; + if (buffer && channel == 3) return BufferAlphaByte; + return Texel[channel]; +} + +const char* DrawnWhat(Mode mode, bool buffer) { + if (mode == Mode::Bda) return " (texture and BDA)"; + if (mode == Mode::MovedBuffer) return " (texture and a buffer moved between two draws)"; + if (mode == Mode::MovedTexture) return " (a texture changed between two draws)"; + if (buffer) return " (texture and buffer)"; + if (mode == Mode::Table) return " (image table)"; + return " (texture)"; +} + +const char* TakenWhat(Mode mode, bool buffer) { + if (buffer) return " reading a buffer"; + if (mode == Mode::Vertex) return " (vertex stage)"; + if (mode == Mode::Bda) return " loading through BDA"; + if (mode == Mode::Table) return " reading an image table"; + if (mode == Mode::VertexTable) return " (vertex stage reading an image table)"; + return ""; +} + +std::array ImageDescriptor(const void* memory) { + const auto address = static_cast(reinterpret_cast(memory)); + return {static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (56u << 20u) | (3u << 30u), + 3u << 14u, + 0xfacu | (9u << 28u), + 0u, + 0u, + 0u, + 0u}; +} + +std::array BufferDescriptor(const void* memory, std::size_t bytes) { + const auto address = static_cast(reinterpret_cast(memory)); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(bytes), + 0xfacu}; +} + +std::vector Draw(const std::uint32_t* pixelCode, Mode mode, const std::uint32_t* vertexCode = nullptr) { + const bool vertexSampling = mode == Mode::Vertex || mode == Mode::VertexTable; + const auto vertex = reinterpret_cast(vertexCode != nullptr ? vertexCode : VertexProgram(mode)); + const auto pixel = reinterpret_cast(pixelCode); + const auto color = reinterpret_cast(ColorMemory.data()); + std::array descriptors{}; + const auto textureImage = ImageDescriptor(TextureMemory.data()); + const auto bufferDescriptor = BufferDescriptor(BufferMemory.data() + 1, sizeof(BufferMemory) - sizeof(float)); + const auto storageImage = ImageDescriptor(StorageMemory.data()); + std::copy(textureImage.begin(), textureImage.end(), descriptors.begin()); + std::copy(bufferDescriptor.begin(), bufferDescriptor.end(), descriptors.begin() + 12); + std::copy(storageImage.begin(), storageImage.end(), descriptors.begin() + 16); + const auto userWords = PixelUserWords(mode, vertexSampling); + const auto otherImage = ImageDescriptor(OtherTextureMemory.data()); + std::copy(otherImage.begin(), otherImage.end(), TableMemory.begin()); + std::copy(descriptors.begin(), descriptors.begin() + 8, TableMemory.begin() + 8); + MaterialMemory = {1u, 0u, 0u, 0u}; + std::array tableWords{}; + const auto table = BufferDescriptor(TableMemory.data(), sizeof(TableMemory)); + const auto material = BufferDescriptor(MaterialMemory.data(), sizeof(MaterialMemory)); + std::copy(table.begin(), table.end(), tableWords.begin()); + std::copy(material.begin(), material.end(), tableWords.begin() + 8); + const std::vector> context{ + {0x2d5, 0x2000}, + {0x1b6, 0}, + {0x207, 0}, + {0x200, 0}, + {0x203, 0x800}, + {0x2dc, 0xaa00}, + {0x2f8, 0}, + {0x293, 0}, + {0x80, 0}, + {0x8d, 0}, + {0x83, 0xffff}, + {0x8c, 0xa}, + {0x2f9, 0x2d}, + {0x313, 0x6000}, + {0x30e, 0xffffffff}, + {0x30f, 0xffffffff}, + {0x206, 0x43f}, + {0x204, 0x80000}, + {0x205, 0x240}, + {0x8e, 0xf}, + {0x8f, 0xf}, + {0x202, 0xcc0010}, + {0x1c4, 0}, + {0x1c5, 9}, + {0x1c3, 4}, + {0x31c, 0x28028}, + {0x31b, 0}, + {0x31d, 0}, + {0x3b0, (63u << 14u) | 3u}, + {0x3b8, 0x9000000}, + {0x1e0, 0}, + {0xc, 0}, + {0xd, 0x40040}, + {0x81, 0x80000000}, + {0x82, 0x40040}, + {0x90, 0x80000000}, + {0x91, 0x40040}, + {0x94, 0x80000000}, + {0x95, 0x40040}, + {0x10f, std::bit_cast(32.0f)}, + {0x110, std::bit_cast(32.0f)}, + {0x111, std::bit_cast(-2.0f)}, + {0x112, std::bit_cast(2.0f)}, + {0x113, std::bit_cast(1.0f)}, + {0x114, 0}, + {0xb4, 0}, + {0xb5, std::bit_cast(1.0f)}, + {0x2fe, 0}, + {0x302, 0}, + {0x306, 0}, + {0x30a, 0}, + {0x318, static_cast(color >> 8u)}, + {0x390, static_cast(color >> 40u)}, + {0x292, 2}, + }; + std::vector> shader{ + {0xc8, static_cast(vertex >> 8u)}, + {0xc9, static_cast(vertex >> 40u)}, + {0x8b, 1u << 1u}, + {0x8c, 0}, + {0x008, static_cast(pixel >> 8u)}, + {0x009, static_cast(pixel >> 40u)}, + {0x00b, userWords << 1u}, + }; + const auto userWord = [&](std::uint32_t index) -> std::uint32_t { + if (mode == Mode::Table) return tableWords[index]; + if (mode == Mode::Bda && index >= 12u) + return static_cast(reinterpret_cast(BufferMemory.data() + 1) >> (index == 12u ? 0u : 32u)); + return descriptors[index]; + }; + for (std::uint32_t index = 0; index < userWords; ++index) shader.emplace_back(0x00cu + index, userWord(index)); + if (vertexSampling) { + shader.emplace_back(0x8bu, 12u << 1u); + for (std::uint32_t index = 0; index < 12u; ++index) + shader.emplace_back(0x08cu + index, mode == Mode::VertexTable ? tableWords[index] : descriptors[index]); + shader.emplace_back(0x00bu, 0u); + } + std::vector commands; + Append(commands, 0x79, {0x242, 4}); + for (const auto& [offset, value] : context) Append(commands, 0x69, {offset, value}); + if (vertexSampling) { + Append(commands, 0x69, {0x1b6, 1}); + Append(commands, 0x69, {0x191, 1u << 10u}); + Append(commands, 0x69, {0x1b1, 0}); + } + for (const auto& [offset, value] : shader) Append(commands, 0x76, {offset, value}); + Append(commands, 0x2d, {3, 2}); + if (mode == Mode::Bda) { + const auto second = static_cast(reinterpret_cast(BufferMemory.data() + 2)); + Append(commands, 0x76, {0x00cu + 12u, static_cast(second)}); + Append(commands, 0x76, {0x00cu + 13u, static_cast(second >> 32u)}); + } + if (mode == Mode::MovedBuffer) { + const auto moved = BufferDescriptor(BufferMemory.data() + 3, sizeof(BufferMemory) - 3 * sizeof(float)); + for (std::uint32_t index = 0; index < 4u; ++index) Append(commands, 0x76, {0x00cu + 12u + index, moved[index]}); + } + if (mode == Mode::MovedTexture) { + for (std::uint32_t index = 0; index < 8u; ++index) Append(commands, 0x76, {0x00cu + index, otherImage[index]}); + } + Append(commands, 0x2d, {3, 2}); + return commands; +} + +std::span PixelProgram(Mode mode, bool buffer, bool vertexSampling) { + switch (mode) { + case Mode::PixelLds: + return LdsPixelCode; + case Mode::Table: + return TablePixelCode; + case Mode::Gds: + return GdsPixelCode; + case Mode::Storage: + return StoragePixelCode; + case Mode::Bda: + return BdaPixelCode; + default: + break; + } + if (buffer) return BufferPixelCode; + if (vertexSampling) return InterpolantPixelCode; + return PixelCode; +} + +void DrawSamplesTheTexture(bool heap, Mode mode) { + const bool buffer = mode == Mode::Buffer || mode == Mode::Bda || mode == Mode::MovedBuffer; + ColorMemory.fill(Kept); + StorageMemory.fill(0u); + BufferMemory.fill(0.0f); + BufferMemory[1] = BufferAlpha; + BufferMemory[2] = SecondAlpha; + BufferMemory[3] = SecondAlpha; + for (std::size_t offset = 0; offset < TextureMemory.size(); offset += Texel.size()) + std::memcpy(TextureMemory.data() + offset, Texel.data(), Texel.size()); + Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2); + for (std::size_t offset = 0; offset < OtherTextureMemory.size(); offset += 4) + OtherTextureMemory[offset] = OtherTextureMemory[offset + 1] = OtherTextureMemory[offset + 2] = OtherTextureMemory[offset + 3] = + 0x33u; + const bool vertexSampling = mode == Mode::Vertex || mode == Mode::VertexTable; + const auto program = PixelProgram(mode, buffer, vertexSampling); + const auto* code = program.data(); + const auto bytes = program.size_bytes(); + if (mode == Mode::Vertex) Registered(VertexHeader, SamplingVertexCode.data(), sizeof(SamplingVertexCode), 2); + if (mode == Mode::VertexTable) Registered(VertexHeader, TableVertexCode.data(), sizeof(TableVertexCode), 2); + Registered(PixelHeader, code, bytes, 1); + const auto takenBefore = AgcDriver::DriverDetail::HeapStagesTaken(); + const auto tablesBefore = AgcDriver::Graphics::ImageTableBuilds(); + const auto pushesBefore = AgcDriver::Graphics::DataAddressPushes(); + auto* gdsCounter = reinterpret_cast(AgcDriver::Pm4::GdsAddress() + 0x10u); + if (mode == Mode::Gds) *gdsCounter = 0u; + auto commands = Draw(code, mode); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw submission failed"); + AgcDriverWaitIdle_nid_postfix(); + std::size_t drawn = 0; + for (std::size_t offset = 0; offset < ColorMemory.size(); offset += 4) { + if (std::all_of(ColorMemory.begin() + static_cast(offset), + ColorMemory.begin() + static_cast(offset + 4), [](std::byte value) { return value == Kept; })) + continue; + for (std::size_t channel = 0; channel < 4; ++channel) { + const auto value = static_cast(ColorMemory[offset + channel]); + const unsigned expected = ExpectedChannel(mode, buffer, channel); + Require(value == expected, "pixel " + std::to_string(offset / 4) + " channel " + std::to_string(channel) + " is " + + std::to_string(value) + ", expected " + std::to_string(expected) + DrawnWhat(mode, buffer) + + (heap ? " through the heap" : "")); + } + ++drawn; + } + Require(drawn != 0, "the draw covered no pixel"); + if (mode == Mode::Table || mode == Mode::VertexTable) + Require(AgcDriver::Graphics::ImageTableBuilds() > tablesBefore, + std::string("the table draw's ") + (mode == Mode::Table ? "fragment" : "vertex") + " stage bound no image table"); + if (mode == Mode::Gds) Require(*gdsCounter != 0u, "the GDS append did not count the drawn lanes"); + if (mode == Mode::Storage) { + for (std::size_t channel = 0; channel < 4; ++channel) + Require(StorageMemory[channel] == Texel[channel], "storage texel (0, 0) channel " + std::to_string(channel) + " is " + + std::to_string(StorageMemory[channel]) + ", expected the sampled " + + std::to_string(Texel[channel]) + (heap ? " through the heap" : "")); + } + if (!heap || std::getenv("APS5_HEAP_SLOT_CAP") != nullptr) return; + if (mode == Mode::Vertex || mode == Mode::VertexTable) + Require(AgcDriver::DriverDetail::HeapStagesTaken() > takenBefore, + "the heap did not take the sampling vertex stage with descriptor memory"); + else + Require(AgcDriver::DriverDetail::HeapStagesTaken() > takenBefore, + std::string("the heap did not take the fragment stage") + TakenWhat(mode, buffer)); + Require(AgcDriver::Graphics::DataAddressPushes() > pushesBefore, + "the heap stages did not read their shader data through pushed addresses"); +} + +void MixedDrawTakesThePerDrawPath() { + using AgcDriver::Graphics::HeapFallback; + using AgcDriver::Graphics::HeapFallbacks; + MixedVertexCode = SamplingVertexCode; + MixedPixelCode = InterpolantPixelCode; + for (std::size_t offset = 0; offset < TextureMemory.size(); offset += Texel.size()) + std::memcpy(TextureMemory.data() + offset, Texel.data(), Texel.size()); + Registered(PixelHeader, MixedPixelCode.data(), sizeof(MixedPixelCode), 1); + { + Registered(VertexHeader, VertexCode.data(), sizeof(VertexCode), 2); + auto commands = Draw(MixedPixelCode.data(), Mode::Texture); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, "the draw putting the fragment stage on the heap failed"); + AgcDriverWaitIdle_nid_postfix(); + Registered(VertexHeader, MixedVertexCode.data(), sizeof(MixedVertexCode), 2); + } + AgcDriver::DriverDetail::SetHeapStageUnmapped(ShaderRecompiler::ShaderStage::Fragment); + for (const char* draw : {"first", "repeated"}) { + const auto takenBefore = AgcDriver::DriverDetail::HeapStagesTaken(); + const auto retriesBefore = HeapFallbacks(HeapFallback::DrawMixedRetry); + const auto refusedBefore = HeapFallbacks(HeapFallback::DrawRefusedPair); + ColorMemory.fill(Kept); + auto commands = Draw(MixedPixelCode.data(), Mode::Vertex, MixedVertexCode.data()); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, std::string("the ") + draw + " mixed draw submission failed"); + AgcDriverWaitIdle_nid_postfix(); + std::size_t drawn = 0; + for (std::size_t offset = 0; offset < ColorMemory.size(); offset += 4) { + if (std::all_of(ColorMemory.begin() + static_cast(offset), + ColorMemory.begin() + static_cast(offset + 4), [](std::byte value) { return value == Kept; })) + continue; + for (std::size_t channel = 0; channel < 4; ++channel) + Require(static_cast(ColorMemory[offset + channel]) == Texel[channel], + std::string("the ") + draw + " mixed draw: pixel " + std::to_string(offset / 4) + " channel " + + std::to_string(channel) + " is " + std::to_string(static_cast(ColorMemory[offset + channel])) + + ", expected " + std::to_string(Texel[channel])); + ++drawn; + } + Require(drawn != 0, std::string("the ") + draw + " mixed draw covered no pixel"); + const bool first = std::strcmp(draw, "first") == 0; + if (first) + Require(AgcDriver::DriverDetail::HeapStagesTaken() > takenBefore, "the heap did not take the vertex stage of the mixed draw"); + Require(HeapFallbacks(HeapFallback::DrawMixedRetry) - retriesBefore == (first ? 1u : 0u), + std::string("the ") + draw + " mixed draw retried on the per-draw path " + + std::to_string(HeapFallbacks(HeapFallback::DrawMixedRetry) - retriesBefore) + " times, expected " + + (first ? "once" : "never")); + Require(HeapFallbacks(HeapFallback::DrawRefusedPair) == refusedBefore, + std::string("the ") + draw + " mixed draw stayed mixed after its per-draw retry"); + } + AgcDriver::DriverDetail::SetHeapStageUnmapped(std::nullopt); +} + +} + +int main() { + SetTestEnvironment("APS5_EAGER_WRITEBACK", "1"); + try { + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + const bool heap = std::getenv("APS5_GPU_HEAP") != nullptr; + if (heap && SkipWithoutDescriptorHeap()) return VulkanTestSkipped; + { + GuestAllocations::Mutation mutation; + mutation.Add(StorageMemory.data(), StorageMemory.size(), true, true, true); + mutation.Add(BufferMemory.data(), sizeof(BufferMemory), true, true, true); + } + for (const auto mode : {Mode::Texture, Mode::Buffer, Mode::Storage, Mode::Gds, Mode::Vertex, Mode::Bda, Mode::Table, Mode::VertexTable, + Mode::MovedBuffer, Mode::MovedTexture, Mode::PixelLds, Mode::Texture}) + DrawSamplesTheTexture(heap, mode); + const bool capped = heap && std::getenv("APS5_HEAP_SLOT_CAP") != nullptr; + if (heap && !capped) MixedDrawTakesThePerDrawPath(); + if (capped) + Require(AgcDriver::DriverDetail::HeapRefused(), "a draw found the capped heap full and the heap was not refused"); + AgcDriverShutdown_nid_postfix(); + if (heap && !capped) + Require(AgcDriver::Graphics::EmptyResourceSets() != 0, "no heap draw bound without a per-draw resource set"); + std::cout << "A sampled texture reached the drawn pixels" << (heap ? " through the heap" : "") << '\n'; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HeapSampleKernel.hpp b/core/libs/prx/libSceAgcDriver/tests/execution/HeapSampleKernel.hpp new file mode 100644 index 000000000000..f256baf2b6c6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HeapSampleKernel.hpp @@ -0,0 +1,69 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_EXECUTION_HEAPSAMPLEKERNEL_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_TESTS_EXECUTION_HEAPSAMPLEKERNEL_HPP + +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +alignas(256) inline constexpr std::array SampleCode{0x7e0002f0u, 0x7e0202f0u, 0xf09c0f08u, 0x00400000u, + 0xbf8c0000u, 0xe0780000u, 0x80030000u, 0xbf810000u}; +inline constexpr std::array SampleTexel{0x10u, 0x80u, 0xc0u, 0xffu}; + +inline std::array SampleUserData(const void* image, const float* output) { + const auto imageAddress = static_cast(reinterpret_cast(image)); + const auto outputAddress = static_cast(reinterpret_cast(output)); + return {static_cast(imageAddress >> 8u), + static_cast((imageAddress >> 40u) & 0xffu) | (56u << 20u) | (3u << 30u), + 3u << 14u, + 0xfacu | (9u << 28u), + 0u, + 0u, + 0u, + 0u, + 0u, + 0u, + 0u, + 0u, + static_cast(outputAddress), + static_cast(outputAddress >> 32u) & 0xffffu, + static_cast(4u * sizeof(float)), + 0x31016facu}; +} + +inline void FillTexels(std::span memory, const std::array& texel) { + for (std::size_t offset = 0; offset < memory.size(); offset += texel.size()) std::memcpy(memory.data() + offset, texel.data(), texel.size()); +} + +inline void RequireSampled(const float* output, const std::array& expected, const std::string& what, bool heap = false) { + for (std::size_t channel = 0; channel < 4; ++channel) { + const float wanted = static_cast(expected[channel]) / 255.0f; + if (!(std::fabs(output[channel] - wanted) < 1e-6f)) + throw std::runtime_error(what + ": channel " + std::to_string(channel) + " is " + std::to_string(output[channel]) + ", expected " + + std::to_string(wanted) + (heap ? " through the heap" : "")); + } +} + +inline ShaderRecompiler::RecompileResult CompileSampleKernel(AgcDriver::VulkanDevice& device, const void* image, const float* output, + bool useCache = true, std::span code = SampleCode, + std::uint32_t ldsDwords = 0u) { + const auto userData = SampleUserData(image, output); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{1u, 1u, 1u}, ldsDwords, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{ + {ShaderRecompiler::ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, + {32u, 0u, userData, compute, std::nullopt, std::nullopt, memory}, + device.ComputeTarget(32u), + {0u, 0u, 0u, 128u}}; + request.useCache = useCache; + return ShaderRecompiler::Recompile(request); +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HeapTableStageReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HeapTableStageReuse.cpp new file mode 100644 index 000000000000..4756ae1f406b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HeapTableStageReuse.cpp @@ -0,0 +1,227 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "RuntimeAbi.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +struct VulkanDeviceTestAccess { + static Graphics::Context ContextOf(const VulkanDevice& device) { return device.graphicsContext(); } +}; + +} + +namespace { + +using AgcDriver::Graphics::CompiledShader; +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::ShaderResources; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t Textures = 4; +constexpr std::uint32_t Stride = 32; +constexpr std::size_t SurfaceBytes = 256; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t SrtOffset = 4096; +constexpr std::size_t InputOffset = 8192; +constexpr std::size_t OutputOffset = 12288; +constexpr std::size_t HeapOffset = 65536; +constexpr std::size_t BlockBytes = (HeapOffset + std::size_t{Textures} * Stride + 65535u) & ~std::size_t{65535}; +constexpr std::size_t SurfaceBlockBytes = (std::size_t{Textures} * SurfaceBytes + 65535u) & ~std::size_t{65535}; + +constexpr std::array Code{ + 0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, + 0x34080082u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xbf8cc07fu, + 0xe0301000u, 0x80030504u, 0xf09c0f08u, 0x00450000u, 0xbf8c3f70u, 0x4a000105u, + 0xe0701000u, 0x80070004u, 0xbf810000u, 0u, 0u}; + +class GuestBlock { +public: + GuestBlock(std::size_t bytes, bool gpu) { +#ifdef _WIN32 + block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, 65536)); + if (block != nullptr) GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); +#else + constexpr std::uintptr_t alignment = 65536; + void* mapped = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (mapped != MAP_FAILED) { + const auto begin = reinterpret_cast(mapped); + const auto aligned = (begin + alignment - 1) & ~(alignment - 1); + if (aligned != begin) munmap(mapped, aligned - begin); + if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast(aligned + bytes), begin + alignment - aligned); + block = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); + } +#endif + Require(block != nullptr, "heap table stage reuse: cannot allocate a guest block"); + std::memset(block, 0, bytes); + GuestAllocations::Mutation().Add(block, bytes, true, true, gpu); + AgcDriver::GuestMemory::CollectWritesUncached(reinterpret_cast(block), bytes); + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint64_t At(std::size_t offset) const { return reinterpret_cast(block + offset); } + std::uint8_t* Bytes(std::size_t offset) const { return block + offset; } + std::uint32_t* Words(std::size_t offset) const { return reinterpret_cast(block + offset); } + +private: + std::uint8_t* block = nullptr; +}; + +std::array BufferDescriptor(std::uint64_t address, std::uint32_t stride, std::uint32_t records) { + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), records, stride == 0u ? 0x31016facu : 0x01016facu}; +} + +void Prepare(const GuestBlock& block, const GuestBlock& surfaces) { + std::memcpy(block.Words(0), Code.data(), sizeof(Code)); + for (std::uint32_t texture = 0; texture < Textures; ++texture) { + const auto offset = static_cast(texture) * SurfaceBytes; + std::memset(surfaces.Bytes(offset), static_cast(texture & 0xffu), SurfaceBytes); + const auto surface = surfaces.At(offset); + const std::array words{static_cast(surface >> 8u), static_cast((surface >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (3u << 30u), 1u | (7u << 14u), 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u}; + std::memcpy(block.Words(HeapOffset + static_cast(texture) * Stride), words.data(), sizeof(words)); + } + auto* srt = block.Words(SrtOffset); + const auto heap = BufferDescriptor(block.At(HeapOffset), Stride, Textures); + const auto input = BufferDescriptor(block.At(InputOffset), 0u, Lanes * 4u); + const auto output = BufferDescriptor(block.At(OutputOffset), 0u, Lanes * 4u); + std::copy(heap.begin(), heap.end(), srt); + std::copy(input.begin(), input.end(), srt + 8); + std::copy(output.begin(), output.end(), srt + 12); + AgcDriver::GuestMemory::CollectWritesUncached(block.At(0), BlockBytes); + AgcDriver::GuestMemory::CollectWritesUncached(surfaces.At(0), SurfaceBlockBytes); +} + +ShaderRecompiler::RecompileResult Compile(const AgcDriver::VulkanDevice& device, const GuestBlock& block) { + const auto srt = block.At(SrtOffset); + const std::array userData{static_cast(srt), static_cast(srt >> 32u)}; + const std::span code(block.Words(0), Code.size()); + const std::array memory{{ + {block.At(0), std::as_bytes(code)}, + {srt, std::as_bytes(std::span(block.Words(SrtOffset), 64))}, + {block.At(HeapOffset), std::as_bytes(std::span(block.Words(HeapOffset), static_cast(Textures) * Stride / 4u))}, + }}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, block.At(0), code, 0, {}}, + {Lanes, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.ComputeTarget(Lanes), + {0, 0, 0, 128} + }; + return ShaderRecompiler::Recompile(request); +} + +void RequireHeapTable(const ShaderRecompiler::RecompileResult& result) { + const auto has = [&](ShaderRecompiler::DescriptorRole role) { + return std::any_of(result.bindings.begin(), result.bindings.end(), [&](const auto& binding) { return binding.role == role; }); + }; + Require(result.gpuResolvedImages && result.dataAddressPush, "heap table stage reuse: the program is not in the heap's GPU-resolved form"); + Require(has(ShaderRecompiler::DescriptorRole::ImageTable) && has(ShaderRecompiler::DescriptorRole::ImageTableMap), "heap table stage reuse: the program does not bind an image table and its map"); +} + +std::uint64_t TableMapRoot(const ShaderResources& resources, const CompiledShader& shader) { + namespace Abi = ShaderRecompiler::RuntimeAbi; + std::array bytes{}; + const auto placements = AgcDriver::Graphics::PushDataPlacements(std::span(&shader, 1)); + resources.PatchDataAddresses(bytes, std::span(&shader, 1), placements, nullptr); + std::uint64_t value = 0; + const auto position = placements.front().Item(Abi::RootItem::TableMap); + Require(position != Abi::NoPushPosition, "heap table stage reuse: the stage's push layout holds no table map"); + std::memcpy(&value, bytes.data() + position * sizeof(std::uint32_t), sizeof(value)); + return value; +} + +void RefreshAfterOriginalDestroyed(const AgcDriver::VulkanDevice& device, const GuestBlock& block, bool sameAddress) { + const auto context = AgcDriver::VulkanDeviceTestAccess::ContextOf(device); + const char* what = sameAddress ? "an equal-content program at the destroyed program's address" : "an equal-content program at another address"; + std::optional slot; + slot.emplace(Compile(device, block)); + RequireHeapTable(*slot); + const auto* original = &*slot; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::Graphics::BindlessHeap::SpanMark lead; + static_cast(AgcDriver::Graphics::BindlessHeap::For(context).Allocate(64, lead)); + const CompiledShader built{ShaderStage::Compute, original, 0}; + auto resources = std::make_shared(context, built); + Require(resources->HeapStage(), "heap table stage reuse: the set was not built on the heap"); + const auto builtMap = TableMapRoot(*resources, built); + Require(builtMap != 0u, "heap table stage reuse: the build left the table map unset"); + const auto bindingBytes = slot->bindings.capacity() * sizeof(ShaderRecompiler::DescriptorBinding); + std::optional other; + std::vector> squatters; + if (sameAddress) { + slot.reset(); + for (int i = 0; i < 16; ++i) { + squatters.push_back(std::make_unique(bindingBytes)); + std::memset(squatters.back().get(), 0xff, bindingBytes); + } + slot.emplace(Compile(device, block)); + Require(&*slot == original, "heap table stage reuse: the reused slot moved"); + } else { + other.emplace(Compile(device, block)); + slot.reset(); + } + const auto& current = sameAddress ? *slot : *other; + RequireHeapTable(current); + const CompiledShader shader{ShaderStage::Compute, ¤t, 0}; + resources->RefreshHeapStage(std::span(&shader, 1)); + const auto refreshed = TableMapRoot(*resources, shader); + Require(refreshed != 0u, std::string("heap table stage reuse: refreshing for ") + what + " left the table map unset"); + Require(refreshed == builtMap, std::string("heap table stage reuse: refreshing for ") + what + " wrote a table map that differs from the build's"); + std::printf("the heap table map refreshed for %s\n", what); +} + +} + +int main() { + SetTestEnvironment("APS5_GPU_HEAP", "1"); + SetTestEnvironment("APS5_VERIFY_GPU_HEAP", "1"); + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock block(BlockBytes, true); + GuestBlock surfaces(SurfaceBlockBytes, false); + Prepare(block, surfaces); + RefreshAfterOriginalDestroyed(*device, block, false); + RefreshAfterOriginalDestroyed(*device, block, true); + std::puts("heap table stage reuse tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp index 1b8cf8e771c5..1f1f0dfc73e6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" @@ -124,11 +125,12 @@ void Run(AgcDriver::VulkanDevice& device, std::uint32_t format) { ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(32), {0, 0, 0, 128} }; request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + Require(result.gpuResolvedImages == AgcDriver::Graphics::DescriptorHeapEnabled(), "the compute heap did not take the image atomic stage"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp index 02436563708e..c0777af38b0c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomics64.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" @@ -123,11 +124,12 @@ void Run(AgcDriver::VulkanDevice& device, std::uint32_t format) { ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(32), {0, 0, 0, 128} }; request.useCache = false; const auto result = ShaderRecompiler::Recompile(request); + Require(result.gpuResolvedImages == AgcDriver::Graphics::DescriptorHeapEnabled(), "the compute heap did not take the image atomic stage"); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp index ca6796d7a2c5..98f8b9c08264 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsBitwise.cpp @@ -127,9 +127,9 @@ int main() { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; FillInput(); - Run(*device, 32, device->Target()); + Run(*device, 32, device->ComputeTarget(32)); Check("wave32"); - Run(*device, 64, device->Target()); + Run(*device, 64, device->ComputeTarget(64)); Check("wave64"); Run(*device, 64, device->ComputeTarget(32)); Check("wave64 split"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp index 78b36356c7db..b4e269aadbbd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageAtomicsFloat.cpp @@ -544,8 +544,8 @@ int main() { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; for (const auto format : {Format32UInt, Format32SInt, Format32Float}) { - Run(*device, 32, device->Target(), "wave32", format); - Run(*device, 64, device->Target(), "wave64", format); + Run(*device, 32, device->ComputeTarget(32), "wave32", format); + Run(*device, 64, device->ComputeTarget(64), "wave64", format); Run(*device, 64, device->ComputeTarget(32), "wave64 split", format); } bool refused = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp index f321df600250..7f60148977e6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageCacheBits.cpp @@ -224,7 +224,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::span co ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index 8e302dd98959..16299080b16a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -247,7 +247,7 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, c ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; @@ -621,7 +621,7 @@ void CheckConvertedPreparation(AgcDriver::VulkanDevice& device, const std::uint8 ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; auto converted = request; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp index cb8958107b62..d5ce832ae322 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp @@ -115,7 +115,7 @@ void Run(AgcDriver::VulkanDevice& device, std::span code, s ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp index eafabe0d7c87..898a73421caf 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp @@ -188,7 +188,7 @@ std::vector Run(AgcDriver::VulkanDevice& device, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = useCache; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreArrayThroughVolume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreArrayThroughVolume.cpp index b410321d5cd0..24cafee2007e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreArrayThroughVolume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreArrayThroughVolume.cpp @@ -126,7 +126,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::size_t ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFaceOnCube.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFaceOnCube.cpp index 148b6e45c74d..d58e1b29b6ff 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFaceOnCube.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFaceOnCube.cpp @@ -140,7 +140,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, const Forma ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFloatRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFloatRounding.cpp index 20bb6f2a1834..5a1a134d2bc1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFloatRounding.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreFloatRounding.cpp @@ -168,7 +168,7 @@ void Store(AgcDriver::VulkanDevice& device, std::uint8_t* texels, const Format& ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(storeCode.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp index 6f6e1cfb2dbb..0b29350010ca 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp @@ -146,7 +146,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp index 26d3e4d1ff99..5327471762b9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlaneArray.cpp @@ -138,7 +138,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp index 427ebc94b078..93262be4318a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp @@ -177,7 +177,7 @@ void Run(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; const auto result = ShaderRecompiler::Recompile(request); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp index 8daddabd90c8..670081b62ca1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp @@ -238,7 +238,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreUnormRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreUnormRounding.cpp index bb6c6f25cf17..ffbcd129b4cb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreUnormRounding.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreUnormRounding.cpp @@ -207,7 +207,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreVolumeThroughArray.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreVolumeThroughArray.cpp index b82fa4ba0725..acd2e7530f40 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreVolumeThroughArray.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreVolumeThroughArray.cpp @@ -126,7 +126,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::size_t ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp index 552994ca1788..15b6419520ca 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTableCacheBounds.cpp @@ -7,6 +7,7 @@ #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/SrtWalker.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" @@ -73,6 +74,17 @@ constexpr std::size_t WideKernelOffset = 2048; constexpr std::size_t WideHeapOffset = 65536; constexpr std::array WideSlots{SrtSlots - 1u, SrtSlots - 2u, SrtSlots - 3u}; constexpr std::array WideTableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x9310b010u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t AtomicRecords = 16; +constexpr std::uint32_t AtomicWidth = 64; +constexpr std::size_t AtomicKernelOffset = 1024; +constexpr std::size_t AtomicSurfaceBytes = AtomicWidth * 4u; +constexpr std::uint32_t AtomicElement = 5; +constexpr std::uint32_t Format16x4Float = 71; +constexpr std::size_t StoreSurfaceBytes = AtomicWidth * 8u; +constexpr std::uint64_t StoredTexel = 0x3c00ull | 0x3800ull << 16u | 0x4000ull << 32u | 0xbc00ull << 48u; +constexpr std::array StoreTableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x2c240086u, 0x4a222485u, 0x7e200511u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0x7e020280u, 0x7e0402f2u, 0x7e0602f0u, 0x7e0802f4u, 0x7e0a02f3u, 0xbf8c3f70u, 0xf0200f08u, 0x00050200u, 0xbf810000u}; +constexpr std::array AtomicTableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x2c240086u, 0x4a222485u, 0x7e200511u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0x7e020280u, 0x7e040281u, 0x34060082u, 0xbf8c3f70u, 0xf0442108u, 0x00050200u, 0xbf8c3f70u, 0xe0701000u, 0x80070203u, 0xbf8c3f70u, 0xbf810000u}; constexpr std::array TableCode{0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}; constexpr std::size_t VertexOffset = 256; @@ -241,12 +253,19 @@ void Prepare(const GuestBlock& block) { void TablesChangingPerDispatch(const GuestBlock& block) { auto& cache = AgcDriver::Graphics::SharedResourceCache(); + const auto heapStages = AgcDriver::Graphics::HeapComputeStages(); std::uint64_t most = 0; for (std::uint32_t dispatch = 0; dispatch < 6; ++dispatch) { SetRecord(block, dispatch * 977u % Records, Elements + dispatch % Spare); Dispatch(block, 0); most = std::max(most, cache.TableElements()); } + const bool tableFitsHeap = AgcDriver::Graphics::BindlessHeap::ImageSlotCapacity() >= Elements + Spare; + if (AgcDriver::Graphics::DescriptorHeapEnabled() && tableFitsHeap) { + Require(AgcDriver::Graphics::HeapComputeStages() > heapStages, "image table cache bounds: the dispatches did not take the descriptor heap"); + return; + } + if (AgcDriver::Graphics::DescriptorHeapEnabled()) Require(AgcDriver::Graphics::HeapComputeStages() == heapStages, "image table cache bounds: a table stage with more images than the heap's slots took the heap"); Require(most >= Elements, "image table cache bounds: no dispatch kept its table in the resource cache"); Require(most <= AgcDriver::Graphics::ResourceCache::TableElementBudget(), "image table cache bounds: tables that changed every dispatch kept " + std::to_string(most) + " table elements in the resource cache, over its budget of " + std::to_string(AgcDriver::Graphics::ResourceCache::TableElementBudget())); } @@ -442,6 +461,68 @@ void VouchedLookups(const GuestBlock& block) { std::puts("image table vouched lookups passed"); } +std::uint32_t AtomicStart(std::uint32_t texel) { return texel * 7u + 3u; } + +template +void PrepareAtomicRecords(const GuestBlock& block, const void* kernel, std::size_t kernelBytes, std::uint32_t format, std::size_t surfaceBytes, FillSurface&& fillSurface) { + std::memcpy(block.Bytes(AtomicKernelOffset), kernel, kernelBytes); + for (std::uint32_t record = 0; record < AtomicRecords; ++record) { + const auto surface = TexturesOffset + static_cast(record) * surfaceBytes; + fillSurface(record, block.Bytes(surface)); + const auto address = block.At(surface); + const std::array words{static_cast(address >> 8u), static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((AtomicWidth - 1u) & 3u) << 30u), (AtomicWidth - 1u) >> 2u, 0xfacu | (Type2D << 28u), 0u, 0u, 0u, 0u}; + std::memcpy(block.Bytes(HeapOffset + static_cast(record) * Stride), words.data(), sizeof(words)); + } + auto* srt = reinterpret_cast(block.Bytes(SrtOffset)); + std::fill(srt, srt + SrtBytes / 4u, 0u); + const auto heap = V(block.At(HeapOffset), Stride, AtomicRecords); + const auto output = V(block.At(OutputOffset), 0u, 4096u); + std::copy(heap.begin(), heap.end(), srt); + std::copy(output.begin(), output.end(), srt + 12); +} + +void AtomicTable(const GuestBlock& block) { + PrepareAtomicRecords(block, AtomicTableCode.data(), sizeof(AtomicTableCode), Format32UInt, AtomicSurfaceBytes, [](std::uint32_t record, std::uint8_t* surface) { + auto* texels = reinterpret_cast(surface); + for (std::uint32_t texel = 0; texel < AtomicWidth; ++texel) texels[texel] = AtomicStart(texel) + record * 1000u; + }); + const auto heapStages = AgcDriver::Graphics::HeapComputeStages(); + for (std::uint32_t dispatch = 0; dispatch < 2; ++dispatch) { + std::memset(block.Bytes(OutputOffset), 0xee, AtomicWidth * 4u); + Dispatch(block, 0, AtomicKernelOffset); + std::array returned{}; + AgcDriver::GuestMemory::Read(block.At(OutputOffset), std::as_writable_bytes(std::span(returned))); + for (std::uint32_t texel = 0; texel < AtomicWidth; ++texel) { + const auto expected = AtomicStart(texel) + AtomicElement * 1000u + dispatch; + Require(returned[texel] == expected, "image table atomics: dispatch " + std::to_string(dispatch) + " texel " + std::to_string(texel) + " returned " + std::to_string(returned[texel]) + ", expected " + std::to_string(expected)); + } + } + if (AgcDriver::Graphics::DescriptorHeapEnabled()) Require(AgcDriver::Graphics::HeapComputeStages() > heapStages, "image table atomics: the dispatches did not take the descriptor heap"); + std::puts("image table atomics passed"); +} + +bool StoreTable(const GuestBlock& block) { + PrepareAtomicRecords(block, StoreTableCode.data(), sizeof(StoreTableCode), Format16x4Float, StoreSurfaceBytes, + [](std::uint32_t, std::uint8_t* surface) { std::memset(surface, 0, StoreSurfaceBytes); }); + const auto heapStages = AgcDriver::Graphics::HeapComputeStages(); + Dispatch(block, 0, AtomicKernelOffset); + if (!AgcDriver::Graphics::DescriptorHeapEnabled()) { + std::puts("image table store skipped: per-dispatch image tables have no uint storage view (docs/dev/debt/libSceAgcDriver.md)"); + return false; + } + std::array texels{}; + AgcDriver::GuestMemory::Read(block.At(TexturesOffset + static_cast(AtomicElement) * StoreSurfaceBytes), std::as_writable_bytes(std::span(texels))); + for (std::uint32_t texel = 0; texel < AtomicWidth; ++texel) { + char text[160]; + std::snprintf(text, sizeof(text), "image table store: texel %u of a 16_16_16_16_FLOAT table element holds 0x%016llx, expected 0x%016llx", texel, + static_cast(texels[texel]), static_cast(StoredTexel)); + Require(texels[texel] == StoredTexel, text); + } + Require(AgcDriver::Graphics::HeapComputeStages() > heapStages, "image table store: the dispatch did not take the descriptor heap"); + std::puts("image table store passed"); + return true; +} + struct QueuedDraw { std::atomic release{false}; @@ -584,6 +665,16 @@ int main(int argc, char** argv) { AgcDriverShutdown_nid_postfix(); return 0; } + if (argc == 2 && std::string(argv[1]) == "--store-table") { + const bool ran = StoreTable(block); + AgcDriverShutdown_nid_postfix(); + return ran ? 0 : VulkanTestSkipped; + } + if (argc == 2 && std::string(argv[1]) == "--atomic-table") { + AtomicTable(block); + AgcDriverShutdown_nid_postfix(); + return 0; + } if (argc == 2 && std::string(argv[1]) == "--queued-draws") { VouchUnderQueuedDraws(block); AgcDriverShutdown_nid_postfix(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp index 319faa1a0675..2c333ba4a6f0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTexelStatus.cpp @@ -231,7 +231,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::span co ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LazyResolvedAbi.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LazyResolvedAbi.cpp index 923ff59054ce..a7785a3cd6be 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LazyResolvedAbi.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LazyResolvedAbi.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" @@ -556,10 +557,11 @@ void LazyResolution(const Layout& layout, const std::filesystem::path& reference Require(unmatched.firstUses - start.firstUses == Drawn && unmatched.own - start.own == Drawn && unmatched.found == start.found, "lazy resolved ABI: draws whose state differs from the resolved ABI did not count as preparing their own artifact (before: " + Uses(start) + "; after: " + Uses(unmatched) + ")"); Require(unmatched.entries > start.entries && unmatched.unmatchedEntries - start.unmatchedEntries == unmatched.entries - start.entries, "lazy resolved ABI: entries prepared for an ABI no draw used do not count as unmatched (before: " + Uses(start) + "; after: " + Uses(unmatched) + ")"); Require(matched.firstUses == unmatched.firstUses + 1 && matched.found == unmatched.found + 1 && matched.own == unmatched.own, "lazy resolved ABI: a draw whose state matches the resolved ABI did not count as finding its artifact (before: " + Uses(unmatched) + "; after: " + Uses(matched) + ")"); - Require(matched.entries > unmatched.entries && matched.unmatchedEntries == unmatched.unmatchedEntries + (matched.entries - unmatched.entries) - 1, "lazy resolved ABI: the entry a matching draw used still counts as unmatched (before: " + Uses(unmatched) + "; after: " + Uses(matched) + ")"); + const std::uint64_t usedVariants = AgcDriver::DriverDetail::HeapFragmentsEnabled() ? 2u : 1u; + Require(matched.entries > unmatched.entries && matched.unmatchedEntries == unmatched.unmatchedEntries + (matched.entries - unmatched.entries) - usedVariants, "lazy resolved ABI: the entry a matching draw used still counts as unmatched (before: " + Uses(unmatched) + "; after: " + Uses(matched) + ")"); for (const auto index : {Matching, Twice, Replaced}) Require(eager.at("drawn " + std::to_string(index)) == eager.at("resolved " + std::to_string(index)), "lazy resolved ABI: shader " + std::to_string(index) + ", drawn with its resolved ABI's state, did not find the ABI's entries with eager preparation"); - for (std::uint32_t index = 0; index < Drawn; ++index) Require(Count(eager.at("drawn " + std::to_string(index))) == Count(eager.at("resolved " + std::to_string(index))) + 1, "lazy resolved ABI: a draw whose state differs from its resolved ABI did not prepare its own entry with eager preparation"); - Require(Count(eager.at("drawn " + std::to_string(Replacement))) == Count(eager.at("registered " + std::to_string(Replacement))) + 1, "lazy resolved ABI: the header registered over a resolved one did not prepare its own entry at its draw with eager preparation"); + for (std::uint32_t index = 0; index < Drawn; ++index) Require(Count(eager.at("drawn " + std::to_string(index))) == Count(eager.at("resolved " + std::to_string(index))) + usedVariants, "lazy resolved ABI: a draw whose state differs from its resolved ABI did not prepare its own entry with eager preparation"); + Require(Count(eager.at("drawn " + std::to_string(Replacement))) == Count(eager.at("registered " + std::to_string(Replacement))) + usedVariants, "lazy resolved ABI: the header registered over a resolved one did not prepare its own entry at its draw with eager preparation"); Require(lazy.at("libraries drawn") == eager.at("libraries drawn"), "lazy resolved ABI: the pipeline libraries built differ from eager preparation (eager " + eager.at("libraries drawn") + "; lazy " + lazy.at("libraries drawn") + ")"); Require(first == expected, "lazy resolved ABI: the shader prepared at its first draw drew other bytes than with eager preparation"); AgcDriverWaitIdle_nid_postfix(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MemoryAccounting.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MemoryAccounting.cpp index 23b07da10799..4444184ad7da 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MemoryAccounting.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MemoryAccounting.cpp @@ -4,6 +4,7 @@ #include "prx/libSceAgcDriver/Execution/include/PqToSdr.hpp" #include "prx/libSceAgcDriver/Graphics/include/DeferredDestruction.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/FirstSampleCopy.hpp" @@ -159,6 +160,7 @@ int main() { auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; auto context = device->graphicsContext(); + if (context.descriptorHeap) static_cast(BindlessHeap::For(context)); context.bufferPool = std::make_shared(context); drain(context); const auto baseline = MemoryAccounting::Read(context.device).allocations.size(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MeshDrawRelocation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MeshDrawRelocation.cpp index 1b64d9d3dff9..5608ab54eae2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MeshDrawRelocation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MeshDrawRelocation.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" #include "../MeshGeometryFixture.hpp" #include "VulkanTestDevice.hpp" @@ -103,6 +104,7 @@ std::vector Setup(const std::array& descriptor) AgcDriver::DriverDetail::DrawEntryCounters Draw(const std::array& descriptor) { Pixels.fill(std::byte{0}); + const auto pushes = AgcDriver::Graphics::DataAddressPushes(); auto commands = Setup(descriptor); const auto address = reinterpret_cast(Indices.data()); commands.insert(commands.end(), {0xc0042700u, 3u, static_cast(address), static_cast(address >> 32u), 3u, 0u}); @@ -110,6 +112,7 @@ AgcDriver::DriverDetail::DrawEntryCounters Draw(const std::array(Pixels[(24u * Width + 16u) * 4u]) > 200u, "the mesh draw did not cover its triangle"); + Require(!AgcDriver::Graphics::DescriptorHeapEnabled() || AgcDriver::Graphics::DataAddressPushes() > pushes, "the submitted mesh draw did not take the descriptor heap"); return AgcDriver::DriverDetail::Driver::Get().DrawCacheCounters(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleReadback.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleReadback.cpp index a6c84ad27f7d..1701dd301d90 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleReadback.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleReadback.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +32,21 @@ struct VulkanDeviceTestAccess { namespace { +template +void RecordStayingOnHeap(const AgcDriver::Graphics::Context& context, VkCommandBuffer commands, const std::string& what, Record&& record) { + if (!AgcDriver::Graphics::HeapPassesActive(context)) { + record(); + return; + } + auto& heap = AgcDriver::Graphics::BindlessHeap::For(context); + heap.BindDescriptorHeaps(commands); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds(); + record(); + heap.BindDescriptorHeaps(commands); + if (AgcDriver::Graphics::DescriptorHeapBinds() != binds) throw std::runtime_error(what + " put its command buffer back in classic descriptor mode, so the heaps were bound again"); +} + + using namespace AgcDriver::Graphics; PFN_vkGetDeviceProcAddr originalDeviceProc = nullptr; @@ -80,7 +97,7 @@ bool SurfaceSupported(const Context& context, VkFormat format, std::uint32_t sam return false; } -struct Surface { +struct Surface : AgcDriver::Graphics::HeapSlotOwner { Context context; VkFormat format; std::uint32_t elementBytes; @@ -234,6 +251,7 @@ struct Surface { Surface& operator=(const Surface&) = delete; void Release() noexcept { + ReleaseHeapSlots(context); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); for (const auto module : modules) if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); @@ -294,7 +312,7 @@ std::vector CheckReadback(TextureDetiler& reader, const Surface& surf CommandBatch batch(surface.context); RecordMemoryBarrier(surface.context, batch.Handle(), VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t repeat = 0u; repeat < repetitions; ++repeat) { - reader.DispatchMultisampleReadback(batch.Handle(), surface.format, output.Handle(), offset, bytes, surface.arrayView, surface.width, surface.height, surface.layers, surface.samples, first, elements); + RecordStayingOnHeap(surface.context, batch.Handle(), "the multisample readback", [&] { reader.DispatchMultisampleReadback(batch.Handle(), surface.format, output.Handle(), offset, bytes, surface.arrayView, surface.width, surface.height, surface.layers, surface.samples, first, elements, &surface); }); if (repeat + 1u < repetitions) RecordMemoryBarrier(surface.context, batch.Handle(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); } RecordMemoryBarrier(surface.context, batch.Handle(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); @@ -373,7 +391,7 @@ void CheckRefusals(TextureDetiler& reader, const Surface& surface) { Buffer output(surface.context, 4096u, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); CommandBatch batch(surface.context); const auto call = [&](VkFormat format, std::uint64_t offset, std::uint64_t bytes, std::uint32_t width, std::uint32_t height, std::uint32_t layers, std::uint32_t fragments, std::uint32_t first, std::uint32_t elements) { - reader.DispatchMultisampleReadback(batch.Handle(), format, output.Handle(), offset, bytes, surface.arrayView, width, height, layers, fragments, first, elements); + RecordStayingOnHeap(surface.context, batch.Handle(), "the multisample readback", [&] { reader.DispatchMultisampleReadback(batch.Handle(), format, output.Handle(), offset, bytes, surface.arrayView, width, height, layers, fragments, first, elements, &surface); }); }; for (auto format : {VK_FORMAT_UNDEFINED, VK_FORMAT_R32_SFLOAT, VK_FORMAT_D16_UNORM, VK_FORMAT_D32_SFLOAT_S8_UINT}) Refused([&] { call(format, 0u, 4u, 1u, 1u, 1u, 4u, 0u, 1u); }, "requires an unsigned element view or D32 depth"); for (auto fragments : {0u, 1u, 3u, 16u}) Refused([&] { call(surface.format, 0u, 4u, 1u, 1u, 1u, fragments, 0u, 1u); }, "fragment count is unsupported"); @@ -406,7 +424,7 @@ std::vector Upload(TextureDetiler& transfer, TextureDetiler& reader, for (std::uint32_t repeat = 0; repeat < repetitions; ++repeat) { for (std::size_t done = 0; done < bytes; done += stride) { const auto count = std::min(stride, bytes - done); - transfer.DispatchMultisampleImage(batch.Handle(), surface.elementBytes, source.Handle(), offset + done, (count + 3u) & ~std::size_t{3}, surface.arrayView, surface.width, surface.height, surface.layers, surface.samples, first + static_cast(done / surface.elementBytes), static_cast(count / surface.elementBytes)); + RecordStayingOnHeap(surface.context, batch.Handle(), "the multisample upload", [&] { transfer.DispatchMultisampleImage(batch.Handle(), surface.elementBytes, source.Handle(), offset + done, (count + 3u) & ~std::size_t{3}, surface.arrayView, surface.width, surface.height, surface.layers, surface.samples, first + static_cast(done / surface.elementBytes), static_cast(count / surface.elementBytes), &surface); }); } RecordMemoryBarrier(surface.context, batch.Handle(), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleRetirement.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleRetirement.cpp index 38d9e56ad90c..0decb5acdc19 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleRetirement.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MultisampleRetirement.cpp @@ -299,8 +299,8 @@ class Run { realCreateImage = context.Function("vkCreateImage"); realDestroyImage = context.Function("vkDestroyImage"); realBindImageMemory = context.Function("vkBindImageMemory"); - realCreateImageView = context.Function("vkCreateImageView"); - realDestroyImageView = context.Function("vkDestroyImageView"); + realCreateImageView = reinterpret_cast(realProc(context.device, "vkCreateImageView")); + realDestroyImageView = reinterpret_cast(realProc(context.device, "vkDestroyImageView")); realClearColorImage = context.Function("vkCmdClearColorImage"); realCopyImage = context.Function("vkCmdCopyImage"); context.deviceProc = CountingDeviceProc; @@ -488,7 +488,7 @@ class Run { { std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); const auto keptBefore = context.recorder->OpenKeptBytes(); - storageView = MultisampleTargetStorageView(context, resource); + storageView = MultisampleTargetStorageView(context, resource).view; promoted = ImageFor(storageView, &promotedImage); Require(promoted.generation != old.generation && Alive(oldImage, old), "MSAA storage promotion did not retain its source image"); Require(ImageFor(sampledView).generation == old.generation, "MSAA storage promotion destroyed a held sampled view"); @@ -496,7 +496,7 @@ class Run { totalBytes = AllocationBytesOf(old) + AllocationBytesOf(promoted); const auto kept = context.recorder->OpenKeptBytes(); Require(kept == keptBefore + totalBytes, "MSAA storage promotion did not account for exactly both allocations"); - Require(MultisampleTargetStorageView(context, resource) == storageView, "a repeated MSAA storage lookup changed its view"); + Require(MultisampleTargetStorageView(context, resource).view == storageView, "a repeated MSAA storage lookup changed its view"); Require(context.recorder->OpenKeptBytes() == kept, "a repeated MSAA storage lookup double-counts its allocations"); } Sync(); @@ -506,10 +506,10 @@ class Run { Require(held->View() == sampledView, "a held sampled MSAA alias changed after promotion completion"); held.reset(); const auto keptBefore = context.recorder->OpenKeptBytes(); - Require(MultisampleTargetStorageView(context, resource) == storageView, "a later MSAA storage lookup changed its promoted image"); + Require(MultisampleTargetStorageView(context, resource).view == storageView, "a later MSAA storage lookup changed its promoted image"); const auto kept = context.recorder->OpenKeptBytes(); Require(kept == keptBefore + totalBytes, "an already-storable MSAA target is missing allocations from the new batch budget"); - Require(MultisampleTargetStorageView(context, resource) == storageView, "a repeated promoted MSAA lookup changed its view"); + Require(MultisampleTargetStorageView(context, resource).view == storageView, "a repeated promoted MSAA lookup changed its view"); Require(context.recorder->OpenKeptBytes() == kept, "an already-storable MSAA target double-counts its allocations in one batch"); } block.Unmap(); @@ -549,7 +549,7 @@ class Run { Sync(); { std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); - promoted = ImageFor(MultisampleTargetStorageView(context, resource), &promotedImage); + promoted = ImageFor(MultisampleTargetStorageView(context, resource).view, &promotedImage); Require(bornStorable == (promoted.generation == rendered.generation), bornStorable ? "a born-storable MSAA target was promoted to a new image" : "MSAA storage promotion kept the rendered image"); const auto promotedTwin = MultisampleTargetView(context, twin); ImageFor(promotedTwin, &promotedTwinImage); @@ -573,9 +573,9 @@ class Run { { std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); MultisampleTargetView(context, target); - storageView = MultisampleTargetStorageView(context, resource); + storageView = MultisampleTargetStorageView(context, resource).view; Require(storageView != VK_NULL_HANDLE, "a UNORM storage access to a multisampled sRGB target has no view"); - Require(MultisampleTargetStorageView(context, resource) == storageView, "a repeated UNORM storage access to a multisampled sRGB target changed its view"); + Require(MultisampleTargetStorageView(context, resource).view == storageView, "a repeated UNORM storage access to a multisampled sRGB target changed its view"); resource.format = 60; bool refused = false; try { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp index 7d9abb917bed..d4d9024a7832 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PipelineJournal.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" @@ -16,6 +17,7 @@ #include "Recompiler.hpp" #include "ShaderDiskCache.hpp" #include "Optimization/ResourceProgram.hpp" +#include "HeapSampleKernel.hpp" #include "VulkanTestDevice.hpp" #ifdef _WIN32 #ifndef NOMINMAX @@ -29,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -306,10 +309,50 @@ Totals Snapshot() { return {AgcDriver::Graphics::PipelineJournalTotals(), AgcDriver::Graphics::PipelineStoreTotals()}; } +alignas(256) std::array SampleTexture{}; +alignas(4096) std::array SampleOutput{}; + +void DispatchSample(AgcDriver::VulkanDevice& device, const char* what) { + SampleOutput.fill(-1.0f); + const auto result = CompileSampleKernel(device, SampleTexture.data(), SampleOutput.data()); + Require(result.gpuResolvedImages, std::string(what) + ": the sampling compute stage is not GPU mode"); + const auto before = AgcDriver::Graphics::HeapComputeStages(); + device.Dispatch(result, 1u, 1u, 1u, {}, reinterpret_cast(SampleCode.data())); + device.WaitIdle(); + Require(AgcDriver::Graphics::HeapComputeStages() > before, std::string(what) + ": the heap did not take the sampling compute stage"); + RequireSampled(SampleOutput.data(), SampleTexel, what); +} + +void HeapJournalTests() { + FillTexels(SampleTexture, SampleTexel); + { + const auto device = OpenVulkanTestDevice(); + Require(device != nullptr, "the first device is required once the second opened"); + device->WaitForPipelineJournalReplay(); + DispatchSample(*device, "first run"); + Require(device->ComputePipelinesCreated() == 1, "the first run did not build exactly one compute pipeline"); + AgcDriver::Graphics::FlushPipelineJournal(); + Require(Snapshot().journal.recordedCompute == 1, "the first run did not record the GPU-mode compute pipeline"); + Require(AgcDriver::Graphics::PipelineJournalPath().filename() == "pipelines-heap.journal", "the GPU-mode pipelines were recorded in the journal the per-draw binding model replays"); + } + { + const auto device = OpenVulkanTestDevice(); + Require(device != nullptr, "the second device could not be opened"); + device->WaitForPipelineJournalReplay(); + const auto replayed = Snapshot(); + Require(replayed.journal.replayedCompute == 1 && replayed.journal.replayFailed == 0 && replayed.journal.replayRejected == 0, "the replay did not build the GPU-mode compute pipeline"); + Require(device->ComputePipelinesCreated() == 1, "the replay did not build the compute pipeline on the new device"); + Require(device->DescriptorHeapComputePipelines() == 1, "the replay did not build the GPU-mode compute pipeline on the descriptor heap"); + DispatchSample(*device, "second run"); + Require(device->ComputePipelinesCreated() == 1, "the second run's dispatch built a compute pipeline the replay had built"); + } +} + void JournalTests() { using namespace NarrowConstantStoreFixture; Require(AgcDriver::Graphics::PipelineJournalEnabled(), "the pipeline journal is disabled although the shader cache directory is set"); const auto journalPath = AgcDriver::Graphics::PipelineJournalPath(); + Require(journalPath.filename() == (AgcDriver::Graphics::GpuHeapEnabled() ? "pipelines-heap.journal" : "pipelines.journal"), "the pipeline journal file does not match the GPU heap mode"); Block compute(BlockBytes); Block target(ColorTargetLayout(Width, Height, ColorTileMode::Linear, 2).Bytes()); const auto computeCode = ComputeCode(0); @@ -804,6 +847,12 @@ int main(int argc, char** argv) { Benchmark(argc >= 3 ? static_cast(std::strtoul(argv[2], nullptr, 10)) : 24u); return 0; } + if (argc >= 2 && std::string(argv[1]) == "--gpu-heap") { + if (AgcDriver::Graphics::GpuHeapEnabled() && SkipWithoutDescriptorHeap()) return VulkanTestSkipped; + HeapJournalTests(); + std::puts("pipeline journal record and replay of a GPU-mode compute stage passed"); + return 0; + } JournalTests(); ReplayRacesDispatchTests(); NewestEntriesTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp index b8793c04b5ae..9827dba6f0fb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" @@ -88,6 +89,7 @@ ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, Emulatio Pixels.fill(Unwritten); auto target = device.Target(); target.fragmentShaderBarycentricEnabled = false; + target.gpuResolvedImages = AgcDriver::Graphics::DescriptorHeapEnabled(); target.fragmentBarycentricBuiltins = builtins; static const auto trianglePositions = ClipPositions(Emulation::Geometry); static const auto rectanglePositions = ClipPositions(Emulation::RectList); @@ -127,6 +129,7 @@ ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, Emulatio }; fragment.useCache = false; auto pixelResult = ShaderRecompiler::Recompile(fragment); + AgcDriver::Graphics::Require(vertexResult.gpuResolvedImages == target.gpuResolvedImages && pixelResult.gpuResolvedImages == target.gpuResolvedImages, "the barycentric draw's stages did not take GPU mode"); std::vector shaders{{ShaderStage::Vertex, &vertexResult, 0}}; ShaderRecompiler::RecompileResult geometry; ShaderRecompiler::BarycentricTessellationShaders patches; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp index ca80e465ebc5..627a934b4c77 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Draw/HeapStages.hpp" #include "VulkanTestDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" #include "SceShaders.hpp" @@ -23,6 +24,12 @@ namespace { +std::vector PreparePerDrawStages(const AgcDriver::DriverDetail::DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) { + auto stages = AgcDriver::DriverDetail::PrepareGraphicsStages(decoded, target); + std::erase_if(stages, [](const AgcDriver::DriverDetail::PreparedGraphicsStage& stage) { return stage.heapVariant; }); + return stages; +} + void Require(bool condition, const char* message) { if (!condition) throw std::runtime_error(message); } @@ -135,7 +142,7 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path target.mesh = MeshTargetLimits{{128, 1, 1}, 128, 32768, 256, 256, 128, 32768, 1, 1}; target.spirvVersion = 0x00010400u; } - const auto stages = PrepareGraphicsStages(prepared, target); + const auto stages = PreparePerDrawStages(prepared, target); Require(stages.size() == (tessellation ? 4u : 2u), "graphics preparation compiled the wrong stages"); if (!mesh && !tessellation && target.registrationModules) { const auto prebuild = [&](const std::vector& group, std::initializer_list picked) { @@ -153,10 +160,17 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path Require(stages.front().registration == nullptr && !GetPreparedArtifact(*stages.front().entry.handle).registrationModule, "registration modules: the vertex stage was prepared in registration form"); const auto pixelOnly = prebuild(stages, {1u}); Require(pixelOnly.second == 1u, "registration modules: the pixel stage alone did not build its fragment library"); - const auto again = PrepareGraphicsStages(prepared, target); + const auto again = PreparePerDrawStages(prepared, target); Require(again.back().registration != stages.back().registration, "registration modules: the second preparation reused the first result object"); const auto repeated = prebuild(again, {1u}); Require(repeated.first == 0u && repeated.second == 0u, "registration modules: another materialization of the same stage did not find the prebuilt library"); + std::size_t heapVertexStages = 0; + for (const auto& stage : AgcDriver::DriverDetail::PrepareGraphicsStages(prepared, target)) { + if (!stage.heapVariant || stage.stage != ShaderRecompiler::ShaderStage::Vertex) continue; + Require(GetPreparedArtifact(*stage.entry.handle).registrationModule, "registration modules: the heap vertex stage was not prepared in registration form"); + ++heapVertexStages; + } + Require(!target.descriptorMemory || heapVertexStages != 0, "registration modules: the descriptor heap prepared no heap vertex stage"); } for (const auto& stage : stages) stage.snapshot->prepared->entries.push_back(stage.entry); const auto& pixelArtifact = GetPreparedArtifact(*stages.back().entry.handle); @@ -192,7 +206,7 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path auto changed = prepared; changed.state.stages.vertexWaveSize = 64; changed.pixel.interpolatorSettings[0] = 0x404u; - const auto newStages = PrepareGraphicsStages(changed, target); + const auto newStages = PreparePerDrawStages(changed, target); front.snapshot->prepared->entries.push_back(newStages.front().entry); ResolvePreparedGraphics(*front.snapshot, {}, 7, target); const auto newVertexId = GetPreparedArtifact(*newStages.front().entry.handle).variantId; @@ -214,7 +228,7 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path Require(front.snapshot->prepared->rectangleProgress.size() == 1 && !front.snapshot->prepared->rectangleProgress.front().fragment.expired(), "expired rectangle progress was not removed"); auto drawOnly = prepared; drawOnly.pixel.interpolatorSettings[0] = 0x405u; - const auto drawStages = PrepareGraphicsStages(drawOnly, target); + const auto drawStages = PreparePerDrawStages(drawOnly, target); auto unlinked = std::make_shared(); unlinked->type = 1; unlinked->prepared->entries.push_back(drawStages.back().entry); @@ -236,10 +250,10 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path } auto incomplete = prepared; incomplete.roles.pop_back(); - Reject([&] { static_cast(PrepareGraphicsStages(incomplete, target)); }, "program roles are incomplete"); + Reject([&] { static_cast(PreparePerDrawStages(incomplete, target)); }, "program roles are incomplete"); auto wrongEntry = prepared; wrongEntry.programs.front().binary.codeAddress += 4; - Reject([&] { static_cast(PrepareGraphicsStages(wrongEntry, target)); }, "entry point differs"); + Reject([&] { static_cast(PreparePerDrawStages(wrongEntry, target)); }, "entry point differs"); std::vector linked; std::vector memory; for (std::size_t index = 0; index < draw.programs.size(); ++index) { @@ -283,7 +297,7 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path resolved.pixel = prepared.pixel; DecodeGraphicsPrograms(resolved, unbound, registry, true, true); Require(resolved.programs.size() == 3u && resolved.roles.front() == ProgramRole::Hull, "an unbound local program was not left to the draw"); - Require(PrepareGraphicsStages(resolved, target).size() == 3u, "graphics preparation without the local program compiled the wrong stages"); + Require(PreparePerDrawStages(resolved, target).size() == 3u, "graphics preparation without the local program compiled the wrong stages"); auto bound = queue; late.Bind(bound, 0x148u, 0x10bu); DrawDecode lateDraw{}; @@ -345,7 +359,7 @@ void PrebuildPolicyReadback(AgcDriver::VulkanDevice& device, bool disabled) { append(ShaderStage::Vertex, vertexCode, {static_cast(address), static_cast(address >> 32u) | (16u << 16u), 3u, 0x01016facu}); append(ShaderStage::Fragment, pixelCode, {}); const auto target = device.Target(); - const auto stages = PrepareGraphicsStages(decoded, target); + const auto stages = PreparePerDrawStages(decoded, target); Require(stages.size() == 2u, "prebuild policy lost a required graphics stage"); for (const auto& stage : stages) { Require(stage.entry.handle != nullptr && stage.snapshot != nullptr, "prebuild policy lost a required source handle"); @@ -447,7 +461,7 @@ void RetainedPixelMatchesNullProgram(AgcDriver::VulkanDevice& device) { Require(AgcDriver::Graphics::DrawKeyCovers(read), "a pixel substitution dependency is absent from the draw key"); Require(read.bank != AgcDriver::Graphics::RegisterBank::Shader || read.offset < 0x00bu || read.offset > 0x02bu, "the substituted program read fragment resources or userdata"); } - const auto stages = PrepareGraphicsStages(draw, device.Target()); + const auto stages = PreparePerDrawStages(draw, device.Target()); Require(stages.size() == 2 && stages.back().entry.handle != nullptr, "a retained-format null program was not prepared"); std::vector linked; std::vector memory; @@ -556,7 +570,23 @@ void CheckMergedHull(AgcDriver::VulkanDevice& device) { DecodeGraphicsPrograms(stale, unmapped, registry, false, true); Require(stale.programs[1].userData[0] == 0x10u && stale.programs[1].userData[1] == 0u, "a merged hull draw did not pass a user-data address it does not need unchecked"); const auto stages = PrepareGraphicsStages(prepared, device.Target()); - Require(stages.size() == 4u && stages[0].entry.handle != stages[1].entry.handle, "a merged hull program was not prepared as separate local and hull stages"); + for (const bool heapVariant : {false, true}) { + std::vector set; + for (const auto& stage : stages) { + if (stage.heapVariant == heapVariant) set.push_back(&stage); + } + if (heapVariant && set.empty()) { + Require(!HeapFragmentsEnabled(), "the heap prepared no heap variants of the merged hull draw"); + continue; + } + const auto handleOf = [&](ShaderRecompiler::ShaderStage wanted) { + const auto found = std::find_if(set.begin(), set.end(), [&](const auto* stage) { return stage->stage == wanted; }); + return found == set.end() ? nullptr : (*found)->entry.handle.get(); + }; + const auto* local = handleOf(ShaderRecompiler::ShaderStage::Local); + const auto* hull = handleOf(ShaderRecompiler::ShaderStage::TessellationControl); + Require((heapVariant || set.size() == 4u) && local != nullptr && hull != nullptr && local != hull, "a merged hull program was not prepared as separate local and hull stages"); + } for (const auto& stage : stages) stage.snapshot->prepared->entries.push_back(stage.entry); std::vector linked; std::vector memory; @@ -704,7 +734,7 @@ void NullPixelMatchesRegistration(AgcDriver::VulkanDevice& device) { DecodeGraphicsPrograms(draw, queue, registry, false, true); const auto& fragment = draw.programs.back(); Require(fragment.snapshot == null && fragment.userData.empty(), "a null pixel draw took the last pixel shader's user SGPRs"); - const auto stages = PrepareGraphicsStages(draw, device.Target()); + const auto stages = PreparePerDrawStages(draw, device.Target()); const auto& handle = stages.back().entry.handle; Require(std::ranges::any_of(registered, [&](const auto& entry) { return entry.handle == handle; }), "a null pixel draw did not match the variant prepared at registration"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index f02dadb11991..576471caf845 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -83,6 +83,7 @@ void RunUnregistered(AgcDriver::VulkanDevice& device, ShaderRecompiler::Recompil Require(snapshot.prepared->entries.size() == 1, "unregistered compute invocation did not cache exactly one artifact"); const auto handle = snapshot.prepared->entries.front().handle; const auto& artifact = ShaderRecompiler::GetPreparedArtifact(*handle); + const std::uint32_t* module = artifact.gpuResolvedImages ? nullptr : artifact.spirv.data(); for (const bool useCache : {false, true}) { request.useCache = useCache; Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request) == handle, "unregistered compute source was prepared again"); @@ -91,7 +92,8 @@ void RunUnregistered(AgcDriver::VulkanDevice& device, ShaderRecompiler::Recompil runtime.userData = request.context.userData; const auto capture = repeated.Capture(runtime); const auto result = repeated.Materialize(*capture); - Require(snapshot.prepared->entries.size() == 1 && result->variantId == artifact.variantId && result->spirv.data() == artifact.spirv.data(), "unregistered compute invocation replaced its cached artifact"); + if (module == nullptr) module = result->spirv.data(); + Require(snapshot.prepared->entries.size() == 1 && result->variantId == artifact.variantId && result->spirv.data() == module, "unregistered compute invocation replaced its cached artifact"); device.Dispatch(*result, 1, 1, 1); } device.WaitIdle(); @@ -134,6 +136,7 @@ void Run(AgcDriver::VulkanDevice& device) { snapshot.prepared->entries.push_back({0, handle}); ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 1, request)); }, "outside the snapshot"); const auto& artifact = ShaderRecompiler::GetPreparedArtifact(*handle); + const std::uint32_t* module = artifact.gpuResolvedImages ? nullptr : artifact.spirv.data(); for (std::uint32_t value = 0; value < 8; ++value) { users.fill(value); request.useCache = value % 2 == 0; @@ -142,7 +145,8 @@ void Run(AgcDriver::VulkanDevice& device) { runtime.userData = users; const auto capture = ShaderRecompiler::CaptureResources(request, runtime, *ready); const auto result = ShaderRecompiler::MaterializeShader(request, *capture, *ready); - Require(result->variantId == artifact.variantId && result->spirv.data() == artifact.spirv.data(), "invocation replaced the prepared artifact"); + if (module == nullptr) module = result->spirv.data(); + Require(result->variantId == artifact.variantId && result->spirv.data() == module, "invocation replaced the prepared artifact"); device.Dispatch(*result, 1, 1, 1); } device.WaitIdle(); @@ -178,7 +182,7 @@ void Run(AgcDriver::VulkanDevice& device) { const auto replacementInvocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, replacementRequest); const auto replacementCapture = replacementInvocation.Capture(preparedRuntime); const auto replacementResult = replacementInvocation.Materialize(*replacementCapture); - Require(replacementResult->spirv.data() == artifact.spirv.data(), "device replacement recompiled the prepared shader"); + Require(replacementResult->spirv.data() == (artifact.gpuResolvedImages ? firstResult->spirv.data() : artifact.spirv.data()), "device replacement recompiled the prepared shader"); replacement->Dispatch(*replacementResult, 1, 1, 1); replacement->WaitIdle(); replacementRequest.target.nonConstantImageOffsets = !replacementRequest.target.nonConstantImageOffsets; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PushOnlyHeapPasses.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PushOnlyHeapPasses.cpp new file mode 100644 index 000000000000..610fc266e510 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PushOnlyHeapPasses.cpp @@ -0,0 +1,116 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::BindlessHeap; +using AgcDriver::Graphics::Buffer; +using AgcDriver::Graphics::Recorder; +using AgcDriver::Graphics::Require; + +void RunChangedCopy(const AgcDriver::VulkanDevice& device) { + const auto context = device.graphicsContext(); + const bool heapMode = AgcDriver::Graphics::DescriptorHeapEnabled() && context.descriptorHeap && BindlessHeap::For(context).DescriptorHeapMode(); + constexpr std::uint32_t words = 300; + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + auto result = std::make_shared(context, words * 4u, usage); + auto snapshot = std::make_shared(context, words * 4u, usage); + auto target = std::make_shared(context, words * 4u, usage); + std::vector shader(words); + for (std::uint32_t i = 0; i < words; ++i) shader[i] = i % 3 == 0 ? 0x9e3779b9u * (i + 1u) : 0u; + std::memset(snapshot->Bytes().data(), 0, words * 4u); + std::memset(target->Bytes().data(), 0, words * 4u); + std::memcpy(result->Bytes().data(), shader.data(), words * 4u); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "push-only heap passes: no active recorder"); + if (heapMode) BindlessHeap::For(context).BindDescriptorHeaps(recorder->Commands()); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds(); + Require(recorder->RecordChangedCopy(recorder->Commands(), result->DeviceAddress(), snapshot->DeviceAddress(), target->DeviceAddress(), words), "push-only heap passes: the changed copy kernel is unavailable"); + if (heapMode) { + BindlessHeap::For(context).BindDescriptorHeaps(recorder->Commands()); + Require(AgcDriver::Graphics::DescriptorHeapBinds() == binds, "push-only heap passes: the changed copy put its command buffer back in classic descriptor mode, so the heaps were bound again"); + } + recorder->Keep(result); + recorder->Keep(snapshot); + recorder->Keep(target); + recorder->Submit(); + recorder->Sync(); + std::vector stored(words); + std::memcpy(stored.data(), target->Bytes().data(), words * 4u); + for (std::uint32_t i = 0; i < words; ++i) { + Require(stored[i] == shader[i], "push-only heap passes: changed copy word " + std::to_string(i) + " is " + std::to_string(stored[i]) + ", the shader wrote " + std::to_string(shader[i])); + } +} + +void RunDispatchArguments(const AgcDriver::VulkanDevice& device) { + const auto context = device.graphicsContext(); + const bool heapMode = AgcDriver::Graphics::DescriptorHeapEnabled() && context.descriptorHeap && BindlessHeap::For(context).DescriptorHeapMode(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT; + auto threads = std::make_shared(context, 16, usage); + auto oversized = std::make_shared(context, 16, usage); + auto grouped = std::make_shared(context, 32, usage); + auto refused = std::make_shared(context, 32, usage); + const std::uint32_t threadCounts[4] = {3u, 5u, 7u, 0u}; + const auto* limit = context.limits.maxComputeWorkGroupCount; + const auto axis = static_cast(std::min_element(limit, limit + 3) - limit); + const bool limited = limit[axis] < std::numeric_limits::max(); + std::uint32_t tooMany[4] = {1u, 1u, 1u, 0u}; + tooMany[axis] = limited ? limit[axis] + 1u : limit[axis]; + std::memcpy(threads->Bytes().data(), threadCounts, sizeof(threadCounts)); + std::memcpy(oversized->Bytes().data(), tooMany, sizeof(tooMany)); + std::memset(grouped->Bytes().data(), 0xff, 32); + std::memset(refused->Bytes().data(), 0xff, 32); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto* recorder = Recorder::Active(); + Require(recorder != nullptr, "push-only heap passes: no active recorder"); + if (heapMode) BindlessHeap::For(context).BindDescriptorHeaps(recorder->Commands()); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds(); + const std::array group{2u, 2u, 2u}; + Require(recorder->RecordDispatchArguments(recorder->Commands(), threads->DeviceAddress(), grouped->DeviceAddress(), &group), "push-only heap passes: the dispatch argument kernel is unavailable"); + Require(recorder->RecordDispatchArguments(recorder->Commands(), oversized->DeviceAddress(), refused->DeviceAddress()), "push-only heap passes: the dispatch argument kernel is unavailable"); + if (heapMode) { + BindlessHeap::For(context).BindDescriptorHeaps(recorder->Commands()); + Require(AgcDriver::Graphics::DescriptorHeapBinds() == binds, "push-only heap passes: the dispatch argument check put its command buffer back in classic descriptor mode, so the heaps were bound again"); + } + for (const auto& kept : {threads, oversized, grouped, refused}) recorder->Keep(kept); + recorder->Submit(); + recorder->Sync(); + std::uint32_t words[6]{}; + std::memcpy(words, grouped->Bytes().data(), sizeof(words)); + const std::uint32_t expected[6] = {2u, 3u, 4u, 3u, 5u, 7u}; + for (std::size_t i = 0; i < 6; ++i) Require(words[i] == expected[i], "push-only heap passes: dispatch argument word " + std::to_string(i) + " is " + std::to_string(words[i]) + ", expected " + std::to_string(expected[i])); + std::memcpy(words, refused->Bytes().data(), 3 * sizeof(std::uint32_t)); + if (limited) Require(words[0] == 0u && words[1] == 0u && words[2] == 0u, "push-only heap passes: a group count past the device limit was dispatched"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + RunChangedCopy(*device); + RunDispatchArguments(*device); + device->WaitIdle(); + std::puts("push-only heap pass tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp index de2e3a8c710e..50fe78d6e35d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampleMultisampleTarget.cpp @@ -2,12 +2,15 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTarget.hpp" #include "prx/libSceAgcDriver/Graphics/include/MultisampleTextureLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/State.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -465,7 +468,7 @@ ShaderRecompiler::RecompileResult CompileProgram(AgcDriver::VulkanDevice& device ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; @@ -597,6 +600,26 @@ void CheckFirstSampleReads(AgcDriver::VulkanDevice& device, const std::arrayCommands()); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds(); + const auto texture = AgcDriver::Graphics::MultisampleTargetFirstSample(context, resource, {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}); + Require(texture != nullptr, "first-sample copy in heap mode: the drawn 4x target has no sample 0 texture"); + heap.BindDescriptorHeaps(recorder->Commands()); + Require(AgcDriver::Graphics::DescriptorHeapBinds() == binds, + "a first-sample copy in heap mode put its command buffer back in classic descriptor mode, so the heaps were bound again"); + recorder->Submit(); + recorder->Sync(); +} + std::array CompressedDescriptor(const Block& block, std::uint32_t tileMode, std::uint64_t htile) { auto texture = TargetDescriptor(block, SampleShift, Format16161616Float, tileMode); const auto meta = htile >> 8u; @@ -1331,6 +1354,7 @@ int main(int argc, char** argv) { const auto drawn = LoadSamples(*device, block); CheckSamples(state, drawn, {}); CheckFirstSampleReads(*device, TargetDescriptor(block, SampleShift), drawn, true, "from the drawn 4x target"); + CheckFirstSampleCopyStaysInHeapMode(*device, block); CheckFirstSampleReads(*device, TargetDescriptor(block, SampleShift), drawn, true, "from the drawn 4x target", AdjustedPlaneCode, "image_sample_a"); std::puts("a DIM 2D image_sample_a read of a 4x color target returns each pixel's sample 0"); Draw(*device, state, 0, 2u, Unorm8PixelCode); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampleOrder.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampleOrder.cpp index ba9346b3e716..e3a921640cd3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampleOrder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampleOrder.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" @@ -61,6 +62,21 @@ struct VulkanDeviceTestAccess { namespace { +template +void RecordStayingOnHeap(const AgcDriver::Graphics::Context& context, VkCommandBuffer commands, const std::string& what, Record&& record) { + if (!AgcDriver::Graphics::HeapPassesActive(context)) { + record(); + return; + } + auto& heap = AgcDriver::Graphics::BindlessHeap::For(context); + heap.BindDescriptorHeaps(commands); + const auto binds = AgcDriver::Graphics::DescriptorHeapBinds(); + record(); + heap.BindDescriptorHeaps(commands); + if (AgcDriver::Graphics::DescriptorHeapBinds() != binds) throw std::runtime_error(what + " put its command buffer back in classic descriptor mode, so the heaps were bound again"); +} + + using AgcDriver::Graphics::Check; using AgcDriver::Graphics::Context; using AgcDriver::Graphics::KeptSampleOrder; @@ -524,7 +540,8 @@ bool ColorSwaps(const Context& context, VkFormat format, std::uint32_t elementBy texels.Storage(1u, values.Handle()); const std::array store{SwapExtent.width, SwapExtent.height, samples, 1u}; const std::array load{SwapExtent.width, SwapExtent.height, samples, 0u}; - const SampleOrderTarget target{image.Handle(), view, alias, SwapExtent, samples, SampleOrderMethod::Storage, nullptr}; + const AgcDriver::Graphics::HeapPassSlots slots(context); + const SampleOrderTarget target{image.Handle(), view, alias, SwapExtent, samples, SampleOrderMethod::Storage, nullptr, &slots}; const SampleOrderPass pass{order, SwapRegion, {}, false}; std::vector> kept; Submit(context, [&](VkCommandBuffer commands) { @@ -536,14 +553,14 @@ bool ColorSwaps(const Context& context, VkFormat format, std::uint32_t elementBy }); Require(Masked(ReadValues(values, count), elementBytes) == before, name + ": the stored samples did not read back"); Submit(context, [&](VkCommandBuffer commands) { - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), true, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), true, kept); }); FullBarrier(context, commands); texels.Dispatch(commands, load.data(), SwapExtent); HostBarrier(context, commands); }); CheckOrdered(before, Masked(ReadValues(values, count), elementBytes), SwapExtent, SwapRegion, samples, order, true, name + " put in the host's order"); Submit(context, [&](VkCommandBuffer commands) { - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), false, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), false, kept); }); FullBarrier(context, commands); texels.Dispatch(commands, load.data(), SwapExtent); HostBarrier(context, commands); @@ -624,14 +641,14 @@ bool DepthSwaps(const Context& context, VkFormat format, std::uint32_t samples, Require(depthHeld && (!stencil || before[index][1] == filled[index][1]), name + ": the filled sample " + std::to_string(index) + " did not read back"); } Submit(context, [&](VkCommandBuffer commands) { - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), true, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), true, kept); }); FullBarrier(context, commands); read.Dispatch(commands, push.data(), SwapExtent); HostBarrier(context, commands); }); CheckOrdered(before, ReadValues(values, count), SwapExtent, SwapRegion, samples, order, true, name + " put in the host's order"); Submit(context, [&](VkCommandBuffer commands) { - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), false, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(&target, 1u), false, kept); }); FullBarrier(context, commands); read.Dispatch(commands, push.data(), SwapExtent); HostBarrier(context, commands); @@ -1442,8 +1459,9 @@ int Bench(AgcDriver::VulkanDevice& device, VkExtent2D extent, std::uint32_t samp Image depth(context, VK_FORMAT_D32_SFLOAT, samples, extent, DepthUsage, located ? VK_IMAGE_CREATE_SAMPLE_LOCATIONS_COMPATIBLE_DEPTH_BIT_EXT : 0u); const auto storage = color.View(alias, VK_IMAGE_ASPECT_COLOR_BIT, VK_IMAGE_USAGE_STORAGE_BIT); const auto attachment = depth.View(VK_FORMAT_D32_SFLOAT, VK_IMAGE_ASPECT_DEPTH_BIT, VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT); + const AgcDriver::Graphics::HeapPassSlots slots(context); const std::array targets{{ - {color.Handle(), storage, alias, extent, samples, SampleOrderMethod::Storage, nullptr}, + {color.Handle(), storage, alias, extent, samples, SampleOrderMethod::Storage, nullptr, &slots}, {depth.Handle(), attachment, VK_FORMAT_D32_SFLOAT, extent, samples, SampleOrderMethod::Depth, nullptr}, }}; VkQueryPoolCreateInfo queryInfo{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; @@ -1466,13 +1484,13 @@ int Bench(AgcDriver::VulkanDevice& device, VkExtent2D extent, std::uint32_t samp context.Function("vkCmdWriteTimestamp")(commands, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, queries, query); }; stamp(0u); - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(targets).subspan(1u), true, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(targets).subspan(1u), true, kept); }); stamp(1u); - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(targets).subspan(1u), false, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, std::span(targets).subspan(1u), false, kept); }); stamp(2u); - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, targets, true, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, targets, true, kept); }); stamp(3u); - AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, targets, false, kept); + RecordStayingOnHeap(context, commands, "the sample order pass", [&] { AgcDriver::Graphics::RecordSampleOrder(context, commands, pass, targets, false, kept); }); stamp(4u); }); std::array ticks{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp index a054a1e69f92..62a4b1c4349f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp @@ -1,5 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" @@ -584,6 +586,47 @@ void viewTests(const Context& context) { Require(heldView.expired() && heldStorage.expired(), "the evicted image outlived its last caller without a sampled lookup"); } +bool heapViewTests(const Context& context) { + if (!GpuHeapEnabled() || !context.descriptorHeap || context.hostImportAlignment == 0) return false; + Block block(context); + if (HostImportFor(context, block.Surface(0), BlockBytes) == nullptr) return false; + auto view = Sampled(context, block.Surface(0)); + Require(view->ViewsStorageImage(), "a sampled texture in host-imported memory is not a view of its storage image"); + const std::weak_ptr viewed = view; + for (std::size_t surface = 1; surface < Surfaces; ++surface) CachedStorageSurface(context, Resource(block.Surface(surface))); + Require(!view->StorageSource()->Cached(), "the heap view's storage image was not evicted"); + auto& heap = BindlessHeap::For(context); + const auto slot = heap.TextureSlot(ShaderRecompiler::RuntimeAbi::FirstImageBinding, view.get(), view->View(), view->Layout()); + Require(slot.has_value(), "the view of an evicted storage image took no heap slot"); + view.reset(); + Require(viewed.expired(), "a heap slot kept the view of an evicted storage image, and with it the image"); + Require(heap.SlotTexture(ShaderRecompiler::RuntimeAbi::FirstImageBinding, *slot) == nullptr, "a destroyed view kept its heap slot"); + return true; +} + +void heapHeldReleaseTests(const Context& context) { + Block block(context); + auto& heap = BindlessHeap::For(context); + std::vector> held; + std::vector slots; + for (std::size_t surface = 0; surface < 2; ++surface) { + auto texture = Sampled(context, block.Surface(surface)); + const auto slot = heap.TextureSlot(ShaderRecompiler::RuntimeAbi::FirstImageBinding, texture.get(), texture->View(), texture->Layout()); + Require(slot.has_value(), "the cached sampled texture took no heap slot"); + held.push_back(texture); + slots.push_back(*slot); + } + Require(TextureCacheUsage().sampledEntries == 2u, "the two sampled textures were not cached"); + Require(!held[0].expired() && !held[1].expired(), "the cache and the heap slots did not keep the sampled textures"); + if (auto* recorder = Recorder::Active()) recorder->Sync(); + const auto released = ReleaseTextureCaches(SurfaceBytes, true); + Require(held[0].expired() != held[1].expired(), "a memory recovery for one texture's bytes did not free exactly one of two textures that only the cache and the GPU heap held"); + Require(released >= SurfaceBytes && TextureCacheUsage().sampledEntries == 1u, "a memory recovery did not count the texture it freed from the cache and the GPU heap, or went past its target"); + const auto gone = held[0].expired() ? 0u : 1u; + Require(heap.SlotTexture(ShaderRecompiler::RuntimeAbi::FirstImageBinding, slots[gone]) == nullptr, "the texture a memory recovery freed kept its heap slot"); + Require(heap.SlotTexture(ShaderRecompiler::RuntimeAbi::FirstImageBinding, slots[1u - gone]) == held[1u - gone].lock().get(), "the texture a memory recovery kept lost its heap slot"); +} + ShaderRecompiler::RecompileResult SampledProgram(std::uint64_t address) { ShaderRecompiler::DescriptorBinding binding{}; binding.kind = ShaderRecompiler::DescriptorKind::SampledImage; @@ -1388,6 +1431,34 @@ int main(int argc, char** argv) { if (!memory && mode != "budget-report-age") setenv("APS5_TEXTURE_CACHE_MIB", recover ? "16" : "1", 1); setenv("APS5_TEMPLATE_AUDIT_MS", "250", 0); #endif + if (mode == "heap-view") { + SetTestEnvironment("APS5_GPU_HEAP", "1"); + auto heapDevice = OpenVulkanTestDevice(); + if (!heapDevice) return VulkanTestSkipped; + if (!heapDevice->graphicsContext().descriptorHeap) { + std::puts("skipped, the device has no VK_EXT_descriptor_heap"); + return VulkanTestSkipped; + } + bool tested = false; + { + const auto context = heapDevice->graphicsContext(); + { + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + heapHeldReleaseTests(context); + ClearCachedTextures(context.device); + tested = heapViewTests(context); + ClearCachedTextures(context.device); + } + heapDevice->WaitIdle(); + } + heapDevice.reset(); + if (!tested) { + std::puts("skipped, host imports unavailable: heap views of evicted storage images not tested"); + return VulkanTestSkipped; + } + std::puts("heap view release tests passed"); + return 0; + } std::unique_ptr device; try { device = std::make_unique(!disableStorageImageMultisample); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScratchPool.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScratchPool.cpp index 0359abebcc3a..d0d92918f951 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScratchPool.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScratchPool.cpp @@ -244,14 +244,16 @@ struct Facts { bool functionArrayAccess = false; }; -Facts Inspect(std::span words) { +Facts Inspect(const ShaderRecompiler::RecompileResult& result) { + const std::span words(result.spirv.data(), result.spirv.size()); + const auto poolSet = result.gpuResolvedImages ? ShaderRecompiler::GpuScratchPoolDescriptorSet : ShaderRecompiler::ScratchPoolDescriptorSet; Facts facts; std::set arrays, pointers, variables; for (std::size_t cursor = 5; cursor < words.size();) { const auto count = words[cursor] >> 16u; const auto op = words[cursor] & 0xffffu; Require(count != 0u, "scratch pool: malformed SPIR-V"); - if (op == spv::OpDecorate && words[cursor + 2u] == spv::DecorationDescriptorSet && words[cursor + 3u] == ShaderRecompiler::ScratchPoolDescriptorSet) facts.poolBinding = true; + if (op == spv::OpDecorate && words[cursor + 2u] == spv::DecorationDescriptorSet && words[cursor + 3u] == poolSet) facts.poolBinding = true; if (op == spv::OpTypeArray) arrays.insert(words[cursor + 1u]); if (op == spv::OpTypePointer && words[cursor + 2u] == spv::StorageClassFunction && arrays.contains(words[cursor + 3u])) pointers.insert(words[cursor + 1u]); if (op == spv::OpVariable && pointers.contains(words[cursor + 1u])) variables.insert(words[cursor + 2u]); @@ -261,15 +263,23 @@ Facts Inspect(std::span words) { return facts; } +void RequireHeapPipeline(const AgcDriver::VulkanDevice& device, const ShaderRecompiler::RecompileResult& result, std::uint64_t before, const std::string& name) { + if (std::getenv("APS5_GPU_HEAP") == nullptr) return; + Require(result.gpuResolvedImages, name + ": the scratch program is not GPU mode"); + Require(device.DescriptorHeapComputePipelines() > before, name + ": the scratch program did not take the descriptor heap"); +} + ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, const Words& code, std::vector& userData, std::uint32_t waveSize, std::uint32_t threads, bool groupId, std::uint32_t scratchDwords) { const std::span codeSpan(code); const std::array memory{{{reinterpret_cast(codeSpan.data()), std::as_bytes(codeSpan)}}}; ShaderRecompiler::ShaderComputeStageInfo compute{{threads, 1, 1}, 0, {groupId, false, false}, false, 1}; compute.scratchDwords = scratchDwords; + auto target = device.Target(); + target.gpuResolvedImages = device.ComputeTarget(waveSize).gpuResolvedImages; ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(codeSpan.data()), codeSpan, 0, {}}, {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + target, {0, 0, 0, 128} }; request.useCache = false; @@ -294,11 +304,13 @@ void RunAlias(AgcDriver::VulkanDevice& device, Form form, bool atomic, std::uint std::copy(tableDescriptor.begin(), tableDescriptor.end(), userData.begin() + 4); const auto code = AliasKernel(form, atomic); const auto result = Compile(device, code, userData, waveSize, waveSize, false, AliasScratchDwords); - const auto facts = Inspect(std::span(result.spirv.data(), result.spirv.size())); + const auto facts = Inspect(result); Require(facts.poolBinding && result.scratchPoolSlotDwords != 0u, name + ": the scratch program has no scratch pool binding"); Require(!facts.functionArrayAccess, name + ": the scratch program indexes a Function array"); + const auto heapPipelines = device.DescriptorHeapComputePipelines(); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); + RequireHeapPipeline(device, result, heapPipelines, name); for (std::uint32_t lane = 0; lane < waveSize; ++lane) { const auto first = output[lane * 8u]; const auto second = output[lane * 8u + 1u]; @@ -326,8 +338,10 @@ void RunCallee(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { const auto code = CalleeKernel(); const auto result = Compile(device, code, userData, waveSize, waveSize, false, AliasScratchDwords); Require(result.scratchPoolSlotDwords != 0u, name + ": the scratch program has no scratch pool binding"); + const auto heapPipelines = device.DescriptorHeapComputePipelines(); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); + RequireHeapPipeline(device, result, heapPipelines, name); for (std::uint32_t lane = 0; lane < waveSize; ++lane) { Require(output[lane * 8u] == OperandA(lane), name + ": lane " + std::to_string(lane) + " read " + Hex(output[lane * 8u]) + " back, expected " + Hex(OperandA(lane))); } @@ -343,7 +357,7 @@ void RunPlain(AgcDriver::VulkanDevice& device) { std::copy(tableDescriptor.begin(), tableDescriptor.end(), userData.begin() + 4); const auto code = PlainKernel(); const auto result = Compile(device, code, userData, 32u, 32u, false, 0u); - const auto facts = Inspect(std::span(result.spirv.data(), result.spirv.size())); + const auto facts = Inspect(result); Require(!facts.poolBinding && result.scratchPoolSlotDwords == 0u, "scratch pool: a program without scratch got a scratch pool binding"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp index 75282a2894b6..a86f4226ef8b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp @@ -1,5 +1,6 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -11,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -74,14 +76,44 @@ void Record(AgcDriver::VulkanDevice& device, std::uint64_t address) { device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); } +bool heapSlots = false; + +struct HeapSlotted { + std::uint32_t slot = 0; + std::uint32_t generation = 0; +}; +std::map heapSlotsByAddress; + +std::shared_ptr heapSlotted(AgcDriver::VulkanDevice& device, std::uint64_t address, std::shared_ptr image) { + if (!heapSlots || image == nullptr) return image; + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + auto& heap = AgcDriver::Graphics::BindlessHeap::For(device.graphicsContext()); + const auto slot = heap.StorageSlot(ShaderRecompiler::RuntimeAbi::FirstStorageImageBinding, image.get(), image->StorageView(0, false)); + Require(slot.has_value(), "the stored image took no GPU heap storage slot"); + heapSlotsByAddress.try_emplace(address, HeapSlotted{*slot, heap.SlotGeneration(*slot)}); + return image; +} + +bool heapSlotFreed(AgcDriver::VulkanDevice& device, std::uint64_t address) { + const auto found = heapSlotsByAddress.find(address); + Require(found != heapSlotsByAddress.end(), "no GPU heap slot was recorded for the surface"); + return AgcDriver::Graphics::BindlessHeap::For(device.graphicsContext()).SlotGeneration(found->second.slot) != found->second.generation; +} + std::shared_ptr Store(AgcDriver::VulkanDevice& device, std::uint64_t base, std::uint32_t surface, bool flush = true) { const auto address = base + surface * Stride; Record(device, address); device.WaitIdle(); - if (!flush) return StorageTexture::FindLive(address, SurfaceBytes); + if (!flush) return heapSlotted(device, address, StorageTexture::FindLive(address, SurfaceBytes)); StorageTexture::FlushPending(address, static_cast(SurfaceBytes), nullptr, "test"); device.WaitIdle(); - return StorageTexture::FindLive(address, SurfaceBytes); + return heapSlotted(device, address, StorageTexture::FindLive(address, SurfaceBytes)); +} + +bool LacksHeapSlots(AgcDriver::VulkanDevice& device) { + if (!heapSlots || device.graphicsContext().descriptorHeap) return false; + std::puts("skipped, the device has no VK_EXT_descriptor_heap"); + return true; } int RetainedBatch() { @@ -94,6 +126,7 @@ int RetainedBatch() { #endif const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (LacksHeapSlots(*device)) return VulkanTestSkipped; std::vector storage((SurfaceBytes + MaxSurfaces * Stride + Stride) / 4u, 0u); auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + Stride - 1u) & ~std::uintptr_t{Stride - 1u}); const auto base = static_cast(reinterpret_cast(texels)); @@ -110,8 +143,9 @@ int RetainedBatch() { }; try { Record(*device, base); - auto retained = StorageTexture::FindLive(base, SurfaceBytes); + auto retained = heapSlotted(*device, base, StorageTexture::FindLive(base, SurfaceBytes)); Require(retained != nullptr, "surface 0 has no cached storage image"); + const std::weak_ptr weak = retained; const auto retainedBytes = retained->BudgetBytes(); auto* recorder = Recorder::Active(); Require(recorder != nullptr && recorder->Recording(), "the surface dispatch was not left in an open batch"); @@ -122,6 +156,11 @@ int RetainedBatch() { for (std::uint32_t surface = 1; surface <= capacity; ++surface) Record(*device, base + surface * Stride); Require(StorageTexture::FindLive(base, SurfaceBytes) == nullptr, "surface 0 remained cached after the storage budget was exceeded"); Require(recorder->OpenKeptBytes() >= beforeEviction + retainedBytes, "an evicted storage image held by a batch without a shared resource-cache entry was not charged to that batch"); + if (heapSlots) { + Require(!weak.expired() && !heapSlotFreed(*device, base), "an evicted storage image lost its GPU heap slot while its open batch still read it"); + device->WaitIdle(); + Require(weak.expired() && heapSlotFreed(*device, base), "an evicted storage image kept its GPU heap slot after the batch that held it finished"); + } } catch (...) { release(); throw; @@ -134,9 +173,12 @@ int RetainedBatch() { int main(int argc, char** argv) { try { - if (argc == 2 && std::string(argv[1]) == "--retained-batch") return RetainedBatch(); - const bool budgetOnly = argc == 2 && std::string(argv[1]) == "--budget-only"; - Require(argc == 1 || budgetOnly, "storage cache budget: unknown arguments"); + const std::string mode = argc == 2 ? argv[1] : ""; + heapSlots = mode == "--heap-slots" || mode == "--retained-batch-heap-slots"; + if (heapSlots) Require(SetTestEnvironment("APS5_GPU_HEAP", "1") == 0, "cannot enable the GPU heap"); + if (mode == "--retained-batch" || mode == "--retained-batch-heap-slots") return RetainedBatch(); + const bool budgetOnly = mode == "--budget-only"; + Require(argc == 1 || budgetOnly || heapSlots, "storage cache budget: unknown arguments"); #ifdef _WIN32 Require(_putenv_s("APS5_TEXTURE_CACHE_MIB", "32") == 0, "cannot set the test cache budget"); #else @@ -144,6 +186,7 @@ int main(int argc, char** argv) { #endif const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + if (LacksHeapSlots(*device)) return VulkanTestSkipped; const auto budget = TestBudget; std::vector storage((SurfaceBytes + MaxSurfaces * Stride + Stride) / 4u, 0u); auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + Stride - 1u) & ~std::uintptr_t{Stride - 1u}); @@ -181,6 +224,10 @@ int main(int argc, char** argv) { const auto capacity = static_cast(budget / held); Require(capacity + 2u <= MaxSurfaces, "the eviction set exceeds the test storage"); for (std::uint32_t surface = reused; surface < capacity + 2u; ++surface) images[surface] = Store(*device, base, surface); + for (std::uint32_t surface = 0; heapSlots && surface < capacity + 2u; ++surface) { + const bool evicted = surface < 2u; + Require(heapSlotFreed(*device, base + surface * Stride) == evicted, "surface " + std::to_string(surface) + (evicted ? " was evicted but kept its GPU heap slot" : " is cached but lost its GPU heap slot")); + } for (std::uint32_t surface = 0; surface < capacity + 2u; ++surface) { const bool cached = StorageTexture::FindLive(base + surface * Stride, SurfaceBytes) != nullptr; Require(cached == (surface >= 2u), "surface " + std::to_string(surface) + (cached ? " is still cached" : " was evicted") + " after " + std::to_string(capacity + 2u) + " surfaces under a budget of " + std::to_string(capacity) + " of them"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp index c0511eead75b..ed83a14f0ef4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageFlushOrdering.cpp @@ -205,7 +205,7 @@ void RunShadowedUnitUnderWriter(AgcDriver::VulkanDevice& device, const GuestBloc void Run(AgcDriver::VulkanDevice& device, const GuestBlock& guest, std::size_t offset, bool submit, const char* name) { const auto* words = guest.Words() + offset / 4u; const auto image = guest.Address() + offset; - const auto target = device.Target(); + const auto target = device.ComputeTarget(64); const auto storeImage = Compile(StoreImageCode, ImageUserData(image), target); const auto storeGlobal = Compile(StoreGlobalCode, GlobalUserData(image), target); device.Dispatch(storeImage, 1u, 1u, 1u, {}, reinterpret_cast(StoreImageCode.data())); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageOneSliceAlias.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageOneSliceAlias.cpp index 78437ac99d4f..cbb2b9b485e3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StorageOneSliceAlias.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageOneSliceAlias.cpp @@ -129,7 +129,7 @@ void Dispatch(AgcDriver::VulkanDevice& device, std::span co ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp index 31ae127eab47..31e495d555df 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Tessellation.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" #include "prx/libc/include/GuestAllocations.hpp" @@ -669,6 +670,8 @@ void Draw(AgcDriver::VulkanDevice& device, const Stages& stages, VkCullModeFlags for (std::size_t i = 0; i < Pixels.size(); i += 4) { for (std::size_t c = 0; c < 4; ++c) Pixels[i + c] = std::byte{Background[c]}; } + const bool heap = AgcDriver::Graphics::DescriptorHeapEnabled(); + Require(stages.local.gpuResolvedImages == heap && stages.hull.gpuResolvedImages == heap && stages.domain.gpuResolvedImages == heap && stages.fragment.gpuResolvedImages == heap, "the tessellation draw's stages did not take GPU mode"); std::uint32_t cursor = 0; const auto offset = [&](const ShaderRecompiler::RecompileResult& result) { const auto at = result.pushConstants.empty() ? 0u : cursor; @@ -718,6 +721,7 @@ void Draw(AgcDriver::VulkanDevice& device, const Stages& stages, VkCullModeFlags void RequireDepthRangeRemap(AgcDriver::VulkanDevice& device) { auto target = device.Target(); + target.gpuResolvedImages = AgcDriver::Graphics::DescriptorHeapEnabled(); Require(device.graphicsContext().depthClamp, "tessellation depth remap needs depth clamping"); target.depthRangeRemap = true; const auto compile = [&](float z, std::uint32_t mode) { @@ -1215,7 +1219,8 @@ int main(int argc, char** argv) { try { auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - const auto target = device->Target(); + auto target = device->Target(); + target.gpuResolvedImages = AgcDriver::Graphics::DescriptorHeapEnabled(); if (!target.tessellation.has_value()) { std::puts("tessellation: skipped, the device has no tessellation shaders"); return VulkanTestSkipped; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/TessellationSubmit.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/TessellationSubmit.cpp new file mode 100644 index 000000000000..a58c70692612 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/TessellationSubmit.cpp @@ -0,0 +1,291 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BindlessHeap.hpp" +#include "prx/libSceAgcDriver/Submit/include/Dcb.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 80; +constexpr std::array Background{16, 24, 40, 255}; +alignas(256) std::array Pixels{}; +alignas(256) std::array HullUserMemory{}; + +alignas(256) constexpr std::array LocalCode{ + 0xd5480004, 0x02090502, 0xd5480005, 0x02090102, 0xd5480006, 0x02110902, 0xd5480007, 0x02111102, 0x7e080d04, 0x7e0a0d05, + 0x7e0c0d06, 0x7e0e0d07, 0x10100cff, 0x3f19999a, 0x061010ff, 0xbf666666, 0x3e1008ff, 0x3e800000, 0xd54b0009, 0x03c5e105, + 0x161406a0, 0xd9340000, 0x0000080a, 0xd8340008, 0x0000070a, 0xd8340010, 0x0000070a, 0xbf8cc07f, 0xbefd2106, +}; + +alignas(256) constexpr std::array HullCode{ + 0xd7650000, 0x000100c1, 0xd7660000, 0x000200c1, 0xbf8a0000, 0x7da206f9, 0x81060000, 0xd5480014, 0x02151101, 0x362402ff, + 0x000000ff, 0x160028a0, 0xd5430015, 0x040224ff, 0x00000060, 0xdbfc0000, 0x02000015, 0x162c2890, 0xd5430016, 0x045a24ff, + 0x00000030, 0xbf8cc07f, 0xe0781000, 0x02020216, 0x7da82881, 0xbe94047e, 0xb06a0060, 0x162602f9, 0x0086066a, 0xd8d80008, + 0x06000013, 0xd8dc1008, 0x07000013, 0xbf8cc07f, 0x7e300306, 0x7e320306, 0x7e340306, 0x7e360306, 0x34142484, 0xbefe0414, + 0xe0781000, 0x0403180a, 0x4a162481, 0x16161690, 0x4c1616ff, 0x00008000, 0xe0741000, 0x0202070b, 0xbf810000, +}; + +alignas(256) constexpr std::array DomainCode{ + 0xf8000941, 0x00000000, 0x34020e84, 0x16020283, 0xe0341000, 0x04030a01, 0xe0341010, 0x04030c01, 0xe0341020, 0x04030e01, + 0xbf8c3f70, 0x08040af2, 0x08040d02, 0x10201505, 0x3e201906, 0x3e201d02, 0x10221705, 0x3e221b06, 0x3e221f02, 0xd5510012, + 0x040a0d05, 0x102424ff, 0x40400000, 0x7e260280, 0x7e2802f2, 0xf80008cf, 0x14131110, 0xf800020f, 0x14131214, 0xbf810000, +}; + +alignas(256) constexpr std::array DomainGeometryCode{ + 0x8f6a9003, 0x94fe6ac1, 0xbf880028, 0x342a0e84, 0x162a2a83, 0xe0341000, 0x04030a15, 0xe0341010, + 0x04030c15, 0xe0341020, 0x04030e15, 0xbf8c3f70, 0x082c0af2, 0x082c0d16, 0x10201505, 0x3e201906, + 0x3e201d16, 0x10221705, 0x3e221b06, 0x3e221f16, 0xd5510012, 0x045a0d05, 0x102424ff, 0x40400000, + 0x7e260280, 0x7e2802f2, 0xd7650009, 0x000100c1, 0xd7660009, 0x000212c1, 0x93ebff03, 0x00040018, + 0xd7460009, 0x04250c6b, 0x34121285, 0xd8380100, 0x00111009, 0xd8380302, 0x00141309, 0xd8380504, + 0x00121409, 0xd8380706, 0x00141309, 0xbf8cc07f, 0xbefe04c1, 0xbf8a0000, 0x938dff02, 0x00090016, + 0x9382ff03, 0x00040018, 0x938cff03, 0x00080008, 0xd7650009, 0x000100c1, 0xd7660009, 0x000212c1, + 0xd746000a, 0x04250c02, 0x7da8120c, 0xbf88002e, 0x361600ff, 0x0000ffff, 0x34161682, 0x2c180090, + 0x34181882, 0x361a02ff, 0x0000ffff, 0x341a1a82, 0xdbfc0000, 0x1000000b, 0xdbfc0010, 0x1400000b, + 0xdbfc0000, 0x1800000c, 0xdbfc0010, 0x1c00000c, 0xdbfc0000, 0x2000000d, 0xdbfc0010, 0x2400000d, + 0xbf8cc07f, 0xbf800000, 0xbf800000, 0xbf800000, 0x161c1483, 0x161e14ff, 0x00000060, 0xbf8c3f70, + 0xdb7c0400, 0x0000100f, 0xdb7c0410, 0x0000140f, 0xdb7c0420, 0x0000180f, 0xdb7c0430, 0x00001c0f, + 0xdb7c0440, 0x0000200f, 0xdb7c0450, 0x0000240f, 0x4a501c81, 0x4a521c82, 0x3450508a, 0x34525294, + 0xd772002a, 0x04a6510e, 0xbf8cc07f, 0xbefe04c1, 0xbf8a0000, 0x930e830d, 0xbf078002, 0xbf850003, + 0x8f0f8c0d, 0x887c0f0e, 0xbf900009, 0x7da8140d, 0xbf880002, 0xf8000941, 0x0000002a, 0xbefe04c1, + 0x7da8140e, 0xbf88000a, 0x34561485, 0xdbfc0400, 0x2c00002b, 0xdbfc0410, 0x3000002b, 0xbf8cc07f, + 0xf80008cf, 0x2f2e2d2c, 0xf800020f, 0x33323130, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0xc8020002, 0xc8060102, 0xc80a0202, 0xc80e0302, 0xf800180f, 0x03020100, 0xbf810000, +}; + +constexpr std::array PatchLevels{1u, 0u, 5u}; + +constexpr std::uint16_t Corner(std::uint32_t patch, std::uint32_t x, std::uint32_t y) { + return static_cast((PatchLevels[patch] << 8u) | (patch << 4u) | (x << 2u) | y); +} + +alignas(256) constexpr std::array Indices{ + Corner(0, 0, 0), Corner(0, 2, 0), Corner(0, 1, 2), Corner(1, 0, 0), Corner(1, 2, 0), + Corner(1, 1, 2), Corner(2, 0, 0), Corner(2, 2, 0), Corner(2, 1, 2), +}; + +struct Header { + Shader shader{}; + ShaderUserData users{}; +}; + +Header LocalHeader{}; +Header HullHeader{}; +Header DomainHeader{}; +Header DomainGeometryHeader{}; +Header PixelHeader{}; + +void Register(Header& header, const std::uint32_t* code, std::size_t size, std::uint8_t type) { + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18u; + header.shader.header_size = sizeof(header); + header.shader.shader_size = static_cast(size); + header.shader.code = code; + header.shader.type = type; + header.shader.user_data = &header.users; + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + +void Write(std::vector& commands, std::uint32_t opcode, std::uint32_t offset, std::uint32_t value) { + commands.insert(commands.end(), {0xc0010000u | (opcode << 8u), offset, value}); +} + +std::uint32_t Lo(const void* address) { + return static_cast(reinterpret_cast(address) >> 8u); +} + +std::uint32_t Hi(const void* address) { + return static_cast(reinterpret_cast(address) >> 40u); +} + +std::vector Commands(bool geometry) { + const auto color = reinterpret_cast(Pixels.data()); + const auto users = reinterpret_cast(HullUserMemory.data()); + const std::vector> context{{0x2d5, geometry ? 0x0000212du : 0x0200210du}, + {0x2d6, 128u | (3u << 8u) | (3u << 14u)}, + {0x2db, 1u | (2u << 2u) | (2u << 5u)}, + {0x286, std::bit_cast(64.0f)}, + {0x287, std::bit_cast(1.0f)}, + {0x1b6, 1}, + {0x191, 0x400}, + {0x1b3, 0}, + {0x1b4, 0}, + {0x260, 0}, + {0x207, 0}, + {0x200, 0}, + {0x203, 0x800}, + {0x2dc, 0xaa00}, + {0x2f8, 0}, + {0x292, 2}, + {0x293, 0}, + {0x80, 0}, + {0x8d, 0}, + {0x83, 0xffff}, + {0x8c, 0xa}, + {0x2f9, 0x2d}, + {0x313, 0x6000}, + {0x30e, 0xffffffff}, + {0x30f, 0xffffffff}, + {0x206, 0x43f}, + {0x204, 0x80000}, + {0x205, 0x240}, + {0x8e, 0xf}, + {0x8f, 0xf}, + {0x202, 0xcc0010}, + {0x1c4, 0}, + {0x1c5, 9}, + {0x1c3, 4}, + {0x31c, 0x28028}, + {0x31b, 0}, + {0x31d, 0}, + {0x3b0, ((Width - 1u) << 14u) | (Height - 1u)}, + {0x3b8, 0x9000000}, + {0x1e0, 0}, + {0xc, 0}, + {0xd, (Height << 16u) | Width}, + {0x81, 0x80000000}, + {0x82, (Height << 16u) | Width}, + {0x90, 0x80000000}, + {0x91, (Height << 16u) | Width}, + {0x94, 0x80000000}, + {0x95, (Height << 16u) | Width}, + {0x318, static_cast(color >> 8u)}, + {0x390, static_cast(color >> 40u)}, + {0x10f, std::bit_cast(Width / 2.0f)}, + {0x110, std::bit_cast(Width / 2.0f)}, + {0x111, std::bit_cast(-(Height / 2.0f))}, + {0x112, std::bit_cast(Height / 2.0f)}, + {0x113, std::bit_cast(1.0f)}, + {0x114, 0}, + {0xb4, 0}, + {0xb5, std::bit_cast(1.0f)}}; + const std::vector> shaders{{0x148, Lo(LocalCode.data())}, + {0x149, Hi(LocalCode.data())}, + {0x108, Lo(HullCode.data())}, + {0x109, Hi(HullCode.data())}, + {0x102, static_cast(users)}, + {0x103, static_cast(users >> 32u)}, + {0x10b, 0}, + {0xc8, geometry ? Lo(DomainGeometryCode.data()) : Lo(DomainCode.data())}, + {0xc9, geometry ? Hi(DomainGeometryCode.data()) : Hi(DomainCode.data())}, + {0x8a, 0}, + {0x8b, geometry ? 0x400010u : 0u}, + {0x8, Lo(PixelCode.data())}, + {0x9, Hi(PixelCode.data())}, + {0xb, 0}}; + std::vector commands; + Write(commands, 0x79, 0x242, 9); + for (const auto& [offset, value] : context) Write(commands, 0x69, offset, value); + if (geometry) { + for (const auto& [offset, value] : std::initializer_list>{{0x1ff, 12u}, {0x2ce, 3u}, {0x2ab, 8u}, {0x29b, 2u}}) Write(commands, 0x69, offset, value); + Write(commands, 0x79, 0x25b, 0x1804u); + } + for (const auto& [offset, value] : shaders) Write(commands, 0x76, offset, value); + commands.insert(commands.end(), {0xc0002a00u, 0u}); + const auto address = reinterpret_cast(Indices.data()); + commands.insert(commands.end(), {0xc0042700u, static_cast(Indices.size()), static_cast(address), + static_cast(address >> 32u), static_cast(Indices.size()), 0u}); + return commands; +} + +std::array PixelAt(std::uint32_t x, std::uint32_t y) { + const auto offset = (static_cast(y) * Width + x) * 4u; + return {std::to_integer(Pixels[offset]), std::to_integer(Pixels[offset + 1]), + std::to_integer(Pixels[offset + 2]), std::to_integer(Pixels[offset + 3])}; +} + +std::string PixelText(const std::array& pixel) { + return "(" + std::to_string(pixel[0]) + ", " + std::to_string(pixel[1]) + ", " + std::to_string(pixel[2]) + ", " + + std::to_string(pixel[3]) + ")"; +} + +std::uint32_t Column(std::uint32_t patch, float offset) { + return static_cast((-0.9f + 0.6f * static_cast(patch) + offset + 1.0f) * 0.5f * Width); +} + +std::uint32_t Row(float y) { + return static_cast((1.0f - y) * 0.5f * Height); +} + +std::uint32_t CoveredPixels(std::uint32_t patch, bool interior) { + std::uint32_t count = 0; + for (std::uint32_t y = Row(0.5f); y <= Row(-0.5f) && y < Height; ++y) { + for (std::uint32_t x = Column(patch, 0.0f); x <= Column(patch, 0.5f) && x < Width; ++x) { + const auto pixel = PixelAt(x, y); + if (pixel == Background) continue; + Require(pixel[0] == 255 && pixel[2] == 0 && pixel[3] == 255, "patch " + std::to_string(patch) + " pixel (" + std::to_string(x) + + ", " + std::to_string(y) + ") is " + PixelText(pixel)); + if (!interior || pixel[1] > 32) ++count; + } + } + return count; +} + +void SubmittedPatches(bool geometry) { + for (std::size_t i = 0; i < Pixels.size(); i += 4) { + for (std::size_t c = 0; c < 4; ++c) Pixels[i + c] = std::byte{Background[c]}; + } + const auto pushes = AgcDriver::Graphics::DataAddressPushes(); + auto commands = Commands(geometry); + const std::string path = geometry ? "tessellation and geometry" : "tessellation"; + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + Require(sceAgcDriverSubmitDcb(&packet) == 0, path + " submission failed"); + AgcDriverWaitIdle_nid_postfix(); + const auto center = Row(-0.5f + 1.0f / 3.0f); + Require(CoveredPixels(0, false) > 100 && CoveredPixels(0, true) == 0, + path + ": the level-1 patch was not drawn as one flat triangle, centre " + PixelText(PixelAt(Column(0, 0.25f), center))); + Require(CoveredPixels(1, false) == 0, path + ": the patch with level 0 was not culled"); + Require(CoveredPixels(2, false) > 100 && PixelAt(Column(2, 0.25f), center)[1] > 32, + path + ": the level-5 patch has no interior domain points at its centre, " + PixelText(PixelAt(Column(2, 0.25f), center))); + Require(PixelAt(0, 0) == Background && PixelAt(Width - 1, Height - 1) == Background, path + ": the corners changed"); + Require(!AgcDriver::Graphics::DescriptorHeapEnabled() || AgcDriver::Graphics::DataAddressPushes() > pushes, + "the submitted " + path + " draw did not take the descriptor heap"); +} + +} + +int main(int argc, char** argv) { + try { + const bool geometry = argc > 1 && std::string_view(argv[1]) == "--geometry"; + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->Target().tessellation.has_value()) { + std::cout << "skipped, the device has no tessellation shaders\n"; + return VulkanTestSkipped; + } + if (geometry && (!device->GeometryLimits().has_value() || (device->SubgroupStages() & VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT) == 0)) { + std::cout << "skipped, the device has no geometry shaders or no subgroup operations in tessellation-evaluation shaders\n"; + return VulkanTestSkipped; + } + } + Register(LocalHeader, LocalCode.data(), sizeof(LocalCode), 5); + Register(HullHeader, HullCode.data(), sizeof(HullCode), 7); + Register(DomainHeader, DomainCode.data(), sizeof(DomainCode), 2); + Register(DomainGeometryHeader, DomainGeometryCode.data(), sizeof(DomainGeometryCode), 2); + Register(PixelHeader, PixelCode.data(), sizeof(PixelCode), 1); + SubmittedPatches(geometry); + AgcDriverShutdown_nid_postfix(); + std::cout << (geometry ? "A submitted tessellation and geometry draw drew its patches\n" : "A submitted tessellation draw drew its patches\n"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp index ec229885ef14..98c56fec3415 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp @@ -64,7 +64,7 @@ void Store(AgcDriver::VulkanDevice& device, const std::uint32_t* texels, std::ui ShaderRecompiler::RecompileRequest request{ {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, - device.Target(), + device.ComputeTarget(64), {0, 0, 0, 128} }; request.useCache = false; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp index 4cbf73216610..9eb435e06832 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp @@ -17,9 +17,25 @@ inline bool TargetHasCapability(const ShaderRecompiler::SpirvTarget& target, spv return std::find(capabilities.begin(), capabilities.end(), static_cast(capability)) != capabilities.end(); } +inline int SetTestEnvironment(const char* name, const char* value) { +#ifdef _WIN32 + return _putenv_s(name, value); +#else + return setenv(name, value, 1); +#endif +} + +inline bool SkipWithoutDescriptorHeap() { + if (AgcDriver::Graphics::DescriptorHeapEnabled()) return false; + std::puts("skipped, the device lacks VK_EXT_descriptor_heap, or the maintenance5, slot room and push data the GPU heap needs with it, so the GPU heap has nothing to run through"); + return true; +} + inline std::unique_ptr OpenVulkanTestDevice() { try { - return std::make_unique(); + auto device = std::make_unique(); + if (std::getenv("APS5_VERIFY_GPU_HEAP") != nullptr && SkipWithoutDescriptorHeap()) return nullptr; + return device; } catch (const AgcDriver::VulkanUnavailable& error) { if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; std::printf("skipped, no usable Vulkan device: %s\n", error.what()); diff --git a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp index a06e99df3eb6..2c6a2acdeefb 100644 --- a/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp +++ b/core/libs/prx/libSceAgcDriver/tools/ShaderReplay.cpp @@ -15,7 +15,9 @@ #include "spirv-tools/libspirv.hpp" #endif +#include #include +#include #include #include #include @@ -54,22 +56,94 @@ bool g_waveLdsMemoryBarriers = false; bool g_deviceScopeLdsMemoryBarriers = false; bool g_ldsLockFences = false; bool g_shaderFma = false; +bool g_gpuHeap = false; +bool g_noGpuHeap = false; +bool g_hasHeapDataUniformDwords = false; +std::uint32_t g_heapDataUniformDwords = 0; + +constexpr std::uint32_t DefaultHeapDataUniformDwords = 16384u; + +struct HeapProfileSelection { + const char* source; + bool heap; +}; + +HeapProfileSelection SelectHeapProfile(ShaderRecompiler::SpirvTarget& target) { + HeapProfileSelection selection{"captured", target.gpuResolvedImages || target.descriptorMemory}; + if (g_noGpuHeap) { + target.gpuResolvedImages = false; + target.descriptorMemory = false; + target.heapDataUniformDwords = 0u; + selection = {"override", false}; + } + if (g_gpuHeap) { + target.gpuResolvedImages = true; + target.descriptorMemory = true; + if (!g_hasHeapDataUniformDwords) target.heapDataUniformDwords = DefaultHeapDataUniformDwords; + selection = {"override", true}; + } + if (g_hasHeapDataUniformDwords) { + target.heapDataUniformDwords = g_heapDataUniformDwords; + selection.source = "override"; + } + return selection; +} bool g_code = false; +bool ParseHeapDataUniformDwords(int index, int argc, char** argv, std::uint32_t& result) { + char* end = nullptr; + const auto value = index + 1 < argc ? std::strtoull(argv[index + 1], &end, 10) : 0ull; + if (index + 1 >= argc || end == argv[index + 1] || *end != '\0' || value > UINT32_MAX) return false; + result = static_cast(value); + return true; +} + +bool HeapFlagsConflict(int argc, char** argv) { + bool gpuHeap = false; + bool noGpuHeap = false; + std::uint32_t dwords = 0; + for (int i = 1; i < argc; ++i) { + const std::string argument = argv[i]; + if (argument == "--gpu-heap") gpuHeap = true; + else if (argument == "--no-gpu-heap") noGpuHeap = true; + else if (argument == "--heap-data-uniform-dwords") ParseHeapDataUniformDwords(i, argc, argv, dwords); + } + if (gpuHeap && noGpuHeap) { + std::fprintf(stderr, "--gpu-heap and --no-gpu-heap conflict\n"); + return true; + } + if (noGpuHeap && dwords != 0u) { + std::fprintf(stderr, "--no-gpu-heap conflicts with --heap-data-uniform-dwords %u: the limit needs a heap profile\n", dwords); + return true; + } + return false; +} + bool Replay(const char* path) { auto request = ShaderRecompiler::RequestSerializer{}.Deserialize(ReadText(path)); - if (g_maintenance8) request.request.target.nonConstantImageOffsets = true; - if (g_robust2) request.request.target.robustBufferAccess2 = true; - if (g_waveLdsMemoryBarriers) request.request.target.waveLdsMemoryBarriers = true; - if (g_deviceScopeLdsMemoryBarriers) request.request.target.deviceScopeLdsMemoryBarriers = true; - if (g_ldsLockFences) request.request.target.ldsLockFences = true; + std::string forced; + const auto force = [&](bool enabled, bool& field, const char* name) { + if (!enabled) return; + if (!forced.empty()) forced += ", "; + forced += field ? std::string(name) + " (already captured)" : name; + field = true; + }; + auto& target = request.request.target; + force(g_maintenance8, target.nonConstantImageOffsets, "maintenance8"); + force(g_robust2, target.robustBufferAccess2, "robust2"); + force(g_waveLdsMemoryBarriers, target.waveLdsMemoryBarriers, "wave-lds-memory-barriers"); + force(g_deviceScopeLdsMemoryBarriers, target.deviceScopeLdsMemoryBarriers, "device-scope-lds-memory-barriers"); + force(g_ldsLockFences, target.ldsLockFences, "lds-lock-fences"); if (g_shaderFma) { request.supportedCapabilities.push_back(spv::CapabilityFMAKHR); request.supportedExtensions.push_back("SPV_KHR_fma"); request.request.target.supportedCapabilities = request.supportedCapabilities; request.request.target.supportedExtensions = request.supportedExtensions; } + const auto heapProfile = SelectHeapProfile(request.request.target); std::printf("%s: %zu code words, %zu user data, %zu memory regions, wave%u\n", path, request.request.shader.code.size(), request.request.context.userData.size(), request.request.context.memory.size(), request.request.context.waveSize); + std::printf(" heap profile (%s): %s, gpuResolvedImages %d, descriptorMemory %d, data uniform dwords %u\n", heapProfile.source, heapProfile.heap ? "heap" : "no heap", request.request.target.gpuResolvedImages, request.request.target.descriptorMemory, request.request.target.heapDataUniformDwords); + if (!forced.empty()) std::printf(" target flags forced on: %s\n", forced.c_str()); if (request.request.context.compute.has_value()) { const auto& compute = *request.request.context.compute; std::printf(" compute: threads %ux%ux%u, lds %u dwords, group ids %d%d%d, tg size %d, thread id components %u\n", compute.numThreads[0], compute.numThreads[1], compute.numThreads[2], compute.ldsSizeDwords, compute.groupIdEnable[0], compute.groupIdEnable[1], compute.groupIdEnable[2], compute.tgSizeEnable, compute.threadIdComponentCount); @@ -219,9 +293,10 @@ bool Replay(const char* path) { int main(int argc, char** argv) { if (argc < 2) { - std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] [--robust2] [--wave-lds-memory-barriers] [--device-scope-lds-memory-barriers] [--lds-lock-fences] [--shader-fma] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set; shader_
.prepare.req holds a request whose preparation failed before any resource capture, so replay it with --prepare\n"); + std::fprintf(stderr, "usage: agc_shader_replay [--dis] [--asm] [--cfg] [--mem] [--spv] [--prepare] [--materialize] [--code] [--maintenance8] [--robust2] [--wave-lds-memory-barriers] [--device-scope-lds-memory-barriers] [--lds-lock-fences] [--shader-fma] [--gpu-heap] [--no-gpu-heap] [--heap-data-uniform-dwords N] ...\n the driver writes shader_
.req files when APS5_DUMP_SHADERS is set; shader_
.prepare.req holds a request whose preparation failed before any resource capture, so replay it with --prepare\n a request replays with the heap profile it was captured with; --gpu-heap forces the heap profile (gpuResolvedImages, descriptorMemory, and 16384 data uniform dwords unless --heap-data-uniform-dwords is given, replacing a captured limit), --no-gpu-heap forces it off, --heap-data-uniform-dwords N forces the limit (0 selects ring-address reads); --gpu-heap with --no-gpu-heap, and --no-gpu-heap with a nonzero limit, exit 2\n --maintenance8, --robust2, --wave-lds-memory-barriers, --device-scope-lds-memory-barriers and --lds-lock-fences force their target flag on over the captured value and are listed on a 'target flags forced on' line\n"); return 2; } + if (HeapFlagsConflict(argc, argv)) return 2; int failures = 0; for (int i = 1; i < argc; ++i) { if (std::string(argv[i]) == "--gcn") { @@ -244,6 +319,23 @@ int main(int argc, char** argv) { g_maintenance8 = true; continue; } + if (std::string(argv[i]) == "--gpu-heap") { + g_gpuHeap = true; + continue; + } + if (std::string(argv[i]) == "--no-gpu-heap") { + g_noGpuHeap = true; + continue; + } + if (std::string(argv[i]) == "--heap-data-uniform-dwords") { + if (!ParseHeapDataUniformDwords(i, argc, argv, g_heapDataUniformDwords)) { + std::fprintf(stderr, "--heap-data-uniform-dwords needs an unsigned 32-bit number\n"); + return 2; + } + g_hasHeapDataUniformDwords = true; + ++i; + continue; + } if (std::string(argv[i]) == "--robust2") { g_robust2 = true; continue; diff --git a/core/shader/recompiler/CMakeLists.txt b/core/shader/recompiler/CMakeLists.txt index c198f07758f0..dd181710a5fc 100644 --- a/core/shader/recompiler/CMakeLists.txt +++ b/core/shader/recompiler/CMakeLists.txt @@ -242,8 +242,17 @@ function(add_shader_recompiler target runtimeTarget) endif() set(shaderCacheReviewedNames BdaAbi::RangesOrdered + RuntimeAbi::GpuHeapStorageKind + RuntimeAbi::StageGroup + StagePushFeatures + RuntimeAbi::UintStorageImageKind + StorageElementPointer + FirstAtomicStorageImageBinding BuildBarycentricGeometryShader BuildBarycentricTessellationShaders + HeapFragmentEligible + CompleteHeapDataWords + HeapTemplateCounts BuildPreparedShaderKey BuildRectListShaders BuildTessellationCaptureGeometryShader @@ -265,6 +274,9 @@ function(add_shader_recompiler target runtimeTarget) PipelineSpecialization::WorkgroupOffsetWords PixelInput PixelInputBit + PlacePushData + PushBytesMask + PushPartOf RayTracingStrict RayTracingMiss ReadWholeFile @@ -281,7 +293,10 @@ function(add_shader_recompiler target runtimeTarget) MaterializeShader RegistrationResult ResolveSource + DescriptorBindingForImage ResourceMaterializer::EvaluateNanoseconds + ResourceMaterializer::TryRuntimeImageMode + ResourceMaterializer::ExpectedDeviceImages ResourceMaterializer::MaterializeNanoseconds ReleaseUndrawnResourcePlan ResourceMaterializer::SpecializationNanoseconds diff --git a/core/shader/recompiler/CacheKey.hpp b/core/shader/recompiler/CacheKey.hpp index 116703feeccd..33b5c8260b4d 100644 --- a/core/shader/recompiler/CacheKey.hpp +++ b/core/shader/recompiler/CacheKey.hpp @@ -6,6 +6,9 @@ #include "prx/libc/include/HostThreadLocal.hpp" #include #include +#include +#include +#include #include #include #include @@ -375,10 +378,15 @@ class RecompileCacheKey { append(key, value.largePixelFunctionLdsBuffer); append(key, value.pixelFunctionStorageDwords); append(key, value.deviceScopeLdsMemoryBarriers); + append(key, value.gpuResolvedImages); + append(key, value.descriptorMemory); + append(key, value.heapDataUniformDwords); + append(key, value.heapBufferDescriptors); append(key, value.ldsLockFences); append(key, value.meshWindowBuffer); append(key, value.hostInterpolationFirst); append(key, value.shaderImageMinLod); + append(key, value.deviceImageTables); } }; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index f14fda01ecbc..953935f6408b 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -898,7 +898,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(53u); + writer.WriteU32(55u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -943,6 +943,16 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const writer.WriteBool(request.target.tessellationEvaluationSubgroups); writer.WriteBool(request.target.hostInterpolationFirst); writer.WriteBool(request.target.shaderImageMinLod); + writer.WriteBool(request.target.gpuResolvedImages); + writer.WriteBool(request.target.descriptorMemory); + writer.WriteU32(request.target.heapDataUniformDwords); + writer.WriteBool(request.target.ldsLockFences); + writer.WriteBool(request.target.meshWindowBuffer); + writer.WriteBool(request.target.waveLdsMemoryBarriers); + writer.WriteBool(request.target.deviceScopeLdsMemoryBarriers); + writer.WriteBool(request.target.exactRobustLoads); + writer.WriteBool(request.target.robustBufferAccess2); + writer.WriteBool(request.target.heapBufferDescriptors); return base64Encode(buffer); } @@ -951,7 +961,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 53u || version == 50u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 55u || version == 50u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); @@ -1014,6 +1024,18 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const if (version >= 51u) result.request.target.tessellationEvaluationSubgroups = reader.ReadBool(); if (version >= 52u) result.request.target.hostInterpolationFirst = reader.ReadBool(); if (version >= 53u) result.request.target.shaderImageMinLod = reader.ReadBool(); + if (version >= 55u) { + result.request.target.gpuResolvedImages = reader.ReadBool(); + result.request.target.descriptorMemory = reader.ReadBool(); + result.request.target.heapDataUniformDwords = reader.ReadU32(); + result.request.target.ldsLockFences = reader.ReadBool(); + result.request.target.meshWindowBuffer = reader.ReadBool(); + result.request.target.waveLdsMemoryBarriers = reader.ReadBool(); + result.request.target.deviceScopeLdsMemoryBarriers = reader.ReadBool(); + result.request.target.exactRobustLoads = reader.ReadBool(); + result.request.target.robustBufferAccess2 = reader.ReadBool(); + result.request.target.heapBufferDescriptors = reader.ReadBool(); + } if (version >= 21u && reader.Remaining() != 0u) throw std::runtime_error("RequestSerializer: unexpected trailing data"); return result; } diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index 5bc2aec69619..4a6ada3e769a 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -227,7 +227,7 @@ int main(int argc, char** argv) { std::cout << "ShaderInfoCollector::Collect OK\n"; BindingAllocator bindingAllocator; - const auto bindings = bindingAllocator.Allocate(program, request.layout, request.target.registrationModules); + const auto bindings = bindingAllocator.Allocate(program, request.layout, {.registrationModules = request.target.registrationModules}); std::cout << "BindingAllocator::Allocate OK, bindings=" << bindings.bindings.size() << "\n"; std::cout << "ALL STAGES OK\n"; diff --git a/core/shader/recompiler/DeviceImageTableAbi.hpp b/core/shader/recompiler/DeviceImageTableAbi.hpp new file mode 100644 index 000000000000..5040d65f6964 --- /dev/null +++ b/core/shader/recompiler/DeviceImageTableAbi.hpp @@ -0,0 +1,120 @@ +#ifndef CORE_SHADER_RECOMPILER_DEVICEIMAGETABLEABI_HPP +#define CORE_SHADER_RECOMPILER_DEVICEIMAGETABLEABI_HPP + +#include +#include +#include +#include + +namespace ShaderRecompiler::DeviceImageTableAbi { + +inline constexpr std::uint32_t Version = 4; + +inline constexpr std::uint32_t DescriptorWords = 8; +inline constexpr std::uint32_t InterfaceWord = 8; +inline constexpr std::uint32_t SamplerClassWord = 10; +inline constexpr std::uint32_t SamplerWord = 11; +inline constexpr std::uint32_t SamplerWords = 4; +inline constexpr std::uint32_t KeyWords = 15; +inline constexpr std::uint32_t StateWord = 15; +inline constexpr std::uint32_t SlotWord = 16; +inline constexpr std::uint32_t ModeWord = 17; +inline constexpr std::uint32_t FaultWord = 18; +inline constexpr std::uint32_t SamplerSlotWord = 19; +inline constexpr std::uint32_t BucketWords = 20; +inline constexpr std::uint32_t BucketBytes = BucketWords * 4u; +inline constexpr std::uint32_t EmptyState = 0; +inline constexpr std::uint32_t LiveState = 1; +inline constexpr std::uint32_t ProbeSteps = 16; + +inline constexpr std::uint32_t HeaderVersion = 0; +inline constexpr std::uint32_t HeaderMask = 1; +inline constexpr std::uint32_t HeaderRingCapacity = 2; +inline constexpr std::uint32_t HeaderWords = 16; + +inline constexpr std::uint32_t RingCountWord = 0; +inline constexpr std::uint32_t RingSeenWord = 4; +inline constexpr std::uint32_t RingSeenSlots = 4096; +inline constexpr std::uint32_t RingSeenProbes = 8; +inline constexpr std::uint32_t RingRecordsWord = RingSeenWord + RingSeenSlots; +inline constexpr std::uint32_t RecordWords = 16; +inline constexpr std::uint32_t RecordPcWord = 15; + +inline constexpr std::uint32_t FaultFlag = 0x40000000u; + +inline constexpr std::uint32_t SamplerFilterMinmax = 1u << 0u; +inline constexpr std::uint32_t SamplerOpaqueBlackBorder = 1u << 1u; +inline constexpr std::uint32_t SamplerNonSeamlessCube = 1u << 2u; +inline constexpr std::uint32_t SamplerFloatBorder = 1u << 3u; +inline constexpr std::uint32_t SamplerIntegerBorder = 1u << 4u; +inline constexpr std::uint32_t SamplerAnisoOverride = 1u << 5u; +inline constexpr std::uint32_t SamplerReduction = 1u << 6u; +inline constexpr std::uint32_t SamplerExactAtBase = 1u << 7u; +inline constexpr std::uint32_t SamplerUnnormalized = 1u << 15u; +inline constexpr std::uint32_t ModeCompareShift = 16u; +inline constexpr std::uint32_t RuntimeSampler = 1u << 31u; + +inline constexpr std::uint32_t HashSeed = 0x9747b28cu; +inline constexpr std::uint32_t HashC1 = 0xcc9e2d51u; +inline constexpr std::uint32_t HashC2 = 0x1b873593u; +inline constexpr std::uint32_t HashRound = 0xe6546b64u; +inline constexpr std::uint32_t HashMix1 = 0x85ebca6bu; +inline constexpr std::uint32_t HashMix2 = 0xc2b2ae35u; + +using Key = std::array; + +inline constexpr std::uint32_t NoExpectedSampler = std::numeric_limits::max(); + +struct Expectation { + Key key{}; + std::uint32_t pc = 0; + std::uint32_t image = 0; + std::uint32_t sampler = NoExpectedSampler; + bool candidate = false; + + bool operator==(const Expectation& other) const = default; +}; + +inline void NameInterfaces(std::span images, std::span interfaces) { + for (auto& image : images) { + const auto id = interfaces[image.image]; + image.key[InterfaceWord] = static_cast(id); + image.key[InterfaceWord + 1u] = static_cast(id >> 32u); + } +} + +inline void NameSamplerClasses(std::span images, std::span classes) { + for (auto& image : images) image.key[SamplerClassWord] = image.sampler == NoExpectedSampler ? 0u : classes[image.sampler]; +} + +constexpr std::uint32_t RotateLeft(std::uint32_t value, std::uint32_t bits) { + return (value << bits) | (value >> (32u - bits)); +} + +constexpr std::uint32_t HashWord(std::uint32_t hash, std::uint32_t word) { + const auto mixed = RotateLeft(word * HashC1, 15u) * HashC2; + return RotateLeft(hash ^ mixed, 13u) * 5u + HashRound; +} + +constexpr std::uint32_t HashFinish(std::uint32_t hash) { + hash ^= KeyWords * 4u; + hash ^= hash >> 16u; + hash *= HashMix1; + hash ^= hash >> 13u; + hash *= HashMix2; + return hash ^ (hash >> 16u); +} + +constexpr std::uint32_t Hash(const Key& key) { + auto hash = HashSeed; + for (const auto word : key) hash = HashWord(hash, word); + return HashFinish(hash); +} + +constexpr bool NullDescriptor(std::uint32_t word0, std::uint32_t word1) { + return word0 == 0u && (word1 & 0xffu) == 0u; +} + +} + +#endif diff --git a/core/shader/recompiler/HeapTemplateBudget.hpp b/core/shader/recompiler/HeapTemplateBudget.hpp new file mode 100644 index 000000000000..ee94bc9741d0 --- /dev/null +++ b/core/shader/recompiler/HeapTemplateBudget.hpp @@ -0,0 +1,106 @@ +#pragma once + +#include "Recompiler.hpp" + +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { + +template +[[nodiscard]] std::uint64_t OwnedBytes(const std::vector& values) { + return static_cast(values.capacity()) * sizeof(TValue); +} + +[[nodiscard]] inline std::uint64_t OwnedBytes(const std::vector& values) { + return (static_cast(values.capacity()) + 7u) / 8u; +} + +[[nodiscard]] inline std::uint64_t OwnedBytes(const DescriptorBinding& binding) { + return OwnedBytes(binding.guestDescriptor) + OwnedBytes(binding.samplerDepthCompare) + OwnedBytes(binding.imageWritten) + OwnedBytes(binding.imageDepthCompare) + OwnedBytes(binding.imageAtomic) + OwnedBytes(binding.imageAtomic64) + + OwnedBytes(binding.imageShapes) + OwnedBytes(binding.bufferAtomic) + OwnedBytes(binding.bufferWritten) + OwnedBytes(binding.samplerUnnormalized) + OwnedBytes(binding.samplerIntegerBorder) + + OwnedBytes(binding.samplerCoordinatesIgnored) + OwnedBytes(binding.imageUnnormalized) + OwnedBytes(binding.imageUnnormalizedBaseLevel) + OwnedBytes(binding.imageSamplers) + OwnedBytes(binding.bufferRead) + + OwnedBytes(binding.bufferExtent); +} + +[[nodiscard]] inline std::uint64_t OwnedBytes(const RecompileResult& result) { + std::uint64_t bytes = sizeof(RecompileResult) + OwnedBytes(result.vertexInputPatches) + OwnedBytes(result.runtimeImageResources) + OwnedBytes(result.vertexInputs) + OwnedBytes(result.parameterExports) + + OwnedBytes(result.fragmentParameters) + OwnedBytes(result.stencilReferences) + OwnedBytes(result.specialization) + OwnedBytes(result.heapSlotWords) + OwnedBytes(result.bindings) + + OwnedBytes(result.pushConstants) + OwnedBytes(result.vertexAttributes) + OwnedBytes(result.imageTablePoison) + OwnedBytes(result.imageTableRanges) + OwnedBytes(result.imageTableReadRanges) + + OwnedBytes(result.heapUserWords) + OwnedBytes(result.shaderDataUserWords); + if (result.meshBarycentrics.has_value()) bytes += OwnedBytes(result.meshBarycentrics->parameters); + for (const auto& binding : result.bindings) bytes += OwnedBytes(binding); + for (const auto& ranges : result.imageTableReadRanges) bytes += OwnedBytes(ranges); + return bytes; +} + +class HeapTemplateLru { +public: + struct Name { + std::uint64_t owner = 0; + std::uint64_t key = 0; + std::uint64_t id = 0; + }; + + struct Evicted { + Name name; + std::weak_ptr owner; + }; + + explicit HeapTemplateLru(std::uint64_t budget) : budget(budget) {} + + [[nodiscard]] std::vector Insert(const Name& name, std::uint64_t bytes, std::weak_ptr owner = {}) { + std::vector evicted; + if (const auto found = index.find(Slot{name.owner, name.key}); found != index.end()) { + total -= found->second->bytes; + order.erase(found->second); + index.erase(found); + } + order.push_back({name, bytes, std::move(owner)}); + index.emplace(Slot{name.owner, name.key}, std::prev(order.end())); + total += bytes; + while (total > budget && !order.empty()) { + auto oldest = std::move(order.front()); + index.erase(Slot{oldest.name.owner, oldest.name.key}); + order.pop_front(); + total -= oldest.bytes; + evicted.push_back({oldest.name, std::move(oldest.owner)}); + } + return evicted; + } + + void Touch(const Name& name) { + const auto found = index.find(Slot{name.owner, name.key}); + if (found == index.end() || found->second->name.id != name.id) return; + order.splice(order.end(), order, found->second); + } + + [[nodiscard]] std::uint64_t Entries() const { return order.size(); } + [[nodiscard]] std::uint64_t Bytes() const { return total; } + +private: + struct Entry { + Name name; + std::uint64_t bytes = 0; + std::weak_ptr owner; + }; + struct Slot { + std::uint64_t owner = 0; + std::uint64_t key = 0; + bool operator==(const Slot&) const = default; + }; + struct SlotHash { + std::size_t operator()(const Slot& slot) const { return std::hash{}(slot.owner * 0x9e3779b97f4a7c15ull ^ slot.key); } + }; + + std::uint64_t budget = 0; + std::uint64_t total = 0; + std::list order; + std::unordered_map::iterator, SlotHash> index; +}; + +} diff --git a/core/shader/recompiler/ImageTableAbi.hpp b/core/shader/recompiler/ImageTableAbi.hpp index b9c9773b5f4b..4ec9a4913e2b 100644 --- a/core/shader/recompiler/ImageTableAbi.hpp +++ b/core/shader/recompiler/ImageTableAbi.hpp @@ -29,6 +29,12 @@ inline constexpr std::uint32_t TableOutsideCode = 6; inline constexpr std::uint32_t TableFault = 7; inline constexpr std::uint32_t TableWords = 8; +inline constexpr std::uint32_t TrailerBucketsLow = 0; +inline constexpr std::uint32_t TrailerBucketsHigh = 1; +inline constexpr std::uint32_t TrailerRingLow = 2; +inline constexpr std::uint32_t TrailerRingHigh = 3; +inline constexpr std::uint32_t TrailerWords = 4; + inline constexpr std::uint32_t ImageRecordFlags = 0; inline constexpr std::uint32_t ImageRecordDword3 = 1; inline constexpr std::uint32_t ImageRecordElement = 2; @@ -121,6 +127,36 @@ constexpr std::uint32_t TableHeader(std::uint32_t table) { return HeaderWords + TableWords * table; } +constexpr std::uint32_t TrailerWord(std::uint32_t elementBase, std::uint32_t field) { + return elementBase - TrailerWords + field; +} + +struct Trailer { + std::uint64_t buckets = 0; + std::uint64_t ring = 0; + bool operator==(const Trailer&) const = default; +}; + +template +std::uint32_t AppendTrailer(Map& map, const Trailer& trailer) { + map.push_back(static_cast(trailer.buckets)); + map.push_back(static_cast(trailer.buckets >> 32u)); + map.push_back(static_cast(trailer.ring)); + map.push_back(static_cast(trailer.ring >> 32u)); + const auto base = static_cast(map.size()); + map[HeaderElementBase] = base; + return base; +} + +template +std::optional ReadTrailer(const Map& map) { + if (map.size() <= HeaderElementBase) return std::nullopt; + const auto base = map[HeaderElementBase]; + if (base < HeaderWords + TrailerWords || base > map.size()) return std::nullopt; + const auto word = [&](std::uint32_t field) { return static_cast(map[TrailerWord(base, field)]); }; + return Trailer{word(TrailerBucketsLow) | word(TrailerBucketsHigh) << 32u, word(TrailerRingLow) | word(TrailerRingHigh) << 32u}; +} + constexpr std::optional ScalarBufferDword(std::uint32_t offset, std::uint32_t immediate, std::uint64_t size) { const std::uint64_t address = static_cast(offset) + immediate; if (address > 0xffffffffull) return std::nullopt; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp index 809c3e0932e3..40215a9ce071 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp @@ -39,12 +39,19 @@ struct IrDescriptorBinding { struct IrBindingLayout { std::uint32_t pushDataStartDword = PushData::NoStart; + bool dataAddress = false; std::uint32_t stencilReferenceDword = PushData::NoStart; std::uint32_t memoryOffsetDword = 0; std::uint32_t memoryOffsetCount = 0; bool dispatchThreadLimit = false; std::uint32_t runtimeImageCount = 0; std::uint32_t runtimeStateDwords = 0; + std::uint32_t samplerSlotDwords = 0; + std::uint32_t imageSlotDwords = 0; + std::uint32_t bufferAddressDwords = 0; + std::vector pushedData; + bool gpuResolvedImages = false; + bool heapStorageImages = false; std::vector userDataRegisters; std::vector descriptors; @@ -54,22 +61,39 @@ struct IrBindingLayout { [[nodiscard]] std::uint32_t ImageMetadataDword() const { return DispatchThreadLimitDword() + (dispatchThreadLimit ? 3u : 0u); } + [[nodiscard]] std::uint32_t ImageFirstElementDword(std::uint32_t image) const { + return ImageMetadataDword() + image * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)) + offsetof(RuntimeAbi::ResourceMetadata, firstElement) / sizeof(std::uint32_t); + } [[nodiscard]] std::uint32_t RuntimeStateDword() const { return ImageMetadataDword() + runtimeImageCount * (sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)); } - [[nodiscard]] std::uint32_t ShaderDataDwords() const { + [[nodiscard]] std::uint32_t SamplerSlotDword() const { return RuntimeStateDword() + runtimeStateDwords; } + [[nodiscard]] std::uint32_t ImageSlotDword() const { + return SamplerSlotDword() + samplerSlotDwords; + } + [[nodiscard]] std::uint32_t BufferAddressDword() const { + return ImageSlotDword() + imageSlotDwords; + } + [[nodiscard]] std::uint32_t ShaderDataDwords() const { + return BufferAddressDword() + bufferAddressDwords; + } + [[nodiscard]] bool HeapImageKind(DescriptorBindingKind kind) const { + const auto value = static_cast(kind); + return (value >= RuntimeAbi::FirstImageBinding && value < RuntimeAbi::FirstStorageImageBinding) || (heapStorageImages && RuntimeAbi::GpuHeapStorageKind(value)); + } [[nodiscard]] bool UsesPushData() const { return pushDataStartDword != PushData::NoStart; } + [[nodiscard]] bool UsesDataAddress() const { + return dataAddress; + } [[nodiscard]] std::uint32_t PushSlotDword() const { return UsesPushData() ? pushDataStartDword / PushData::DwordCount * PushData::DwordCount : 0u; } void AdvancePushData(std::uint32_t& cursor) const { - if (UsesPushData()) { - cursor = pushDataStartDword + ShaderDataDwords(); - } + if (UsesPushData()) cursor = pushDataStartDword + ShaderDataDwords(); } bool operator==(const IrBindingLayout& other) const = default; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index cbc6fc798654..52efdfe8b256 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -124,6 +124,7 @@ struct ImageResource { std::uint32_t table = NoTable; TableOperation tableOperation = TableOperation::Supported; bool targetMerge = false; + bool deviceTable = false; bool operator==(const ImageResource& other) const = default; }; @@ -171,6 +172,17 @@ inline bool MirrorsIfUnnormalized(const SamplerResource& sampler) { return sampler.table == NoTable && !sampler.depthCompare && sampler.uses != 0u && ((sampler.uses & ~static_cast(SamplerUseExplicitLod | SamplerUseOffset | SamplerUseNonZeroLod)) == 0u || SamplesBaseLevelIfUnnormalized(sampler)); } +inline constexpr std::uint32_t NoSampler = std::numeric_limits::max(); + +struct DeviceImageUse { + std::uint32_t image = 0; + std::uint32_t sampler = NoSampler; + std::uint32_t descriptor = 0; + std::uint32_t firstUsePc = 0; + + bool operator==(const DeviceImageUse& other) const = default; +}; + inline bool AppliesLodBiasInShader(const SamplerResource& sampler) { return sampler.table == NoTable && (sampler.uses & (SamplerUseExplicitLod | SamplerUseImplicitLod | SamplerUseGradient)) != 0u; } diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index 40f197a264e5..ed755a041410 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -21,6 +21,7 @@ struct ShaderInfo { std::vector> runtimeImageModes; std::vector samplers; std::vector sampledPairs; + std::vector deviceImageUses; std::vector inputs; std::vector outputs; std::array vertexFetchComponents {}; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp index 69abc3bb2b47..9a50f8f4cfd4 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrProgram.hpp @@ -145,6 +145,9 @@ IrValue* ResolveInvariantPhi(const IrResourcePlan& program, IrValue* value); bool IsAddressResourceKind(ResourceKind kind); PositionExportComponent DecodePositionExportComponent(std::uint32_t control, std::uint32_t positionIndex, std::uint32_t component); std::uint32_t NativeBinding(IrShaderStage stage, DescriptorBindingKind kind); +bool SharedHeapArray(const IrBindingLayout& layout, DescriptorBindingKind kind, bool gpuResolved); +std::uint32_t DescriptorBindingNumber(const IrBindingLayout& layout, DescriptorBindingKind kind, IrShaderStage stage, bool gpuResolved); +std::uint32_t DescriptorSetNumber(const IrBindingLayout& layout, DescriptorBindingKind kind, IrShaderStage stage, bool gpuResolved, bool fragmentDescriptorSet, bool plan); ImageResourceClass ImageBindingResourceClass(DescriptorBindingKind kind); std::uint32_t ImageBindingIndex(DescriptorBindingKind kind); DescriptorBindingKind DescriptorBindingForImage(const ImageResource& image); diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp index b7633ca3b20f..fbc9605a744e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrProgram.cpp @@ -964,6 +964,24 @@ std::uint32_t NativeBinding(IrShaderStage stage, DescriptorBindingKind kind) { return RuntimeAbi::BindingNumber(static_cast(group), kind); } +bool SharedHeapArray(const IrBindingLayout& layout, DescriptorBindingKind kind, bool gpuResolved) { + return gpuResolved && (kind == DescriptorBindingKind::Samplers || layout.HeapImageKind(kind)); +} + +std::uint32_t DescriptorBindingNumber(const IrBindingLayout& layout, DescriptorBindingKind kind, IrShaderStage stage, bool gpuResolved) { + return NativeBinding(SharedHeapArray(layout, kind, gpuResolved) ? IrShaderStage::Pixel : stage, kind); +} + +std::uint32_t DescriptorSetNumber(const IrBindingLayout& layout, DescriptorBindingKind kind, IrShaderStage stage, bool gpuResolved, bool fragmentDescriptorSet, bool plan) { + if (kind == DescriptorBindingKind::Gds && layout.UsesDataAddress()) return RuntimeAbi::GpuGdsSet; + if (gpuResolved) { + if (SharedHeapArray(layout, kind, true)) return RuntimeAbi::FragmentDescriptorSet; + if (kind == DescriptorBindingKind::ShaderData || kind == DescriptorBindingKind::FlattenedSrt) return plan ? RuntimeAbi::FragmentDescriptorSet : RuntimeAbi::GpuDataSet; + return RuntimeAbi::DescriptorSet; + } + return fragmentDescriptorSet && stage == IrShaderStage::Pixel ? RuntimeAbi::FragmentDescriptorSet : RuntimeAbi::DescriptorSet; +} + ImageResourceClass ImageBindingResourceClass(DescriptorBindingKind kind) { const auto value = static_cast(kind); if (value >= FirstImageBinding && value < FirstStorageImageBinding) { diff --git a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp index 7396e8e96d78..0710b88619b9 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/BindingAllocator.hpp @@ -22,9 +22,16 @@ struct BindingAllocationResult : CompiledBindingLayout { std::vector shaderDataUserWords; }; +struct AllocationOptions { + bool registrationModules = false; + bool stencilReplay = false; + bool gpuResolvedImages = false; + bool descriptorMemory = false; +}; + class BindingAllocator { public: - [[nodiscard]] BindingAllocationResult Allocate(IrProgram& program, const BindingLayout& layout, bool registrationModules = false, bool stencilReplay = false) const; + [[nodiscard]] BindingAllocationResult Allocate(IrProgram& program, const BindingLayout& layout, const AllocationOptions& options = {}) const; [[nodiscard]] const IrDescriptorBinding& FindBinding(const IrBindingLayout& layout, DescriptorBindingKind kind) const; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp index 7163cd216712..98b1103303d6 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/DescriptorBindingBuilder.hpp @@ -6,9 +6,11 @@ #include "Optimization/ResourceMaterializer.hpp" #include "Recompiler.hpp" #include +#include namespace ShaderRecompiler { +std::uint32_t DeviceSamplerClass(std::span words, bool integerBorder); std::uint32_t PointFilteredSamplerWord(std::uint32_t word0, std::uint32_t filter); struct PreparedDescriptorBinding { DescriptorBinding descriptor; @@ -45,7 +47,10 @@ struct DescriptorBindingPlan { std::vector imageMetadata; std::vector specialization; std::vector runtimeState; + std::vector heapSlotWords; bool storageData = false; + bool addressedBuffers = false; + bool addressedData = false; }; class DescriptorBindingBuilder { diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp index 8c1cc7eff0df..62700dc378fb 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp @@ -3,11 +3,13 @@ #include "IntermediateRepresentation/IrProgram.hpp" #include "Optimization/SrtWalker.hpp" +#include "DeviceImageTableAbi.hpp" #include "ImageTableAbi.hpp" #include #include #include #include +#include #include #include #include @@ -61,6 +63,12 @@ struct RuntimeImageModeMatch { std::string detail; }; +struct DeviceImageInterface { + std::uint64_t id = 0; + ImageResource image; + std::vector modes; +}; + class ConvertedSampleRequired : public std::runtime_error { public: using std::runtime_error::runtime_error; @@ -84,6 +92,12 @@ class ResourceMaterializer { static std::shared_ptr ResolveImageTablesShared(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey); static bool NullImageDescriptor(const ImageResource& image, const DescriptorValue& descriptor); static ResolvedImageTables ResolveImageTables(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint64_t memoKey = 0); + static std::uint64_t DeviceImageInterfaceId(const ImageResource& image, std::span modes); + static std::uint64_t RegisterDeviceImageInterface(const ImageResource& image, std::span modes); + static void RegisterDeviceImageInterfaces(const ShaderInfo& info); + static std::shared_ptr FindDeviceImageInterface(std::uint64_t id); + static constexpr std::size_t MaxDeviceImageCandidates = 16; + static std::vector ExpectedDeviceImages(const IrResourcePlan& plan, const SrtRuntime& runtime, bool& complete); [[nodiscard]] IrResourcePlan ExtractPlan(const IrProgram& program) const; static void CompactPlan(IrResourcePlan& plan); void Materialize(const IrResourcePlan& program, const SrtRuntime& runtime, ResourceSnapshot& snapshot) const; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 7067a1fc6cec..9d1af4624486 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -62,16 +62,23 @@ class PreparedShaderInvocation { public: static std::optional TryCreate(const RecompileRequest& request, const std::shared_ptr& handle); static std::optional TryCreate(const RecompileRequest& request, const std::shared_ptr& handle, std::span key); + static std::optional TryCreate(RecompileRequest&& request, const std::shared_ptr& handle, std::span key); const RecompileRequest& Request() const { return request; } std::shared_ptr Capture(const SrtRuntime& runtime) const; std::shared_ptr Materialize(const ResourceCapture& capture) const; + std::shared_ptr MaterializeHeap(const ResourceCapture& capture, HeapStageWords& words) const; + const std::shared_ptr& Plan() const; + std::shared_ptr LoadPlan() const; + std::uint64_t VariantId() const; private: - PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle); + PreparedShaderInvocation(RecompileRequest request, const std::shared_ptr& handle); RecompileRequest request; std::shared_ptr handle; }; [[nodiscard]] std::span GetPreparedCode(const SourceHandle& handle); +[[nodiscard]] bool HeapFragmentEligible(const IrResourcePlan& plan); +void CompleteHeapDataWords(const RecompileResult& shared, HeapStageWords& words); [[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request); [[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request, ShaderPreparationContext* preparation); [[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceTracker.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceTracker.hpp index fae68d51d4a3..3f280ec5295b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceTracker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceTracker.hpp @@ -7,7 +7,7 @@ namespace ShaderRecompiler { class ResourceTracker { public: - void Track(IrProgram& program) const; + void Track(IrProgram& program, bool deviceImageTables = false) const; }; } diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index 1f6c140f8d78..42da8b0e968c 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -2,6 +2,7 @@ #include "Optimization/ResourceMaterializer.hpp" #include #include +#include #include #include #include @@ -13,6 +14,23 @@ namespace { throw std::runtime_error(message); } +bool PushHeapData() { + static const bool enabled = std::getenv("APS5_NO_HEAP_PUSHED_DATA") == nullptr; + return enabled; +} + +void placePushedData(IrBindingLayout& layout) { + const auto capacity = RuntimeAbi::PushedDataWantedLimit; + const auto take = [&](std::uint32_t dword) { + if (layout.pushedData.size() < capacity && std::ranges::find(layout.pushedData, dword) == layout.pushedData.end()) layout.pushedData.push_back(dword); + }; + for (auto dword = layout.SamplerSlotDword(); dword < layout.BufferAddressDword(); ++dword) take(dword); + for (std::uint32_t image = 0; image < layout.runtimeImageCount; ++image) take(layout.ImageFirstElementDword(image)); + for (std::uint32_t dword = 0; dword < layout.userDataRegisters.size(); ++dword) take(dword); + for (auto dword = layout.memoryOffsetDword; dword < layout.memoryOffsetDword + (layout.memoryOffsetCount + 3u) / 4u; ++dword) take(dword); + for (auto dword = layout.ImageMetadataDword(); dword < layout.SamplerSlotDword(); ++dword) take(dword); +} + std::vector collectUserData(const IrProgram& program) { std::array registers {}; for (const auto& block : program.Blocks()) { @@ -68,7 +86,8 @@ bool usesGds(const IrProgram& program) { } -BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const BindingLayout& layout, bool registrationModules, bool stencilReplay) const { +BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const BindingLayout& layout, const AllocationOptions& options) const { + const auto [registrationModules, stencilReplay, gpuResolvedImages, descriptorMemory] = options; IrProgramMetadata& metadata = program.Metadata(); if (!metadata.shaderInfoComplete || metadata.bindingLayoutComplete) { fail(metadata.shaderInfoComplete ? "shader binding layout failed: binding layout already allocated" @@ -113,9 +132,14 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } next.runtimeStateDwords = (attributes + RuntimeAbi::VertexSelectorsPerDword - 1u) / RuntimeAbi::VertexSelectorsPerDword; } + if (gpuResolvedImages) { + next.samplerSlotDwords = static_cast(info.samplers.size()) * RuntimeAbi::SamplerSlotWords; + next.gpuResolvedImages = true; + } const auto pushSize = layout.pushConstantSizeBytes / 4u; - next.pushDataStartDword = next.runtimeImageCount == 0u && next.ShaderDataDwords() != 0u && next.ShaderDataDwords() <= pushSize ? layout.pushConstantOffsetBytes / 4u : PushData::NoStart; - if (stencilReplay) { + next.pushDataStartDword = !gpuResolvedImages && next.runtimeImageCount == 0u && next.ShaderDataDwords() != 0u && next.ShaderDataDwords() <= pushSize ? layout.pushConstantOffsetBytes / 4u : PushData::NoStart; + next.dataAddress = descriptorMemory && gpuResolvedImages; + if (stencilReplay && !next.UsesDataAddress()) { const auto offset = next.UsesPushData() ? next.ShaderDataDwords() : 0u; if (offset >= pushSize) fail("stencil replay requires an unused push constant dword"); next.stencilReferenceDword = offset; @@ -153,6 +177,15 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } } + next.heapStorageImages = gpuResolvedImages && descriptorMemory; + if (gpuResolvedImages && !registrationModules) { + for (const auto& descriptor : next.descriptors) { + if (next.HeapImageKind(descriptor.kind)) next.imageSlotDwords += static_cast(descriptor.resources.size()); + } + } + if (descriptorMemory && gpuResolvedImages) next.bufferAddressDwords = static_cast(info.buffers.size()) * RuntimeAbi::BufferAddressWords; + if (next.UsesDataAddress() && PushHeapData()) placePushedData(next); + std::vector samplers; for (std::uint32_t i = 0; i < info.samplers.size(); i++) { if (info.samplers[i].table == NoTable) { @@ -204,7 +237,7 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin result.pushConstantOffsetBytes = result.layout.pushDataStartDword * static_cast(sizeof(std::uint32_t)); result.pushConstantSizeBytes = result.layout.ShaderDataDwords() * static_cast(sizeof(std::uint32_t)); } - if (stencilReplay) { + if (stencilReplay && !result.layout.UsesDataAddress()) { result.pushConstantOffsetBytes = layout.pushConstantOffsetBytes; result.pushConstantSizeBytes += sizeof(std::uint32_t); } diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 36e92b6c4a3f..c8979df75fb7 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -3,6 +3,7 @@ #include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" +#include "DeviceImageTableAbi.hpp" #include #include #include @@ -360,6 +361,27 @@ bool ReducesBetweenTexels(std::uint32_t word0, std::uint32_t filter) { return ((word0 >> 29u) & 3u) != 0u && (((filter >> 20u) & 0xfu) != 0u || ((filter >> 26u) & 3u) == 2u); } +std::uint32_t DeviceSamplerClass(std::span words, bool integerBorder) { + namespace Abi = DeviceImageTableAbi; + const auto linear = [](std::uint32_t filter) { return filter == 1u || filter == 3u; }; + const auto anisotropic = [](std::uint32_t filter) { return filter == 2u || filter == 3u; }; + const auto magFilter = (words[2] >> 20u) & 3u; + const auto minFilter = (words[2] >> 22u) & 3u; + bool border = false; + for (std::uint32_t axis = 0; axis < 3u; ++axis) { + const auto clamp = (words[0] >> (axis * 3u)) & 7u; + border = border || clamp == 4u || clamp == 6u; + } + std::uint32_t bits = 0u; + if (((words[0] >> 29u) & 3u) != 0u && (linear(magFilter) || linear(minFilter))) bits |= Abi::SamplerFilterMinmax; + if (border && ((words[3] >> 30u) & 3u) == 1u) bits |= Abi::SamplerOpaqueBlackBorder; + if (((words[0] >> 28u) & 1u) != 0u) bits |= Abi::SamplerNonSeamlessCube; + if (border) bits |= integerBorder ? Abi::SamplerIntegerBorder : Abi::SamplerFloatBorder; + if (((words[2] >> 29u) & 1u) != 0u && (anisotropic(magFilter) || anisotropic(minFilter))) bits |= Abi::SamplerAnisoOverride; + if (ReducesBetweenTexels(words[0], words[2])) bits |= Abi::SamplerReduction; + return bits; +} + std::uint32_t PointFilteredSamplerWord(std::uint32_t word0, std::uint32_t filter) { if (ReducesBetweenTexels(word0, filter)) { fail("DescriptorBindingBuilder: a min or max reduction sampler that filters between texels or mip levels samples an image that needs point filtering (sint, converted or depth-bits format), which is not implemented"); @@ -395,6 +417,8 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l if (stage == IrShaderStage::Pixel && exportMappings.size() != 8u) fail("fragment runtime export mappings are missing"); if (info.images.size() > ShaderInfo::MaxImages) fail("runtime image count exceeds the static capacity"); DescriptorBindingPlan plan; + plan.addressedBuffers = layout.bufferAddressDwords != 0u; + plan.addressedData = layout.UsesDataAddress(); const auto unnormalized = ProveUnnormalized(info, snapshot); for (std::uint32_t index = 0; index < info.samplers.size(); ++index) { if (FoldsTexelOffsets(info.samplers[index])) plan.specialization.push_back({PipelineSpecialization::FoldOffsetBase + index, unnormalized.samplers[index] ? 1u : 0u}); @@ -513,6 +537,17 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l samplerModes.at(pair.sampler) |= 3u; } } + for (std::uint32_t sampler = 0; sampler < info.samplers.size(); ++sampler) { + if (std::ranges::none_of(info.sampledPairs, [&](const SampledResourcePair& pair) { return pair.sampler == sampler && info.images.at(pair.image).deviceTable; })) continue; + const auto& words = snapshot.samplers.at(sampler).dwords; + std::uint32_t bits = 0u; + for (const auto mode : {1u, 2u}) { + if ((samplerModes.at(sampler) & mode) == 0u) continue; + const bool integerBorder = (samplerIntegerModes.at(sampler) & mode) != 0u && (samplerFloatModes.at(sampler) & mode) == 0u; + bits |= DeviceSamplerClass(std::span(words.data(), 4u), integerBorder); + } + plan.specialization.push_back({PipelineSpecialization::DeviceSamplerClassBase + sampler, bits}); + } for (std::uint32_t index = 0; index < info.images.size(); ++index) { if (info.images[index].table == NoTable) { const auto& mode = info.runtimeImageModes.at(index).at(imageModes[index]); @@ -525,10 +560,14 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l if (last < first || last - first >= RuntimeAbi::StorageMipSlots) fail("invalid dynamic storage mip range"); if (!registrationModules) plan.specialization.push_back({PipelineSpecialization::MipCountBase + index, last - first + 1u}); } + std::vector> activeElements(info.images.size(), {UINT32_MAX, 0u}); plan.bindings.reserve(layout.descriptors.size()); + const bool slotWords = layout.gpuResolvedImages && !registrationModules; + std::uint32_t imageSlotBase = layout.ImageSlotDword(); for (const auto& logical : layout.descriptors) { const bool imageHeap = ImageBindingResourceClass(logical.kind) != ImageResourceClass::None; const bool samplerHeap = logical.kind == DescriptorBindingKind::Samplers; + const bool sampledSlots = slotWords && layout.HeapImageKind(logical.kind); IrDescriptorBinding compact{logical.kind, {}}; std::vector originals; std::vector inactive; @@ -551,18 +590,21 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l active = (samplerModes.at(resource) & (1u << (element & 1u))) != 0u; } if ((imageHeap || samplerHeap) && !registrationModules) plan.specialization.push_back({PipelineSpecialization::DescriptorIndex(static_cast(logical.kind), element), active ? static_cast(compact.resources.size()) : 0u}); + if (active && sampledSlots) plan.heapSlotWords.push_back({imageSlotBase + element, static_cast(logical.kind), static_cast(compact.resources.size())}); + if (active && slotWords && samplerHeap) plan.heapSlotWords.push_back({layout.SamplerSlotDword() + element, static_cast(logical.kind), static_cast(compact.resources.size())}); if (active || ((imageHeap || samplerHeap) && registrationModules)) { compact.resources.push_back(resource); originals.push_back(element); inactive.push_back(!active); } } + if (sampledSlots) imageSlotBase += static_cast(logical.resources.size()); if ((imageHeap || samplerHeap) && !registrationModules) plan.specialization.push_back({PipelineSpecialization::HeapCountBase + static_cast(logical.kind), std::max(1u, static_cast(compact.resources.size()))}); if ((imageHeap || samplerHeap) && compact.resources.empty()) continue; PreparedDescriptorBinding entry{}; auto& physical = entry.descriptor; - physical.descriptorSet = fragmentDescriptorSet && stage == IrShaderStage::Pixel ? RuntimeAbi::FragmentDescriptorSet : RuntimeAbi::DescriptorSet; - physical.binding = NativeBinding(stage, compact.kind); + physical.descriptorSet = DescriptorSetNumber(layout, compact.kind, stage, layout.gpuResolvedImages, fragmentDescriptorSet, true); + physical.binding = DescriptorBindingNumber(layout, compact.kind, stage, layout.gpuResolvedImages); physical.count = compact.resources.empty() ? 1u : static_cast(compact.resources.size()); physical.kind = PhysicalKindFor(compact.kind); physical.role = RoleFor(compact.kind); @@ -593,6 +635,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l physical.imageUnnormalizedBaseLevel.push_back(unnormalized.imagesAtBaseLevel.at(resource)); physical.imageSamplers.push_back(0u); physical.imageAtomic64.push_back(image.atomic64); + if (layout.gpuResolvedImages && !inactive[element] && activeElements[resource].first == UINT32_MAX && DescriptorBindingForImage(mode) == compact.kind) activeElements[resource] = {static_cast(compact.kind), static_cast(element)}; } } else if (physical.role == DescriptorRole::GuestSamplers) { for (std::uint32_t element = 0; element < compact.resources.size(); ++element) { @@ -623,6 +666,10 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l RuntimeAbi::ResourceMetadata metadata{}; if (descriptor.dwordCount != metadata.descriptor.size()) fail("invalid runtime image descriptor width"); metadata.elementCount = 1u; + if (layout.gpuResolvedImages && activeElements[resource].first != UINT32_MAX) { + metadata.binding = activeElements[resource].first; + metadata.firstElement = activeElements[resource].second; + } if (imageModes[resource] > RuntimeAbi::ImageModeMask) fail("runtime image mode exceeds its metadata field"); metadata.flags = imageModes[resource] << 1u | (registrationModules ? (compareStates[resource] & RuntimeAbi::CompareStateMask) << RuntimeAbi::CompareStateShift : 0u); if (descriptor.dwords[0] == 0u && (descriptor.dwords[1] & 0xffu) == 0u) metadata.flags |= 1u; @@ -655,15 +702,40 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l return plan; } +namespace { + +bool IsAddressedRole(const DescriptorBindingPlan& plan, DescriptorRole role) { + switch (role) { + case DescriptorRole::GuestBuffers: return plan.addressedBuffers; + case DescriptorRole::ShaderData: + case DescriptorRole::FlattenedSrt: + case DescriptorRole::BdaPagetable: + case DescriptorRole::FaultBuffer: + case DescriptorRole::ImageTable: + case DescriptorRole::ImageStorageTable: + case DescriptorRole::SamplerTable: + case DescriptorRole::ImageTableMap: return plan.addressedData; + default: return false; + } +} + +} + DescriptorBindingPlan DescriptorBindingBuilder::Select(const DescriptorBindingPlan& plan, std::span bindings) const { DescriptorBindingPlan selected; std::set remaining(bindings.begin(), bindings.end()); selected.bindings.reserve(remaining.size()); for (const auto& binding : plan.bindings) { - if (remaining.erase(binding.descriptor.binding) == 0u) continue; + const bool addressed = IsAddressedRole(plan, binding.descriptor.role); + if (remaining.erase(binding.descriptor.binding) == 0u && !addressed) continue; selected.storageData |= binding.descriptor.role == DescriptorRole::ShaderData; selected.bindings.push_back(binding); } + if (plan.addressedData) { + for (const auto kind : {RuntimeAbi::FirstImageBinding, RuntimeAbi::FirstStorageImageBinding, RuntimeAbi::FirstAtomicImageBinding, RuntimeAbi::FirstAtomic64ImageBinding, static_cast(DescriptorBindingKind::Samplers)}) { + remaining.erase(NativeBinding(IrShaderStage::Pixel, static_cast(kind))); + } + } if (!remaining.empty()) fail("specialized module bindings disagree with the prepared plan"); const bool hasSamplers = std::any_of(selected.bindings.begin(), selected.bindings.end(), [](const auto& binding) { return binding.descriptor.role == DescriptorRole::GuestSamplers; }); if (!hasSamplers) { @@ -674,6 +746,16 @@ DescriptorBindingPlan DescriptorBindingBuilder::Select(const DescriptorBindingPl } selected.imageMetadata = plan.imageMetadata; selected.runtimeState = plan.runtimeState; + constexpr auto bindingCount = static_cast(RuntimeAbi::Binding::Count); + std::set droppedKinds; + for (const auto& binding : plan.bindings) droppedKinds.insert(binding.descriptor.binding % bindingCount); + for (const auto& binding : selected.bindings) droppedKinds.erase(binding.descriptor.binding % bindingCount); + selected.heapSlotWords.reserve(plan.heapSlotWords.size()); + for (const auto& word : plan.heapSlotWords) { + if (!droppedKinds.contains(word.kind)) selected.heapSlotWords.push_back(word); + } + selected.addressedBuffers = plan.addressedBuffers; + selected.addressedData = plan.addressedData; return selected; } @@ -730,8 +812,8 @@ void DescriptorBindingBuilder::Populate(BindingAllocationResult& allocation, con physical.imageShapes.push_back(ImageShapeForResource(shape)); physical.imageDepthCompare.push_back(element.depthCompare); physical.imageWritten.push_back(ImageBindingResourceClass(element.kind) == ImageResourceClass::Storage); - physical.imageAtomic.push_back(element.atomic); - physical.imageAtomic64.push_back(element.atomic64); + physical.imageAtomic.push_back(element.atomic || RuntimeAbi::AtomicImageKind(static_cast(element.kind))); + physical.imageAtomic64.push_back(element.atomic64 || RuntimeAbi::Atomic64ImageKind(static_cast(element.kind))); std::uint64_t samplers = 0; for (std::uint32_t sampler = 0; sampler < entry.tableSamplerMasks.size(); ++sampler) { if (((element.samplers >> sampler) & 1u) != 0u) samplers |= entry.tableSamplerMasks[sampler]; diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 3c562da2bc9d..953498d190fa 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -8,6 +8,7 @@ #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" #include "ThreadOwned.hpp" +#include "DeviceImageTableAbi.hpp" #include "ImageTableAbi.hpp" #include "prx/libc/include/HostThreadLocal.hpp" #include @@ -761,6 +762,10 @@ class TableSnapshotter { if (count == 0u) { return; } + const bool snapshotted = std::ranges::any_of(plan.info.images, [](const ImageResource& image) { return image.table != NoTable && !image.deviceTable; }) || std::ranges::any_of(plan.info.samplers, [](const SamplerResource& sampler) { return sampler.table != NoTable; }); + tables.shader = plan.shaderHash; + tables.tables.resize(count); + if (!snapshotted) return; if (runtime.readMemory == nullptr) { throw std::runtime_error("image table snapshot requires runtime memory access"); } @@ -781,7 +786,7 @@ class TableSnapshotter { const SrtForceSyncScope forceSync(runtime, checkEpoch && attempt == 2u); const auto invalidationsBefore = checkEpoch ? runtime.descriptorScope(runtime.userContext, 0u) : 0u; for (const auto& image : plan.info.images) { - if (image.table != NoTable) Snapshot(image.source, image.table); + if (image.table != NoTable && !image.deviceTable) Snapshot(image.source, image.table); } for (const auto& sampler : plan.info.samplers) { if (sampler.table != NoTable) Snapshot(sampler.source, sampler.table); @@ -2158,7 +2163,7 @@ ResolvedImageTables ResourceMaterializer::ResolveImageTables(const ShaderInfo& i } }; for (std::uint32_t resource = 0; resource < info.images.size(); resource++) { - if (info.images[resource].table != NoTable) fill(fill, resource, info.images[resource].table, false, tables.tables.at(info.images[resource].table), Abi::TableHeader(info.images[resource].table)); + if (info.images[resource].table != NoTable && !info.images[resource].deviceTable) fill(fill, resource, info.images[resource].table, false, tables.tables.at(info.images[resource].table), Abi::TableHeader(info.images[resource].table)); } for (std::uint32_t resource = 0; resource < info.samplers.size(); resource++) { if (info.samplers[resource].table != NoTable) fill(fill, resource, info.samplers[resource].table, true, tables.tables.at(info.samplers[resource].table), Abi::TableHeader(info.samplers[resource].table)); @@ -2579,7 +2584,7 @@ void ResourceMaterializer::ApplyStaticInterface(IrProgram& program, bool nativeS tables = true; const auto modes = RuntimeImageModes(image); const bool storage = image.resourceClass == ImageResourceClass::Storage; - if (modes.empty() || modes.size() > ImageTableAbi::MaxModes || (storage && image.mipMode == ImageMipMode::DynamicStorage)) image.tableOperation = TableOperation::Unsupported; + if (modes.empty() || (!image.deviceTable && modes.size() > ImageTableAbi::MaxModes) || (storage && image.mipMode == ImageMipMode::DynamicStorage)) image.tableOperation = TableOperation::Unsupported; } for (const auto& pair : resources.info.sampledPairs) { if (pair.image >= images.size() || pair.sampler >= resources.info.samplers.size()) throw std::runtime_error("static sampled pair is out of range"); @@ -2831,10 +2836,10 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const if (image.source >= plan.descriptorSources.size()) { throw std::runtime_error("ResourceMaterializer::ExtractPlan image references an unknown descriptor source"); } - if (image.table != NoTable) { - plan.requiresSpecializationMemory = true; - } else { + if (image.table == NoTable) { addSource(image.source); + } else if (!image.deviceTable) { + plan.requiresSpecializationMemory = true; } } for (const auto& sampler : plan.info.samplers) { @@ -2939,4 +2944,228 @@ std::uint64_t ResourceMaterializer::MaterializeNanoseconds() { return materializeNanoseconds.load(std::memory_order_relaxed); } +namespace { + +ImageResource deviceInterfaceImage(const ImageResource& image) { + auto normalized = image; + normalized.source = 0u; + normalized.firstUsePc = 0u; + normalized.adjustedSample = false; + normalized.optionalDescriptor = false; + if (normalized.table != NoTable) normalized.table = 0u; + return normalized; +} + +std::uint64_t mixImageInterface(std::uint64_t hash, const ImageResource& image) { + const auto mix = [&](std::uint64_t value) { + hash ^= value; + hash *= 0x100000001b3ull; + }; + mix(static_cast(image.resourceClass)); + mix(static_cast(image.numericClass)); + mix(static_cast(image.dimension)); + mix(static_cast(image.mipMode)); + mix(image.mipCount); + mix(static_cast(image.conversionFormat)); + mix(image.shaderSwizzle); + mix((image.read ? 1u : 0u) | (image.written ? 2u : 0u) | (image.atomic ? 4u : 0u) | (image.atomic64 ? 8u : 0u) | (image.depthCompare ? 16u : 0u) | (image.cube ? 32u : 0u) | (image.r128 ? 64u : 0u) | (image.srgbDecode ? 128u : 0u)); + mix((image.srgbDecodeCompatible ? 1u : 0u) | (image.depthBits ? 2u : 0u) | (image.depthUnorm16 ? 4u : 0u) | (image.packed ? 8u : 0u) | (image.fmaskCompatible ? 16u : 0u) | (image.depthBitsCompatible ? 32u : 0u) | (image.constantSwizzle ? 64u : 0u) | (image.constantSwizzleCompatible ? 128u : 0u)); + mix((image.flatVolumeCompatible ? 1u : 0u) | (image.flatLineCompatible ? 2u : 0u) | (image.foldTexelOffsetsOnly ? 4u : 0u) | (image.table != NoTable ? 8u : 0u) | (image.deviceTable ? 16u : 0u) | (image.singleMipPlaneBiasCompatible ? 32u : 0u) | (image.flatVolumeLineCompatible ? 64u : 0u) | (image.cubeFaceCompatible ? 128u : 0u)); + mix((image.volumeArraySampleCompatible ? 1u : 0u) | (image.convertedSamples ? 2u : 0u) | (image.remappedDepth1D ? 4u : 0u) | (image.dimensionQueryOnly ? 8u : 0u) | (image.sampledOnly ? 16u : 0u) | (image.targetMerge ? 32u : 0u)); + mix(image.srgbDecodeFormats); + mix(image.byElements); + mix(image.byComponents); + mix(static_cast(image.packedFormat)); + mix(image.emulatedCompare); + mix(static_cast(image.tableOperation)); + return hash; +} + +struct DeviceImageInterfaceRegistry { + std::mutex mutex; + std::unordered_map> interfaces; +}; + +DeviceImageInterfaceRegistry& deviceImageInterfaces() { + static auto& registry = *new DeviceImageInterfaceRegistry(); + return registry; +} + +} + +std::uint64_t ResourceMaterializer::DeviceImageInterfaceId(const ImageResource& image, std::span modes) { + auto hash = mixImageInterface(0xcbf29ce484222325ull, deviceInterfaceImage(image)); + hash = (hash ^ modes.size()) * 0x100000001b3ull; + for (const auto& mode : modes) hash = mixImageInterface(hash, deviceInterfaceImage(mode)); + return hash; +} + +std::uint64_t ResourceMaterializer::RegisterDeviceImageInterface(const ImageResource& image, std::span modes) { + const auto id = DeviceImageInterfaceId(image, modes); + const auto same = [&](const DeviceImageInterface& entry) { + if (entry.image != deviceInterfaceImage(image) || entry.modes.size() != modes.size()) return false; + for (std::size_t index = 0; index < modes.size(); ++index) { + if (entry.modes[index] != deviceInterfaceImage(modes[index])) return false; + } + return true; + }; + auto& registry = deviceImageInterfaces(); + std::lock_guard lock(registry.mutex); + if (const auto found = registry.interfaces.find(id); found != registry.interfaces.end()) { + if (!same(*found->second)) throw std::runtime_error("device image interface: two image interfaces share the id 0x" + hexText(id)); + return id; + } + auto entry = std::make_shared(); + entry->id = id; + entry->image = deviceInterfaceImage(image); + for (const auto& mode : modes) entry->modes.push_back(deviceInterfaceImage(mode)); + registry.interfaces.emplace(id, std::move(entry)); + return id; +} + +void ResourceMaterializer::RegisterDeviceImageInterfaces(const ShaderInfo& info) { + for (std::uint32_t resource = 0; resource < info.images.size(); ++resource) { + if (!info.images[resource].deviceTable) continue; + if (resource >= info.runtimeImageModes.size()) throw std::runtime_error("device image interface: the runtime image modes are missing"); + static_cast(RegisterDeviceImageInterface(info.images[resource], info.runtimeImageModes[resource])); + } +} + +std::shared_ptr ResourceMaterializer::FindDeviceImageInterface(std::uint64_t id) { + auto& registry = deviceImageInterfaces(); + std::lock_guard lock(registry.mutex); + const auto found = registry.interfaces.find(id); + return found != registry.interfaces.end() ? found->second : nullptr; +} + +std::vector ResourceMaterializer::ExpectedDeviceImages(const IrResourcePlan& plan, const SrtRuntime& runtime, bool& complete) { + namespace Abi = DeviceImageTableAbi; + std::vector expected; + complete = true; + if (plan.info.deviceImageUses.empty()) return expected; + Detail::Evaluator evaluator(plan, runtime); + std::vector active(plan.info.images.size(), 1u); + std::vector> taken(plan.controlFlow.size()); + std::vector visited(plan.controlFlow.size()); + if (!plan.controlFlow.empty()) { + for (const auto& block : plan.controlFlow) { + for (const auto image : block.images) active.at(image) = 0u; + } + std::vector pending{0}; + while (!pending.empty()) { + const auto index = pending.back(); + pending.pop_back(); + if (visited.at(index) != 0u) continue; + visited[index] = 1u; + const auto& block = plan.controlFlow[index]; + for (const auto image : block.images) active[image] = 1u; + std::uint32_t condition = 0; + if (block.condition != nullptr && block.successors.size() == 2u && evaluator.Evaluate(Detail::IrNode(block.condition), condition)) { + taken[index] = {block.successors[condition != 0u ? 0u : 1u]}; + } else { + taken[index] = block.successors; + } + pending.insert(pending.end(), taken[index].begin(), taken[index].end()); + } + } + using Words = std::vector; + using Values = std::vector; + const auto unwrapped = [](IrValue* word) { + IrValue* value = word->Resolve(); + while (value->Opcode() == IrOpcode::ReadFirstLane) value = value->Argument(0)->Resolve(); + return value; + }; + const auto joining = [](const IrValue* value) { return value->IsPhi() || (value->Opcode() == IrOpcode::SelectU32 && !value->Argument(0)->Resolve()->HasImmediate()); }; + const auto incoming = [&](const Words& words) { + Words start; + for (auto* word : words) start.push_back(unwrapped(word)); + std::vector sets{start}; + for (std::uint32_t round = 0; round < 4u && sets.size() <= MaxDeviceImageCandidates; ++round) { + std::vector next; + bool expanded = false; + for (const auto& set : sets) { + const auto found = std::ranges::find_if(set, joining); + if (found == set.end()) { + next.push_back(set); + continue; + } + expanded = true; + const IrValue* join = *found; + if (!join->IsPhi()) { + const IrValue* condition = join->Argument(0)->Resolve(); + for (const std::size_t operand : {1u, 2u}) { + auto candidate = set; + for (auto*& word : candidate) { + if (word->Opcode() == IrOpcode::SelectU32 && word->Argument(0)->Resolve() == condition) word = unwrapped(word->Argument(operand)); + } + next.push_back(candidate); + } + continue; + } + for (std::size_t edge = 0; edge < join->ArgumentCount(); ++edge) { + auto candidate = set; + for (auto*& word : candidate) { + if (!word->IsPhi() || word->ArgumentCount() != join->ArgumentCount()) continue; + bool joined = true; + for (std::size_t index = 0; joined && index < join->ArgumentCount(); ++index) joined = word->PhiBlock(index) == join->PhiBlock(index); + if (joined) word = unwrapped(word->Argument(edge)); + } + next.push_back(candidate); + } + } + sets = std::move(next); + if (!expanded) break; + } + return sets; + }; + const auto evaluated = [&](const Words& words, Values& values) { + values.assign(words.size(), 0u); + for (std::size_t word = 0; word < words.size(); ++word) { + if (!evaluator.Evaluate(Detail::IrNode(words[word]), values[word])) return false; + } + return true; + }; + const auto valuesOf = [&](const Words& words, bool& joined) { + std::vector values(1); + joined = false; + if (evaluated(words, values.front())) return values; + values.clear(); + joined = true; + const auto sets = incoming(words); + if (sets.size() > MaxDeviceImageCandidates) return values; + for (const auto& candidate : sets) { + Values value; + if (!evaluated(candidate, value)) return std::vector{}; + if (std::ranges::find(values, value) == values.end()) values.push_back(std::move(value)); + } + return values; + }; + expected.reserve(plan.info.deviceImageUses.size()); + for (const auto& use : plan.info.deviceImageUses) { + if (active.at(use.image) == 0u) continue; + const auto& descriptor = plan.descriptorSources.at(use.descriptor); + if (descriptor.dwordCount != Abi::DescriptorWords || std::any_of(descriptor.dwords.begin(), descriptor.dwords.begin() + Abi::DescriptorWords, [](const IrValue* word) { return word == nullptr; })) { + complete = false; + continue; + } + const Words words(descriptor.dwords.begin(), descriptor.dwords.begin() + Abi::DescriptorWords); + bool joined = false; + const auto images = valuesOf(words, joined); + if (images.empty()) complete = false; + for (const auto& value : images) { + Abi::Expectation expectation; + expectation.pc = use.firstUsePc; + expectation.image = use.image; + expectation.sampler = use.sampler; + expectation.candidate = joined; + std::copy_n(value.begin(), Abi::DescriptorWords, expectation.key.begin()); + if (Abi::NullDescriptor(expectation.key[0], expectation.key[1])) continue; + const auto same = std::ranges::find_if(expected, [&](const Abi::Expectation& held) { return held.key == expectation.key && held.pc == expectation.pc && held.image == expectation.image && held.sampler == expectation.sampler; }); + if (same == expected.end()) expected.push_back(expectation); + else same->candidate = same->candidate && expectation.candidate; + } + } + return expected; +} + } diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index 3d06941873fe..374fa1a71cdb 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -29,6 +29,7 @@ namespace { constexpr std::uint32_t samplerBorderClampMask = (1u << 2u) | (1u << 5u) | (1u << 8u); constexpr std::uint32_t samplerDword3ReservedMask = 0x3ffff000u; +constexpr std::uint32_t samplerForceUnnormalizedBit = 1u << 15u; [[noreturn]] void fail(const std::string& message) { throw std::runtime_error(message); @@ -105,11 +106,12 @@ std::uint32_t byteExtent(const MemoryInfo& memory) { class Tracker { public: - explicit Tracker(IrProgram& program) : m_program(program), m_info(program.Resources().info), m_builder(program) { + Tracker(IrProgram& program, bool deviceImageTables) : m_program(program), m_info(program.Resources().info), m_builder(program), m_deviceImageTables(deviceImageTables) { m_info.buffers.clear(); m_info.images.clear(); m_info.samplers.clear(); m_info.sampledPairs.clear(); + m_info.deviceImageUses.clear(); m_info.usesDma = false; m_info.bdaWrites = false; } @@ -144,6 +146,9 @@ class Tracker { if (m_bdaTraces > bdaTraceLimit) std::fprintf(stderr, "[bda] %u more address accesses in this program not shown\n", m_bdaTraces - bdaTraceLimit); CheckTableStores(); AssignTables(); + for (const auto& use : m_imageUses) { + if (m_info.images.at(use.image).deviceTable) m_info.deviceImageUses.push_back(use); + } LinkImageAliases(); for (const auto& patch : m_handlePatches) { patch.handle->SetFlags(patch.resource); @@ -157,6 +162,7 @@ class Tracker { } std::vector referenced; for (const auto& plan : m_tableColumns) { + if (DeviceColumn(plan)) continue; const auto& column = *m_sources[plan.source].tableColumn; IrValue* offset = column.selected ? &m_builder.Constant(column.addend) : plan.offset; for (std::uint32_t argument = 0; argument < plan.handle->ArgumentCount(); argument++) { @@ -2009,11 +2015,14 @@ class Tracker { } } } + PlanDeviceColumns(); for (const auto& plan : m_tableColumns) { + if (DeviceColumn(plan)) continue; for (std::size_t index = 0; index < plan.reads.size(); index++) { const IrValue* read = plan.reads[index]; const bool planned = std::ranges::all_of(read->Uses(), [&](const IrValue* user) { - return FindTableColumn(*user) != nullptr; + const auto* column = FindTableColumn(*user); + return column != nullptr && !DeviceColumn(*column); }); if (planned && std::ranges::find(m_tablePlanningReads, read) == m_tablePlanningReads.end()) { m_tablePlanningReads.push_back(read); @@ -2024,6 +2033,7 @@ class Tracker { for (bool marked = true; marked;) { marked = false; for (const auto& plan : m_tableColumns) { + if (DeviceColumn(plan)) continue; for (std::size_t index = 0; index < plan.hopReads.size(); index++) { const IrValue* read = plan.hopReads[index]; if (std::ranges::find(m_tablePlanningReads, read) != m_tablePlanningReads.end()) { @@ -2046,6 +2056,56 @@ class Tracker { m_memoryUsers.clear(); } + bool DeviceAccess(const IrValue& inst) { + switch (inst.Opcode()) { + case IrOpcode::ImageSampleRaw: + case IrOpcode::ImageGatherRaw: + case IrOpcode::ImageRead: + case IrOpcode::ImageQueryDimensions: + case IrOpcode::ImageQueryLod: + break; + default: + return false; + } + const auto flags = inst.Flags(); + if (flags.index >= m_program.Resources().memoryInfo.size()) return false; + const auto& memory = m_program.Resources().memoryInfo[flags.index]; + if (memory.imagePacked || memory.imageByElements != 0u || (memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) return false; + if (!ImageOpcodeInfoOf(inst.Opcode()).needsSampler) return true; + if (inst.ArgumentCount() < 2u) return false; + const IrValue& sampler = *inst.Argument(1)->Resolve(); + if (!DirectSampler(sampler)) return false; + const IrValue* word0 = sampler.Argument(0)->Resolve(); + return !(word0->HasImmediate() && word0->Type() == IrType::U32 && (word0->ImmediateU32() & samplerForceUnnormalizedBit) != 0u); + } + + void PlanDeviceColumns() { + m_deviceSources.assign(m_sources.size(), 0u); + if (!m_deviceImageTables) return; + for (const auto& plan : m_tableColumns) { + const auto& column = *m_sources[plan.source].tableColumn; + if (!column.sampler && !column.address && !column.indirect && column.pointerHops.empty()) m_deviceSources[plan.source] = 1u; + } + for (auto& block : m_program.Blocks()) { + for (IrValue* inst : block->Instructions()) { + if (ImageOpcodeInfoOf(inst->Opcode()).access == ImageAccess::None || inst->ArgumentCount() == 0u) continue; + const auto* column = FindTableColumn(*inst->Argument(0)->Resolve()); + if (column != nullptr && !DeviceAccess(*inst)) m_deviceSources[column->source] = 0u; + } + } + for (std::uint32_t source = 0; source < m_sources.size(); ++source) { + if (m_deviceSources[source] != 0u) m_sources[source].tableColumn->keyDomains.clear(); + } + } + + bool DeviceSource(std::uint32_t source) const { + return source < m_deviceSources.size() && m_deviceSources[source] != 0u; + } + + bool DeviceColumn(const TableColumnPlan& plan) const { + return DeviceSource(plan.source); + } + void KeepSourceAlive(IrValue& position, const DescriptorSource& source, std::vector& referenced) { for (std::uint32_t dword = 0; dword < source.dwordCount; dword++) { IrValue* value = source.dwords[dword]->Resolve(); @@ -2197,6 +2257,9 @@ class Tracker { continue; } image.table = next++; + image.deviceTable = DeviceSource(image.source); + const bool storage = image.resourceClass == ImageResourceClass::Storage || image.written || image.atomic; + if (image.deviceTable && (storage || image.packed || image.byElements != 0u)) fail("a device-table image is not a read-only sampled image"); if (image.packed || image.byElements != 0u) { image.tableOperation = TableOperation::Unsupported; } @@ -2416,6 +2479,26 @@ class Tracker { m_info.sampledPairs.push_back({image, sampler, pc, volumeGather}); } + std::uint32_t ElementSource(const TableColumnPlan& column) { + if (const auto found = m_elementSources.find(column.handle); found != m_elementSources.end()) return found->second; + DescriptorSource element; + MakeSource(*column.handle, 8u, false, false, element); + const auto index = static_cast(m_sources.size()); + m_sources.push_back(std::move(element)); + m_elementSources.emplace(column.handle, index); + return index; + } + + void AddImageUse(std::uint32_t image, std::uint32_t sampler, std::uint32_t descriptor, std::uint32_t pc) { + for (auto& use : m_imageUses) { + if (use.image == image && use.sampler == sampler && use.descriptor == descriptor) { + use.firstUsePc = std::min(use.firstUsePc, pc); + return; + } + } + m_imageUses.push_back({image, sampler, descriptor, pc}); + } + void AddHandlePatch(IrValue* handle, std::uint32_t resource) { for (const auto& patch : m_handlePatches) { if (patch.handle == handle) { @@ -2544,6 +2627,7 @@ class Tracker { AddHandlePatch(samplerHandle, sampler); AddSampledPair(resource, sampler, flags.pc, op == IrOpcode::ImageGatherRaw && m_info.images[resource].dimension == RdnaImageDimension::Dim3D); } + if (column != nullptr && DeviceColumn(*column)) AddImageUse(resource, imageInfo.needsSampler ? sampler : NoSampler, ElementSource(*column), flags.pc); AddMemoryPatch(flags.index, resource, sampler, imageInfo.needsSampler); } @@ -2591,6 +2675,10 @@ class Tracker { std::vector m_tableColumns; std::vector m_tablePlanningReads; std::vector m_tablePlanningMemory; + std::vector m_deviceSources; + std::vector m_imageUses; + std::unordered_map m_elementSources; + bool m_deviceImageTables = false; std::vector m_unselectedHandles; struct EdgeSelectorEntry { IrBlock* block = nullptr; @@ -2604,8 +2692,8 @@ class Tracker { } -void ResourceTracker::Track(IrProgram& program) const { - Tracker(program).Run(); +void ResourceTracker::Track(IrProgram& program, bool deviceImageTables) const { + Tracker(program, deviceImageTables).Run(); } } diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index 8a367c5c1832..109d62b7a99e 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -35,10 +35,14 @@ inline constexpr std::uint32_t StencilReferenceOffset = 8235u; inline constexpr std::uint32_t ReversedBlend = 8236u; inline constexpr std::uint32_t MeshLayerBase = 8237u; inline constexpr std::uint32_t MeshLayerCount = 8238u; +inline constexpr std::uint32_t HeapImageSlots = 8239u; inline constexpr std::uint32_t DepthRemapBase = 8240u; inline constexpr std::uint32_t DepthRemapWords = 1u; inline constexpr std::uint32_t WorkgroupOffsetBase = 8241u; inline constexpr std::uint32_t WorkgroupOffsetWords = 3u; +inline constexpr std::uint32_t PushedDataBase = 8244u; +inline constexpr std::uint32_t PushedDataCount = 8245u; +inline constexpr std::uint32_t RootItemPositionBase = 8246u; inline constexpr std::uint32_t HeapCountBase = 8256u; inline constexpr std::uint32_t MipCountBase = 8320u; inline constexpr std::uint32_t StoreQuantizeBase = 12288u; @@ -53,11 +57,16 @@ inline constexpr std::uint32_t MinLodBase = 57600u; inline constexpr std::uint32_t VolumeGatherBase = 57664u; inline constexpr std::uint32_t SamplerLodBiasBase = 57728u; inline constexpr float SamplerLodBiasShaderThreshold = 15.0f; +inline constexpr std::uint32_t DeviceSamplerClassBase = 58368u; inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) { return DescriptorIndexBase + binding * DescriptorIndexStride + element; } +inline constexpr bool IsPushPlacement(std::uint32_t id) { + return id == PushedDataBase || id == PushedDataCount || (id >= RootItemPositionBase && id < HeapCountBase); +} + inline constexpr bool IsWorkgroupOffset(std::uint32_t id) { return id >= WorkgroupOffsetBase && id < WorkgroupOffsetBase + WorkgroupOffsetWords; } diff --git a/core/shader/recompiler/PushDataLayout.hpp b/core/shader/recompiler/PushDataLayout.hpp new file mode 100644 index 000000000000..0bccc9015eee --- /dev/null +++ b/core/shader/recompiler/PushDataLayout.hpp @@ -0,0 +1,201 @@ +#ifndef CORE_SHADER_RECOMPILER_PUSHDATALAYOUT_HPP +#define CORE_SHADER_RECOMPILER_PUSHDATALAYOUT_HPP + +#include "PipelineSpecialization.hpp" +#include "Recompiler.hpp" +#include "RuntimeAbi.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { + +enum class PushPart : std::uint32_t { PreRasterization, Fragment, Compute }; + +inline PushPart PushPartOf(ShaderStage stage) { + if (stage == ShaderStage::Fragment) return PushPart::Fragment; + return stage == ShaderStage::Compute ? PushPart::Compute : PushPart::PreRasterization; +} + +struct PushStage { + PushPart part = PushPart::PreRasterization; + std::uint32_t items = 0; + std::uint32_t pushedWanted = 0; +}; + +struct StagePushPlacement { + std::array items{}; + RuntimeAbi::PushedPlacement pushed; + + [[nodiscard]] std::uint32_t Item(RuntimeAbi::RootItem item) const { return items[static_cast(item)]; } + bool operator==(const StagePushPlacement&) const = default; +}; + +constexpr std::uint64_t PushBytesMask(std::uint32_t offset, std::uint32_t bytes) { + std::uint64_t mask = 0; + for (auto dword = offset / 4u; dword < (offset + bytes + 3u) / 4u && dword < RuntimeAbi::PushDataDwords; ++dword) mask |= 1ull << dword; + return mask; +} + +constexpr std::uint64_t PushFeatureDwords(bool meshDraw, bool tessellationLevels, bool depthRemap) { + std::uint64_t mask = 0; + if (meshDraw) mask |= PushBytesMask(MeshDrawPushOffsetBytes, MeshDrawPushBytes); + if (tessellationLevels) mask |= PushBytesMask(TessellationLevelPushOffsetBytes, TessellationLevelPushBytes); + if (depthRemap) mask |= PushBytesMask(DepthRemapPushOffsetBytes, DepthRemapPushBytes); + return mask; +} + +inline constexpr std::uint32_t RootItemMask = (1u << RuntimeAbi::RootItemCount) - 1u; + +constexpr std::uint64_t StagePushFeatures(ShaderStage stage, std::uint32_t pushItems) { + const bool depthRemap = (pushItems & RuntimeAbi::DepthRemapPushBit) != 0u; + std::uint64_t mask = PushFeatureDwords(stage == ShaderStage::Mesh || stage == ShaderStage::Geometry, stage == ShaderStage::TessellationControl || stage == ShaderStage::TessellationEvaluation, depthRemap); + if (stage == ShaderStage::Compute) mask |= PushBytesMask(GpuScratchPoolAddressPushByte - 8u, GpuScratchPoolDwordsPushByte + 4u - (GpuScratchPoolAddressPushByte - 8u)); + return mask; +} + +inline std::vector PlacePushData(std::span stages, std::uint64_t occupied) { + namespace Abi = RuntimeAbi; + constexpr std::uint32_t total = Abi::PushDataDwords; + std::vector placements(stages.size()); + for (auto& placement : placements) placement.items.fill(Abi::NoPushPosition); + auto used = occupied; + const auto isFree = [&](std::uint32_t first, std::uint32_t width) { + for (auto dword = first; dword < first + width; ++dword) { + if (dword >= total || (used >> dword & 1ull) != 0u) return false; + } + return true; + }; + const auto take = [&](std::uint32_t width, bool aligned, bool top) { + for (std::uint32_t step = 0; step + width <= total; ++step) { + const auto first = top ? total - width - step : step; + if ((aligned && first % 2u != 0u) || !isFree(first, width)) continue; + for (auto dword = first; dword < first + width; ++dword) used |= 1ull << dword; + return first; + } + throw std::runtime_error("the push data has no room for a pipeline's root"); + }; + const auto placeRoot = [&](std::uint32_t items, bool top, std::array& positions) { + for (const bool addresses : {true, false}) { + for (std::uint32_t index = 0; index < Abi::RootItemCount; ++index) { + const auto item = static_cast(index); + if ((items & Abi::RootItemBit(item)) == 0u || Abi::RootAddressItem(item) != addresses) continue; + positions[index] = take(addresses ? 2u : 1u, addresses, top); + } + } + }; + constexpr auto globalItems = Abi::RootItemBit(Abi::RootItem::ShaderData) - 1u; + for (const auto part : {PushPart::Fragment, PushPart::Compute, PushPart::PreRasterization}) { + const bool top = part != PushPart::PreRasterization; + std::uint32_t globals = 0; + for (const auto& stage : stages) { + if (stage.part == part) globals |= stage.items & globalItems; + } + std::array shared{}; + shared.fill(Abi::NoPushPosition); + placeRoot(globals, top, shared); + for (std::size_t index = 0; index < stages.size(); ++index) { + if (stages[index].part != part) continue; + for (std::uint32_t item = 0; item < Abi::RootItemCount; ++item) { + if ((stages[index].items & globalItems & (1u << item)) != 0u) placements[index].items[item] = shared[item]; + } + placeRoot(stages[index].items & ~globalItems, top, placements[index].items); + } + } + std::vector> runs; + for (std::uint32_t dword = 0; dword < total;) { + if ((used >> dword & 1ull) != 0u) { + ++dword; + continue; + } + auto end = dword; + while (end < total && (used >> end & 1ull) == 0u) ++end; + runs.emplace_back(dword, end); + dword = end; + } + std::uint32_t room = 0; + std::uint32_t largest = 0; + for (const auto& [first, end] : runs) { + room += end - first; + largest = std::max(largest, end - first); + } + std::vector wanted(stages.size(), 0u); + std::uint32_t sum = 0; + for (std::size_t index = 0; index < stages.size(); ++index) { + wanted[index] = std::min({stages[index].pushedWanted, Abi::PushedDataWantedLimit, largest}); + sum += wanted[index]; + } + auto counts = wanted; + if (sum > room) { + const auto sharedAt = [&](std::uint32_t level) { + std::uint32_t shared = 0; + for (const auto want : wanted) shared += std::min(want, level); + return shared; + }; + std::uint32_t level = 0; + while (sharedAt(level + 1u) <= room) ++level; + auto left = room - sharedAt(level); + for (std::size_t index = 0; index < counts.size(); ++index) { + counts[index] = std::min(wanted[index], level); + if (left != 0u && wanted[index] > level) { + ++counts[index]; + --left; + } + } + } + const auto claim = [&](std::size_t index, bool top) { + auto count = counts[index]; + if (count == 0u) return; + auto chosen = runs.end(); + if (top) { + for (auto run = runs.rbegin(); run != runs.rend(); ++run) { + if (run->second - run->first >= count) { + chosen = std::prev(run.base()); + break; + } + } + } else { + chosen = std::find_if(runs.begin(), runs.end(), [&](const auto& run) { return run.second - run.first >= count; }); + } + if (chosen == runs.end()) { + chosen = std::max_element(runs.begin(), runs.end(), [](const auto& left, const auto& right) { return left.second - left.first < right.second - right.first; }); + if (chosen == runs.end()) return; + count = chosen->second - chosen->first; + } + if (top) { + placements[index].pushed = {chosen->second - count, count}; + chosen->second -= count; + } else { + placements[index].pushed = {chosen->first, count}; + chosen->first += count; + } + }; + for (std::size_t index = 0; index < stages.size(); ++index) { + if (stages[index].part == PushPart::Fragment) claim(index, true); + } + for (std::size_t index = 0; index < stages.size(); ++index) { + if (stages[index].part != PushPart::Fragment) claim(index, false); + } + return placements; +} + +inline StagePushPlacement StandalonePushPlacement(ShaderStage stage, std::uint32_t pushItems, std::size_t pushedWanted) { + const PushStage pushStage{PushPartOf(stage), pushItems & RootItemMask, static_cast(pushedWanted)}; + return PlacePushData(std::span(&pushStage, 1), StagePushFeatures(stage, pushItems)).front(); +} + +inline void AppendPushSpecialization(std::vector& constants, const StagePushPlacement& placement) { + for (std::uint32_t item = 0; item < RuntimeAbi::RootItemCount; ++item) { + if (placement.items[item] != RuntimeAbi::NoPushPosition) constants.push_back({PipelineSpecialization::RootItemPositionBase + item, placement.items[item]}); + } + constants.push_back({PipelineSpecialization::PushedDataBase, placement.pushed.base}); + constants.push_back({PipelineSpecialization::PushedDataCount, placement.pushed.count}); +} + +} + +#endif diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index a5c968873caa..f094deed5184 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -1,5 +1,7 @@ #include "prx/libc/include/HostThreadLocal.hpp" #include "Recompiler.hpp" +#include "PushDataLayout.hpp" +#include "HeapTemplateBudget.hpp" #include "RdnaDecoder/DiagnosticSink.hpp" #include "StencilReplay.hpp" #include @@ -362,8 +364,10 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparat deadCodeEliminator.Eliminate(program); dumpIr("srt"); + const bool heapStage = request.shader.stage == ShaderStage::Fragment || request.shader.stage == ShaderStage::Vertex || request.shader.stage == ShaderStage::Compute; + const bool gpuModeStage = request.target.gpuResolvedImages && request.target.descriptorMemory && heapStage; constexpr ResourceTracker resourceTracker; - resourceTracker.Track(program); + resourceTracker.Track(program, request.target.deviceImageTables && gpuModeStage && supports(spv::CapabilityInt64) && supports(spv::CapabilityPhysicalStorageBufferAddresses)); deadCodeEliminator.Eliminate(program); dumpIr("resources"); program.Resources().srgbDecodeFormats = request.target.srgbDecodeFormats; @@ -410,8 +414,37 @@ constexpr std::size_t ResultMemoSets = 64; constexpr std::size_t ResultMemoWays = 4; using ResultMemoTable = std::array; +std::uint64_t NextId(std::atomic& counter) { + return counter.fetch_add(1, std::memory_order_relaxed) + 1u; +} + +std::uint64_t NextSourceId() { + static std::atomic ids{0}; + return NextId(ids); +} + +std::uint64_t NextTemplateId() { + static std::atomic ids{0}; + return NextId(ids); +} + +struct UsedBindingPlan { + std::shared_ptr owner; + const DescriptorBindingPlan* plan = nullptr; +}; + +struct HeapTemplate { + std::shared_ptr result; + UsedBindingPlan plan; + std::vector buffers; + std::vector images; + std::vector samplers; + std::uint64_t bytes = 0; +}; + struct SourceEntry : std::enable_shared_from_this { std::mutex mutex; + const std::uint64_t id = NextSourceId(); // The code the entry was built for: the key carries only a hash of it, so a candidate entry is // accepted only when its code matches word for word. Owned here because the request's span // points into a registration the driver may replace while the entry lives on. @@ -428,10 +461,77 @@ struct SourceEntry : std::enable_shared_from_this { std::vector emissionFailures; std::unique_ptr memo; std::unique_ptr maskedMemo; + std::unordered_map> heapTemplates; }; namespace { +std::uint64_t HeapTemplateBudget() { + static const std::uint64_t budget = [] { + const char* value = std::getenv("APS5_HEAP_TEMPLATE_MIB"); + const auto mib = value != nullptr ? std::strtoull(value, nullptr, 10) : 256ull; + return std::max(mib, 1u) << 20u; + }(); + return budget; +} + +struct HeapTemplateStore { + std::mutex mutex; + HeapTemplateLru order{HeapTemplateBudget()}; + std::uint64_t evicted = 0; +}; + +HeapTemplateStore& HeapTemplates() { + static auto* store = new HeapTemplateStore(); + return *store; +} + +std::uint64_t HeapTemplateBytes(const HeapTemplate& entry) { + return sizeof(HeapTemplate) + OwnedBytes(*entry.result) + OwnedBytes(entry.buffers) + OwnedBytes(entry.images) + OwnedBytes(entry.samplers); +} + +HeapTemplateLru::Name HeapTemplateName(const SourceEntry& source, std::uint64_t key, const HeapTemplate& entry) { + return {source.id, key, entry.result->heapTemplateId}; +} + +void KeepHeapTemplate(const std::shared_ptr& source, std::uint64_t key, const HeapTemplate& entry) { + auto& store = HeapTemplates(); + std::vector evicted; + { + std::lock_guard lock(store.mutex); + evicted = store.order.Insert(HeapTemplateName(*source, key, entry), entry.bytes, source); + } + std::uint64_t erased = 0; + for (const auto& gone : evicted) { + const auto owner = std::static_pointer_cast(gone.owner.lock()); + if (owner == nullptr) continue; + std::lock_guard lock(owner->mutex); + const auto found = owner->heapTemplates.find(gone.name.key); + if (found == owner->heapTemplates.end() || found->second->result->heapTemplateId != gone.name.id) continue; + owner->heapTemplates.erase(found); + ++erased; + } + if (erased == 0) return; + std::lock_guard lock(store.mutex); + store.evicted += erased; +} + +void TouchHeapTemplate(const HeapTemplateLru::Name& name) { + auto& store = HeapTemplates(); + std::lock_guard lock(store.mutex); + store.order.Touch(name); +} + +} + +HeapTemplateCacheCounts HeapTemplateCounts() { + auto& store = HeapTemplates(); + std::lock_guard lock(store.mutex); + return {store.order.Entries(), store.order.Bytes(), std::exchange(store.evicted, 0)}; +} + +namespace { + class PrimedSharedMutex : public std::shared_mutex { public: PrimedSharedMutex() { @@ -587,7 +687,7 @@ std::array partialThreads(const RecompileRequest& request) { std::uint64_t nextVariantId() { static std::atomic variants{0}; - return variants.fetch_add(1, std::memory_order_relaxed) + 1; + return NextId(variants); } template @@ -627,10 +727,12 @@ void prepareInterface(const RecompileRequest& request, IrProgram& program) { CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) { const auto inputInfo = RequestInputInfo(request); constexpr BindingAllocator bindingAllocator; - bool registration = request.target.registrationModules && request.shader.stage == ShaderStage::Fragment; + const bool heapVertex = request.shader.stage == ShaderStage::Vertex && request.target.gpuResolvedImages && request.target.descriptorMemory; + bool registration = request.target.registrationModules && (request.shader.stage == ShaderStage::Fragment || heapVertex); program.Metadata().bindingLayoutComplete = false; const bool stencilReplay = request.shader.stage == ShaderStage::Fragment && std::ranges::any_of(program.Info().outputs, [](const StageOutput& output) { return output.kind == StageOutputKind::StencilRef; }) && std::ranges::find(request.target.supportedCapabilities, static_cast(spv::CapabilityStencilExportEXT)) == request.target.supportedCapabilities.end(); - auto bindings = bindingAllocator.Allocate(program, request.layout, registration, stencilReplay); + AllocationOptions allocation {.registrationModules = registration, .stencilReplay = stencilReplay, .gpuResolvedImages = request.target.gpuResolvedImages, .descriptorMemory = request.target.descriptorMemory}; + auto bindings = bindingAllocator.Allocate(program, request.layout, allocation); SpirvTargetOptions targetOptions {}; targetOptions.vulkanVersion = request.target.vulkanVersion; @@ -690,6 +792,9 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) } targetOptions.waveLdsMemoryBarriers = request.target.waveLdsMemoryBarriers; targetOptions.deviceScopeLdsMemoryBarriers = request.target.deviceScopeLdsMemoryBarriers; + targetOptions.gpuResolvedImages = allocation.gpuResolvedImages; + targetOptions.heapDataUniformDwords = request.target.heapDataUniformDwords; + targetOptions.heapBufferDescriptors = request.target.descriptorMemory && request.target.heapBufferDescriptors; targetOptions.ldsLockFences = request.target.ldsLockFences; targetOptions.meshWindowBuffer = request.target.meshWindowBuffer; @@ -707,7 +812,8 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) if (registration && result.spirv.size() > request.target.registrationWordLimit) { registration = false; program.Metadata().bindingLayoutComplete = false; - bindings = bindingAllocator.Allocate(program, request.layout, false, stencilReplay); + allocation.registrationModules = false; + bindings = bindingAllocator.Allocate(program, request.layout, allocation); targetOptions.registrationModules = false; result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions, &emitted); } @@ -720,6 +826,16 @@ CompiledVariant emitVariant(const RecompileRequest& request, IrProgram& program) result.stencilReferences = StencilReferenceValues(program, request.context.pixel.has_value() && request.context.pixel->stencilOperationValueExport ? 8u : 0u); result.stencilReferenceDword = bindings.layout.stencilReferenceDword; } + result.gpuResolvedImages = targetOptions.gpuResolvedImages; + result.samplerSlotDword = result.gpuResolvedImages ? bindings.layout.SamplerSlotDword() : 0u; + result.deviceImageTables = std::ranges::any_of(program.Info().images, [](const ImageResource& image) { return image.deviceTable; }); + if (result.deviceImageTables && !(result.gpuResolvedImages && bindings.layout.UsesDataAddress())) throw std::logic_error("a device image table column was planned for a stage that is not a GPU-mode stage reading its table map by address"); + result.bufferAddressDword = bindings.layout.bufferAddressDwords != 0u ? bindings.layout.BufferAddressDword() : 0u; + result.bufferAddressCount = bindings.layout.bufferAddressDwords / RuntimeAbi::BufferAddressWords; + result.dataAddressPush = bindings.layout.UsesDataAddress(); + result.pushedData = bindings.layout.pushedData; + result.bdaStores = emitted.bdaStores; + result.pushItems = emitted.pushItems; result.bdaAbiVersion = program.Info().usesDma || program.Info().usesFaultBuffer ? request.target.bdaAbiVersion : 0u; result.windowLaneGuards = 0; @@ -806,6 +922,14 @@ struct SpecializedSpirv { SharedSpirv spirv; }; +bool StencilInPush(const CompiledShaderArtifact& artifact, const IrBindingLayout& layout) { + return !artifact.stencilReferences.empty() && !layout.UsesDataAddress(); +} + +bool UsesRootOrPush(const IrBindingLayout& layout) { + return layout.UsesPushData() || layout.UsesDataAddress(); +} + std::vector specializeWords(const CompiledShaderArtifact& artifact, std::span classes, std::span constants, const SpirvTarget& target) { auto source = SpecializeVertexInputTypes(artifact, classes); std::map supplied; @@ -823,7 +947,7 @@ std::vector specializeWords(const CompiledShaderArtifact& artifac const auto count = words[cursor] >> 16u; const auto op = static_cast(words[cursor] & 0xffffu); if (count == 0u || count > words.size() - cursor) throw std::runtime_error("truncated prepared specialization instruction"); - if (!artifact.stencilReferences.empty() && op == spv::OpTypePointer && count == 4u && words[cursor + 2u] == spv::StorageClassPushConstant) pushTypes.insert(words[cursor + 3u]); + if (!artifact.stencilReferences.empty() && !artifact.dataAddressPush && op == spv::OpTypePointer && count == 4u && words[cursor + 2u] == spv::StorageClassPushConstant) pushTypes.insert(words[cursor + 3u]); if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId) { if (PipelineSpecialization::IsWorkgroupOffset(words[cursor + 3u])) { if (supplied.contains(words[cursor + 3u])) throw std::runtime_error("a workgroup offset is supplied at dispatch, not prepared"); @@ -833,7 +957,7 @@ std::vector specializeWords(const CompiledShaderArtifact& artifac } const auto found = supplied.find(words[cursor + 3u]); if (found == supplied.end()) throw std::runtime_error("missing prepared specialization value"); - if (found->first == PipelineSpecialization::MeshLayerBase || found->first == PipelineSpecialization::MeshLayerCount) { + if (found->first == PipelineSpecialization::MeshLayerBase || found->first == PipelineSpecialization::MeshLayerCount || found->first == PipelineSpecialization::HeapImageSlots || PipelineSpecialization::IsPushPlacement(found->first)) { retained.insert(words[cursor + 1u]); cursor += count; continue; @@ -920,7 +1044,7 @@ SpecializedSpirv buildSpecializedModule(const CompiledShaderArtifact& artifact, const auto op = static_cast(materialized[cursor] & 0xffffu); if (op == spv::OpEntryPoint && count >= 2u) compute = materialized[cursor + 1u] == spv::ExecutionModelGLCompute; if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationBinding) bindingNumbers.emplace(materialized[cursor + 1u], materialized[cursor + 3u]); - if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationDescriptorSet && (materialized[cursor + 3u] == PixelFunctionLdsDescriptorSet || materialized[cursor + 3u] == MeshWindowDescriptorSet || (compute && materialized[cursor + 3u] == WorkgroupMemoryDescriptorSet))) ldsBuffers.insert(materialized[cursor + 1u]); + if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationDescriptorSet && (materialized[cursor + 3u] == PixelFunctionLdsDescriptorSet || materialized[cursor + 3u] == MeshWindowDescriptorSet || (compute && !target.gpuResolvedImages && materialized[cursor + 3u] == WorkgroupMemoryDescriptorSet))) ldsBuffers.insert(materialized[cursor + 1u]); if (op == spv::OpDecorate && count == 4u && materialized[cursor + 2u] == spv::DecorationSpecId) { const auto id = materialized[cursor + 3u]; if (!PipelineSpecialization::IsWorkgroupOffset(id)) module->specialization.push_back({id, supplied.at(id)}); @@ -1015,33 +1139,42 @@ bool bufferSubstitutesEnabled() { return enabled; } -std::shared_ptr preparedBindingPlan(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, bool fragmentDescriptorSet, std::span substitutes, bool shaderImageMinLod) { - struct BindingPlanKeyStorage {}; - auto& key = HostThreadLocal, BindingPlanKeyStorage>(); - key.clear(); - key.push_back(variant.artifact.variantId); +template +void visitBindingPlanKey(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, std::span substitutes, TVisit&& visit) { + visit(variant.artifact.variantId); for (std::size_t index = 0; index < variant.info.info.buffers.size(); ++index) { const auto& descriptor = snapshot.buffers.at(index); - key.push_back(descriptor.dwordCount); + visit(descriptor.dwordCount); const auto fields = BufferSpecializationFields(descriptor, substitutes, index); - key.push_back(fields.word1); - key.push_back(fields.word3); - key.push_back(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); + visit(fields.word1); + visit(fields.word3); + visit(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); } for (std::size_t index = 0; index < variant.info.info.images.size(); ++index) { const auto& descriptor = snapshot.images.at(index); - key.push_back(descriptor.dwordCount); - key.push_back(descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffu) != 0u); - key.push_back(descriptor.dwords[1] & ~0xffu); - for (std::size_t word = 2; word < 8; ++word) key.push_back(descriptor.dwords[word]); + visit(descriptor.dwordCount); + visit(descriptor.dwords[0] != 0u || (descriptor.dwords[1] & 0xffu) != 0u); + visit(descriptor.dwords[1] & ~0xffu); + for (std::size_t word = 2; word < 8; ++word) visit(descriptor.dwords[word]); } for (std::size_t index = 0; index < variant.info.info.samplers.size(); ++index) { const auto& descriptor = snapshot.samplers.at(index); - key.push_back(descriptor.dwordCount); - for (std::size_t word = 0; word < 4; ++word) key.push_back(descriptor.dwords[word]); + visit(descriptor.dwordCount); + for (std::size_t word = 0; word < 4; ++word) visit(descriptor.dwords[word]); } - key.push_back(exports.size()); - for (const auto mapping : exports) key.push_back(mapping); + visit(exports.size()); + for (const auto mapping : exports) visit(mapping); +} + +void bindingPlanKey(std::vector& key, const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, std::span substitutes) { + key.clear(); + visitBindingPlanKey(variant, snapshot, exports, substitutes, [&](std::uint64_t word) { key.push_back(word); }); +} + +std::shared_ptr preparedBindingPlan(const CompiledVariant& variant, const ResourceSnapshot& snapshot, std::span exports, bool fragmentDescriptorSet, std::span substitutes, bool shaderImageMinLod) { + struct BindingPlanKeyStorage {}; + auto& key = HostThreadLocal, BindingPlanKeyStorage>(); + bindingPlanKey(key, variant, snapshot, exports, substitutes); static PrimedSharedMutex mutex; static std::map, std::shared_ptr> plans; std::shared_ptr plan; @@ -1136,13 +1269,14 @@ void recordMemoPatch(MemoPatch& patch, const DescriptorBindingPlan& plan, const } } patch.pushData = layout.UsesPushData(); + if (layout.UsesDataAddress()) return; if (patch.pushData || plan.storageData) { for (const auto reg : layout.userDataRegisters) patch.userWords.push_back(reg - variant.info.userDataBase); } patch.valid = layout.userDataRegisters.size() <= layout.DispatchThreadLimitDword() && (patch.userWords.empty() || patch.pushData || patch.shaderData != MemoPatch::NoBinding); } -RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false, MemoPatch* patch = nullptr, const std::vector* substituteSnapshot = nullptr) { +RecompileResult materializeResult(const CompiledVariant& variant, const RecompileRequest& request, const ResourceSnapshot& captured, bool registrationDefaults = false, UsedBindingPlan* usedPlan = nullptr, MemoPatch* patch = nullptr, const std::vector* substituteSnapshot = nullptr) { const auto localSubstitutes = substituteSnapshot == nullptr && bufferSubstitutesEnabled() && variant.substitutes != nullptr ? variant.substitutes->Slots() : std::vector{}; const auto substitutes = substituteSnapshot != nullptr ? std::span(*substituteSnapshot) : std::span(localSubstitutes); ResourceSnapshot merged; @@ -1156,6 +1290,15 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil .runtimeImageCount = artifact.runtimeImageCount, .runtimeImageResources = artifact.runtimeImageResources, .registrationModule = artifact.registrationModule, + .gpuResolvedImages = artifact.gpuResolvedImages, + .samplerSlotDword = artifact.samplerSlotDword, + .bufferAddressDword = artifact.bufferAddressDword, + .bufferAddressCount = artifact.bufferAddressCount, + .pushedData = artifact.pushedData, + .pushItems = artifact.pushItems, + .dataAddressPush = artifact.dataAddressPush, + .bdaStores = artifact.bdaStores, + .deviceImageTables = artifact.deviceImageTables, .bdaAbiVersion = artifact.bdaAbiVersion, .windowLaneGuards = artifact.windowLaneGuards, .descriptorFaults = artifact.descriptorFaults, @@ -1194,7 +1337,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil auto& moduleKey = HostThreadLocal, LocalModuleKeyStorage>(); moduleKey.clear(); if (variant.bindings.layout.UsesPushData()) moduleKey.push_back(request.layout.pushConstantOffsetBytes / 4u); - if (!artifact.stencilReferences.empty()) moduleKey.push_back(request.layout.pushConstantOffsetBytes / 4u + artifact.stencilReferenceDword); + if (StencilInPush(artifact, variant.bindings.layout)) moduleKey.push_back(request.layout.pushConstantOffsetBytes / 4u + artifact.stencilReferenceDword); if (request.context.pixel) { for (const auto packing : request.context.pixel->targetExportPacking) moduleKey.push_back(static_cast(packing)); moduleKey.push_back(request.context.pixel->dualSourceBlend ? 1u : 0u); @@ -1243,7 +1386,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil if (artifact.registrationModule) runtimeSelectors.at(input.location / RuntimeAbi::VertexSelectorsPerDword) |= (selectors & 0xfffu) << (input.location % RuntimeAbi::VertexSelectorsPerDword * 16u); } } - if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty() || request.context.pixel || depthRemap || request.shader.stage == ShaderStage::Mesh) { + if (!plan->bindings.specialization.empty() || UsesRootOrPush(variant.bindings.layout) || !artifact.vertexInputPatches.empty() || request.context.pixel || depthRemap || request.shader.stage == ShaderStage::Mesh) { { std::shared_lock lock(plan->mutex); if (const auto found = plan->modules.find(moduleKey); found != plan->modules.end()) entry = found->second; @@ -1257,9 +1400,11 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil auto constants = plan->bindings.specialization; constants.push_back({PipelineSpecialization::MeshLayerBase, 0u}); constants.push_back({PipelineSpecialization::MeshLayerCount, 0xffffffffu}); + constants.push_back({PipelineSpecialization::HeapImageSlots, RuntimeAbi::GpuImageHeapSlots}); + if (artifact.dataAddressPush) AppendPushSpecialization(constants, StandalonePushPlacement(request.shader.stage, artifact.pushItems, artifact.pushedData.size())); std::size_t index = 0; if (variant.bindings.layout.UsesPushData()) constants.push_back({PipelineSpecialization::PushDataOffset, moduleKey[index++]}); - if (!artifact.stencilReferences.empty()) constants.push_back({PipelineSpecialization::StencilReferenceOffset, moduleKey[index++]}); + if (StencilInPush(artifact, variant.bindings.layout)) constants.push_back({PipelineSpecialization::StencilReferenceOffset, moduleKey[index++]}); if (request.context.pixel) { for (std::uint32_t target = 0; target < request.context.pixel->targetExportPacking.size(); ++target) constants.push_back({PipelineSpecialization::ExportPackingBase + target, moduleKey[index++]}); constants.push_back({PipelineSpecialization::DualSourceBlend, moduleKey[index++]}); @@ -1286,7 +1431,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil const auto prepare = [&] { auto specialized = specialize(); const auto& module = specialized.module; - if (variant.bindings.layout.UsesPushData() && !module->pushData) throw std::runtime_error("specialization removed the prepared push constant interface"); + if (UsesRootOrPush(variant.bindings.layout) && !module->pushData) throw std::runtime_error("specialization removed the prepared push constant interface"); auto selected = DescriptorBindingBuilder{}.Select(plan->bindings, module->bindings); entry->bindings = std::move(selected); entry->module = module; @@ -1312,6 +1457,20 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil } const auto resolved = ResourceMaterializer::ResolveImageTablesShared(variant.info.info, snapshot, artifact.variantId); const auto& tables = *resolved; + if (artifact.deviceImageTables) { + const auto& info = variant.info.info; + ResourceMaterializer::RegisterDeviceImageInterfaces(info); + result.deviceImageInterfaces.assign(info.images.size(), 0u); + for (std::size_t image = 0; image < info.images.size(); ++image) { + if (info.images[image].deviceTable) result.deviceImageInterfaces[image] = ResourceMaterializer::DeviceImageInterfaceId(info.images[image], info.runtimeImageModes.at(image)); + } + result.deviceSamplerClasses.assign(info.samplers.size(), 0u); + for (const auto& constant : plan->bindings.specialization) { + const auto sampler = constant.id - PipelineSpecialization::DeviceSamplerClassBase; + if (constant.id >= PipelineSpecialization::DeviceSamplerClassBase && sampler < info.samplers.size()) result.deviceSamplerClasses[sampler] = constant.value; + } + } + if (usedPlan != nullptr) *usedPlan = {entry != nullptr ? std::shared_ptr(entry) : std::shared_ptr(plan), bindingPlan}; BindingAllocationResult bindings; DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request), &tables); if (patch != nullptr) recordMemoPatch(*patch, *bindingPlan, variant); @@ -1332,10 +1491,12 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil if (std::ranges::none_of(result.bindings, [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::BdaPagetable || binding.role == DescriptorRole::FaultBuffer; })) result.bdaAbiVersion = 0u; result.pushConstants = std::move(bindings.pushConstants); result.readsDepthRemapPush = depthRemap; + if ((result.pushItems & RuntimeAbi::DepthRemapPushBit) != 0u && !depthRemap) throw std::runtime_error("a stage reads the depth remap push words its request does not remap"); if (request.shader.stage == ShaderStage::Compute) { for (std::uint32_t component = 0; component < PipelineSpecialization::WorkgroupOffsetWords; ++component) result.specialization.push_back({PipelineSpecialization::WorkgroupOffsetBase + component, 0u}); } - if (!artifact.stencilReferences.empty()) { + result.heapSlotWords = bindingPlan->heapSlotWords; + if (StencilInPush(artifact, variant.bindings.layout)) { if (artifact.stencilReferenceDword * sizeof(std::uint32_t) != result.pushConstants.size() || result.pushConstants.size() + sizeof(std::uint32_t) > request.layout.pushConstantSizeBytes) throw std::runtime_error("stencil replay push constant lies outside the stage range"); result.pushConstants.resize(result.pushConstants.size() + sizeof(std::uint32_t)); } @@ -1599,6 +1760,8 @@ ResultMemoEntry* memoSet(const std::unique_ptr& table, std::uin return table->data() + static_cast((index ^ (index >> 32u)) % ResultMemoSets) * ResultMemoWays; } + + struct ResultMemoCounters { std::atomic hits{0}, patched{0}, misses{0}, evictions{0}, trimmed{0}, populateNanoseconds{0}, patchNanoseconds{0}, mismatches{0}; std::atomic lastReport{0}; @@ -1800,7 +1963,13 @@ void reportResultMemo() { // Everything materializeResult reads besides the variant: the snapshot (the descriptor words, the // flattened SRT, the user data, the uniform fill) and, for the vertex family, the V# table the // attributes are resolved from. -std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot, bool masked, std::span substitutes = {}) { +struct SnapshotHashScope { + bool masked = false; + bool perDrawData = true; + bool descriptors = true; +}; + +std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapshot& snapshot, SnapshotHashScope scope = {}, std::span substitutes = {}) { std::uint64_t hash = 0xcbf29ce484222325ull; const auto mix = [&](std::uint64_t value) { hash ^= value; @@ -1818,29 +1987,31 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh for (std::uint32_t i = 0; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); } }; - if (masked) { - mix(snapshot.buffers.size()); - for (std::size_t index = 0; index < snapshot.buffers.size(); ++index) { - const auto& value = snapshot.buffers[index]; - mix(value.dwordCount); - mix(value.availableDwordMask); - const auto fields = BufferSpecializationFields(value, substitutes, index); - mix(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); - mix(fields.word1); - if (value.dwordCount > 3u) mix(fields.word3); - for (std::uint32_t i = 4; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); - } - } else { - mixDescriptors(snapshot.buffers); - for (std::size_t index = 0; index < substitutes.size() && index < snapshot.buffers.size(); ++index) { - const auto fields = BufferSpecializationFields(snapshot.buffers[index], substitutes, index); - mix(fields.word1); - mix(fields.word3); - mix(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); + if (scope.descriptors) { + if (scope.masked) { + mix(snapshot.buffers.size()); + for (std::size_t index = 0; index < snapshot.buffers.size(); ++index) { + const auto& value = snapshot.buffers[index]; + mix(value.dwordCount); + mix(value.availableDwordMask); + const auto fields = BufferSpecializationFields(value, substitutes, index); + mix(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); + mix(fields.word1); + if (value.dwordCount > 3u) mix(fields.word3); + for (std::uint32_t i = 4; i < value.dwordCount && i < value.dwords.size(); ++i) mix(value.dwords[i]); + } + } else { + mixDescriptors(snapshot.buffers); + for (std::size_t index = 0; index < substitutes.size() && index < snapshot.buffers.size(); ++index) { + const auto fields = BufferSpecializationFields(snapshot.buffers[index], substitutes, index); + mix(fields.word1); + mix(fields.word3); + mix(fields.known ? 4u | fields.misalignment | fields.tail << 3u : 0u); + } } + mixDescriptors(snapshot.images); + mixDescriptors(snapshot.samplers); } - mixDescriptors(snapshot.images); - mixDescriptors(snapshot.samplers); mix(snapshot.registeredImages.size()); for (const auto& view : snapshot.registeredImages) { mix(view.resource); @@ -1858,10 +2029,10 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh for (const auto word : view.provenance.dwords) mix(word); for (const auto word : view.effective.dwords) mix(word); } - if (masked) { + if (scope.masked) { mix(snapshot.flattenedSrt.size()); mix(snapshot.userData.size()); - } else { + } else if (scope.perDrawData) { mixWords(snapshot.flattenedSrt); mixWords(snapshot.userData); } @@ -1926,7 +2097,11 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh const auto count = std::min(vertex.resourcesNum, ShaderVertexStageInfo::MaxResources); mix(count); for (std::uint32_t i = 0; i < count; ++i) { - for (const auto field : vertex.resources[i].fields) mix(field); + if (scope.perDrawData) { + for (const auto field : vertex.resources[i].fields) mix(field); + } else { + mix(vertex.resources[i].fields[3]); + } mix(vertex.resourcesDst[i].fetchIndex); } } else { @@ -1942,7 +2117,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; const auto variantId = variant->artifact.variantId; const auto substitutes = bufferSubstitutesEnabled() && variant->substitutes != nullptr ? variant->substitutes->Slots() : std::vector{}; - const auto hash = snapshotHash(request, snapshot, false, substitutes); + const auto hash = snapshotHash(request, snapshot, {}, substitutes); const auto index = (variantId * 0x9e3779b97f4a7c15ull) ^ hash; auto& counters = resultMemoCounters(); { @@ -1960,7 +2135,7 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& } } const bool masking = MaskedResultMemo(); - const auto maskedHash = masking ? snapshotHash(request, snapshot, true, substitutes) : 0u; + const auto maskedHash = masking ? snapshotHash(request, snapshot, {.masked = true}, substitutes) : 0u; const auto maskedIndex = (variantId * 0x9e3779b97f4a7c15ull) ^ maskedHash; std::shared_ptr donor; std::shared_ptr donorPatch; @@ -1983,16 +2158,17 @@ std::shared_ptr materializePreparedMemoized(SourceEntry& if (donor != nullptr) { result = std::make_shared(*donor); applyMemoPatch(*result, *donorPatch, snapshot); - if (VerifyMaskedResultMemo()) verifyPatchedResult(*result, materializeResult(*variant, request, snapshot, false, nullptr, &substitutes), variantId); + if (VerifyMaskedResultMemo()) verifyPatchedResult(*result, materializeResult(*variant, request, snapshot, false, nullptr, nullptr, &substitutes), variantId); if (profile) counters.patchNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); counters.patched.fetch_add(1, std::memory_order_relaxed); } else { if (masking) patch = std::make_shared(); - result = std::make_shared(materializeResult(*variant, request, snapshot, false, patch.get(), &substitutes)); + result = std::make_shared(materializeResult(*variant, request, snapshot, false, nullptr, patch.get(), &substitutes)); if (profile) counters.populateNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); counters.misses.fetch_add(1, std::memory_order_relaxed); } result->cacheHit = cacheHit; + if (result->deviceImageTables) result->deviceImagePlan = source.plan; std::shared_ptr shared = std::move(result); const auto bytes = resultBytes(*shared); const auto insert = [&](std::unique_ptr& table, std::uint64_t at, std::uint64_t key, std::shared_ptr recipe, bool count) { @@ -2051,9 +2227,15 @@ std::shared_ptr RecompileImpl(const RecompileRequest& req if (!request.useCache || capture.source == nullptr) { auto program = PrepareResourceProgram(request); const auto variant = CountedCompile([&] { return compileVariant(request, std::move(program)); }); - return std::make_shared(materializeResult(variant, request, capture.snapshot)); + auto built = materializeResult(variant, request, capture.snapshot); + if (built.deviceImageTables) built.deviceImagePlan = capture.plan; + return std::make_shared(std::move(built)); + } + if (!ResultMemo()) { + auto built = materializeVariant(*capture.source, request, capture.snapshot); + if (built.deviceImageTables) built.deviceImagePlan = capture.plan; + return std::make_shared(std::move(built)); } - if (!ResultMemo()) return std::make_shared(materializeVariant(*capture.source, request, capture.snapshot)); return materializeMemoized(*capture.source, request, capture.snapshot, memoHit); } @@ -2171,9 +2353,12 @@ bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& if (layout.descriptorSet != handle.artifact->layout.descriptorSet || layout.firstBinding != handle.artifact->layout.firstBinding) return false; if (layout.pushConstantOffsetBytes % 4u != 0u || layout.pushConstantSizeBytes % 4u != 0u || layout.pushConstantOffsetBytes > NativePushConstantSize || layout.pushConstantSizeBytes > NativePushConstantSize - layout.pushConstantOffsetBytes) return false; const auto words = prepared.ShaderDataDwords(); - const bool usesPush = prepared.runtimeImageCount == 0u && words != 0u && words <= layout.pushConstantSizeBytes / 4u; + const bool usesPush = !prepared.gpuResolvedImages && prepared.runtimeImageCount == 0u && words != 0u && words <= layout.pushConstantSizeBytes / 4u; if (prepared.UsesPushData() != usesPush) return false; - if (!handle.artifact->artifact.stencilReferences.empty() && (prepared.stencilReferenceDword >= layout.pushConstantSizeBytes / 4u || prepared.stencilReferenceDword != (usesPush ? words : 0u))) return false; + if (!handle.artifact->artifact.stencilReferences.empty()) { + const bool misplaced = prepared.UsesDataAddress() ? prepared.stencilReferenceDword != PushData::NoStart : prepared.stencilReferenceDword >= layout.pushConstantSizeBytes / 4u || prepared.stencilReferenceDword != (usesPush ? words : 0u); + if (misplaced) return false; + } if (!std::ranges::equal(key, handle.staticKey) || handle.source->code.size() != request.shader.code.size()) return false; return handle.source->code.data() == request.shader.code.data() || std::ranges::equal(handle.source->code, request.shader.code); } @@ -2183,7 +2368,7 @@ std::span GetPreparedCode(const SourceHandle& handle) { return handle.source->code; } -PreparedShaderInvocation::PreparedShaderInvocation(const RecompileRequest& request, const std::shared_ptr& handle) : request(request), handle(handle) { +PreparedShaderInvocation::PreparedShaderInvocation(RecompileRequest request, const std::shared_ptr& handle) : request(std::move(request)), handle(handle) { this->request.context.linkedCallFallbacks = handle->source->linkedCallFallbacks; } @@ -2200,6 +2385,12 @@ std::optional PreparedShaderInvocation::TryCreate(cons return PreparedShaderInvocation(request, handle); } +std::optional PreparedShaderInvocation::TryCreate(RecompileRequest&& request, const std::shared_ptr& handle, std::span key) { + if (handle == nullptr || !MatchesPreparedShader(request, *handle, key)) return std::nullopt; + if (request.shader.stage != ShaderStage::Compute && request.shader.stage != ShaderStage::Fragment) static_cast(RequestInputInfo(request)); + return PreparedShaderInvocation(std::move(request), handle); +} + std::shared_ptr PreparedShaderInvocation::Capture(const SrtRuntime& runtime) const { auto capture = std::make_shared(); capture->source = handle->source; @@ -2208,14 +2399,190 @@ std::shared_ptr PreparedShaderInvocation::Capture(const S return capture; } +namespace { + +bool AllEmpty(const std::vector>& words) { + return std::all_of(words.begin(), words.end(), [](const std::vector& descriptor) { return descriptor.empty(); }); +} + +std::uint64_t HeapTemplateKey(const RecompileRequest& request, const CompiledVariant& variant, const ResourceSnapshot& snapshot, bool perDrawBindings, std::span substitutes) { + std::uint64_t key = snapshotHash(request, snapshot, {.perDrawData = false, .descriptors = !perDrawBindings}, substitutes) ^ variant.artifact.variantId * 0x9e3779b97f4a7c15ull; + if (perDrawBindings) { + visitBindingPlanKey(variant, snapshot, request.context.pixel ? std::span(request.context.pixel->targetExportMapping) : std::span{}, substitutes, [&](std::uint64_t word) { key = (key ^ word) * 0x100000001b3ull; }); + } + return key; +} + +void HeapVertexAttributes(const RecompileRequest& request, const RecompileResult& result, HeapStageWords& words) { + words.vertexAttributes.clear(); + if (!request.context.vertex) return; + const auto& vertex = *request.context.vertex; + words.vertexAttributes = result.vertexAttributes; + for (auto& attribute : words.vertexAttributes) { + if (attribute.location >= vertex.resourcesNum || attribute.location >= vertex.resources.size()) throw std::runtime_error("ShaderRecompiler: vertex attribute outside the draw's vertex resources"); + attribute.resource = vertex.resources[attribute.location]; + if (vertex.fetchEmbedded) attribute.fetchIndex = vertex.resourcesDst[attribute.location].fetchIndex; + } +} + +void HeapBindingWords(const RecompileRequest& request, const CompiledVariant& variant, const ResourceSnapshot& snapshot, const HeapTemplate& shared, HeapStageWords& words) { + const auto& layout = variant.bindings.layout; + if (shared.plan.plan == nullptr) throw std::runtime_error("ShaderRecompiler: a heap template has no binding plan"); + const bool sameBuffers = snapshot.buffers == shared.buffers; + const bool sameImages = snapshot.images == shared.images && snapshot.samplers == shared.samplers; + if (sameBuffers && sameImages) return; + const auto count = shared.result->bindings.size(); + if (shared.plan.plan->bindings.size() != count) throw std::runtime_error("ShaderRecompiler: a heap template's binding plan differs from its bindings"); + if (sameImages) { + words.bindings.assign(count, {}); + for (std::size_t index = 0; index < count; ++index) { + const auto& entry = shared.plan.plan->bindings[index]; + if (entry.descriptor.role != DescriptorRole::GuestBuffers) continue; + auto& descriptor = words.bindings[index]; + descriptor.reserve(entry.resources.size() * 4u); + for (const auto resource : entry.resources) { + if (resource >= snapshot.buffers.size() || snapshot.buffers[resource].dwordCount != 4u) throw std::runtime_error("ShaderRecompiler: a heap stage's guest buffer descriptor is out of range"); + descriptor.insert(descriptor.end(), snapshot.buffers[resource].dwords.begin(), snapshot.buffers[resource].dwords.begin() + 4); + } + if (descriptor == shared.result->bindings[index].guestDescriptor) descriptor.clear(); + } + if (AllEmpty(words.bindings)) words.bindings.clear(); + return; + } + const auto tables = ResourceMaterializer::ResolveImageTables(variant.info.info, snapshot, variant.artifact.variantId); + BindingAllocationResult bindings; + DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *shared.plan.plan, variant.info.userDataBase, snapshot, partialThreads(request), &tables); + if (bindings.bindings.size() != count) throw std::runtime_error("ShaderRecompiler: a heap stage's per-draw bindings differ from its template"); + words.bindings.reserve(count); + for (auto& binding : bindings.bindings) words.bindings.push_back(std::move(binding.guestDescriptor)); + if (variant.artifact.registrationModule && request.shader.stage != ShaderStage::Fragment && layout.runtimeStateDwords != 0u) { + const auto first = layout.RuntimeStateDword() + RuntimeAbi::StateVertexSelectors; + for (std::size_t index = 0; index < words.bindings.size(); ++index) { + if (shared.result->bindings[index].role != DescriptorRole::ShaderData) continue; + const auto& kept = shared.result->bindings[index].guestDescriptor; + if (first + layout.runtimeStateDwords > kept.size() || first + layout.runtimeStateDwords > words.bindings[index].size()) throw std::runtime_error("ShaderRecompiler: vertex selector state lies outside a heap stage's shader data"); + std::copy_n(kept.begin() + first, layout.runtimeStateDwords, words.bindings[index].begin() + first); + } + } + for (std::size_t index = 0; index < count; ++index) { + if (words.bindings[index] == shared.result->bindings[index].guestDescriptor) words.bindings[index].clear(); + } + if (AllEmpty(words.bindings)) words.bindings.clear(); +} + +} + +std::shared_ptr PreparedShaderInvocation::MaterializeHeap(const ResourceCapture& capture, HeapStageWords& words) const { + if (capture.source != handle->source || capture.plan != handle->source->plan) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); + const auto& variant = handle->artifact; + if (!variant->artifact.gpuResolvedImages || !variant->artifact.registrationModule || variant->bindings.layout.UsesPushData()) return nullptr; + const auto& snapshot = capture.snapshot; + const auto& layout = variant->bindings.layout; + const auto userDataBase = request.context.userDataBaseRegister; + words.userData.resize(layout.userDataRegisters.size()); + for (std::size_t i = 0; i < layout.userDataRegisters.size(); ++i) { + const auto reg = layout.userDataRegisters[i]; + if (reg < userDataBase || reg - userDataBase >= snapshot.userData.size()) throw std::runtime_error("ShaderRecompiler: shader user-data register is out of range"); + words.userData[i] = snapshot.userData[reg - userDataBase]; + } + words.flattenedSrt.assign(snapshot.flattenedSrt.begin(), snapshot.flattenedSrt.end()); + words.bindings.clear(); + const bool tables = std::ranges::any_of(layout.descriptors, [](const IrDescriptorBinding& descriptor) { return descriptor.kind == DescriptorBindingKind::ImageTable || descriptor.kind == DescriptorBindingKind::SamplerTable || descriptor.kind == DescriptorBindingKind::ImageTableMap; }); + const bool perDrawBindings = (request.shader.stage == ShaderStage::Fragment || request.shader.stage == ShaderStage::Vertex) && !tables; + auto& source = *handle->source; + const auto substitutes = bufferSubstitutesEnabled() && variant->substitutes != nullptr ? variant->substitutes->Slots() : std::vector{}; + const std::uint64_t key = HeapTemplateKey(request, *variant, snapshot, perDrawBindings, substitutes); + const auto complete = [&](const std::shared_ptr& shared) { + HeapVertexAttributes(request, *shared->result, words); + if (perDrawBindings) HeapBindingWords(request, *variant, snapshot, *shared, words); + CompleteHeapDataWords(*shared->result, words); + }; + std::shared_ptr hit; + { + std::lock_guard lock(source.mutex); + const auto found = source.heapTemplates.find(key); + if (found != source.heapTemplates.end()) hit = found->second; + } + if (hit != nullptr) { + complete(hit); + TouchHeapTemplate({source.id, key, hit->result->heapTemplateId}); + return hit->result; + } + UsedBindingPlan usedPlan; + auto built = std::make_shared(materializeResult(*variant, request, snapshot, false, &usedPlan, nullptr, &substitutes)); + built->cacheHit = true; + if (built->deviceImageTables) built->deviceImagePlan = capture.plan; + for (const auto reg : layout.userDataRegisters) built->heapUserWords.push_back(reg - userDataBase); + built->heapTemplateId = NextTemplateId(); + auto created = std::make_shared(HeapTemplate{std::move(built), std::move(usedPlan), perDrawBindings ? snapshot.buffers : std::vector{}, perDrawBindings ? snapshot.images : std::vector{}, perDrawBindings ? snapshot.samplers : std::vector{}}); + created->bytes = HeapTemplateBytes(*created); + std::shared_ptr shared = std::move(created); + bool inserted = false; + { + std::lock_guard lock(source.mutex); + auto [at, added] = source.heapTemplates.emplace(key, shared); + inserted = added; + shared = at->second; + } + if (inserted) KeepHeapTemplate(source.shared_from_this(), key, *shared); + else TouchHeapTemplate(HeapTemplateName(source, key, *shared)); + complete(shared); + return shared->result; +} + std::shared_ptr PreparedShaderInvocation::Materialize(const ResourceCapture& capture) const { if (capture.source != handle->source) throw std::runtime_error("ShaderRecompiler: resource capture belongs to another prepared shader"); if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle->source, handle->artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle->artifact, request, capture.snapshot)); result->cacheHit = true; + if (result->deviceImageTables) result->deviceImagePlan = capture.plan; return result; } +const std::shared_ptr& PreparedShaderInvocation::Plan() const { + static const std::shared_ptr none; + return handle != nullptr && handle->source != nullptr ? handle->source->plan : none; +} + +std::shared_ptr PreparedShaderInvocation::LoadPlan() const { + if (handle == nullptr || handle->source == nullptr) return nullptr; + return acquirePlan(request, handle->source, false, handle.get()); +} + +std::uint64_t PreparedShaderInvocation::VariantId() const { + return handle != nullptr && handle->artifact != nullptr ? handle->artifact->artifact.variantId : 0u; +} + +void CompleteHeapDataWords(const RecompileResult& shared, HeapStageWords& words) { + const auto count = shared.bindings.size(); + if (!words.bindings.empty() && words.bindings.size() != count) throw std::runtime_error("ShaderRecompiler: a heap stage's per-draw bindings differ in number from its template"); + for (std::size_t index = 0; index < count; ++index) { + const auto& binding = shared.bindings[index]; + if (binding.role != DescriptorRole::ShaderData && binding.role != DescriptorRole::FlattenedSrt) continue; + if (!words.bindings.empty() && !words.bindings[index].empty()) continue; + std::vector descriptor; + if (binding.role == DescriptorRole::FlattenedSrt) { + if (words.flattenedSrt.empty() || words.flattenedSrt == binding.guestDescriptor) continue; + descriptor = words.flattenedSrt; + } else { + const auto patched = std::min(words.userData.size(), binding.guestDescriptor.size()); + if (std::equal(words.userData.begin(), words.userData.begin() + static_cast(patched), binding.guestDescriptor.begin())) continue; + descriptor = binding.guestDescriptor; + std::copy_n(words.userData.begin(), patched, descriptor.begin()); + } + if (words.bindings.empty()) words.bindings.assign(count, {}); + words.bindings[index] = std::move(descriptor); + } +} + +bool HeapFragmentEligible(const IrResourcePlan& plan) { + if (plan.stage == IrShaderStage::Unknown || plan.stage == IrShaderStage::Fetch) return false; + for (const auto& image : plan.info.images) { + if (image.source >= plan.descriptorSources.size()) return false; + } + return true; +} + const CompiledShaderArtifact& GetPreparedArtifact(const SourceHandle& handle) { if (handle.artifact == nullptr) throw std::runtime_error("ShaderRecompiler: prepared artifact is missing"); return handle.artifact->artifact; @@ -2241,6 +2608,7 @@ std::shared_ptr MaterializeShader(const RecompileRequest& if (request.useCache && ResultMemo()) return materializePreparedMemoized(*handle.source, handle.artifact, request, capture.snapshot, true, nullptr); auto result = std::make_shared(materializeResult(*handle.artifact, request, capture.snapshot)); result->cacheHit = true; + if (result->deviceImageTables) result->deviceImagePlan = capture.plan; return result; } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index aca3fdd71b7d..94779932cfe6 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -21,6 +21,8 @@ namespace ShaderRecompiler { +struct IrResourcePlan; + enum class ShaderStage { Compute, Vertex, @@ -357,11 +359,16 @@ struct SpirvTarget { bool largePixelFunctionLdsBuffer = false; std::uint32_t pixelFunctionStorageDwords = 4096u; bool deviceScopeLdsMemoryBarriers = false; + bool gpuResolvedImages = false; + bool descriptorMemory = false; + std::uint32_t heapDataUniformDwords = 0; + bool heapBufferDescriptors = false; bool ldsLockFences = false; bool fragmentBarycentricBuiltins = false; bool meshWindowBuffer = false; bool hostInterpolationFirst = false; bool shaderImageMinLod = false; + bool deviceImageTables = false; }; struct BindingLayout { @@ -438,7 +445,10 @@ inline constexpr std::uint32_t TessellationCaptureArgumentAddressDword = 2; inline constexpr std::uint32_t MeshIndexBufferUserWord = 4; inline constexpr std::uint32_t WorkgroupMemoryDescriptorSet = 1; inline constexpr std::uint32_t ScratchPoolDescriptorSet = 2; - +inline constexpr std::uint32_t GpuWorkgroupMemoryDescriptorSet = 3; +inline constexpr std::uint32_t GpuScratchPoolDescriptorSet = 5; +inline constexpr std::uint32_t GpuScratchPoolAddressPushByte = 136; +inline constexpr std::uint32_t GpuScratchPoolDwordsPushByte = 144; struct GraphicsDrawParameters { std::uint64_t indexAddress; std::uint32_t indexCount; @@ -481,6 +491,22 @@ struct GraphicsCompileContext { std::optional meshBarycentrics; }; +struct VertexAttribute; + +struct HeapTemplateCacheCounts { + std::uint64_t entries = 0; + std::uint64_t bytes = 0; + std::uint64_t evicted = 0; +}; +HeapTemplateCacheCounts HeapTemplateCounts(); + +struct HeapStageWords { + std::vector userData; + std::vector flattenedSrt; + std::vector vertexAttributes; + std::vector> bindings; +}; + struct RecompileRequest { ShaderBinary shader; GuestContext context; @@ -755,6 +781,15 @@ struct CompiledShaderArtifact { std::uint32_t runtimeImageCount = 0; std::vector runtimeImageResources; bool registrationModule = false; + bool gpuResolvedImages = false; + std::uint32_t samplerSlotDword = 0; + std::uint32_t bufferAddressDword = 0; + std::uint32_t bufferAddressCount = 0; + std::vector pushedData; + std::uint32_t pushItems = 0; + bool dataAddressPush = false; + bool bdaStores = false; + bool deviceImageTables = false; std::uint32_t bdaAbiVersion = 0; std::uint32_t windowLaneGuards = 0; std::uint32_t descriptorFaults = 0; @@ -785,6 +820,7 @@ struct CompiledShaderArtifact { struct ShaderInvocation { std::vector specialization; + std::vector heapSlotWords; std::uint64_t specializationId = 0; std::vector bindings; std::vector pushConstants; @@ -797,6 +833,8 @@ struct ShaderInvocation { std::vector>> imageTableReadRanges; std::uint64_t imageTableShader = 0; std::uint32_t imageTableFaults = 0; + std::vector deviceImageInterfaces; + std::vector deviceSamplerClasses; }; // What a Vulkan pipeline built from a result depends on besides the fixed-function state and the @@ -824,10 +862,13 @@ struct ResultKeyMemo { }; struct RecompileResult : CompiledShaderArtifact, ShaderInvocation { + std::uint64_t heapTemplateId = 0; + std::vector heapUserWords; ResultKeyMemo drawKeyMemo; bool cacheHit = false; std::uint32_t workgroupMemoryDwords = 0; std::vector shaderDataUserWords; + std::shared_ptr deviceImagePlan; [[nodiscard]] std::uint64_t PipelineVariantId() const { return specializationId != 0 ? specializationId : variantId; } [[nodiscard]] PipelineIdentity Identity() const { const auto hash = spirv.ContentHash(); diff --git a/core/shader/recompiler/RuntimeAbi.hpp b/core/shader/recompiler/RuntimeAbi.hpp index 000fb41204af..5912f183af30 100644 --- a/core/shader/recompiler/RuntimeAbi.hpp +++ b/core/shader/recompiler/RuntimeAbi.hpp @@ -9,7 +9,7 @@ namespace ShaderRecompiler::RuntimeAbi { -inline constexpr std::uint32_t Version = 14u; +inline constexpr std::uint32_t Version = 15u; inline constexpr std::uint32_t DescriptorSet = 0u; inline constexpr std::uint32_t FragmentDescriptorSet = 1u; inline constexpr std::uint32_t StageCount = 4u; @@ -18,6 +18,37 @@ inline constexpr std::uint32_t FirstImageBinding = 1u; inline constexpr std::uint32_t FirstComparisonImageBinding = 22u; inline constexpr std::uint32_t FirstStorageImageBinding = 29u; inline constexpr std::uint32_t ImageBindingCount = 56u; +inline constexpr bool GpuHeapStorageKind(std::uint32_t kind) { return kind >= FirstStorageImageBinding && kind < FirstImageBinding + ImageBindingCount; } +inline constexpr std::uint32_t StorageKindStride = 7u; +inline constexpr std::uint32_t FirstUintStorageImageBinding = FirstStorageImageBinding + StorageKindStride; +inline constexpr std::uint32_t FirstAtomicImageBinding = FirstStorageImageBinding + 2u * StorageKindStride; +inline constexpr std::uint32_t FirstAtomic64ImageBinding = FirstStorageImageBinding + 3u * StorageKindStride; +inline constexpr std::uint32_t SamplerSlotWords = 2u; +inline constexpr std::uint32_t BufferAddressWords = 4u; +inline constexpr bool AtomicImageKind(std::uint32_t kind) { return kind >= FirstAtomicImageBinding && kind < FirstAtomic64ImageBinding; } +inline constexpr bool Atomic64ImageKind(std::uint32_t kind) { return kind >= FirstAtomic64ImageBinding && kind < FirstImageBinding + ImageBindingCount; } +inline constexpr bool UintStorageImageKind(std::uint32_t kind) { return kind >= FirstUintStorageImageBinding && kind < FirstAtomicImageBinding; } + +enum class HeapPart : std::uint32_t { Sampled, Storage, Sampler, Atomic, Atomic64, UintStorage }; + +inline constexpr HeapPart StorageHeapPart(std::uint32_t kind) { + if (AtomicImageKind(kind)) return HeapPart::Atomic; + if (Atomic64ImageKind(kind)) return HeapPart::Atomic64; + if (UintStorageImageKind(kind)) return HeapPart::UintStorage; + return HeapPart::Storage; +} + +inline constexpr std::uint32_t HeapPartFirstBinding(HeapPart part) { + switch (part) { + case HeapPart::Sampled: return FirstImageBinding; + case HeapPart::Storage: return FirstStorageImageBinding; + case HeapPart::Atomic: return FirstAtomicImageBinding; + case HeapPart::Atomic64: return FirstAtomic64ImageBinding; + case HeapPart::UintStorage: return FirstUintStorageImageBinding; + case HeapPart::Sampler: break; + } + return 0u; +} enum class Binding : std::uint32_t { Buffers = 0u, @@ -36,6 +67,13 @@ enum class Binding : std::uint32_t { enum class Stage : std::uint32_t { Main, Fragment, TessellationControl, TessellationEvaluation }; +inline constexpr Stage StageGroup(bool fragment, bool tessellationControl, bool tessellationEvaluation) { + if (fragment) return Stage::Fragment; + if (tessellationControl) return Stage::TessellationControl; + if (tessellationEvaluation) return Stage::TessellationEvaluation; + return Stage::Main; +} + inline constexpr std::uint32_t UserDataCapacity = 128u; inline constexpr std::uint32_t BufferCapacity = 128u; inline constexpr std::uint32_t ImageCapacity = 256u; @@ -44,6 +82,35 @@ inline constexpr std::uint32_t BindlessTableSlots = 16u; inline constexpr std::uint32_t StorageMipSlots = 16u; inline constexpr std::uint32_t StorageHeapCapacity = 64u; inline constexpr std::uint32_t SamplerHeapCapacity = 64u; +inline constexpr std::uint32_t GpuImageHeapSlots = static_cast(Binding::Samplers) * 4096u; +inline constexpr std::uint32_t GpuSamplerHeapSlots = 2048u; +inline constexpr std::uint32_t GpuDataSet = 2u; +inline constexpr std::uint32_t GpuGdsSet = 4u; +inline constexpr std::uint32_t GdsBytes = 0x10000u; +inline constexpr std::uint32_t PushDataDwords = 64u; +inline constexpr std::uint32_t PushedDataWantedLimit = 48u; +enum class RootItem : std::uint32_t { PagetableAddress, FaultAddress, GdsAddress, PagetableDwords, StencilReference, ShaderData, TableMap, SrtAddress, FlattenedSrt, SrtDwords, Count }; +inline constexpr std::uint32_t RootItemCount = static_cast(RootItem::Count); +inline constexpr std::uint32_t RootItemBit(RootItem item) { return 1u << static_cast(item); } +inline constexpr bool RootAddressItem(RootItem item) { + return item == RootItem::PagetableAddress || item == RootItem::FaultAddress || item == RootItem::GdsAddress || item == RootItem::ShaderData || item == RootItem::TableMap || + item == RootItem::SrtAddress; +} +inline constexpr std::uint32_t DepthRemapPushBit = 1u << 16u; +inline constexpr std::uint32_t NoPushPosition = 0xffffffffu; +struct PushedPlacement { + std::uint32_t base = 0; + std::uint32_t count = 0; + + bool operator==(const PushedPlacement&) const = default; +}; +struct HeapSlotWord { + std::uint32_t dword = 0; + std::uint32_t kind = 0; + std::uint32_t element = 0; + + bool operator==(const HeapSlotWord& other) const = default; +}; inline constexpr std::uint32_t StateExportMappings = 0u; inline constexpr std::uint32_t StateDwords = 2u; inline constexpr std::uint32_t StateVertexSelectors = 0u; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 082e55eb8363..fca7a433c698 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -85,29 +85,29 @@ std::filesystem::path ShaderCacheDirectory() { namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) -static_assert(sizeof(CompiledShaderArtifact) == 288, "CompiledShaderArtifact changed: update the artifact encoder"); -static_assert(sizeof(ShaderInvocation) == 216, "ShaderInvocation changed: update the invocation encoder"); +static_assert(sizeof(CompiledShaderArtifact) == 328, "CompiledShaderArtifact changed: update the artifact encoder"); +static_assert(sizeof(ShaderInvocation) == 288, "ShaderInvocation changed: update the invocation encoder"); #if defined(__cpp_lib_atomic_shared_ptr) -static_assert(sizeof(RecompileResult) == 552, "RecompileResult changed: update EncodeResult and DecodeResult"); +static_assert(sizeof(RecompileResult) == 712, "RecompileResult changed: update EncodeResult and DecodeResult"); #endif static_assert(sizeof(DescriptorBinding) == 656, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 16, "FragmentParameter changed: update the parameter encoder"); -static_assert(sizeof(CompiledShaderInfo) == 352, "CompiledShaderInfo changed: update the info encoder"); -static_assert(sizeof(ShaderInfo) == 232, "ShaderInfo changed: update the info encoder"); +static_assert(sizeof(CompiledShaderInfo) == 424, "CompiledShaderInfo changed: update the info encoder"); +static_assert(sizeof(ShaderInfo) == 256, "ShaderInfo changed: update the info encoder"); static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder"); static_assert(sizeof(ImageResource) == 108, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 24, "SamplerResource changed: update the info encoder"); static_assert(sizeof(SampledResourcePair) == 16, "SampledResourcePair changed: update the info encoder"); static_assert(sizeof(StageInput) == 56, "StageInput changed: update the info encoder"); static_assert(sizeof(StageOutput) == 48, "StageOutput changed: update the info encoder"); -static_assert(sizeof(IrBindingLayout) == 80, "IrBindingLayout changed: update the layout encoder"); +static_assert(sizeof(IrBindingLayout) == 128, "IrBindingLayout changed: update the layout encoder"); static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: update the layout encoder"); -static_assert(sizeof(BindingAllocationResult) == 184, "BindingAllocationResult changed: update the allocation encoder"); -static_assert(sizeof(CompiledBindingLayout) == 88, "CompiledBindingLayout changed: update the allocation encoder"); +static_assert(sizeof(BindingAllocationResult) == 232, "BindingAllocationResult changed: update the allocation encoder"); +static_assert(sizeof(CompiledBindingLayout) == 136, "CompiledBindingLayout changed: update the allocation encoder"); static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey"); -static_assert(sizeof(IrResourcePlan) == 728, "IrResourcePlan changed: update EncodePlan and DecodePlan"); +static_assert(sizeof(IrResourcePlan) == 752, "IrResourcePlan changed: update EncodePlan and DecodePlan"); static_assert(sizeof(IrValue) == 144, "IrValue changed: update the plan value encoder"); static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); @@ -141,11 +141,12 @@ constexpr std::size_t fieldCount() { } } -static_assert(fieldCount() == 24, "ShaderInfo changed: update the info encoder"); +static_assert(fieldCount() == 25, "ShaderInfo changed: update the info encoder"); static_assert(fieldCount() == 12, "BufferResource changed: update the info encoder"); -static_assert(fieldCount() == 50, "ImageResource changed: update the info encoder"); +static_assert(fieldCount() == 51, "ImageResource changed: update the info encoder"); static_assert(fieldCount() == 8, "SamplerResource changed: update the info encoder"); static_assert(fieldCount() == 4, "SampledResourcePair changed: update the info encoder"); +static_assert(fieldCount() == 4, "DeviceImageUse changed: update the info encoder"); static_assert(fieldCount() == 5, "StageInput changed: update the info encoder"); static_assert(fieldCount() == 4, "StageOutput changed: update the info encoder"); static_assert(fieldCount() == 25, "IrResourcePlan changed: update EncodePlan and DecodePlan"); @@ -523,6 +524,15 @@ void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { writer.Value(result.runtimeImageCount); writer.Values(std::span(result.runtimeImageResources)); writer.Value(result.registrationModule); + writer.Value(result.gpuResolvedImages); + writer.Value(result.samplerSlotDword); + writer.Value(result.bufferAddressDword); + writer.Value(result.bufferAddressCount); + writer.Values(std::span(result.pushedData)); + writer.Value(result.pushItems); + writer.Value(result.deviceImageTables); + writer.Value(result.dataAddressPush); + writer.Value(result.bdaStores); writer.List(result.vertexInputPatches, [](Writer& out, const VertexInputPatch& patch) { out.Value(patch.location); out.Value(patch.word); @@ -590,6 +600,15 @@ void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { reader.Value(result.runtimeImageCount); reader.Values(result.runtimeImageResources); reader.Value(result.registrationModule); + reader.Value(result.gpuResolvedImages); + reader.Value(result.samplerSlotDword); + reader.Value(result.bufferAddressDword); + reader.Value(result.bufferAddressCount); + reader.Values(result.pushedData); + reader.Value(result.pushItems); + reader.Value(result.deviceImageTables); + reader.Value(result.dataAddressPush); + reader.Value(result.bdaStores); reader.List(result.vertexInputPatches, 20, [](Reader& in, VertexInputPatch& patch) { in.Value(patch.location); in.Value(patch.word); @@ -656,6 +675,11 @@ void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { out.Value(constant.value); }); writer.List(invocation.bindings, encodeBinding); + writer.List(invocation.heapSlotWords, [](Writer& out, const RuntimeAbi::HeapSlotWord& word) { + out.Value(word.dword); + out.Value(word.kind); + out.Value(word.element); + }); writer.Value(invocation.pushConstants.size()); for (const auto byte : invocation.pushConstants) writer.Value(static_cast(byte)); writer.Value(invocation.readsDepthRemapPush); @@ -686,6 +710,8 @@ void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { }); writer.Value(invocation.imageTableShader); writer.Value(invocation.imageTableFaults); + writer.List(invocation.deviceImageInterfaces, [](Writer& out, std::uint64_t id) { out.Value(id); }); + writer.List(invocation.deviceSamplerClasses, [](Writer& out, std::uint32_t bits) { out.Value(bits); }); } void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { @@ -695,6 +721,11 @@ void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { in.Value(constant.value); }); reader.List(invocation.bindings, 8, decodeBinding); + reader.List(invocation.heapSlotWords, 12, [](Reader& in, RuntimeAbi::HeapSlotWord& word) { + in.Value(word.dword); + in.Value(word.kind); + in.Value(word.element); + }); std::vector pushConstants; reader.Values(pushConstants); invocation.pushConstants.resize(pushConstants.size()); @@ -727,16 +758,25 @@ void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { }); reader.Value(invocation.imageTableShader); reader.Value(invocation.imageTableFaults); + reader.List(invocation.deviceImageInterfaces, 8, [](Reader& in, std::uint64_t& id) { in.Value(id); }); + reader.List(invocation.deviceSamplerClasses, 4, [](Reader& in, std::uint32_t& bits) { in.Value(bits); }); } void encodeLayout(Writer& writer, const IrBindingLayout& layout) { writer.Value(layout.pushDataStartDword); + writer.Value(layout.dataAddress); writer.Value(layout.stencilReferenceDword); writer.Value(layout.memoryOffsetDword); writer.Value(layout.memoryOffsetCount); writer.Value(layout.dispatchThreadLimit); writer.Value(layout.runtimeImageCount); writer.Value(layout.runtimeStateDwords); + writer.Value(layout.samplerSlotDwords); + writer.Value(layout.imageSlotDwords); + writer.Value(layout.bufferAddressDwords); + writer.Values(std::span(layout.pushedData)); + writer.Value(layout.heapStorageImages); + writer.Value(layout.gpuResolvedImages); writer.Values(std::span(layout.userDataRegisters)); writer.List(layout.descriptors, [](Writer& out, const IrDescriptorBinding& descriptor) { out.Value(descriptor.kind); @@ -746,12 +786,19 @@ void encodeLayout(Writer& writer, const IrBindingLayout& layout) { void decodeLayout(Reader& reader, IrBindingLayout& layout) { reader.Value(layout.pushDataStartDword); + reader.Value(layout.dataAddress); reader.Value(layout.stencilReferenceDword); reader.Value(layout.memoryOffsetDword); reader.Value(layout.memoryOffsetCount); reader.Value(layout.dispatchThreadLimit); reader.Value(layout.runtimeImageCount); reader.Value(layout.runtimeStateDwords); + reader.Value(layout.samplerSlotDwords); + reader.Value(layout.imageSlotDwords); + reader.Value(layout.bufferAddressDwords); + reader.Values(layout.pushedData); + reader.Value(layout.heapStorageImages); + reader.Value(layout.gpuResolvedImages); reader.Values(layout.userDataRegisters); reader.List(layout.descriptors, 12, [](Reader& in, IrDescriptorBinding& descriptor) { in.Value(descriptor.kind); @@ -827,6 +874,7 @@ void encodeShaderInfo(Writer& writer, const ShaderInfo& info) { out.Value(image.table); out.Value(image.tableOperation); out.Value(image.targetMerge); + out.Value(image.deviceTable); }); writer.Value(!info.runtimeImageModes.empty()); writer.List(info.samplers, [](Writer& out, const SamplerResource& sampler) { @@ -845,6 +893,12 @@ void encodeShaderInfo(Writer& writer, const ShaderInfo& info) { out.Value(pair.firstUsePc); out.Value(pair.volumeGather); }); + writer.List(info.deviceImageUses, [](Writer& out, const DeviceImageUse& use) { + out.Value(use.image); + out.Value(use.sampler); + out.Value(use.descriptor); + out.Value(use.firstUsePc); + }); writer.List(info.inputs, [](Writer& out, const StageInput& input) { out.Value(input.kind); out.Value(input.location); @@ -943,6 +997,7 @@ void decodeShaderInfo(Reader& reader, ShaderInfo& info) { in.Value(image.table); in.Value(image.tableOperation); in.Value(image.targetMerge); + in.Value(image.deviceTable); }); bool preparedImageModes = false; reader.Value(preparedImageModes); @@ -963,6 +1018,12 @@ void decodeShaderInfo(Reader& reader, ShaderInfo& info) { in.Value(pair.firstUsePc); in.Value(pair.volumeGather); }); + reader.List(info.deviceImageUses, 16, [](Reader& in, DeviceImageUse& use) { + in.Value(use.image); + in.Value(use.sampler); + in.Value(use.descriptor); + in.Value(use.firstUsePc); + }); reader.List(info.inputs, 21, [](Reader& in, StageInput& input) { in.Value(input.kind); in.Value(input.location); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 3e43cbb22130..6ab937e6b971 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -13,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 56; +inline constexpr std::uint32_t FormatVersion = 62; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp index 5938cf31fc26..ee04616929e3 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitter.hpp @@ -44,6 +44,9 @@ struct SpirvTargetOptions { bool largePixelFunctionLdsBuffer = false; std::uint32_t pixelFunctionStorageDwords = 4096u; bool deviceScopeLdsMemoryBarriers = false; + bool gpuResolvedImages = false; + std::uint32_t heapDataUniformDwords = 0; + bool heapBufferDescriptors = false; bool ldsLockFences = false; bool meshWindowBuffer = false; }; @@ -51,7 +54,9 @@ struct SpirvTargetOptions { struct SpirvEmitInfo { bool fullSubgroups = false; bool functionLdsBuffer = false; + bool bdaStores = false; bool meshWindowBuffer = false; + std::uint32_t pushItems = 0; std::uint32_t scratchPoolSlotDwords = 0; }; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 777eda1ea357..b701e078c67f 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -41,6 +41,9 @@ std::uint32_t TypeStorageBufferU8Block(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU8Pointer(SpirvEmitterState& state); std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state); std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state); +std::uint32_t TypePhysicalBufferPointer(SpirvEmitterState& state, bool wide); +std::uint32_t ReadOnlyStorageBufferType(SpirvEmitterState& state, std::uint32_t element, std::uint32_t stride); +std::uint32_t TypePhysicalReadOnlyBufferPointer(SpirvEmitterState& state, bool wide); std::uint32_t TypePushConstantElementPointer(SpirvEmitterState& state); std::uint32_t TypeU32ArrayPointer(SpirvEmitterState& state, std::uint32_t storageClass, std::uint32_t dwords); std::uint32_t TypeU32ElementPointer(SpirvEmitterState& state, std::uint32_t storageClass); @@ -111,13 +114,17 @@ std::uint32_t EmitBallotLaneActiveBool(SpirvEmitterState& state, std::uint32_t b std::uint32_t EmitSubgroupLaneActiveBool(SpirvEmitterState& state, std::uint32_t lane); std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); +bool WritesGuestBuffers(const IrProgram& program); +bool WritesGuestMemory(const IrProgram& program); void EmitMemoryOffsets(SpirvEmitterState& state); +void EmitHeapSlotPreload(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); bool LdsInDeviceMemory(const SpirvEmitterState& state); std::uint32_t LdsMemorySemantics(const SpirvEmitterState& state); std::uint32_t LdsMemoryBarrierScope(const SpirvEmitterState& state); void EmitLdsBufferBase(SpirvEmitterState& state); bool ScratchPoolActive(const SpirvEmitterState& state); +bool GpuScratchPool(const SpirvEmitterState& state); std::uint32_t ScratchPoolSlotDwords(const SpirvEmitterState& state); void DefineScratchPool(SpirvEmitterState& state); void EmitScratchPoolAcquire(SpirvEmitterState& state); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index faba616acf81..25ee7c5760a5 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -71,6 +71,8 @@ struct MemoryResourceAccess { bool addIndexOffset = false; std::uint32_t memoryAccess = 0; std::uint32_t misalignment = 0; + bool physical = false; + bool clampedPhysical = false; }; struct SpirvTableImageVariable { @@ -84,12 +86,17 @@ struct SpirvTableAccess { std::uint32_t samplerElement = 0; bool active = false; bool uniform = false; + bool heap = false; + bool device = false; }; struct SpirvEmitterState { const ImageResource* runtimeImage = nullptr; SpirvTableAccess tableAccess; SpirvEmitterState(const IrProgram& program, const ShaderStageInputInfo& inputInfo); + [[nodiscard]] bool BdaAddressed() const; + [[nodiscard]] bool TableMapAddressed() const; + [[nodiscard]] bool TableHeap() const; SpirvModule module; const IrProgram& program; @@ -110,6 +117,11 @@ struct SpirvEmitterState { bool depthRangeRemap = false; std::uint32_t depthRemapClipIndex = 0; bool registrationModules = false; + bool gpuResolvedImages = false; + std::uint32_t heapDataUniformDwords = 0; + bool heapBufferDescriptors = false; + bool addressedBuffers = false; + std::optional guestBuffersWritten; bool fragmentDescriptorSet = false; bool float16DenormPreserve = false; bool robustBufferAccess2 = false; @@ -222,7 +234,11 @@ struct SpirvEmitterState { std::uint32_t gdsLength = 0; std::uint32_t pushConstantVariable = 0; std::uint32_t shaderDataStorageVariable = 0; + std::uint32_t shaderDataUniformVariable = 0; + std::uint32_t pushItems = 0; + std::unordered_map heapSlotVariables; std::uint32_t flattenedSrtVariable = 0; + bool flattenedSrtAddressed = false; std::uint32_t ldsVariable = 0; std::uint32_t ldsBufferVariable = 0; std::uint32_t ldsBufferU64Variable = 0; @@ -252,6 +268,7 @@ struct SpirvEmitterState { std::vector tableImageVariables; std::uint32_t tableSamplerVariable = 0; std::uint32_t tableMapVariable = 0; + std::uint32_t deviceImageLookupFunction = 0; std::uint32_t mainFunc = 0; std::uint32_t meshGuestFunc = 0; std::uint32_t pixelGuestFunc = 0; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvImageEmitter.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvImageEmitter.hpp index b471cdb6c823..b6dc73a3aa09 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvImageEmitter.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvImageEmitter.hpp @@ -13,6 +13,8 @@ void EmitImageOperation(SpirvModule& module, const IrValue& value, std::uint32_t void EmitImageOperation(SpirvValueEmitContext& context, const IrValue& value, std::uint32_t resultId); +void DefineDeviceImageLookup(SpirvEmitterState& state); + } #endif diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp index 3fbdbef112bd..a74d1097a92b 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp @@ -4,18 +4,33 @@ #include "SpirvBackend/SpirvEmitterState.hpp" #include "IntermediateRepresentation/IrProgram.hpp" #include +#include namespace ShaderRecompiler { std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std::uint32_t lhs, std::uint32_t rhs); std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); +std::uint32_t EmitShaderDataWordLoad(SpirvEmitterState& state, std::uint32_t index); +std::uint32_t ComposeAddress64(SpirvEmitterState& state, const std::function& loadLow, const std::function& loadHigh); +std::uint32_t EmitAlignedLoadU32(SpirvEmitterState& state, std::uint32_t pointer); +std::uint32_t EmitRootItemDword(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t word); +std::uint32_t EmitRootAddressWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index); +std::uint32_t EmitRootAddressReadOnlyWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index); +std::uint32_t EmitRootAddress(SpirvEmitterState& state, RuntimeAbi::RootItem item); +std::uint32_t EmitStageRingWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index); +std::uint32_t EmitAddressWordPointer(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index); +std::uint32_t EmitFlattenedSrtLoad(SpirvEmitterState& state, std::uint32_t index); +bool HasFlattenedSrt(const SpirvEmitterState& state); void EmitMemoryOffsets(SpirvEmitterState& state); +std::uint32_t LoadTableMap(SpirvEmitterState& state, std::uint32_t index); +std::uint32_t LoadHeapSlotWord(SpirvEmitterState& state, std::uint32_t dword); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); bool LdsInDeviceMemory(const SpirvEmitterState& state); std::uint32_t LdsMemorySemantics(const SpirvEmitterState& state); std::uint32_t LdsMemoryBarrierScope(const SpirvEmitterState& state); void EmitLdsBufferBase(SpirvEmitterState& state); bool ScratchPoolActive(const SpirvEmitterState& state); +bool GpuScratchPool(const SpirvEmitterState& state); std::uint32_t ScratchPoolSlotDwords(const SpirvEmitterState& state); void DefineScratchPool(SpirvEmitterState& state); void EmitScratchPoolAcquire(SpirvEmitterState& state); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvConstants.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvConstants.hpp index fb59644f8bed..aece716687b0 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvConstants.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvConstants.hpp @@ -14,6 +14,7 @@ std::uint32_t ConstantF32Value(SpirvEmitterState& state, float value); std::uint32_t ConstantBool(SpirvEmitterState& state, bool value); std::uint32_t ConstantU64(SpirvEmitterState& state, std::uint64_t value); std::uint32_t ConstantU32CompositeZero(SpirvEmitterState& state, std::uint32_t components); +std::uint32_t HeapImageSlotCount(SpirvEmitterState& state); } diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp index 5ff71524180d..0a90241fae9e 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvDescriptors.hpp @@ -18,6 +18,7 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t sampler); std::uint32_t MakeSampledImage(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t sampler); void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t id); +std::uint32_t TableTrailerWord(SpirvEmitterState& state, std::uint32_t field); bool TablePointSampled(const ImageResource& image); bool TableModeEmitted(const ImageResource& mode); std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint32_t resource); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index cb6d2a7cdacd..b4f864a5ec2d 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -1,6 +1,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include #include #include @@ -12,6 +13,8 @@ std::uint32_t BdaConstant(SpirvEmitterState& state, std::uint64_t value) { } std::uint32_t BdaWord(SpirvEmitterState& state, std::uint32_t variable, std::uint32_t index) { + if (state.BdaAddressed() && variable == state.bdaPagetableVariable) return EmitRootAddressReadOnlyWordPointer(state, RuntimeAbi::RootItem::PagetableAddress, index); + if (state.BdaAddressed() && variable == state.faultBufferVariable) return EmitRootAddressWordPointer(state, RuntimeAbi::RootItem::FaultAddress, index); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, variable, ConstantU32(state, 0u), index); return pointer; @@ -19,7 +22,8 @@ std::uint32_t BdaWord(SpirvEmitterState& state, std::uint32_t variable, std::uin std::uint32_t BdaLoadWord(SpirvEmitterState& state, std::uint32_t index) { const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), value, BdaWord(state, state.bdaPagetableVariable, index)); + if (state.BdaAddressed()) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, BdaWord(state, state.bdaPagetableVariable, index), spv::MemoryAccessAlignedMask, 4u); + else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, BdaWord(state, state.bdaPagetableVariable, index)); return value; } @@ -60,7 +64,8 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { const auto won = Binary(state, spv::OpIEqual, TypeBool(state), previous, ConstantU32(state, 0u)); EmitIfCondition(state, won, [&] { const auto store = [&](std::uint32_t index, std::uint32_t value) { - state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value); + if (state.BdaAddressed()) state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value, spv::MemoryAccessAlignedMask, 4u); + else state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value); }; store(1, reason); store(2, words ? address : Unary(state, spv::OpUConvert, TypeU32(state), address)); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index d1889a85fe02..c1ae3df21d13 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -1,6 +1,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include #include #include @@ -64,8 +65,8 @@ std::uint32_t DefineBdaLookup(SpirvEmitterState& state, const char* name, bool r } leaveIf(binary(spv::OpINotEqual, boolean, reason, constant(0u)), reason); }; - const auto length = state.module.AllocateId(); - state.module.AddFunction(spv::OpArrayLength, u32, length, state.bdaPagetableVariable, 0u); + auto length = state.BdaAddressed() ? EmitRootItemDword(state, RuntimeAbi::RootItem::PagetableDwords, 0u) : state.module.AllocateId(); + if (!state.BdaAddressed()) state.module.AddFunction(spv::OpArrayLength, u32, length, state.bdaPagetableVariable, 0u); fail(binary(spv::OpULessThan, boolean, length, constant(4)), BdaAbi::FaultReason::InvalidTable); const auto count = BdaLoadWord(state, constant(1)); const auto available = binary(spv::OpISub, u32, length, constant(4)); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp index 591fac5fd98f..6bcd776df5f5 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvEmitter.cpp @@ -80,6 +80,18 @@ std::uint32_t WaveLdsScope(const IrProgram& program, const ShaderWorkgroupInputI SpirvEmitterState::SpirvEmitterState(const IrProgram& program, const ShaderStageInputInfo& inputInfo) : module(program.Resources().stage == IrShaderStage::Mesh ? 0x00010400u : 0x00010300u), program(program), inputInfo(inputInfo), requirements(AnalyzeProgramRequirements(program)) { } +bool SpirvEmitterState::BdaAddressed() const { + return (bdaPagetableVariable != 0 || faultBufferVariable != 0) && program.Metadata().bindings.UsesDataAddress(); +} + +bool SpirvEmitterState::TableMapAddressed() const { + return tableMapVariable != 0 && program.Metadata().bindings.UsesDataAddress(); +} + +bool SpirvEmitterState::TableHeap() const { + return gpuResolvedImages && program.Metadata().bindings.UsesDataAddress(); +} + SpirvValueEmitContext::SpirvValueEmitContext(SpirvEmitterState& state) : state(state) { } @@ -268,6 +280,9 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh state.float64DenormPreserve = target.float64DenormPreserve; state.depthRangeRemap = target.depthRangeRemap; state.registrationModules = target.registrationModules; + state.gpuResolvedImages = target.gpuResolvedImages; + state.heapDataUniformDwords = target.heapDataUniformDwords; + state.heapBufferDescriptors = target.heapBufferDescriptors; state.fragmentDescriptorSet = target.fragmentDescriptorSet; state.meshBarycentricPixel = target.meshBarycentricPixel; state.meshBarycentricVertex = target.meshBarycentricVertex; @@ -341,6 +356,8 @@ std::vector SpirvEmitter::Emit(const IrProgram& program, const Sh if (info != nullptr) { info->fullSubgroups = state.spannedWave; info->functionLdsBuffer = state.pixelFunctionLdsBuffer; + info->bdaStores = state.bdaWritePointerFunction != 0; + info->pushItems = state.pushItems; info->meshWindowBuffer = state.meshWindowBuffer; info->scratchPoolSlotDwords = ScratchPoolActive(state) ? ScratchPoolSlotDwords(state) : 0u; } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index 20dec7d4d048..3c1b600f982a 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -3,6 +3,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvRuntimeBuffer.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" #include #include @@ -102,11 +103,17 @@ void EmitReturnTerminator(SpirvValueEmitContext& ctx) { state.module.AddFunction(spv::OpLoad, TypeI32(state), value, state.stencilRefVariable); const auto reference = Unary(state, spv::OpBitcast, TypeU32(state), value); const auto masked = Binary(state, spv::OpBitwiseAnd, TypeU32(state), reference, ConstantU32(state, 0xffu)); - const auto offset = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::StencilReferenceOffset, state.program.Metadata().bindings.stencilReferenceDword); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0u), offset); - const auto expected = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), expected, pointer); + const auto& layout = state.program.Metadata().bindings; + auto expected = 0u; + if (layout.UsesDataAddress()) { + expected = EmitRootItemDword(state, RuntimeAbi::RootItem::StencilReference, 0u); + } else { + const auto offset = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::StencilReferenceOffset, layout.stencilReferenceDword); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0u), offset); + expected = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), expected, pointer); + } EmitKillIfBoolFalse(state, Binary(state, spv::OpIEqual, TypeBool(state), masked, expected)); } EmitReturnReleasingScratch(state); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index 422dfbd3576a..637a3a4a9117 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -7,7 +7,9 @@ #include "SpirvBackend/SpirvMemory/SpirvDescriptors.hpp" #include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "RdnaDecoder/RdnaImageOpDecoder.hpp" +#include "DeviceImageTableAbi.hpp" #include "ImageTableAbi.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include #include #include @@ -2062,7 +2064,14 @@ ScalarBufferDword EmitScalarBufferDword(SpirvEmitterState& state, std::uint32_t return dword; } +} + std::uint32_t LoadTableMap(SpirvEmitterState& state, std::uint32_t index) { + if (state.TableMapAddressed()) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, EmitStageRingWordPointer(state, RuntimeAbi::RootItem::TableMap, index), spv::MemoryAccessAlignedMask, 4u); + return value; + } const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.tableMapVariable, ConstantU32(state, 0u), index); const auto value = state.module.AllocateId(); @@ -2070,6 +2079,8 @@ std::uint32_t LoadTableMap(SpirvEmitterState& state, std::uint32_t index) { return value; } +namespace { + std::uint32_t TableMapAt(SpirvEmitterState& state, std::uint32_t base, std::uint32_t offset) { return LoadTableMap(state, offset == 0u ? base : Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, offset))); } @@ -2274,7 +2285,8 @@ TableAccessResult EmitTableAccess(SpirvValueEmitContext& ctx, const IrValue& ins result.valid = both(checked, negate(driverRejected)); result.flags = flags; live = selected; - state.tableAccess.element = Binary(state, spv::OpIAdd, u32, LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementBase)), element); + state.tableAccess.element = state.TableHeap() ? Select(state, u32, checked, element, ConstantU32(state, 0u)) : Binary(state, spv::OpIAdd, u32, LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementBase)), element); + state.tableAccess.heap = state.TableHeap(); const auto& sources = state.program.Resources().descriptorSources; state.tableAccess.uniform = base.source < sources.size() && sources[base.source].tableColumn.has_value() && sources[base.source].tableColumn->selected; if (!state.tableAccess.uniform) DecorateTableNonUniform(state, state.tableAccess.element); @@ -2307,6 +2319,239 @@ TableAccessResult EmitTableAccess(SpirvValueEmitContext& ctx, const IrValue& ins return result; } +std::uint32_t DeviceTableWordPointer(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index) { + return EmitAddressWordPointer(state, base, index); +} + +std::uint32_t LoadDeviceTableWord(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, DeviceTableWordPointer(state, base, index), spv::MemoryAccessAlignedMask, 4u); + return value; +} + +std::uint32_t TrailerAddress(SpirvEmitterState& state, std::uint32_t lowField) { + const auto u64 = TypeScalarU64(state); + const auto low = Unary(state, spv::OpUConvert, u64, TableTrailerWord(state, lowField)); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, TableTrailerWord(state, lowField + 1u)), BdaConstant(state, 32u)); + return Binary(state, spv::OpBitwiseOr, u64, low, high); +} + +std::uint32_t RotateLeftU32(SpirvEmitterState& state, std::uint32_t value, std::uint32_t bits) { + const auto u32 = TypeU32(state); + return Binary(state, spv::OpBitwiseOr, u32, Binary(state, spv::OpShiftLeftLogical, u32, value, ConstantU32(state, bits)), Binary(state, spv::OpShiftRightLogical, u32, value, ConstantU32(state, 32u - bits))); +} + +std::uint32_t DeviceKeyHash(SpirvEmitterState& state, std::span key) { + namespace Abi = DeviceImageTableAbi; + const auto u32 = TypeU32(state); + const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; + auto hash = constant(Abi::HashSeed); + for (const auto word : key) { + const auto mixed = Binary(state, spv::OpIMul, u32, RotateLeftU32(state, Binary(state, spv::OpIMul, u32, word, constant(Abi::HashC1)), 15u), constant(Abi::HashC2)); + hash = Binary(state, spv::OpIAdd, u32, Binary(state, spv::OpIMul, u32, RotateLeftU32(state, Binary(state, spv::OpBitwiseXor, u32, hash, mixed), 13u), constant(5u)), constant(Abi::HashRound)); + } + const auto shiftXor = [&](std::uint32_t value, std::uint32_t bits) { return Binary(state, spv::OpBitwiseXor, u32, value, Binary(state, spv::OpShiftRightLogical, u32, value, constant(bits))); }; + hash = Binary(state, spv::OpBitwiseXor, u32, hash, constant(Abi::KeyWords * 4u)); + hash = Binary(state, spv::OpIMul, u32, shiftXor(hash, 16u), constant(Abi::HashMix1)); + hash = Binary(state, spv::OpIMul, u32, shiftXor(hash, 13u), constant(Abi::HashMix2)); + return shiftXor(hash, 16u); +} + +std::uint32_t CompositeU32x4(SpirvEmitterState& state, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint32_t w) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), value, x, y, z, w); + return value; +} + +std::uint32_t EmitDeviceTableAccess(SpirvValueEmitContext& ctx, const IrValue& inst, const MemoryInfo& memory, const ImageResource& base, std::span modes, bool tableSampler) { + namespace Abi = DeviceImageTableAbi; + auto& state = ctx.state; + if (!state.TableHeap() || !state.TableMapAddressed()) ctx.Fail(inst, "reads a device-table image outside a GPU-mode stage that reads its table map by address"); + if (state.deviceImageLookupFunction == 0u) ctx.Fail(inst, "reads a device-table image without the device image lookup"); + if (tableSampler) ctx.Fail(inst, "samples a device-table image through a sampler table"); + if (base.tableOperation == TableOperation::Unsupported || modes.empty()) ctx.Fail(inst, "reads a device-table image without runtime modes"); + const IrValue* handle = inst.Argument(0)->Resolve(); + if (handle->ArgumentCount() != Abi::DescriptorWords) ctx.Fail(inst, "has a device-table image handle without its eight descriptor dwords"); + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto zero = ConstantU32(state, 0u); + const auto both = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpLogicalAnd, boolean, left, right); }; + const auto equal = [&](std::uint32_t left, std::uint32_t right) { return Binary(state, spv::OpIEqual, boolean, left, right); }; + std::array words{}; + for (std::uint32_t word = 0; word < words.size(); ++word) words[word] = ctx.Def(handle->Argument(word)); + const auto active = Unary(state, spv::OpLogicalNot, boolean, both(equal(words[0], zero), equal(Binary(state, spv::OpBitwiseAnd, u32, words[1], ConstantU32(state, 0xffu)), zero))); + const auto exec = ctx.Arg(inst, inst.ArgumentCount() - 1u); + const auto samplerClass = ImageOpcodeInfoOf(inst.Opcode()).needsSampler ? state.module.SpecializationConstant(u32, PipelineSpecialization::DeviceSamplerClassBase + memory.sampler, 0u) : zero; + const auto id = ResourceMaterializer::DeviceImageInterfaceId(base, modes); + const auto pc = ConstantU32(state, inst.Flags().pc); + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 4u), result, state.deviceImageLookupFunction, + CompositeU32x4(state, words[0], words[1], words[2], words[3]), CompositeU32x4(state, words[4], words[5], words[6], words[7]), + CompositeU32x4(state, ConstantU32(state, static_cast(id)), ConstantU32(state, static_cast(id >> 32u)), samplerClass, pc), active, both(exec, active)); + const auto extract = [&](std::uint32_t index) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, value, result, index); + return value; + }; + const auto slot = extract(0u); + const auto mode = extract(1u); + const auto fault = extract(2u); + const auto hit = Binary(state, spv::OpINotEqual, boolean, extract(3u), zero); + const auto valid = both(both(both(active, hit), equal(fault, zero)), Binary(state, spv::OpULessThan, boolean, slot, ConstantU32(state, RuntimeAbi::GpuImageHeapSlots))); + state.tableAccess = {}; + state.tableAccess.element = Select(state, u32, valid, slot, zero); + DecorateTableNonUniform(state, state.tableAccess.element); + state.tableAccess.dword3 = words[3]; + state.tableAccess.device = true; + state.tableAccess.active = true; + return Select(state, u32, valid, mode, ConstantU32(state, InvalidTableSelector)); +} + +} + +void DefineDeviceImageLookup(SpirvEmitterState& state) { + namespace Abi = DeviceImageTableAbi; + const auto& images = state.program.Info().images; + if (state.deviceImageLookupFunction != 0u || !state.TableMapAddressed() || std::none_of(images.begin(), images.end(), [](const ImageResource& image) { return image.deviceTable; })) return; + const auto u32 = TypeU32(state); + const auto uvec4 = TypeU32Vector(state, 4u); + const auto boolean = TypeBool(state); + const auto constant = [&](std::uint32_t value) { return ConstantU32(state, value); }; + const auto binary = [&](std::uint32_t op, std::uint32_t type, std::uint32_t left, std::uint32_t right) { return Binary(state, op, type, left, right); }; + const auto negate = [&](std::uint32_t value) { return Unary(state, spv::OpLogicalNot, boolean, value); }; + const auto extract = [&](std::uint32_t vector, std::uint32_t index) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, value, vector, index); + return value; + }; + const auto load = [&](std::uint32_t variable, std::uint32_t type) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, type, value, variable); + return value; + }; + const auto store = [&](std::uint32_t pointer, std::uint32_t value) { state.module.AddFunction(spv::OpStore, pointer, value, spv::MemoryAccessAlignedMask, 4u); }; + const auto function = state.module.AllocateId(); + state.module.AddName(function, "device_image_lookup"); + state.module.AddFunction(spv::OpFunction, uvec4, function, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, uvec4, uvec4, uvec4, uvec4, boolean, boolean)); + const auto low = state.module.AllocateId(); + const auto high = state.module.AllocateId(); + const auto extra = state.module.AllocateId(); + const auto active = state.module.AllocateId(); + const auto record = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, uvec4, low); + state.module.AddFunction(spv::OpFunctionParameter, uvec4, high); + state.module.AddFunction(spv::OpFunctionParameter, uvec4, extra); + state.module.AddFunction(spv::OpFunctionParameter, boolean, active); + state.module.AddFunction(spv::OpFunctionParameter, boolean, record); + EmitLabel(state, state.module.AllocateId()); + const auto stepVariable = state.module.AllocateId(); + const auto doneVariable = state.module.AllocateId(); + const auto resultVariable = state.module.AllocateId(); + const auto seenStepVariable = state.module.AllocateId(); + const auto seenDoneVariable = state.module.AllocateId(); + const auto appendVariable = state.module.AllocateId(); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), stepVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), doneVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, uvec4), resultVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), seenStepVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), seenDoneVariable, spv::StorageClassFunction); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), appendVariable, spv::StorageClassFunction); + const auto buckets = TrailerAddress(state, ImageTableAbi::TrailerBucketsLow); + const auto ring = TrailerAddress(state, ImageTableAbi::TrailerRingLow); + const auto attached = binary(spv::OpLogicalAnd, boolean, binary(spv::OpINotEqual, boolean, buckets, BdaConstant(state, 0u)), binary(spv::OpINotEqual, boolean, ring, BdaConstant(state, 0u))); + std::array key{}; + for (std::uint32_t word = 0; word < 4u; ++word) { + key[word] = extract(low, word); + key[word + 4u] = extract(high, word); + } + for (std::uint32_t word = 0; word < 3u; ++word) key[Abi::InterfaceWord + word] = extract(extra, word); + for (std::uint32_t word = 0; word < Abi::SamplerWords; ++word) key[Abi::SamplerWord + word] = constant(0u); + const auto pc = extract(extra, 3u); + const auto hash = DeviceKeyHash(state, key); + const auto searching = binary(spv::OpLogicalAnd, boolean, active, attached); + state.module.AddFunction(spv::OpStore, stepVariable, constant(0u)); + state.module.AddFunction(spv::OpStore, doneVariable, negate(searching)); + state.module.AddFunction(spv::OpStore, resultVariable, ConstantU32CompositeZero(state, 4u)); + EmitIfCondition(state, searching, [&] { + const auto mask = LoadDeviceTableWord(state, buckets, constant(Abi::HeaderMask)); + const auto home = binary(spv::OpBitwiseAnd, u32, hash, mask); + const auto loopHeader = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, loopHeader); + EmitLabel(state, loopHeader); + const auto step = load(stepVariable, u32); + const auto done = load(doneVariable, boolean); + const auto probing = binary(spv::OpLogicalAnd, boolean, negate(done), binary(spv::OpULessThan, boolean, step, constant(Abi::ProbeSteps))); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, probing, body, merge); + EmitLabel(state, body); + const auto first = binary(spv::OpIAdd, u32, constant(Abi::HeaderWords), binary(spv::OpIMul, u32, binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, home, step), mask), constant(Abi::BucketWords))); + const auto at = [&](std::uint32_t word) { return LoadDeviceTableWord(state, buckets, binary(spv::OpIAdd, u32, first, constant(word))); }; + const auto live = binary(spv::OpIEqual, boolean, at(Abi::StateWord), constant(Abi::LiveState)); + auto same = live; + for (std::uint32_t word = 0; word < Abi::KeyWords; ++word) same = binary(spv::OpLogicalAnd, boolean, same, binary(spv::OpIEqual, boolean, at(word), key[word])); + const auto found = CompositeU32x4(state, at(Abi::SlotWord), at(Abi::ModeWord), at(Abi::FaultWord), constant(1u)); + const auto previous = load(resultVariable, uvec4); + const auto sameLanes = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(state, 4u), sameLanes, same, same, same, same); + state.module.AddFunction(spv::OpStore, resultVariable, Select(state, uvec4, sameLanes, found, previous)); + state.module.AddFunction(spv::OpStore, doneVariable, binary(spv::OpLogicalOr, boolean, negate(live), same)); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpStore, stepVariable, binary(spv::OpIAdd, u32, step, constant(1u))); + state.module.AddFunction(spv::OpBranch, loopHeader); + EmitLabel(state, merge); + }); + const auto result = load(resultVariable, uvec4); + const auto missed = binary(spv::OpIEqual, boolean, extract(result, 3u), constant(0u)); + const auto faulted = binary(spv::OpLogicalAnd, boolean, negate(missed), binary(spv::OpINotEqual, boolean, extract(result, 2u), constant(0u))); + const auto recording = binary(spv::OpLogicalAnd, boolean, record, attached); + state.module.AddFunction(spv::OpStore, appendVariable, binary(spv::OpLogicalAnd, boolean, recording, faulted)); + EmitIfCondition(state, binary(spv::OpLogicalAnd, boolean, recording, missed), [&] { + const auto mark = binary(spv::OpBitwiseOr, u32, hash, constant(1u)); + const auto seenHome = binary(spv::OpBitwiseAnd, u32, hash, constant(Abi::RingSeenSlots - 1u)); + state.module.AddFunction(spv::OpStore, appendVariable, ConstantBool(state, true)); + state.module.AddFunction(spv::OpStore, seenStepVariable, constant(0u)); + state.module.AddFunction(spv::OpStore, seenDoneVariable, ConstantBool(state, false)); + const auto seenHeader = state.module.AllocateId(); + const auto seenBody = state.module.AllocateId(); + const auto seenContinue = state.module.AllocateId(); + const auto seenMerge = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, seenHeader); + EmitLabel(state, seenHeader); + const auto seenStep = load(seenStepVariable, u32); + const auto probing = binary(spv::OpLogicalAnd, boolean, negate(load(seenDoneVariable, boolean)), binary(spv::OpULessThan, boolean, seenStep, constant(Abi::RingSeenProbes))); + state.module.AddFunction(spv::OpLoopMerge, seenMerge, seenContinue, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, probing, seenBody, seenMerge); + EmitLabel(state, seenBody); + const auto slot = binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, seenHome, seenStep), constant(Abi::RingSeenSlots - 1u)); + const auto seen = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicCompareExchange, u32, seen, DeviceTableWordPointer(state, ring, binary(spv::OpIAdd, u32, constant(Abi::RingSeenWord), slot)), constant(spv::ScopeDevice), constant(spv::MemorySemanticsMaskNone), constant(spv::MemorySemanticsMaskNone), mark, constant(0u)); + const auto own = binary(spv::OpIEqual, boolean, seen, mark); + state.module.AddFunction(spv::OpStore, appendVariable, negate(own)); + state.module.AddFunction(spv::OpStore, seenDoneVariable, binary(spv::OpLogicalOr, boolean, own, binary(spv::OpIEqual, boolean, seen, constant(0u)))); + state.module.AddFunction(spv::OpBranch, seenContinue); + EmitLabel(state, seenContinue); + state.module.AddFunction(spv::OpStore, seenStepVariable, binary(spv::OpIAdd, u32, seenStep, constant(1u))); + state.module.AddFunction(spv::OpBranch, seenHeader); + EmitLabel(state, seenMerge); + }); + EmitIfCondition(state, load(appendVariable, boolean), [&] { + const auto index = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicIAdd, u32, index, DeviceTableWordPointer(state, ring, constant(Abi::RingCountWord)), constant(spv::ScopeDevice), constant(spv::MemorySemanticsMaskNone), constant(1u)); + const auto capacity = LoadDeviceTableWord(state, buckets, constant(Abi::HeaderRingCapacity)); + EmitIfCondition(state, binary(spv::OpULessThan, boolean, index, capacity), [&] { + const auto recordFirst = binary(spv::OpIAdd, u32, constant(Abi::RingRecordsWord), binary(spv::OpIMul, u32, index, constant(Abi::RecordWords))); + for (std::uint32_t word = 0; word < Abi::KeyWords; ++word) store(DeviceTableWordPointer(state, ring, binary(spv::OpIAdd, u32, recordFirst, constant(word))), key[word]); + const auto pcWord = Select(state, u32, faulted, binary(spv::OpBitwiseOr, u32, pc, constant(Abi::FaultFlag)), pc); + store(DeviceTableWordPointer(state, ring, binary(spv::OpIAdd, u32, recordFirst, constant(Abi::RecordPcWord))), pcWord); + }); + }); + state.module.AddFunction(spv::OpReturnValue, result); + state.module.AddFunction(spv::OpFunctionEnd); + state.deviceImageLookupFunction = function; } void EmitImageOperation(SpirvModule& module, const IrValue& value, std::uint32_t resultId) { @@ -2405,7 +2650,9 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto modes = base.table != NoTable ? state.program.Info().runtimeImageModes.at(memory.resource) : ResourceMaterializer::RuntimeImageModes(base); std::uint32_t tableSelector = 0u; std::uint32_t samplerGuard = 0u; - if (tables) { + if (tables && base.deviceTable) { + tableSelector = EmitDeviceTableAccess(ctx, inst, memory, base, modes, tableSampler); + } else if (tables) { const auto access = EmitTableAccess(ctx, inst, memory, base, tableSampler); if (access.unsupported) { state.tableAccess = {}; @@ -2752,8 +2999,7 @@ void EmitImageBvhIntersectRay(SpirvValueEmitContext& ctx, const IrValue& inst) { for (std::uint32_t i = 0; i < 4u; ++i) { const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, op(spv::OpIAdd, u64, physical, BdaConstant(state, (first + i) * 4u))); - values[i] = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, u32, values[i], pointer, spv::MemoryAccessAlignedMask, 4u); + values[i] = EmitAlignedLoadU32(state, pointer); } return compose(values); }); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 9a2b7a33c87e..c466c0843140 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -3,7 +3,9 @@ #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvDescriptors.hpp" +#include "SpirvBackend/SpirvBda.hpp" #include +#include #include #include #include @@ -32,8 +34,158 @@ std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std: return result; } +std::uint32_t EmitAlignedLoadU32(SpirvEmitterState& state, std::uint32_t pointer) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, spv::MemoryAccessAlignedMask, 4u); + return value; +} + +std::uint32_t EmitRootItemDword(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t word) { + if (!state.program.Metadata().bindings.UsesDataAddress()) FailEmit("the root is read only by stages that take their data through it"); + state.pushItems |= RuntimeAbi::RootItemBit(item); + const auto u32 = TypeU32(state); + const auto position = state.module.SpecializationConstant(u32, PipelineSpecialization::RootItemPositionBase + static_cast(item), 0u); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0), word == 0u ? position : Binary(state, spv::OpIAdd, u32, position, ConstantU32(state, word))); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, value, pointer); + return value; +} + +namespace { + +std::uint32_t EmitPushedDataSelect(SpirvEmitterState& state, std::uint32_t position, std::uint32_t ring) { + const auto u32 = TypeU32(state); + const auto base = state.module.SpecializationConstant(u32, PipelineSpecialization::PushedDataBase, 0u); + const auto count = state.module.SpecializationConstant(u32, PipelineSpecialization::PushedDataCount, 0u); + const auto inside = state.module.AllocateId(); + state.module.AddFunction(spv::OpULessThan, TypeBool(state), inside, ConstantU32(state, position), count); + const auto index = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelect, u32, index, inside, Binary(state, spv::OpIAdd, u32, base, ConstantU32(state, position)), ConstantU32(state, 0u)); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0), index); + const auto pushedValue = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, pushedValue, pointer); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelect, u32, value, inside, pushedValue, ring); + return value; +} + +} + +std::uint32_t ComposeAddress64(SpirvEmitterState& state, const std::function& loadLow, const std::function& loadHigh) { + const auto u64 = TypeScalarU64(state); + const auto low = Unary(state, spv::OpUConvert, u64, loadLow()); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, loadHigh()), BdaConstant(state, 32u)); + return Binary(state, spv::OpBitwiseOr, u64, low, high); +} + +namespace { + +std::uint32_t rootAddress(SpirvEmitterState& state, RuntimeAbi::RootItem item) { + return ComposeAddress64(state, [&] { return EmitRootItemDword(state, item, 0u); }, [&] { return EmitRootItemDword(state, item, 1u); }); +} + +std::uint32_t wordPointer(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index) { + const auto u64 = TypeScalarU64(state); + const auto offset = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, index), BdaConstant(state, 2u)); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, Binary(state, spv::OpIAdd, u64, base, offset)); + return pointer; +} + +std::uint32_t readOnlyWordPointer(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index) { + const auto words = ReadOnlyStorageBufferType(state, TypeU32(state), 4u); + const auto block = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, words), block, base); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePhysicalU32Pointer(state), pointer, block, ConstantU32(state, 0u), index); + return pointer; +} + +} + +std::uint32_t EmitAddressWordPointer(SpirvEmitterState& state, std::uint32_t base, std::uint32_t index) { + return wordPointer(state, base, index); +} + +std::uint32_t EmitRootAddressWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index) { + return wordPointer(state, rootAddress(state, item), index); +} + +std::uint32_t EmitRootAddressReadOnlyWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index) { + return readOnlyWordPointer(state, rootAddress(state, item), index); +} + +std::uint32_t EmitRootAddress(SpirvEmitterState& state, RuntimeAbi::RootItem item) { + return rootAddress(state, item); +} + +std::uint32_t EmitStageRingWordPointer(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index) { + return readOnlyWordPointer(state, rootAddress(state, item), index); +} + +namespace { + +std::uint32_t dataAddressLoad(SpirvEmitterState& state, RuntimeAbi::RootItem item, std::uint32_t index) { + if (item == RuntimeAbi::RootItem::ShaderData && state.shaderDataUniformVariable != 0u) { + const auto u32 = TypeU32(state); + const auto vector = Binary(state, spv::OpShiftRightLogical, u32, index, ConstantU32(state, 2u)); + const auto component = Binary(state, spv::OpBitwiseAnd, u32, index, ConstantU32(state, 3u)); + const auto u32x4 = TypeU32Vector(state, 4); + const auto element = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassUniform, u32x4), element, state.shaderDataUniformVariable, ConstantU32(state, 0u), vector); + const auto words = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32x4, words, element); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpVectorExtractDynamic, u32, value, words, component); + return value; + } + const auto pointer = EmitStageRingWordPointer(state, item, index); + return EmitAlignedLoadU32(state, pointer); +} + +} + +bool HasFlattenedSrt(const SpirvEmitterState& state) { + return state.flattenedSrtVariable != 0 || state.flattenedSrtAddressed; +} + +std::uint32_t EmitShaderDataWordLoad(SpirvEmitterState& state, std::uint32_t index) { + if (state.program.Metadata().bindings.UsesDataAddress()) return dataAddressLoad(state, RuntimeAbi::RootItem::ShaderData, index); + if (state.shaderDataStorageVariable == 0) FailEmit("shader data is neither in push constants nor in a storage buffer"); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0u), index); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; +} + +std::uint32_t EmitFlattenedSrtLoad(SpirvEmitterState& state, std::uint32_t index) { + if (state.flattenedSrtAddressed) { + const auto u32 = TypeU32(state); + const auto dwords = EmitRootItemDword(state, RuntimeAbi::RootItem::SrtDwords, 0u); + const auto inBounds = Binary(state, spv::OpULessThan, TypeBool(state), index, dwords); + const auto last = Select(state, u32, Binary(state, spv::OpIEqual, TypeBool(state), dwords, ConstantU32(state, 0u)), ConstantU32(state, 0u), Binary(state, spv::OpISub, u32, dwords, ConstantU32(state, 1u))); + const auto clamped = Select(state, u32, inBounds, index, last); + return Select(state, u32, inBounds, dataAddressLoad(state, RuntimeAbi::RootItem::SrtAddress, clamped), ConstantU32(state, 0u)); + } + if (state.flattenedSrtVariable == 0) FailEmit("the flattened SRT descriptor is missing"); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), index); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; +} + std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex) { const IrBindingLayout& layout = state.program.Metadata().bindings; + if (layout.UsesDataAddress()) { + const auto pushed = std::ranges::find(layout.pushedData, dwordIndex); + const auto ring = dataAddressLoad(state, RuntimeAbi::RootItem::ShaderData, ConstantU32(state, dwordIndex)); + if (pushed == layout.pushedData.end()) return ring; + return EmitPushedDataSelect(state, static_cast(pushed - layout.pushedData.begin()), ring); + } const auto pointer = state.module.AllocateId(); const auto value = state.module.AllocateId(); if (layout.UsesPushData()) { @@ -59,6 +211,29 @@ void EmitMemoryOffsets(SpirvEmitterState& state) { } } +void EmitHeapSlotPreload(SpirvEmitterState& state) { + if (!state.gpuResolvedImages || (state.shaderDataStorageVariable == 0 && !state.program.Metadata().bindings.UsesDataAddress())) return; + const IrBindingLayout& layout = state.program.Metadata().bindings; + std::vector dwords; + for (auto dword = layout.SamplerSlotDword(); dword < layout.ShaderDataDwords(); ++dword) dwords.push_back(dword); + for (std::uint32_t image = 0; image < layout.runtimeImageCount; ++image) dwords.push_back(layout.ImageFirstElementDword(image)); + const auto pointerType = TypePointer(state, spv::StorageClassPrivate, TypeU32(state)); + for (const auto dword : dwords) { + if (dword >= layout.ShaderDataDwords() || state.heapSlotVariables.contains(dword)) continue; + const auto variable = state.module.DefineGlobalVariable(pointerType, spv::StorageClassPrivate); + state.module.AddFunction(spv::OpStore, variable, EmitShaderDataDwordLoad(state, dword)); + state.heapSlotVariables.emplace(dword, variable); + } +} + +std::uint32_t LoadHeapSlotWord(SpirvEmitterState& state, std::uint32_t dword) { + const auto found = state.heapSlotVariables.find(dword); + if (found == state.heapSlotVariables.end()) return EmitShaderDataDwordLoad(state, dword); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, found->second); + return value; +} + std::uint32_t LdsDwordCount(const SpirvEmitterState& state) { const auto* workgroup = ShaderWorkgroupInput(state); if (workgroup != nullptr) return workgroup->ldsSizeDwords; @@ -248,13 +423,14 @@ std::uint32_t LdsStorageDwords(const SpirvEmitterState& state) { bool LoadsSelectable(const SpirvEmitterState& state, const MemoryResourceAccess& access) { if (access.kind == ResourceKind::Scratch) return true; + if (access.physical) return access.clampedPhysical; if (access.kind == ResourceKind::Lds) return false; return state.robustBufferAccess2; } std::uint32_t GuardedElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t condition, std::uint32_t index) { if (access.kind == ResourceKind::Lds) FailEmit("LDS loads keep their branch and have no guarded element"); - const auto fallback = access.kind == ResourceKind::Scratch ? 0u : 0xfffffffeu; + const auto fallback = access.kind == ResourceKind::Scratch || access.physical ? 0u : 0xfffffffeu; return Select(state, TypeU32(state), condition, index, ConstantU32(state, fallback)); } @@ -278,6 +454,10 @@ std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryRes } std::uint32_t EmitStorageBufferElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index, std::uint32_t pointerType) { + if (access.physical) { + if (pointerType == TypeStorageBufferU64ElementPointer(state)) pointerType = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeScalarU64(state)); + else pointerType = TypePhysicalU32Pointer(state); + } const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, access.objectPointer, ConstantU32(state, 0), index); return pointer; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp index c4a0e26277db..5862b43cb8a7 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvConstants.cpp @@ -1,6 +1,8 @@ #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" +#include "PipelineSpecialization.hpp" +#include "RuntimeAbi.hpp" #include #include #include @@ -36,6 +38,10 @@ std::uint32_t ConstantU64(SpirvEmitterState& state, std::uint64_t value) { return state.module.Constant(spv::OpConstantComposite, TypeU64(state), ConstantU32(state, static_cast(value)), ConstantU32(state, static_cast(value >> 32u))); } +std::uint32_t HeapImageSlotCount(SpirvEmitterState& state) { + return state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::HeapImageSlots, RuntimeAbi::GpuImageHeapSlots); +} + std::uint32_t ConstantU32CompositeZero(SpirvEmitterState& state, std::uint32_t components) { if (components < 2u || components > 4u) { FailEmit("u32 composite component count is out of range"); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp index 594574b8131a..60556b56d57b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvDescriptors.cpp @@ -2,9 +2,12 @@ #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "PipelineSpecialization.hpp" +#include "ImageTableAbi.hpp" #include #include +#include #include #include @@ -57,11 +60,7 @@ std::uint32_t DescriptorElementPointer(SpirvEmitterState& state, std::uint32_t r std::uint32_t RuntimeImageDword(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t member) { const auto index = ConstantU32(state, resource); const auto dword = Binary(state, spv::OpIAdd, TypeU32(state), ConstantU32(state, state.program.Metadata().bindings.ImageMetadataDword() + member), Binary(state, spv::OpIMul, TypeU32(state), index, ConstantU32(state, sizeof(RuntimeAbi::ResourceMetadata) / sizeof(std::uint32_t)))); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0u), dword); - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; + return EmitShaderDataWordLoad(state, dword); } std::uint32_t RuntimeMipCount(SpirvEmitterState& state, std::uint32_t resource) { @@ -79,11 +78,44 @@ void DecorateTableNonUniform(SpirvEmitterState& state, std::uint32_t id) { state.module.AddAnnotation(spv::OpDecorate, id, spv::DecorationNonUniform); } +namespace { + +std::uint32_t tableHeapSlot(SpirvEmitterState& state, RuntimeAbi::HeapPart part, std::uint32_t element, std::uint32_t slots) { + using RuntimeAbi::HeapPart; + const auto partIndex = static_cast(part); + const auto u32 = TypeU32(state); + const auto base = LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementBase)); + const auto count = LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementCount)); + std::uint32_t partBase = base; + if (part == HeapPart::Storage || part == HeapPart::Sampler) { + partBase = Binary(state, spv::OpIAdd, u32, base, Binary(state, spv::OpIMul, u32, count, ConstantU32(state, partIndex))); + } else if (part >= HeapPart::Atomic) { + const auto samplers = LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderSamplerCount)); + const auto samplerSpan = Select(state, u32, Binary(state, spv::OpULessThan, TypeBool(state), count, samplers), samplers, count); + const auto extra = Binary(state, spv::OpIMul, u32, count, ConstantU32(state, partIndex - 1u)); + partBase = Binary(state, spv::OpIAdd, u32, Binary(state, spv::OpIAdd, u32, base, samplerSpan), extra); + } + const auto slot = LoadTableMap(state, Binary(state, spv::OpIAdd, u32, partBase, element)); + const auto fallback = RuntimeAbi::HeapPartFirstBinding(part); + const auto clamped = Select(state, u32, Binary(state, spv::OpULessThan, TypeBool(state), slot, slots), slot, ConstantU32(state, fallback)); + DecorateTableNonUniform(state, clamped); + return clamped; +} + +} + +std::uint32_t TableTrailerWord(SpirvEmitterState& state, std::uint32_t field) { + const auto base = LoadTableMap(state, ConstantU32(state, ImageTableAbi::HeaderElementBase)); + return LoadTableMap(state, Binary(state, spv::OpISub, TypeU32(state), base, ConstantU32(state, ImageTableAbi::TrailerWords - field))); +} + std::uint32_t TableStorageImagePointer(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource, std::uint32_t pointerType) { const auto found = std::find_if(state.tableImageVariables.begin(), state.tableImageVariables.end(), [&](const SpirvTableImageVariable& variable) { return variable.kind == kind; }); if (found == state.tableImageVariables.end() || !state.tableAccess.active) ExitDescriptorBindingFailure(state, kind, resource, "image storage table array was not emitted for the mode"); const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, found->variable, state.tableAccess.element); + const auto value = static_cast(kind); + const auto part = RuntimeAbi::StorageHeapPart(value); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, found->variable, state.tableAccess.heap ? tableHeapSlot(state, part, state.tableAccess.element, HeapImageSlotCount(state)) : state.tableAccess.element); DecorateTableNonUniform(state, pointer); return pointer; } @@ -153,6 +185,43 @@ std::uint32_t ImageViewSizeType(SpirvEmitterState& state, RdnaImageDimension dim } } +namespace { + +void RequireDynamicIndexing(SpirvEmitterState& state, std::uint32_t capability, DescriptorBindingKind kind, std::uint32_t resource, const char* reason) { + const auto supported = std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), capability) != state.supportedCapabilities.end(); + if (!supported) ExitDescriptorBindingFailure(state, kind, resource, reason); + state.module.EmitCapability(capability); +} + +} + +std::uint32_t HeapImageSlot(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource) { + RequireDynamicIndexing(state, spv::CapabilitySampledImageArrayDynamicIndexing, kind, resource, "the image heap needs image array dynamic indexing, which the device lacks"); + return LoadHeapSlotWord(state, state.program.Metadata().bindings.ImageFirstElementDword(resource)); +} + +std::uint32_t HeapImageSlotWord(SpirvEmitterState& state, DescriptorBindingKind kind, std::uint32_t resource, std::uint32_t arrayIndex) { + RequireDynamicIndexing(state, spv::CapabilitySampledImageArrayDynamicIndexing, kind, resource, "the image heap needs image array dynamic indexing, which the device lacks"); + const auto& layout = state.program.Metadata().bindings; + auto dword = layout.ImageSlotDword(); + for (const auto& descriptor : layout.descriptors) { + if (descriptor.kind == kind) break; + if (state.program.Metadata().bindings.HeapImageKind(descriptor.kind)) dword += static_cast(descriptor.resources.size()); + } + dword += arrayIndex; + if (dword >= layout.ShaderDataDwords()) ExitDescriptorBindingFailure(state, kind, resource, "image heap slot word lies outside the shader data"); + return LoadHeapSlotWord(state, dword); +} + +std::uint32_t StorageElementPointer(SpirvEmitterState& state, std::uint32_t pointerType, std::uint32_t variable, std::uint32_t arrayIndex, DescriptorBindingKind kind, std::uint32_t resource) { + if (!state.gpuResolvedImages || !state.program.Metadata().bindings.HeapImageKind(kind)) return DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "storage image descriptor array was not emitted"); + if (variable == 0) ExitDescriptorBindingFailure(state, kind, resource, "storage image heap array was not emitted"); + RequireDynamicIndexing(state, spv::CapabilityStorageImageArrayDynamicIndexing, kind, resource, "the storage image heap needs storage image array dynamic indexing, which the device lacks"); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, state.registrationModules ? HeapImageSlot(state, kind, resource) : HeapImageSlotWord(state, kind, resource, arrayIndex)); + return pointer; +} + std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t resource) { const auto& imageResource = state.runtimeImage != nullptr ? *state.runtimeImage : state.program.Info().images.at(resource); if (imageResource.resourceClass != ImageResourceClass::Sampled) { @@ -165,7 +234,7 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t const auto found = std::find_if(state.tableImageVariables.begin(), state.tableImageVariables.end(), [&](const SpirvTableImageVariable& variable) { return variable.kind == kind; }); if (found == state.tableImageVariables.end() || !state.tableAccess.active) ExitDescriptorBindingFailure(state, kind, resource, "image table array was not emitted for the mode"); const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, found->variable, state.tableAccess.element); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, found->variable, state.tableAccess.heap ? tableHeapSlot(state, RuntimeAbi::HeapPart::Sampled, state.tableAccess.element, HeapImageSlotCount(state)) : state.tableAccess.element); if (!state.tableAccess.uniform) DecorateTableNonUniform(state, pointer); const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); @@ -177,7 +246,13 @@ std::uint32_t LoadSampledImageDescriptor(SpirvEmitterState& state, std::uint32_t if (variable == 0) { ExitDescriptorBindingFailure(state, kind, resource, "sampled image descriptor array was not emitted"); } - const auto pointer = DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "sampled image descriptor array was not emitted"); + std::uint32_t pointer = 0; + if (state.gpuResolvedImages) { + pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, variable, state.registrationModules ? HeapImageSlot(state, kind, resource) : HeapImageSlotWord(state, kind, resource, arrayIndex)); + } else { + pointer = DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "sampled image descriptor array was not emitted"); + } const auto image = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, image, pointer); return image; @@ -195,7 +270,7 @@ std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t samp const auto variant = image != nullptr && image->depthCompare ? 2u : 0u; const auto element = Binary(state, spv::OpIAdd, TypeU32(state), state.tableAccess.samplerElement, ConstantU32(state, variant + (point ? 1u : 0u))); const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, state.tableSamplerVariable, element); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, state.tableSamplerVariable, state.TableHeap() ? tableHeapSlot(state, RuntimeAbi::HeapPart::Sampler, element, ConstantU32(state, RuntimeAbi::GpuSamplerHeapSlots)) : element); DecorateTableNonUniform(state, pointer); const auto loaded = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, samplerType, loaded, pointer); @@ -203,7 +278,16 @@ std::uint32_t LoadSamplerDescriptor(SpirvEmitterState& state, std::uint32_t samp return loaded; } const auto arrayIndex = ResourceForDescriptor(state, DescriptorBindingKind::Samplers, sampler) + (point ? 1u : 0u); - const auto pointer = DescriptorElementPointer(state, pointerType, state.samplerVariable, arrayIndex, DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); + std::uint32_t pointer = 0; + if (state.gpuResolvedImages) { + if (state.samplerVariable == 0) ExitDescriptorBindingFailure(state, DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); + state.module.EmitCapability(spv::CapabilitySampledImageArrayDynamicIndexing); + const auto slot = LoadHeapSlotWord(state, state.program.Metadata().bindings.SamplerSlotDword() + arrayIndex); + pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, pointerType, pointer, state.samplerVariable, slot); + } else { + pointer = DescriptorElementPointer(state, pointerType, state.samplerVariable, arrayIndex, DescriptorBindingKind::Samplers, sampler, "sampler descriptor array was not emitted"); + } const auto samplerId = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, samplerType, samplerId, pointer); return samplerId; @@ -230,7 +314,7 @@ std::uint32_t StorageImageDescriptorPointer(SpirvEmitterState& state, std::uint3 if (image.table != NoTable) return TableStorageImagePointer(state, kind, resource, pointerType); const auto arrayIndex = ResourceForDescriptor(state, kind, resource); const auto variable = state.imageVariables.at(ImageBindingIndex(kind)); - return DescriptorElementPointer(state, pointerType, variable, arrayIndex, kind, resource, "storage image descriptor array was not emitted"); + return StorageElementPointer(state, pointerType, variable, arrayIndex, kind, resource); } std::uint32_t EmitStorageImageRead(SpirvEmitterState& state, std::uint32_t resource, std::uint32_t coord, std::uint32_t sample) { @@ -276,7 +360,7 @@ void EmitStorageImageWrite(SpirvEmitterState& state, std::uint32_t resource, std const auto arrayIndex = ResourceForDescriptor(state, kind, resource); const auto variable = state.imageVariables.at(ImageBindingIndex(kind)); const auto loadAt = [&](std::uint32_t index) { - const auto pointer = DescriptorElementPointer(state, pointerType, variable, index, kind, resource, "storage image descriptor array was not emitted"); + const auto pointer = StorageElementPointer(state, pointerType, variable, index, kind, resource); const auto descriptor = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, imageType, descriptor, pointer); return descriptor; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index e7cc195fa0af..ee376662ce47 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -141,7 +141,7 @@ std::uint32_t StorageBufferU64BlockType(SpirvEmitterState& state) { } std::uint32_t PushConstantArrayType(SpirvEmitterState& state) { - const auto count = ConstantU32(state, PushData::DwordCount); + const auto count = ConstantU32(state, state.program.Metadata().bindings.UsesDataAddress() || GpuScratchPool(state) ? RuntimeAbi::PushDataDwords : PushData::DwordCount); return state.module.DecoratedType(spv::OpTypeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, static_cast(sizeof(std::uint32_t))}}}, TypeU32(state), count); @@ -168,7 +168,7 @@ void CheckBindings(const IrProgram& program, const CompiledBindingLayout& bindin void EmitBaseHeader(SpirvModule& module, const IrProgram& program) { module.EmitCapability(spv::CapabilityShader); - const bool physical = program.Info().usesDma || program.Resources().stage == IrShaderStage::Mesh; + const bool physical = program.Info().usesDma || program.Resources().stage == IrShaderStage::Mesh || program.Metadata().bindings.bufferAddressDwords != 0u || program.Metadata().bindings.UsesDataAddress(); if (physical) { module.EmitCapability(spv::CapabilityInt64); module.EmitCapability(spv::CapabilityPhysicalStorageBufferAddresses); @@ -470,32 +470,52 @@ void DefineOutputs(SpirvEmitterState& state) { } } +namespace { + +std::uint32_t defineHeapArray(SpirvEmitterState& state, std::uint32_t slotCount, std::uint32_t elementType, const char* name, std::uint32_t heapKind) { + const auto type = state.module.Type(spv::OpTypeArray, elementType, slotCount); + const auto variable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassUniformConstant, type), spv::StorageClassUniformConstant); + state.module.AddName(variable, name); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, RuntimeAbi::FragmentDescriptorSet); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, NativeBinding(IrShaderStage::Pixel, static_cast(heapKind))); + return variable; +} + +} + void DefineDescriptors(SpirvEmitterState& state) { const IrBindingLayout& layout = state.program.Metadata().bindings; const IrShaderStage stage = state.program.Resources().stage; - if (layout.UsesPushData() || state.stencilReplay || stage == IrShaderStage::Mesh || stage == IrShaderStage::TessellationControl || (state.depthRangeRemap && (stage == IrShaderStage::Vertex || stage == IrShaderStage::TessellationEvaluation))) { + if (layout.UsesPushData() || layout.UsesDataAddress() || GpuScratchPool(state) || state.stencilReplay || stage == IrShaderStage::Mesh || stage == IrShaderStage::TessellationControl || (state.depthRangeRemap && (stage == IrShaderStage::Vertex || stage == IrShaderStage::TessellationEvaluation))) { const auto type = PushConstantBlockType(state); state.pushConstantVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassPushConstant, type), spv::StorageClassPushConstant); state.module.AddName(type, "BufferResource"); state.module.AddName(state.pushConstantVariable, "vsharp"); } for (const IrDescriptorBinding& binding : layout.descriptors) { + const bool gpuHeap = SharedHeapArray(layout, binding.kind, state.gpuResolvedImages); const auto Define = [&](std::uint32_t type, const char* name, std::uint32_t storage = spv::StorageClassStorageBuffer) { const auto variable = state.module.DefineGlobalVariable(TypePointer(state, storage, type), storage); state.module.AddName(variable, name); - state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, state.fragmentDescriptorSet && stage == IrShaderStage::Pixel ? RuntimeAbi::FragmentDescriptorSet : RuntimeAbi::DescriptorSet); - state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, NativeBinding(stage, binding.kind)); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationDescriptorSet, DescriptorSetNumber(layout, binding.kind, stage, state.gpuResolvedImages, state.fragmentDescriptorSet, false)); + state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationBinding, DescriptorBindingNumber(layout, binding.kind, stage, state.gpuResolvedImages)); return variable; }; const auto ArrayType = [&](std::uint32_t type) { const bool heap = binding.kind == DescriptorBindingKind::Samplers || ImageBindingResourceClass(binding.kind) != ImageResourceClass::None; - const auto count = static_cast(binding.resources.size()); - const auto length = heap && !state.registrationModules ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::HeapCountBase + static_cast(binding.kind), count) : ConstantU32(state, count); + const bool samplers = binding.kind == DescriptorBindingKind::Samplers; + const auto count = gpuHeap ? (samplers ? RuntimeAbi::GpuSamplerHeapSlots : RuntimeAbi::GpuImageHeapSlots) : static_cast(binding.resources.size()); + const auto length = gpuHeap && !samplers ? HeapImageSlotCount(state) : heap && !gpuHeap && !state.registrationModules ? state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::HeapCountBase + static_cast(binding.kind), count) : ConstantU32(state, count); return state.module.Type(spv::OpTypeArray, type, length); }; switch (binding.kind) { - case DescriptorBindingKind::Buffers: + case DescriptorBindingKind::Buffers: { + if (layout.bufferAddressDwords != 0u && !state.heapBufferDescriptors) { + state.addressedBuffers = true; + break; + } state.storageBufferVariable = Define(ArrayType(StorageBufferBlockType(state)), "buffers"); + if (layout.UsesDataAddress()) state.pushItems |= RuntimeAbi::RootItemBit(RuntimeAbi::RootItem::FlattenedSrt); if (state.requirements.coherentBufferAlias) state.storageBufferCoherentVariable = Define(ArrayType(StorageBufferBlockType(state)), "buffers_coherent"); if (state.requirements.bufferU64View) state.storageBufferU64Variable = Define(ArrayType(StorageBufferU64BlockType(state)), "buffers_u64"); if (state.storageBufferU64Variable != 0u || state.storageBufferCoherentVariable != 0u) { @@ -508,25 +528,48 @@ void DefineDescriptors(SpirvEmitterState& state) { if (state.storageBufferU64Variable != 0u) state.module.AddAnnotation(spv::OpDecorate, state.storageBufferU64Variable, spv::DecorationCoherent); } break; + } case DescriptorBindingKind::BdaPagetable: - state.bdaPagetableVariable = Define(StorageBufferBlockType(state), "bda_pagetable"); - state.module.AddAnnotation(spv::OpDecorate, state.bdaPagetableVariable, spv::DecorationNonWritable); + if (layout.UsesDataAddress()) { + state.bdaPagetableVariable = state.module.AllocateId(); + } else { + state.bdaPagetableVariable = Define(StorageBufferBlockType(state), "bda_pagetable"); + state.module.AddAnnotation(spv::OpDecorate, state.bdaPagetableVariable, spv::DecorationNonWritable); + } break; case DescriptorBindingKind::FaultBuffer: - state.faultBufferVariable = Define(StorageBufferBlockType(state), "fault_buffer"); + if (layout.UsesDataAddress()) { + state.faultBufferVariable = state.module.AllocateId(); + } else { + state.faultBufferVariable = Define(StorageBufferBlockType(state), "fault_buffer"); + } break; case DescriptorBindingKind::ShaderData: - state.shaderDataStorageVariable = Define(StorageBufferBlockType(state), "shader_data"); + if (!layout.UsesDataAddress()) { + state.shaderDataStorageVariable = Define(StorageBufferBlockType(state), "shader_data"); + state.module.AddAnnotation(spv::OpDecorate, state.shaderDataStorageVariable, spv::DecorationNonWritable); + } else if (layout.ShaderDataDwords() != 0u && layout.ShaderDataDwords() <= state.heapDataUniformDwords) { + const auto vectors = (layout.ShaderDataDwords() + 3u) / 4u; + const auto array = state.module.DecoratedType(spv::OpTypeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, 16u}}}, TypeU32Vector(state, 4), ConstantU32(state, vectors)); + const auto block = state.module.DecoratedType(spv::OpTypeStruct, {{spv::OpDecorate, {spv::DecorationBlock}}, {spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}}, array); + state.shaderDataUniformVariable = Define(block, "shader_data", spv::StorageClassUniform); + state.pushItems |= RuntimeAbi::RootItemBit(RuntimeAbi::RootItem::ShaderData); + } break; case DescriptorBindingKind::FlattenedSrt: - state.flattenedSrtVariable = Define(StorageBufferBlockType(state), "flattened_srt"); - state.module.AddAnnotation(spv::OpDecorate, state.flattenedSrtVariable, spv::DecorationNonWritable); + if (layout.UsesDataAddress() && !state.heapBufferDescriptors) state.flattenedSrtAddressed = true; + else { + state.flattenedSrtVariable = Define(StorageBufferBlockType(state), "flattened_srt"); + if (layout.UsesDataAddress()) state.pushItems |= RuntimeAbi::RootItemBit(RuntimeAbi::RootItem::FlattenedSrt); + if (!layout.UsesDataAddress()) state.module.AddAnnotation(spv::OpDecorate, state.flattenedSrtVariable, spv::DecorationNonWritable); + } break; case DescriptorBindingKind::Samplers: state.samplerVariable = Define(ArrayType(state.module.Type(spv::OpTypeSampler)), "samplers", spv::StorageClassUniformConstant); break; case DescriptorBindingKind::Gds: state.gdsVariable = Define(StorageBufferBlockType(state), "gds"); + if (layout.UsesDataAddress()) state.pushItems |= RuntimeAbi::RootItemBit(RuntimeAbi::RootItem::GdsAddress); state.module.AddAnnotation(spv::OpDecorate, state.gdsVariable, spv::DecorationCoherent); break; case DescriptorBindingKind::ImageTable: @@ -542,18 +585,35 @@ void DefineDescriptors(SpirvEmitterState& state) { if ((ImageBindingResourceClass(kind) == ImageResourceClass::Storage) != (binding.kind == DescriptorBindingKind::ImageStorageTable)) continue; if (std::any_of(state.tableImageVariables.begin(), state.tableImageVariables.end(), [&](const SpirvTableImageVariable& variable) { return variable.kind == kind; })) continue; const auto name = "table_images_" + std::to_string(static_cast(kind)); - state.tableImageVariables.push_back({kind, Define(state.module.Type(spv::OpTypeRuntimeArray, ImageType(state, mode)), name.c_str(), spv::StorageClassUniformConstant)}); + if (state.TableHeap()) { + const auto value = static_cast(kind); + const auto part = RuntimeAbi::StorageHeapPart(value); + const auto heapKind = binding.kind != DescriptorBindingKind::ImageStorageTable ? RuntimeAbi::FirstImageBinding : part == RuntimeAbi::HeapPart::Atomic || part == RuntimeAbi::HeapPart::Atomic64 ? RuntimeAbi::HeapPartFirstBinding(part) : RuntimeAbi::FirstStorageImageBinding; + const auto slotCount = HeapImageSlotCount(state); + state.tableImageVariables.push_back({kind, defineHeapArray(state, slotCount, ImageType(state, mode), name.c_str(), heapKind)}); + } else { + state.tableImageVariables.push_back({kind, Define(state.module.Type(spv::OpTypeRuntimeArray, ImageType(state, mode)), name.c_str(), spv::StorageClassUniformConstant)}); + } if (mode.dimension == RdnaImageDimension::Dim1D || mode.dimension == RdnaImageDimension::Dim1DArray) state.module.EmitCapability(binding.kind == DescriptorBindingKind::ImageStorageTable ? spv::CapabilityImage1D : spv::CapabilitySampled1D); } } break; case DescriptorBindingKind::SamplerTable: RequireTableIndexing(state); - state.tableSamplerVariable = Define(state.module.Type(spv::OpTypeRuntimeArray, state.module.Type(spv::OpTypeSampler)), "table_samplers", spv::StorageClassUniformConstant); + if (state.gpuResolvedImages && layout.UsesDataAddress()) { + const auto slotCount = ConstantU32(state, RuntimeAbi::GpuSamplerHeapSlots); + state.tableSamplerVariable = defineHeapArray(state, slotCount, state.module.Type(spv::OpTypeSampler), "table_samplers", static_cast(DescriptorBindingKind::Samplers)); + } else { + state.tableSamplerVariable = Define(state.module.Type(spv::OpTypeRuntimeArray, state.module.Type(spv::OpTypeSampler)), "table_samplers", spv::StorageClassUniformConstant); + } break; case DescriptorBindingKind::ImageTableMap: - state.tableMapVariable = Define(StorageBufferBlockType(state), "table_map"); - state.module.AddAnnotation(spv::OpDecorate, state.tableMapVariable, spv::DecorationNonWritable); + if (layout.UsesDataAddress()) { + state.tableMapVariable = state.module.AllocateId(); + } else { + state.tableMapVariable = Define(StorageBufferBlockType(state), "table_map"); + state.module.AddAnnotation(spv::OpDecorate, state.tableMapVariable, spv::DecorationNonWritable); + } break; default: { if (ImageBindingResourceClass(binding.kind) == ImageResourceClass::None) { @@ -576,13 +636,13 @@ void DefineDescriptors(SpirvEmitterState& state) { if (LdsInDeviceMemory(state)) { state.ldsBufferVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferBlockType(state)), spv::StorageClassStorageBuffer); state.module.AddName(state.ldsBufferVariable, "workgroup_memory"); - state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationDescriptorSet, WorkgroupMemoryDescriptorSet); + state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationDescriptorSet, state.gpuResolvedImages ? GpuWorkgroupMemoryDescriptorSet : WorkgroupMemoryDescriptorSet); state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationBinding, 0u); state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationCoherent); if (state.requirements.ldsBufferInt64Atomics) { state.ldsBufferU64Variable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferU64BlockType(state)), spv::StorageClassStorageBuffer); state.module.AddName(state.ldsBufferU64Variable, "workgroup_memory_u64"); - state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferU64Variable, spv::DecorationDescriptorSet, WorkgroupMemoryDescriptorSet); + state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferU64Variable, spv::DecorationDescriptorSet, state.gpuResolvedImages ? GpuWorkgroupMemoryDescriptorSet : WorkgroupMemoryDescriptorSet); state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferU64Variable, spv::DecorationBinding, 0u); state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferU64Variable, spv::DecorationCoherent); for (const auto variable : {state.ldsBufferVariable, state.ldsBufferU64Variable}) state.module.AddAnnotation(spv::OpDecorate, variable, spv::DecorationAliased); @@ -617,7 +677,7 @@ void DefineScratchPool(SpirvEmitterState& state) { if (!ScratchPoolActive(state)) return; state.scratchPoolVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferBlockType(state)), spv::StorageClassStorageBuffer); state.module.AddName(state.scratchPoolVariable, "scratch_pool"); - state.module.AddAnnotation(spv::OpDecorate, state.scratchPoolVariable, spv::DecorationDescriptorSet, ScratchPoolDescriptorSet); + state.module.AddAnnotation(spv::OpDecorate, state.scratchPoolVariable, spv::DecorationDescriptorSet, GpuScratchPool(state) ? GpuScratchPoolDescriptorSet : ScratchPoolDescriptorSet); state.module.AddAnnotation(spv::OpDecorate, state.scratchPoolVariable, spv::DecorationBinding, 0u); state.module.AddAnnotation(spv::OpDecorate, state.scratchPoolVariable, spv::DecorationCoherent); state.scratchPoolSlotVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassWorkgroup, TypeU32(state)), spv::StorageClassWorkgroup); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 009d1cddde5f..b05c83306e1b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -3,6 +3,7 @@ #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "RdnaDecoder/RdnaInstructionDecoder.hpp" #include #include @@ -1547,7 +1548,7 @@ std::uint32_t AtomicFloatCompareSwap(SpirvEmitterState& state, std::uint32_t old std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { auto& state = ctx.state; - if (state.flattenedSrtVariable == 0) { + if (!HasFlattenedSrt(state)) { ctx.Fail(inst, "requires the flattened SRT descriptor"); } const auto& resources = state.program.Resources(); @@ -1563,13 +1564,7 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { ctx.Fail(inst, "reads a guarded SRT slot without a fault buffer"); } const auto u32 = TypeU32(state); - const auto load = [&](std::uint32_t index) { - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), index); - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, u32, value, pointer); - return value; - }; + const auto load = [&](std::uint32_t index) { return EmitFlattenedSrtLoad(state, index); }; const auto guard = static_cast(found - guarded.begin()); const auto flag = load(ConstantU32(state, resources.srtGuardOffset + guard)); const auto poisoned = Binary(state, spv::OpINotEqual, TypeBool(state), flag, ConstantU32(state, 0u)); @@ -1593,11 +1588,7 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { StopBdaInvocationIf(state, poisoned); } } - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), ctx.Arg(inst, 1)); - const auto value = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); - return value; + return EmitFlattenedSrtLoad(state, ctx.Arg(inst, 1)); } void EmitReadConstBuffer(SpirvValueEmitContext& ctx, const IrValue& inst) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index 7816224bbdf6..a7de1f6e2784 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -307,12 +307,12 @@ void PlanMeshWindowBuffer(SpirvEmitterState& state) { } } -bool WritesGuestMemory(const IrProgram& program) { +} + +bool WritesGuestBuffers(const IrProgram& program) { const auto& info = program.Info(); - const auto writtenImage = [](const ImageResource& image) { return !image.targetMerge && (image.written || image.atomic || image.atomic64); }; - const auto writtenModes = [&](const std::vector& modes) { return std::ranges::any_of(modes, writtenImage); }; const auto writtenBuffer = [](const BufferResource& buffer) { return buffer.written || buffer.atomic; }; - if (info.bdaWrites || std::ranges::any_of(info.buffers, writtenBuffer) || std::ranges::any_of(info.images, writtenImage) || std::ranges::any_of(info.runtimeImageModes, writtenModes)) return true; + if (info.bdaWrites || std::ranges::any_of(info.buffers, writtenBuffer)) return true; const auto& memory = program.Resources().memoryInfo; for (const auto& block : program.Blocks()) { for (const IrValue* inst : block->Instructions()) { @@ -326,6 +326,11 @@ bool WritesGuestMemory(const IrProgram& program) { return false; } +bool WritesGuestMemory(const IrProgram& program) { + const auto& info = program.Info(); + const auto writtenImage = [](const ImageResource& image) { return !image.targetMerge && (image.written || image.atomic || image.atomic64); }; + const auto writtenModes = [&](const std::vector& modes) { return std::ranges::any_of(modes, writtenImage); }; + return std::ranges::any_of(info.images, writtenImage) || std::ranges::any_of(info.runtimeImageModes, writtenModes) || WritesGuestBuffers(program); } void DefineModule(SpirvEmitterState& state) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvScratchPool.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvScratchPool.cpp index cf40d62fc2ef..b003e8b35514 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvScratchPool.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvScratchPool.cpp @@ -52,6 +52,10 @@ bool ScratchPoolActive(const SpirvEmitterState& state) { return state.requirements.functionScratch && state.program.Resources().stage == IrShaderStage::Compute; } +bool GpuScratchPool(const SpirvEmitterState& state) { + return state.gpuResolvedImages && ScratchPoolActive(state); +} + std::uint32_t ScratchPoolSlotDwords(const SpirvEmitterState& state) { const auto* workgroup = ShaderWorkgroupInput(state); if (workgroup == nullptr) FailEmit("scratch pool needs a compute workgroup size"); @@ -69,7 +73,14 @@ void EmitScratchPoolAcquire(SpirvEmitterState& state) { const auto device = ConstantU32(state, spv::ScopeDevice); const auto acquire = ConstantU32(state, spv::MemorySemanticsAcquireMask | spv::MemorySemanticsUniformMemoryMask); const auto length = state.module.AllocateId(); - state.module.AddFunction(spv::OpArrayLength, TypeU32(state), length, state.scratchPoolVariable, 0u); + if (GpuScratchPool(state)) { + if (state.pushConstantVariable == 0u) FailEmit("a GPU-mode scratch pool has no push data for its length"); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0u), ConstantU32(state, GpuScratchPoolDwordsPushByte / 4u)); + state.module.AddFunction(spv::OpLoad, TypeU32(state), length, pointer); + } else { + state.module.AddFunction(spv::OpArrayLength, TypeU32(state), length, state.scratchPoolVariable, 0u); + } const auto slotDwords = ScratchPoolSlotDwords(state); const auto slots = EmitBinaryU32(state, spv::OpUDiv, length, ConstantU32(state, slotDwords + HeaderWordsPerSlot)); state.scratchPoolSlotCount = slots; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp index 1265fd1b0280..dbc57f23ae0e 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvTypes.cpp @@ -7,6 +7,15 @@ namespace ShaderRecompiler { +std::uint32_t ReadOnlyStorageBufferType(SpirvEmitterState& state, std::uint32_t element, std::uint32_t stride) { + const auto array = state.module.DecoratedType(spv::OpTypeRuntimeArray, {{spv::OpDecorate, {spv::DecorationArrayStride, stride}}}, element); + return state.module.DecoratedType(spv::OpTypeStruct, + {{spv::OpDecorate, {spv::DecorationBlock}}, + {spv::OpMemberDecorate, {0, spv::DecorationOffset, 0}}, + {spv::OpMemberDecorate, {0, spv::DecorationNonWritable}}}, + array); +} + namespace { std::uint32_t StorageBufferType(SpirvEmitterState& state) { @@ -137,6 +146,14 @@ std::uint32_t TypeStorageBufferU8ElementPointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassStorageBuffer, TypeU8(state)); } +std::uint32_t TypePhysicalReadOnlyBufferPointer(SpirvEmitterState& state, bool wide) { + return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, wide ? ReadOnlyStorageBufferType(state, TypeScalarU64(state), 8u) : ReadOnlyStorageBufferType(state, TypeU32(state), 4u)); +} + +std::uint32_t TypePhysicalBufferPointer(SpirvEmitterState& state, bool wide) { + return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, wide ? StorageBufferU64Type(state) : StorageBufferType(state)); +} + std::uint32_t TypePhysicalU32Pointer(SpirvEmitterState& state) { return TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 5460e4850ffe..16c6c1afb6dc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -1,10 +1,12 @@ #include "SpirvBackend/SpirvModuleEmitter.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvFlowEmitter.hpp" #include "SpirvBackend/SpirvBda.hpp" #include +#include "SpirvBackend/SpirvImageEmitter.hpp" #include #include #include "PipelineSpecialization.hpp" @@ -890,6 +892,7 @@ std::uint32_t RemapPositionDepth(SpirvEmitterState& state, std::uint32_t positio state.module.AddFunction(spv::OpCompositeExtract, f32, components.at(i), position, i); } const auto mode = state.module.SpecializationConstant(u32, PipelineSpecialization::DepthRemapBase, remap.mode); + state.pushItems |= RuntimeAbi::DepthRemapPushBit; if (state.pushConstantVariable == 0u) throw std::runtime_error("depth remap push constants were not declared"); const auto pushFloat = [&](std::uint32_t dword) { const auto pointer = state.module.AllocateId(); @@ -1818,6 +1821,7 @@ void EmitProgram(SpirvEmitterState& state) { state.labels.emplace(block, label); } DefineGetBdaPointer(state); + DefineDeviceImageLookup(state); DefineDot2F32F16Function(state); DefineIntegerFloat64Functions(state); DefineReciprocalCorrectionFunction(state); @@ -1887,7 +1891,9 @@ void EmitProgram(SpirvEmitterState& state) { }); state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state))); } - if (state.gdsVariable != 0u) { + if (state.gdsVariable != 0u && state.program.Metadata().bindings.UsesDataAddress()) { + state.gdsLength = ConstantU32(state, RuntimeAbi::GdsBytes / 4u); + } else if (state.gdsVariable != 0u) { state.gdsLength = state.module.AllocateId(); state.module.AddFunction(spv::OpArrayLength, TypeU32(state), state.gdsLength, state.gdsVariable, 0u); } @@ -1925,6 +1931,7 @@ void EmitProgram(SpirvEmitterState& state) { state.module.AddFunction(spv::OpBeginInvocationInterlockEXT); } EmitMemoryOffsets(state); + EmitHeapSlotPreload(state); if (state.program.Resources().stage == IrShaderStage::Pixel && state.inputInfo.pixel != nullptr && state.inputInfo.pixel->QuadMasked()) { const auto& pixel = *state.inputInfo.pixel; const auto quad = EmitQuadPixel(state); @@ -2070,9 +2077,7 @@ std::uint32_t LoadMeshArgumentWord(SpirvEmitterState& state, std::uint32_t addre const auto element = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, Binary(state, spv::OpShiftLeftLogical, TypeScalarU64(state), Unary(state, spv::OpUConvert, TypeScalarU64(state), dword), ConstantU32(state, 2u))); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state)), pointer, element); - const auto loaded = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), loaded, pointer, spv::MemoryAccessAlignedMask, 4u); - return loaded; + return EmitAlignedLoadU32(state, pointer); } } @@ -2119,8 +2124,7 @@ std::uint32_t EmitMeshArgument(SpirvValueEmitContext& ctx, const IrValue& inst) const auto element = Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, 4ull * index)); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, TypeU32(state)), pointer, element); - const auto loaded = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), loaded, pointer, spv::MemoryAccessAlignedMask, 4u); + const auto loaded = EmitAlignedLoadU32(state, pointer); state.module.AddFunction(spv::OpBranch, merge); EmitLabel(state, merge); const auto value = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 46a1842e819e..d91059161cd9 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -3,6 +3,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvBufferFormat.hpp" #include "SpirvBackend/SpirvMemory/SpirvSubgroup.hpp" +#include "SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp" #include "SpirvBackend/SpirvMemory/SpirvInputOutput.hpp" #include "PipelineSpecialization.hpp" #include "RdnaDecoder/RdnaDescriptorFormat.hpp" @@ -219,9 +220,36 @@ RuntimeAddress address(SpirvValueEmitContext& context, const IrValue& instructio return {AddBdaAddress(context, instruction, resource.base, Unary(state, spv::OpUConvert, TypeScalarU64(state), bytesAddress.guest), false), bytesAddress.inBounds}; } -MemoryResourceAccess directAccess(SpirvValueEmitContext& context, const IrValue& instruction, bool wide = false, bool plainLoad = false) { +bool readOnlyLoad(SpirvValueEmitContext& context, const IrValue& instruction) { + auto& state = context.state; + if (context.Memory(instruction).coherent || (state.requirements.coherentBuffers && state.program.Metadata().invalidatesCaches)) return false; + if (!state.guestBuffersWritten.has_value()) state.guestBuffersWritten = WritesGuestBuffers(state.program); + return !*state.guestBuffersWritten; +} + +MemoryResourceAccess directAccess(SpirvValueEmitContext& context, const IrValue& instruction, bool wide = false, bool plainLoad = false, bool readOnly = false) { auto& state = context.state; const auto& memory = context.Memory(instruction); + if (state.addressedBuffers) { + const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, memory.resource); + const auto dword = state.program.Metadata().bindings.BufferAddressDword() + element * RuntimeAbi::BufferAddressWords; + const auto u64 = TypeScalarU64(state); + auto address = ComposeAddress64(state, [&] { return EmitShaderDataDwordLoad(state, dword); }, [&] { return EmitShaderDataDwordLoad(state, dword + 1u); }); + MemoryResourceAccess access; + access.kind = memory.kind; + access.physical = true; + access.length = Binary(state, spv::OpShiftRightLogical, TypeU32(state), EmitShaderDataDwordLoad(state, dword + 2u), ConstantU32(state, wide ? 3u : 2u)); + if (state.program.Metadata().bindings.UsesDataAddress()) { + const auto empty = Binary(state, spv::OpIEqual, TypeBool(state), access.length, ConstantU32(state, 0u)); + address = Select(state, u64, empty, EmitRootAddress(state, RuntimeAbi::RootItem::ShaderData), address); + access.clampedPhysical = true; + } + access.objectPointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, readOnly ? TypePhysicalReadOnlyBufferPointer(state, wide) : TypePhysicalBufferPointer(state, wide), access.objectPointer, address); + access.byteOffset = state.memoryByteOffsets.at(element); + access.memoryAccess = memory.coherent ? spv::MemoryAccessVolatileMask : 0u; + return access; + } const auto variable = wide ? state.storageBufferU64Variable : plainLoad || state.storageBufferCoherentVariable == 0u ? state.storageBufferVariable : state.storageBufferCoherentVariable; if (variable == 0u) context.Fail(instruction, "direct buffer descriptor array is missing"); const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, memory.resource); @@ -261,23 +289,30 @@ std::uint32_t directWordIndex(SpirvEmitterState& state, const MemoryResourceAcce return EmitAddU32(state, relative, base); } +bool addressedCoherent(SpirvValueEmitContext& context, const IrValue& instruction) { + const auto& state = context.state; + return context.Memory(instruction).coherent || (state.requirements.coherentBuffers && state.program.Metadata().invalidatesCaches); +} + std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t address, std::uint32_t bits, std::uint32_t enable = 0u, std::uint32_t boundedEnable = 0u) { if (context.Memory(instruction).gpuDescriptor) return EmitBdaRead(context, instruction, address, bits); auto& state = context.state; - const auto access = directAccess(context, instruction, false, !context.Memory(instruction).coherent); + const auto access = directAccess(context, instruction, false, !context.Memory(instruction).coherent, readOnlyLoad(context, instruction)); const auto index = directWordIndex(state, access, address); const auto misalignment = baseMisalignment(context, instruction); const auto unaligned = nonzero(state, misalignment); const auto next = EmitAddU32(state, index, ConstantU32(state, 1u)); const auto last = Select(state, TypeU32(state), unaligned, next, index); - const bool exact = state.exactRobustLoads && LoadsSelectable(state, access); + const bool exact = state.exactRobustLoads && !access.physical && LoadsSelectable(state, access); if (!exact && boundedEnable != 0u) enable = boundedEnable; const auto bounds = exact ? 0u : EmitMemoryElementInBounds(state, access, last); const auto inBounds = exact ? 0u : (enable != 0u ? both(state, enable, bounds) : bounds); const auto load = [&](std::uint32_t word) { const auto value = state.module.AllocateId(); const auto pointer = EmitMemoryElementPointer(state, access, word); - if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); + if (access.physical && addressedCoherent(context, instruction)) state.module.AddFunction(spv::OpAtomicLoad, TypeU32(state), value, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone)); + else if (access.physical) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess | spv::MemoryAccessAlignedMask, 4u); + else if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer, access.memoryAccess); else state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); return value; }; @@ -315,10 +350,12 @@ std::uint32_t readBuffer(SpirvValueEmitContext& context, const IrValue& instruct const auto low = Binary(state, spv::OpShiftRightLogical, TypeU32(state), load(lowIndex), shift); const auto high = EmitValueOrZero(state, unaligned, Binary(state, spv::OpShiftLeftLogical, TypeU32(state), load(highIndex), highShift)); const auto value = combine(low, high); - return (access.memoryAccess & spv::MemoryAccessVolatileMask) != 0u || (state.requirements.readOnlyStorage && access.memoryAccess == 0u) ? value : EmitValueOrZero(state, inBounds, value); + const bool robustZero = !access.physical && ((access.memoryAccess & spv::MemoryAccessVolatileMask) != 0u || (state.requirements.readOnlyStorage && access.memoryAccess == 0u)); + return robustZero ? value : EmitValueOrZero(state, inBounds, value); } std::uint32_t byteBuffers(SpirvEmitterState& state) { + if (state.addressedBuffers) return 0u; if (state.storageBufferU8Variable != 0u) return state.storageBufferU8Variable; if (state.program.Resources().stage != IrShaderStage::Compute) return 0u; if (std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(spv::CapabilityStorageBuffer8BitAccess)) == state.supportedCapabilities.end()) return 0u; @@ -387,7 +424,9 @@ void writeBuffer(SpirvValueEmitContext& context, const IrValue& instruction, std if (bits == 32u) { EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { const auto pointer = EmitMemoryElementPointer(state, access, index); - if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess); + if (access.physical && addressedCoherent(context, instruction)) state.module.AddFunction(spv::OpAtomicStore, pointer, ConstantU32(state, spv::ScopeDevice), ConstantU32(state, spv::MemorySemanticsMaskNone), value); + else if (access.physical) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess | spv::MemoryAccessAlignedMask, 4u); + else if (access.memoryAccess != 0u) state.module.AddFunction(spv::OpStore, pointer, value, access.memoryAccess); else state.module.AddFunction(spv::OpStore, pointer, value); }); } @@ -962,7 +1001,8 @@ struct CoherentPair { std::optional coherentPair(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components) { auto& state = context.state; const auto& memory = context.Memory(instruction); - if (components != 2u || !memory.coherent || memory.gpuDescriptor || memory.dataBits != 32u || state.storageBufferU64Variable == 0u) return std::nullopt; + const bool wideView = state.addressedBuffers ? state.requirements.bufferU64View : state.storageBufferU64Variable != 0u; + if (components != 2u || !memory.coherent || memory.gpuDescriptor || memory.dataBits != 32u || !wideView) return std::nullopt; const auto low = address(context, instruction, resource, 0u, 4u); const auto high = address(context, instruction, resource, 4u, 4u); const auto wide = directAccess(context, instruction, true); diff --git a/core/shader/recompiler/tests/HeapTemplateBudgetTests.cpp b/core/shader/recompiler/tests/HeapTemplateBudgetTests.cpp new file mode 100644 index 000000000000..93b380a42883 --- /dev/null +++ b/core/shader/recompiler/tests/HeapTemplateBudgetTests.cpp @@ -0,0 +1,77 @@ +#include "HeapTemplateBudget.hpp" + +#include +#include +#include +#include +#include + +namespace { + +using ShaderRecompiler::HeapTemplateLru; + +void Require(bool condition, const std::string& what) { + if (!condition) throw std::runtime_error(what); +} + +std::string Names(const std::vector& evicted) { + std::string names; + for (const auto& gone : evicted) names += " " + std::to_string(gone.name.owner) + ":" + std::to_string(gone.name.key); + return names.empty() ? " none" : names; +} + +void LeastRecentlyUsedAcrossShaders() { + HeapTemplateLru order(200); + const HeapTemplateLru::Name a{1, 7, 1}, b{2, 7, 2}, c{3, 7, 3}; + Require(order.Insert(a, 100).empty() && order.Insert(b, 100).empty(), "two templates that fit the budget evicted one"); + order.Touch(a); + order.Touch(b); + const auto evicted = order.Insert(c, 100); + Require(evicted.size() == 1 && evicted[0].name.owner == a.owner && evicted[0].name.id == a.id, "insert A, insert B, touch A, touch B, insert C evicted" + Names(evicted) + " instead of A, the least recently used"); + Require(order.Entries() == 2 && order.Bytes() == 200, "the budget does not hold B and C after A was evicted"); + order.Touch(b); + const auto next = order.Insert({4, 7, 4}, 100); + Require(next.size() == 1 && next[0].name.owner == c.owner, "after touching B, the next insert evicted" + Names(next) + " instead of C"); +} + +void StaleTouchAndReplacedTemplate() { + HeapTemplateLru order(200); + Require(order.Insert({1, 5, 1}, 100).empty() && order.Insert({2, 5, 2}, 100).empty(), "two templates that fit the budget evicted one"); + order.Touch({1, 5, 9}); + const auto evicted = order.Insert({3, 5, 3}, 100); + Require(evicted.size() == 1 && evicted[0].name.owner == 1, "a touch naming a template id that is not kept moved the kept template, evicted" + Names(evicted)); + Require(order.Insert({2, 5, 4}, 150).size() == 1 && order.Bytes() == 150 && order.Entries() == 1, "a template replacing the one kept under the same shader and key was charged on top of it"); +} + +void ChargesOwnedArrays() { + ShaderRecompiler::RecompileResult result; + const auto empty = ShaderRecompiler::OwnedBytes(result); + result.imageTableReadRanges.resize(64); + for (auto& ranges : result.imageTableReadRanges) ranges.resize(1024); + const auto readRanges = std::uint64_t{64} * 1024u * sizeof(std::pair); + Require(ShaderRecompiler::OwnedBytes(result) >= empty + readRanges, "a template whose image table read ranges hold " + std::to_string(readRanges) + " bytes was charged " + std::to_string(ShaderRecompiler::OwnedBytes(result) - empty)); + const auto withRanges = ShaderRecompiler::OwnedBytes(result); + result.heapSlotWords.resize(4096); + result.imageTableRanges.resize(4096); + result.imageTablePoison.resize(1024); + const auto arrays = ShaderRecompiler::OwnedBytes(result.heapSlotWords) + ShaderRecompiler::OwnedBytes(result.imageTableRanges) + ShaderRecompiler::OwnedBytes(result.imageTablePoison); + Require(ShaderRecompiler::OwnedBytes(result) >= withRanges + arrays, "a template's heap slot words, image table ranges and poison were not charged"); + result.bindings.resize(8); + for (auto& binding : result.bindings) binding.guestDescriptor.resize(4096); + Require(ShaderRecompiler::OwnedBytes(result) >= withRanges + arrays + 8u * 4096u * sizeof(std::uint32_t), "the guest descriptor words of a template's bindings were not charged"); +} + +} + +int main() { + try { + LeastRecentlyUsedAcrossShaders(); + StaleTouchAndReplacedTemplate(); + ChargesOwnedArrays(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + std::cout << "heap template budget tests passed\n"; + return 0; +} diff --git a/core/shader/recompiler/tests/RequestSerializerTests.cpp b/core/shader/recompiler/tests/RequestSerializerTests.cpp index 135e1969a129..2de7ca0bb6e3 100644 --- a/core/shader/recompiler/tests/RequestSerializerTests.cpp +++ b/core/shader/recompiler/tests/RequestSerializerTests.cpp @@ -174,6 +174,24 @@ std::string EncodeBase64(const std::vector& bytes) { return text; } +std::vector WithoutHeapBufferDescriptors(std::vector bytes) { + Require(bytes.size() > 1u && bytes[4] == 55u && bytes.back() == 0u); + bytes.pop_back(); + return bytes; +} + +std::vector WithHeapBufferDescriptors(std::vector bytes) { + Require(bytes.size() > 4u && bytes[4] == 55u); + bytes.push_back(0u); + return bytes; +} + +std::string EncodeWithDefaultVersion55Tail(std::vector bytes) { + bytes[4] = 55u; + bytes.insert(bytes.end(), 14u, 0u); + return EncodeBase64(bytes); +} + } namespace { @@ -193,6 +211,102 @@ void VerifyTessellationInstancePatches() { } +void VerifyHeapProfile(const ShaderRecompiler::RequestSerializer& serializer) { + struct Profile { + bool gpuResolvedImages; + bool descriptorMemory; + std::uint32_t heapDataUniformDwords; + }; + constexpr std::array profiles{{{false, false, 0u}, {true, true, 0u}, {true, true, 4096u}, {true, true, 16384u}, {true, false, 16384u}, {false, true, 16384u}}}; + for (const auto& profile : profiles) { + ShaderRecompiler::RecompileRequest request{}; + request.target.gpuResolvedImages = profile.gpuResolvedImages; + request.target.descriptorMemory = profile.descriptorMemory; + request.target.heapDataUniformDwords = profile.heapDataUniformDwords; + const auto encoded = serializer.Serialize(request); + const auto replay = serializer.Deserialize(encoded); + Require(replay.request.target.gpuResolvedImages == profile.gpuResolvedImages); + Require(replay.request.target.descriptorMemory == profile.descriptorMemory); + Require(replay.request.target.heapDataUniformDwords == profile.heapDataUniformDwords); + Require(serializer.Serialize(replay.request) == encoded); + auto bytes = DecodeBase64(encoded); + Require(bytes[4] == 55u && bytes.size() > 13u); + bytes.resize(bytes.size() - 13u); + for (const std::uint8_t version : {54u, 53u, 52u}) { + if (version == 52u) bytes.pop_back(); + bytes[4] = version; + const auto older = serializer.Deserialize(EncodeBase64(bytes)).request.target; + Require(!older.gpuResolvedImages && !older.descriptorMemory && older.heapDataUniformDwords == 0u); + } + bytes.push_back(0u); + bytes[4] = 55u; + bool truncated = false; + try { + static_cast(serializer.Deserialize(EncodeBase64(bytes))); + } catch (const std::runtime_error& error) { + truncated = std::string_view(error.what()) == "RequestSerializer: truncated data"; + } + Require(truncated); + bytes = DecodeBase64(encoded); + bytes.push_back(0u); + bool trailing = false; + try { + static_cast(serializer.Deserialize(EncodeBase64(bytes))); + } catch (const std::runtime_error& error) { + trailing = std::string_view(error.what()) == "RequestSerializer: unexpected trailing data"; + } + Require(trailing); + } + ShaderRecompiler::RecompileRequest limits{}; + limits.target.gpuResolvedImages = true; + limits.target.descriptorMemory = true; + limits.target.heapDataUniformDwords = 16384u; + ShaderRecompiler::RecompileRequest addressed = limits; + addressed.target.heapDataUniformDwords = 0u; + Require(serializer.Serialize(limits) != serializer.Serialize(addressed)); + for (const bool descriptors : {false, true}) { + auto request = limits; + request.target.heapBufferDescriptors = descriptors; + const auto encoded = serializer.Serialize(request); + Require(serializer.Deserialize(encoded).request.target.heapBufferDescriptors == descriptors); + Require((encoded == serializer.Serialize(limits)) == !descriptors); + auto bytes = DecodeBase64(encoded); + bytes.resize(bytes.size() - 13u); + bytes[4] = 54u; + Require(!serializer.Deserialize(EncodeBase64(bytes)).request.target.heapBufferDescriptors); + } +} + +void VerifyKeyedTargetFields(const ShaderRecompiler::RequestSerializer& serializer) { + using Field = bool ShaderRecompiler::SpirvTarget::*; + constexpr std::array fields{&ShaderRecompiler::SpirvTarget::ldsLockFences, &ShaderRecompiler::SpirvTarget::meshWindowBuffer, &ShaderRecompiler::SpirvTarget::waveLdsMemoryBarriers, &ShaderRecompiler::SpirvTarget::deviceScopeLdsMemoryBarriers, &ShaderRecompiler::SpirvTarget::exactRobustLoads, &ShaderRecompiler::SpirvTarget::robustBufferAccess2}; + const ShaderRecompiler::RecompileRequest defaults{}; + const auto defaultEncoded = serializer.Serialize(defaults); + for (std::size_t chosen = 0; chosen <= fields.size(); ++chosen) { + ShaderRecompiler::RecompileRequest request{}; + for (std::size_t index = 0; index < fields.size(); ++index) request.target.*fields[index] = chosen == fields.size() || index == chosen; + const auto encoded = serializer.Serialize(request); + Require(chosen == fields.size() ? encoded != defaultEncoded : encoded != defaultEncoded && encoded.size() == defaultEncoded.size()); + const auto replay = serializer.Deserialize(encoded); + for (std::size_t index = 0; index < fields.size(); ++index) Require(replay.request.target.*fields[index] == (chosen == fields.size() || index == chosen)); + Require(serializer.Serialize(replay.request) == encoded); + auto bytes = DecodeBase64(encoded); + Require(bytes[4] == 55u && bytes.size() > 13u); + bytes.resize(bytes.size() - 13u); + for (const std::uint8_t version : {54u, 53u, 52u}) { + if (version == 52u) bytes.pop_back(); + bytes[4] = version; + const auto older = serializer.Deserialize(EncodeBase64(bytes)).request.target; + for (const auto field : fields) Require(!(older.*field)); + } + for (std::size_t index = 0; index < fields.size(); ++index) { + auto other = request; + other.target.*fields[index] = !(other.target.*fields[index]); + Require(serializer.Serialize(other) != encoded); + } + } +} + int main() { VerifyMemoryView(); VerifyTessellationInstancePatches(); @@ -328,8 +442,7 @@ int main() { upgraded.push_back(0u); upgraded.push_back(1u); upgraded.push_back(0u); - upgraded.push_back(0u); - Require(serializer.Serialize(request) == EncodeBase64(upgraded)); + Require(serializer.Serialize(request) == EncodeWithDefaultVersion55Tail(upgraded)); auto truncated = bytes; truncated.pop_back(); bool rejected = false; @@ -343,7 +456,7 @@ int main() { auto vertex23 = bytes; vertex23[4] = 23u; const auto vertexReplay = serializer.Deserialize(EncodeBase64(vertex23)); - Require(serializer.Serialize(vertexReplay.request) == EncodeBase64(upgraded)); + Require(serializer.Serialize(vertexReplay.request) == EncodeWithDefaultVersion55Tail(upgraded)); continue; } for (const bool layer : {false, true}) { @@ -375,8 +488,7 @@ int main() { layered.push_back(0u); layered.push_back(1u); layered.push_back(0u); - layered.push_back(0u); - Require(serializer.Serialize(layerReplay.request) == EncodeBase64(layered)); + Require(serializer.Serialize(layerReplay.request) == EncodeWithDefaultVersion55Tail(layered)); } } } @@ -407,8 +519,7 @@ int main() { bytes.push_back(0u); bytes.push_back(1u); bytes.push_back(0u); - bytes.push_back(0u); - Require(serializer.Serialize(replay.request) == EncodeBase64(bytes)); + Require(serializer.Serialize(replay.request) == EncodeWithDefaultVersion55Tail(bytes)); auto truncated23 = DecodeBase64(encoded23); truncated23.pop_back(); @@ -547,14 +658,13 @@ int main() { upgraded.push_back(0u); upgraded.push_back(1u); upgraded.push_back(0u); - upgraded.push_back(0u); - Require(serializer.Serialize(request) == EncodeBase64(upgraded)); + Require(serializer.Serialize(request) == EncodeWithDefaultVersion55Tail(upgraded)); const auto replay38 = serializer.Deserialize(EncodeBase64(main38)); Require(replay38.request.graphics.has_value() && replay38.request.graphics->tessellation.has_value() == fixture.tessellation); Require(!fixture.tessellation || (!replay38.request.graphics->tessellation->mergedHull && replay38.request.graphics->tessellation->primitiveGroupSize == 37u && replay38.request.graphics->tessellation->breakAtInstance)); Require(replay38.request.graphics->draw.indexAddress == 0xabcdef3400u && replay38.request.graphics->draw.indexCount == 57u && replay38.request.graphics->draw.indexElementBytes == 4u && replay38.request.graphics->draw.instanceCount == 9u); Require(replay38.request.target.float16DenormPreserve == preserve); - Require(serializer.Serialize(replay38.request) == EncodeBase64(upgraded)); + Require(serializer.Serialize(replay38.request) == EncodeWithDefaultVersion55Tail(upgraded)); if (fixture.tessellation) { for (const bool merged : {false, true}) { auto pointed = request; @@ -563,8 +673,12 @@ int main() { pointed.context.vertex->pointSize = 3.0f; pointed.context.vertex->pointSizeMin = 1.0f; pointed.context.vertex->pointSizeMax = 8.0f; - auto version53 = DecodeBase64(serializer.Serialize(pointed)); - Require(version53.size() > 19u && version53[4] == 53u && version53.back() == 0u && version53[version53.size() - 2u] == 0u); + auto version55 = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(pointed))); + Require(version55.size() > 31u && version55[4] == 55u); + auto version53 = version55; + version53.resize(version53.size() - 12u); + version53[4] = 53u; + Require(version53.back() == 0u && version53[version53.size() - 2u] == 0u); auto version52 = version53; version52.pop_back(); version52[4] = 52u; @@ -590,7 +704,7 @@ int main() { version43.pop_back(); version43[4] = 43u; const auto replay43 = serializer.Deserialize(EncodeBase64(version43)); - Require(serializer.Serialize(replay43.request) == EncodeBase64(version53)); + Require(serializer.Serialize(replay43.request) == EncodeBase64(WithHeapBufferDescriptors(version55))); Require(replay43.request.graphics->tessellation->mergedHull == merged && replay43.request.context.vertex->pointList && replay43.request.context.vertex->pointSize == 3.0f && replay43.request.context.vertex->pointSizeMin == 1.0f && replay43.request.context.vertex->pointSizeMax == 8.0f); auto version42 = version43; version42.resize(version42.size() - 13u); @@ -631,8 +745,7 @@ int main() { main25.push_back(0u); main25.push_back(1u); main25.push_back(0u); - main25.push_back(0u); - Require(serializer.Serialize(replay25.request) == EncodeBase64(main25)); + Require(serializer.Serialize(replay25.request) == EncodeWithDefaultVersion55Tail(main25)); } } } @@ -648,11 +761,10 @@ int main() { const auto encoded = serializer.Serialize(request); const auto replay = serializer.Deserialize(encoded); Require(replay.request.target.float16DenormPreserve == preserve && replay.request.target.float32FlushToZero == !preserve && replay.request.target.float32DenormPreserve == preserve && replay.request.target.largePixelFunctionLdsBuffer == !preserve && replay.request.target.pixelFunctionStorageDwords == request.target.pixelFunctionStorageDwords); - auto bytes = DecodeBase64(encoded); - Require(bytes.size() >= 8u && bytes[4] == 53u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); + auto bytes = WithoutHeapBufferDescriptors(DecodeBase64(encoded)); + Require(bytes.size() >= 8u && bytes[4] == 55u && bytes[5] == 0u && bytes[6] == 0u && bytes[7] == 0u); auto version40 = bytes; - version40.pop_back(); - version40.pop_back(); + version40.resize(version40.size() - 14u); version40.resize(version40.size() - sizeof(std::uint32_t)); version40.pop_back(); version40.pop_back(); @@ -663,7 +775,7 @@ int main() { const auto replay40 = serializer.Deserialize(EncodeBase64(version40)); Require(!replay40.request.target.float32DenormPreserve && replay40.request.target.float32FlushToZero == !preserve); auto future = bytes; - future[4] = 54u; + future[4] = 56u; bool futureRejected = false; try { static_cast(serializer.Deserialize(EncodeBase64(future))); @@ -705,7 +817,8 @@ int main() { request.context.pixel = pixel; const auto orderEncoded = serializer.Serialize(request); Require(serializer.Deserialize(orderEncoded).request.context.pixel->sampleOrder == pixel.sampleOrder); - auto version42 = DecodeBase64(orderEncoded); + auto version42 = WithoutHeapBufferDescriptors(DecodeBase64(orderEncoded)); + version42.resize(version42.size() - 12u); const auto orderOffset = std::search(version42.begin(), version42.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.end()) - version42.begin() + static_cast(pixel.sampleLocations.size() + 1u); Require(version42.end() - version42.begin() > orderOffset + static_cast(sizeof(pixel.sampleOrder)) && version42[static_cast(orderOffset)] == 0x10u); EraseDefaultTargetMerge(version42, orderOffset + static_cast(sizeof(pixel.sampleOrder))); @@ -733,9 +846,8 @@ int main() { request.target.spannedWaves = spanned; Require(serializer.Deserialize(serializer.Serialize(request)).request.target.spannedWaves == spanned); request.target.float32FlushToZero = true; - auto version35 = DecodeBase64(serializer.Serialize(request)); - version35.pop_back(); - version35.pop_back(); + auto version35 = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(request))); + version35.resize(version35.size() - 14u); version35.resize(version35.size() - sizeof(std::uint32_t)); version35.pop_back(); version35.pop_back(); @@ -786,8 +898,8 @@ int main() { for (const bool spanned : {false, true}) { for (const std::uint8_t version : {32u, 33u, 34u, 35u, 36u, 37u, 38u}) { pointRequest.target.spannedWaves = spanned; - auto legacy = DecodeBase64(serializer.Serialize(pointRequest)); - legacy.resize(legacy.size() - (version >= 36u ? 24u : version >= 35u ? 25u : 30u)); + auto legacy = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(pointRequest))); + legacy.resize(legacy.size() - (version >= 36u ? 36u : version >= 35u ? 37u : 42u)); legacy[4] = version; const auto replayLegacy = serializer.Deserialize(EncodeBase64(legacy)); Require(replayLegacy.request.target.spannedWaves == spanned); @@ -798,8 +910,8 @@ int main() { Require(replayLegacy.request.context.vertex->paClVsOutCntl == pointRequest.context.vertex->paClVsOutCntl); Require(!replayLegacy.request.context.vertex->pointList && replayLegacy.request.context.vertex->pointSize == 1.0f); Require(replayLegacy.request.context.vertex->pointSizeMin == 0.0f && replayLegacy.request.context.vertex->pointSizeMax == 0.0f); - const auto upgradedLegacy = DecodeBase64(serializer.Serialize(replayLegacy.request)); - Require(upgradedLegacy[4] == 53u && upgradedLegacy.size() == legacy.size() + (version >= 36u ? 24u : version >= 35u ? 25u : 30u)); + const auto upgradedLegacy = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(replayLegacy.request))); + Require(upgradedLegacy[4] == 55u && upgradedLegacy.size() == legacy.size() + (version >= 36u ? 36u : version >= 35u ? 37u : 42u)); } } @@ -807,7 +919,8 @@ int main() { pointRequest.context.pixel = pixel; const auto sampledPoint = serializer.Deserialize(serializer.Serialize(pointRequest)); Require(sampledPoint.request.context.vertex->pointList && sampledPoint.request.context.pixel->sampleOrder == pixel.sampleOrder); - auto legacy43 = DecodeBase64(serializer.Serialize(pointRequest)); + auto legacy43 = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(pointRequest))); + legacy43.resize(legacy43.size() - 12u); const auto mergeOffset = std::search(legacy43.begin(), legacy43.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.end()) - legacy43.begin() + static_cast(pixel.sampleLocations.size() + 1u + sizeof(pixel.sampleOrder)); EraseDefaultTargetMerge(legacy43, mergeOffset); legacy43.pop_back(); @@ -828,8 +941,8 @@ int main() { Require(pointReplay42.request.context.pixel->sampleOrder == pixel.sampleOrder && !pointReplay42.request.context.vertex->pointList && pointReplay42.request.context.vertex->pointSize == 1.0f); pixel.sampleOrder = {}; pointRequest.context.pixel = pixel; - auto legacy34 = DecodeBase64(serializer.Serialize(pointRequest)); - legacy34.resize(legacy34.size() - 30u); + auto legacy34 = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(pointRequest))); + legacy34.resize(legacy34.size() - 42u); const auto stencil34 = std::search(legacy34.begin(), legacy34.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.begin() + 4) + pixel.sampleLocations.size(); const auto legacyPixelTail = static_cast(1u + sizeof(pixel.sampleOrder)); Require(legacy34.end() - stencil34 > legacyPixelTail && std::all_of(stencil34, stencil34 + legacyPixelTail, [](std::uint8_t value) { return value == 0u; })); @@ -843,10 +956,9 @@ int main() { for (const bool fused : {false, true}) { request.target.fusedExactArithmetic = fused; request.target.largePixelFunctionLdsBuffer = false; - auto bytes = DecodeBase64(serializer.Serialize(request)); - Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.fusedExactArithmetic == fused && bytes[bytes.size() - 10u] == (fused ? 1u : 0u) && bytes[bytes.size() - 9u] == 0u); - bytes.pop_back(); - bytes.pop_back(); + auto bytes = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(request))); + Require(serializer.Deserialize(EncodeBase64(WithHeapBufferDescriptors(bytes))).request.target.fusedExactArithmetic == fused && bytes[bytes.size() - 22u] == (fused ? 1u : 0u) && bytes[bytes.size() - 21u] == 0u); + bytes.resize(bytes.size() - 14u); bytes.pop_back(); bytes.pop_back(); const auto fusedMerge = std::search(bytes.begin(), bytes.end(), pixel.sampleLocations.begin(), pixel.sampleLocations.end()) - bytes.begin() + static_cast(pixel.sampleLocations.size() + 1u + sizeof(pixel.sampleOrder)); @@ -863,12 +975,12 @@ int main() { Require(request.context.pixel.has_value()); request.context.pixel->ditheredAlphaToCoverage = true; request.context.pixel->ditheredAlphaSamples = 2u; - auto twoSample = DecodeBase64(serializer.Serialize(request)); - const auto twoReplay = serializer.Deserialize(EncodeBase64(twoSample)); + auto twoSample = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(request))); + const auto twoReplay = serializer.Deserialize(EncodeBase64(WithHeapBufferDescriptors(twoSample))); Require(twoReplay.request.context.pixel->ditheredAlphaToCoverage && twoReplay.request.context.pixel->ditheredAlphaSamples == 2u); request.context.pixel->ditheredAlphaSamples = 8u; - const auto eightSample = DecodeBase64(serializer.Serialize(request)); - const auto eightReplay = serializer.Deserialize(EncodeBase64(eightSample)); + const auto eightSample = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(request))); + const auto eightReplay = serializer.Deserialize(EncodeBase64(WithHeapBufferDescriptors(eightSample))); Require(eightReplay.request.context.pixel->ditheredAlphaToCoverage && eightReplay.request.context.pixel->ditheredAlphaSamples == 8u); Require(twoSample.size() == eightSample.size()); std::size_t ditherByte = 0u; @@ -888,6 +1000,7 @@ int main() { return false; }; auto version46Dither = twoSample; + version46Dither.resize(version46Dither.size() - 12u); version46Dither.pop_back(); version46Dither.pop_back(); version46Dither.pop_back(); @@ -905,9 +1018,10 @@ int main() { for (const bool subgroups : {false, true}) { domain.target.tessellationEvaluationSubgroups = subgroups; const auto encoded = serializer.Serialize(domain); - auto bytes = DecodeBase64(encoded); - Require(bytes.size() > 9u && bytes[4] == 53u && bytes.back() == 0u && bytes[bytes.size() - 2u] == 0u && bytes[bytes.size() - 3u] == (subgroups ? 1u : 0u)); + auto bytes = WithoutHeapBufferDescriptors(DecodeBase64(encoded)); + Require(bytes.size() > 15u && bytes[4] == 55u && bytes[bytes.size() - 15u] == (subgroups ? 1u : 0u)); Require(serializer.Deserialize(encoded).request.target.tessellationEvaluationSubgroups == subgroups); + bytes.resize(bytes.size() - 12u); bytes.pop_back(); bytes[4] = 52u; Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.tessellationEvaluationSubgroups == subgroups); @@ -929,8 +1043,7 @@ int main() { upgraded[4] = 53u; upgraded.push_back(1u); upgraded.push_back(0u); - upgraded.push_back(0u); - Require(serializer.Serialize(replay49.request) == EncodeBase64(upgraded)); + Require(serializer.Serialize(replay49.request) == EncodeWithDefaultVersion55Tail(upgraded)); for (auto skipped : {bytes, upgraded}) { skipped[4] = 50u; bool skippedRejected = false; @@ -948,9 +1061,9 @@ int main() { hostFirstEncoded[hostFirst] = serializer.Serialize(request); const auto replay = serializer.Deserialize(hostFirstEncoded[hostFirst]); Require(replay.request.target.hostInterpolationFirst == hostFirst && serializer.Serialize(replay.request) == hostFirstEncoded[hostFirst]); - auto bytes = DecodeBase64(hostFirstEncoded[hostFirst]); - Require(bytes[4] == 53u && bytes.back() == 0u && bytes[bytes.size() - 2u] == (hostFirst ? 1u : 0u)); - bytes.pop_back(); + auto bytes = WithoutHeapBufferDescriptors(DecodeBase64(hostFirstEncoded[hostFirst])); + Require(bytes[4] == 55u && bytes[bytes.size() - 13u] == 0u && bytes[bytes.size() - 14u] == (hostFirst ? 1u : 0u)); + bytes.resize(bytes.size() - 13u); bytes[4] = 52u; Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.hostInterpolationFirst == hostFirst); bytes.pop_back(); @@ -961,13 +1074,20 @@ int main() { request.target.hostInterpolationFirst = false; for (const bool minLod : {false, true}) { request.target.shaderImageMinLod = minLod; - auto bytes = DecodeBase64(serializer.Serialize(request)); - Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.shaderImageMinLod == minLod && bytes[4] == 53u && bytes.back() == (minLod ? 1u : 0u)); + auto bytes = WithoutHeapBufferDescriptors(DecodeBase64(serializer.Serialize(request))); + Require(serializer.Deserialize(EncodeBase64(WithHeapBufferDescriptors(bytes))).request.target.shaderImageMinLod == minLod && bytes[4] == 55u && bytes[bytes.size() - 13u] == (minLod ? 1u : 0u)); + bytes.resize(bytes.size() - 12u); + for (const std::uint8_t version : {53u, 54u}) { + bytes[4] = version; + Require(serializer.Deserialize(EncodeBase64(bytes)).request.target.shaderImageMinLod == minLod); + } bytes.pop_back(); bytes[4] = 52u; Require(!serializer.Deserialize(EncodeBase64(bytes)).request.target.shaderImageMinLod); } request.target.shaderImageMinLod = false; + VerifyHeapProfile(serializer); + VerifyKeyedTargetFields(serializer); return 0; } diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index d59caa10c4e0..38a221bd141e 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -141,17 +141,18 @@ flowchart LR - [`Recompiler.cpp`](../../core/shader/recompiler/Recompiler.cpp) runs the stages in this order. With `ANYPS5_ENABLE_SPIRV_TOOLS`, the SPIR-V is also validated and optimized with SPIRV-Tools. - `ShaderRecompiler::Recompile` keeps compiled variants in memory, and `ShaderDiskCache` stores them on disk so later runs reuse them. Before the writable cache, a miss is also looked up in read-only prebuilt packs and loose entry folders under `shader_cache/prebuilt`; pack entries are ordinary encoded cache entries behind a sorted key-hash index, validated by magic, compiler source version and index checksum when mapped, and a miss or rejected entry falls through to the writable cache and the compiler. The cache format (`ShaderDiskCache::FormatVersion`) stores, among other things, the finite stencil-reference set, its replay-safety proof and reserved push-data word in the artifact and binding layouts. An entry of any other format, or written by another compiler source version, is rejected. -- The Vulkan pipelines are journaled as well ([`PipelineJournal.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp)): a graphics pipeline the store builds (`CachedPipeline`) or a compute pipeline a dispatch builds is appended to `pipelines.journal` next to the shader cache entries, as the materialized result of each stage (the specialized SPIR-V, bindings and push constants, through `ShaderDiskCache::EncodeResult`) plus the recipe the pipeline was built from (`GraphicsPipelineRecipe`: the stages' identities, vertex input layout, descriptor set layout keys and the state fields the `Pipeline` constructor reads, `CanonicalPipelineState`). When a `VulkanDevice` is created, the journal is loaded and its pipelines are built on a pool of `hardware_concurrency() / 2` threads (`APS5_PIPELINE_JOURNAL_THREADS`) while the title loads. The load parses the file a record at a time and keeps it open, and a replay thread reads an entry's modules from it when it builds that entry (a record that no longer reads back as loaded rejects the entry), so the replay holds the modules of the entries being built rather than the file, so the first draw or dispatch of a configuration seen on an earlier run finds its pipeline resident. The pipeline stores key on `RecompileResult::Identity`, a hash of the module words, specialization constants, subgroup width and bindings that is the same on every run (`SharedSpirv::ContentHash`, computed once per shared module), rather than on the process-lifetime variant ids, and a graphics pipeline is built outside the store's mutex, so the replay threads and the queue workers build in parallel. A draw that arrives before its replayed entry builds the pipeline itself; the first inserted wins. A replayed entry enters the store at its least recently used end and a full store takes none, so the title's own pipelines are never evicted for a replayed one, and the optimized relink of a replayed pipeline (`OptimizedPipeline::Schedule`) waits for the first draw that binds it, so the relink thread serves the title's pipelines first. A device created without a window starts its replay at its first draw or dispatch rather than at creation, since `Driver::AttachWindow` usually replaces it before the title draws. The journal records a pipeline once (modules and recipes are deduplicated by content), keeps at most `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024), replays the last `APS5_PIPELINE_JOURNAL_ENTRIES` recorded (4096, the pipeline store's largest bound: a replayed pipeline that finds the store full grows its bound from 1024 instead of being dropped), and is disabled with `APS5_NO_PIPELINE_JOURNAL=1` or without a shader cache directory. The file header names the device (`PipelineJournalDeviceFingerprint`, the properties the driver's pipeline cache file is named by), so a journal written for another GPU or driver is started over rather than replayed; appends and reads hold `pipelines.journal.lock` against another process sharing the directory, and a device cancels its replay before it is replaced (`PrepareForReplacement`) or destroyed, so a replaced headless device never inserts entries the window's device then sweeps. `APS5_PROFILE_DRAW=1` prints a `[pipeline-journal]` summary when the replay finishes and at teardown. +- The Vulkan pipelines are journaled as well ([`PipelineJournal.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp)): a graphics pipeline the store builds (`CachedPipeline`) or a compute pipeline a dispatch builds is appended to `pipelines.journal` (`pipelines-heap.journal` under `GpuHeapEnabled`, so the two binding models never replay each other's pipelines) next to the shader cache entries, as the materialized result of each stage (the specialized SPIR-V, bindings and push constants, through `ShaderDiskCache::EncodeResult`) plus the recipe the pipeline was built from (`GraphicsPipelineRecipe`: the stages' identities, vertex input layout, descriptor set layout keys and the state fields the `Pipeline` constructor reads, `CanonicalPipelineState`). When a `VulkanDevice` is created, the journal is loaded and its pipelines are built on a pool of `hardware_concurrency() / 2` threads (`APS5_PIPELINE_JOURNAL_THREADS`) while the title loads. The load parses the file a record at a time and keeps it open, and a replay thread reads an entry's modules from it when it builds that entry (a record that no longer reads back as loaded rejects the entry), so the replay holds the modules of the entries being built rather than the file, so the first draw or dispatch of a configuration seen on an earlier run finds its pipeline resident. The pipeline stores key on `RecompileResult::Identity`, a hash of the module words, specialization constants, subgroup width and bindings that is the same on every run (`SharedSpirv::ContentHash`, computed once per shared module), rather than on the process-lifetime variant ids, and a graphics pipeline is built outside the store's mutex, so the replay threads and the queue workers build in parallel. A draw that arrives before its replayed entry builds the pipeline itself; the first inserted wins. A replayed entry enters the store at its least recently used end and a full store takes none, so the title's own pipelines are never evicted for a replayed one, and the optimized relink of a replayed pipeline (`OptimizedPipeline::Schedule`) waits for the first draw that binds it, so the relink thread serves the title's pipelines first. A device created without a window starts its replay at its first draw or dispatch rather than at creation, since `Driver::AttachWindow` usually replaces it before the title draws. The journal records a pipeline once (modules and recipes are deduplicated by content), keeps at most `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024), replays the last `APS5_PIPELINE_JOURNAL_ENTRIES` recorded (4096, the pipeline store's largest bound: a replayed pipeline that finds the store full grows its bound from 1024 instead of being dropped), and is disabled with `APS5_NO_PIPELINE_JOURNAL=1` or without a shader cache directory. The file header names the device (`PipelineJournalDeviceFingerprint`, the properties the driver's pipeline cache file is named by), so a journal written for another GPU or driver is started over rather than replayed; appends and reads hold `pipelines.journal.lock` against another process sharing the directory, and a device cancels its replay before it is replaced (`PrepareForReplacement`) or destroyed, so a replaced headless device never inserts entries the window's device then sweeps. `APS5_PROFILE_DRAW=1` prints a `[pipeline-journal]` summary when the replay finishes and at teardown. - A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory. - `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` threads (at least 2, 64 MiB stacks), which `Driver::stop` joins when it closes the pool. - A thread that needs the GPU mutex, the committer's next item or the end of the committer's queue spins before it blocks (`HandoffSpin.hpp`): `GpuMutexType::lock` retries `try_lock` for the window, the committer polls its item count, and `DrawPipeline::Drain` polls it for twice the window, so a hand-off that completes within the window costs no kernel wake-up. The window is 150 us when the process may run on more than 8 logical processors (its affinity mask, via `CpuTopology`) and 0 otherwise, where the spinning threads would compete with the holder; `APS5_HANDOFF_SPIN_US=` sets it (0 turns it off, at most 10000). Ordering and the lock's semantics are unchanged. - Queue 0's packet worker decodes, captures and prepares a draw and hands its recording (`Graphics::Draw` under the GPU lock) to a committer thread in [`DrawPipeline.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp), so the next packet is decoded while the previous draw is recorded. Each queued item carries the guest ranges it will write (`DrawWriteRanges`: written buffers and storage images, color targets with their DCC and CMASK, depth and stencil). A later packet that reads or writes one of those ranges, or that the pipeline can't order, drains it first; labels, submission ends, buffer fills and DMA packets are queued as items instead. With `APS5_PIPELINE_DISPATCH=1` (off by default), a `DISPATCH_DIRECT` whose dispatch-cache entry hits with a recipe is queued as an item too, carrying the dispatch's written buffers and storage images (`DispatchWriteRanges`); the capture and the cache lookup stay on the worker and the recipe is recorded on the committer. A dispatch still drains when its lookup misses, when what it reads overlaps a write still pending, when the hit is verified against data or when its writes cannot be ranged (bindless access, GDS, fault buffer). An indirect draw resolved on the CPU (its records read by the worker) drains only when its records or its count overlap a queued write; the records it draws inline drain first, and its last drawn record is queued like a direct draw (`APS5_NO_PIPELINED_INDIRECT=1` drains at every indirect draw). An exception thrown by a commit is rethrown on the worker at its next enqueue or drain, as an inline draw error would be. On by default; `APS5_PIPELINED_DRAWS=0` records every draw inline, and `APS5_EXACT_DRAW_WRITES=0` widens the color and depth ranges to estimates. - A dispatch's resource build runs stage A (binding plan, buffers, descriptor set) without the GPU lock and stage B (image lookups) under it. Between them `VulkanDevice::presync` waits, still without the lock, for recorded work that writes a surface stage B would read on the CPU and for recorded work that writes the DCC keys of an image the dispatch binds (`ShaderResources::PresyncSurfaces`), so the locked `WaitForKeyWriters` finds nothing pending to synchronize with. A key range is waited for only when recorded work writes it, the key-store memo does not give its value (`DccKeyWritersPending`, without the mutex) and the pending stores do not give it either (`DccKeyWriterNeedsSync`, under the mutex, the same test `WaitForKeyWriters` makes before it synchronizes). Stage A's own key scan runs before that wait, and stage B compares it with the keys it reads under the lock as before. `APS5_NO_DCC_PRESYNC=1` leaves the key ranges out; `APS5_NO_PRESYNC=1` disables the whole pre-sync. - `APS5_DRAW_PIPELINE` (any value turns it on, even `0`; off by default, meant with `APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1`) takes more of a draw off the worker. A `DrawFinisher` thread runs the second half of a direct pipelined draw (stage materialization, the draw cache insert and the hand-off to the committer) while the worker reads the next packets; the worker drains it before the next draw's cache lookup, before packets other than register writes and draws, and before guest reads, and finishes in place a draw whose stages may write memory (`ProgramWritesMemory`) or write through the BDA page table. The committer skips the fragment-output validation and storage-image layer scans whose inputs match its last commit (`CommitTrims`), and a resource or storage image proved current keeps its proof for the rest of the collect epoch until `GuestMemory::BreakProofs` (a CPU write, a write-back, a recorded write) moves `ProofSerial`. Color metadata passes go through the pipeline with their write ranges (`PipelinedColorMetadataPassWrites`) instead of draining it. A data-only hit whose shape entry belongs to an older draw replaces that entry in place at the second sight of the same read user words. On Linux the main image's pages get a page table entry, and capture pages are pooled across threads. `APS5_VERIFY_DRAW_PIPELINE=1` runs a full check next to 1 in 64 reused proofs and prints `[epoch-proofs]` counts. +- `APS5_GPU_HEAP=1` (off by default) binds the sampled images and samplers of eligible stages through the device's `VK_EXT_descriptor_heap` resource and sampler heaps ([`BindlessHeap.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp): 233472 image slots that every image binding kind shares, the first 57 holding each kind's absent descriptor, and 2048 sampler slots, per-draw shader data and flattened SRT in a ring the shaders reach by address) instead of per-draw descriptor sets. Without the extension, or when the device's heaps cannot hold those slots, every stage takes the per-draw path. A fragment or vertex stage whose plan allows it is recompiled in GPU mode (`gpuResolvedImages`): its image metadata names the binding element and the shader reads the heap slots from the shader data once, at its entry; a stage stays GPU-mode only if its compiled module maps onto the heap and every stage of its draw is GPU-mode too; otherwise it is rebuilt on the per-draw path. A draw whose stages come out mixed is rebuilt once with what the first pass learned, and a stage pair that is still mixed takes the per-draw path from then on. A heap fragment or vertex stage keeps one compiled template per binding plan shape; each draw carries its own descriptor words, so draws whose buffers or textures change share the template. An address-based draw whose BDA accesses only read reuses its address space while the collect epoch and the host imports are unchanged (space reuse; with `APS5_DRAW_PIPELINE=1` its refreshed mirrors are reused within the epoch too). The pipeline journal describes the heap set layouts, so GPU-mode pipelines are replayed on the next run. `APS5_VERIFY_GPU_HEAP=1` checks the heap slots against the bound textures, rebuilds 1 in N GPU-mode stages on the per-draw path and compares them (`[heap-slots]`, `[gpu-heap]`). The storage images of GPU-mode stages take heap slots of their own. A heap slot holds no reference to its texture, storage image, view or sampler: the owner frees its slots when it is destroyed, and every batch that reads a slot keeps the owner alive through its draw's or dispatch's resources, so a slot is never reused while a batch can still read it. With fewer free image or sampler slots than the margin, stages take the per-draw path until released slots leave twice the margin free. A GPU-mode stage reads its shader data and flattened SRT through five push constant dwords at its push offset (both GPU addresses in the heap ring and the SRT's dword count, the offset a specialization like the existing push data), which the driver writes at every draw, so a draw binds no data set. The same five words are followed by the BDA page table's address and dword count and the fault buffer's address, so a heap draw binds no descriptor set. Sampling compute stages are GPU-mode stages under the switch too and bind the same way, with their workgroup memory reached by a pushed address; one whose module does not map onto the heap is recompiled on the per-dispatch path. Every vertex or fragment stage whose images all come from descriptors is a heap stage under the switch, stages without images included: a vertex stage reads its SRT constants through the pushed flattened SRT. Image, storage image and sampler tables index the heap: the table arrays alias the heap's first sampled, first storage and sampler bindings, the driver appends four trailer words (a device image table's bucket and miss ring addresses, zero without one) and then a sampled and a storage slot per element and a slot per table sampler to the map (from `HeaderElementBase`, right after the trailer), and the map goes into the heap ring per draw with its address in two more pushed dwords. Tables with image atomics index the heap too. A heap stage writes its resolved image and sampler slots into its shader data without registering lookup keys, and reads its guest buffers by device address (`addressedBuffers`: four shader data words per buffer, 64-bit address and byte size, after the slot words) instead of storage buffer descriptors; a draw whose buffers are snapshotted gets its own copy of the words with the snapshot addresses. - The shader registry maps a code address to every header registered for that code, in registration order. Static, graphics and rectangle ABI resolution pick the header at the bound address (or one equal to it, `SameHeader`, which also takes a hull header fused by `sceAgcFuseShaderHalves` (type 3) as the registered back half (type 7) it copies, the fused registers aside; graphics ABI resolution then uses the fused header's shader registers); draws and dispatches pick the program by the stage's binary type (`RegisteredProgram`), and a front entry matching both type 2 and type 4 throws. Within a type the last registration wins; re-registering an identical earlier header makes its snapshot current again without preparing it again, and changed code drops the older headers. - The snapshot is published as pending. `SourceHandleFor`, `InvocationFor` and `ShaderPreparationTransaction::Edit` / `Read` wait for the pending job before reading `PreparedShaders`, and rethrow its failure. `APS5_SYNC_SHADER_PREPARE=1` prepares on the registering thread instead. - `APS5_LAZY_SHADER_PREPARE=1` retains registration validation and decoding but discards the temporary plan without queueing preparation. The snapshot is deferred, not pending; draws and dispatches prepare and cache it on first use. A compute or pixel stage's explicit ABI resolution (`ResolveShaderAbi`) plans the stage with the ABI's state, records the ABI and leaves its preparation to the shader's first use, which prepares it with that state (`PreparedShaders::deferredAbis`), so a shader that never draws or dispatches stays unprepared; a graphics-stage ABI is still prepared when it is resolved. This opt-in mode takes precedence over synchronous registration preparation and trades background CPU/memory use for possible first-use stalls; disk caches and shader output are unchanged. -- On devices with graphics pipeline libraries (`SpirvTarget::registrationModules`), a pixel artifact is emitted in registration form: the buffer V# words, image modes, swizzles and mip ranges, depth-compare state and export mappings it used to specialize on per draw are read at run time (image `ResourceMetadata` and a few state dwords in the shader data), descriptor heaps keep every element in place (an unused one gets a null or placeholder descriptor), and the pixel stage's bindings live in descriptor set 1 so each stage's library depends only on its own set layout (`INDEPENDENT_SETS`). A vertex artifact is emitted specialized: in registration form each vertex buffer load would decode the V# stride, swizzle, bounds mode and format at run time for every vertex, so its pre-rasterization library is built at its first draw instead. An artifact above `registrationWordLimit` SPIR-V words is re-emitted specialized, and `CompiledShaderArtifact::registrationModule` records which form it has. For artifacts in registration form, `RegistrationResult` materializes the module once, and `PrebuildStageLibrary` builds its fragment-shader library on a separate pool of `max(2, hardware_concurrency() / 2)` threads (`APS5_PREBUILD_THREADS` sets the count, at least 1) with a fixed default state, pairing it with a stub vertex stage. A later draw with that state finds the fragment-shader library in the store. Ahead-of-time dispatch resolution (`resolveGroupAhead` in [`DispatchAhead.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp)) scans the upcoming dispatch run up to the next wait or synchronization packet, moves still-pending registered compute shaders to the front of the registration preparation queue (`PrioritizeRegistrationPreparation`) and queues preparation (`SubmitRegistrationPreparation`) for registered compute shaders that have no prepared entries yet (such as under `APS5_LAZY_SHADER_PREPARE=1`) with the previewed register state and entry offset; when a dispatch resolved ahead misses the dispatch cache in [`Dispatch.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp), `PrebuildComputePipeline` is submitted to the same `APS5_PREBUILD_THREADS` pool so the compute `VkPipeline` is built before the worker reaches the packet. +- On devices with graphics pipeline libraries (`SpirvTarget::registrationModules`), a pixel artifact is emitted in registration form: the buffer V# words, image modes, swizzles and mip ranges, depth-compare state and export mappings it used to specialize on per draw are read at run time (image `ResourceMetadata` and a few state dwords in the shader data), descriptor heaps keep every element in place (an unused one gets a null or placeholder descriptor), and the pixel stage's bindings live in descriptor set 1 so each stage's library depends only on its own set layout (`INDEPENDENT_SETS`). A vertex artifact is emitted specialized: in registration form each vertex buffer load would decode the V# stride, swizzle, bounds mode and format at run time for every vertex, so its pre-rasterization library is built at its first draw instead. A vertex artifact for a descriptor heap stage (`gpuResolvedImages` with `descriptorMemory`) stays in registration form, because the heap stage shares one template module across draws and reads those words from the heap. An artifact above `registrationWordLimit` SPIR-V words is re-emitted specialized, and `CompiledShaderArtifact::registrationModule` records which form it has. For artifacts in registration form, `RegistrationResult` materializes the module once, and `PrebuildStageLibrary` builds its fragment-shader library on a separate pool of `max(2, hardware_concurrency() / 2)` threads (`APS5_PREBUILD_THREADS` sets the count, at least 1) with a fixed default state, pairing it with a stub vertex stage. A later draw with that state finds the fragment-shader library in the store. Ahead-of-time dispatch resolution (`resolveGroupAhead` in [`DispatchAhead.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp)) scans the upcoming dispatch run up to the next wait or synchronization packet, moves still-pending registered compute shaders to the front of the registration preparation queue (`PrioritizeRegistrationPreparation`) and queues preparation (`SubmitRegistrationPreparation`) for registered compute shaders that have no prepared entries yet (such as under `APS5_LAZY_SHADER_PREPARE=1`) with the previewed register state and entry offset; when a dispatch resolved ahead misses the dispatch cache in [`Dispatch.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp), `PrebuildComputePipeline` is submitted to the same `APS5_PREBUILD_THREADS` pool so the compute `VkPipeline` is built before the worker reaches the packet. - A draw or dispatch whose state matches no artifact prepared at registration (a title that builds its static ABI state itself, without `sceAgcDriverResolve*Abi`) gets one prepared at use by `SourceHandleFor` or `InvocationFor`, logged once per artifact and kept in the snapshot for the next use. - A stage that `sceAgcDriverResolve*Abi` cannot prepare for the state the title creates its pipeline with (a compute shader that calls user-data pointers, a descriptor the resource plan rejects) is left to its first use like a stage deferred at registration: `ResolveShaderAbi` and `PrepareGraphicsStages` catch the failure inside their own preparation transaction, so the `ShaderPreparationScope` the title's call runs in still commits, log it once per shader, mark the shader deferred and record the ABI key, so a repeated call prepares nothing; `ResolvePreparedGraphics` skips a deferred stage's rectangles until a draw has prepared it. A draw or dispatch that cannot prepare the stage either fails as before. Driver failures and `ProcessShutdown` still propagate from the ABI call. A stage whose registration preparation stored a failure is logged once with that reason and marked deferred (`DeferFailedRegistration`); the draw or dispatch that uses the stage rethrows the stored failure. - A compute shader whose `s_swappc_b64` calls a user-data pointer cannot be linked at registration, because the callee's address is only known at dispatch. `InvocationFor` in [`ShaderRegistry.cpp`](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) reads each callee from guest memory, appends one copy per call after the caller (`LinkUserDataCalls`) and prepares that linked code, so a different callee gets its own prepared entry. diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index 88a3b4a485f7..53fa7ed1192b 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -48,6 +48,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_CODE_HASH_KEY` | presence | `core/shader/recompiler/CacheKey.hpp` | | appends every code word to the cache key instead of hashing the code | | `APS5_NO_FAILURE_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | rebuilds a program whose compilation failed instead of reusing the memoized failure | | `APS5_NO_FAULT_CHECK_DEDUP` | presence | `core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp` | | emits every invalid-descriptor fault check of a GPU-descriptor buffer access instead of skipping one that an identical check on the same descriptor under the same execution guard already covers in the same IR block or a dominating block; read once per program | +| `APS5_NO_HEAP_PUSHED_DATA` | presence | `core/shader/recompiler/Optimization/src/BindingAllocator.cpp` | | reads all of a descriptor-heap stage's shader data from the ring instead of pushing its heap slot indices and then its user data (each pipeline packs its stages' root items and pushed dwords into the 64 push dwords) | | `APS5_NO_IMAGE_TABLES` | presence | `core/shader/recompiler/Optimization/src/ResourceTracker.cpp` | | disables image tables, leaving table-loaded descriptors unplanned | | `APS5_NO_MASKED_RESULT_MEMO` | presence | `core/shader/recompiler/Recompiler.cpp` | | materializes every result memo miss instead of patching the buffer descriptors, user data and flattened SRT of a cached result whose snapshot differs only in those words | | `APS5_NO_PERF_DISK_PLANS` | presence | `core/shader/recompiler/Recompiler.cpp` | | disables the disk cache of planning results | @@ -192,12 +193,20 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_FULL_BARRIERS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | records every barrier instead of merging redundant ones | | `APS5_GPU_CHECKPOINTS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | attaches device diagnostic checkpoints through VK_NV_device_diagnostic_checkpoints, or buffer markers through VK_AMD_buffer_marker on a device without it | | `APS5_GPU_COPY_MAX_KIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | 1024 | size cap in KiB on a descriptor sub-range copied through the GPU path, larger ones staying on the CPU | +| `APS5_GPU_HEAP` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp` | | binds the sampled and storage images and samplers of eligible stages through the device's VK_EXT_descriptor_heap heaps; without the extension it does nothing | +| `APS5_GPU_HEAP_EXCLUDE` | string | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp` | | comma list of stage classes that take the per-draw path while the GPU heap stays on: stage names (compute, vertex, hull, domain, geometry, pixel, local, mesh), or tables, storage or sampled for stages with image tables, storage images or sampled images | +| `APS5_GPU_PATH` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp` | | with `APS5_GPU_HEAP` on a device with VK_EXT_descriptor_heap, GPU-mode stages read image table columns at run time through the device image table on the GPU heap; a program the table cannot serve takes the CPU table map; without `APS5_GPU_HEAP` it does nothing | | `APS5_GPU_NO_WRITEBACK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | skips writing GPU results back into guest memory | | `APS5_GPU_WAIT_TIMEOUT_MS` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp` | 1000 | milliseconds a GPU wait may run before a warning is printed | | `APS5_GROUP_CAPTURE_ALL_QUEUES` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp` | | resolves group-ahead capture on queue 0 as well | | `APS5_HANDOFF_SPIN_US` | integer | `core/libs/prx/libSceAgcDriver/Execution/include/HandoffSpin.hpp` | | microseconds a thread spins for a handed-off draw's work or for the GPU mutex before blocking, clamped to 0..10000; unset spins 150 us when the process has more than 8 usable processors and not at all otherwise | | `APS5_HEADLESS_PIPELINE_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | loads the pipeline cache file on a device without a window too; `agc_frame_replay` sets it by default and clears it with `--cold-pipelines`, including inherited values | | `APS5_HEAP_MIRROR_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | 24576 | budget in MiB for the heap mirrors held for registered allocations | +| `APS5_HEAP_TEMPLATE_MIB` | integer | `core/shader/recompiler/Recompiler.cpp` | 256 | budget in MiB for the GPU heap's stage templates kept across all shaders; the least recently used go first | +| `APS5_HEAP_WALK_MEMO_MB` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp` | 256 | MiB each thread's GPU heap walk memo may hold across both its generations before the previous generation and then the oldest walks are dropped (0 keeps them until the 16,384-key cap rotates them out) | +| `APS5_HEAP_SLOT_CAP` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp` | device | number of shared image slots the GPU heap hands out, after the 57 slots reserved for each binding kind's absent descriptor; 0 keeps the default (233415, grown into resource heap space the shader data ring leaves unused, up to 524231); a stage needing more image slots than this takes the per-draw path | +| `APS5_HEAP_RING_MIB` | integer | `core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp` | 16 | MiB of the GPU heap's per-draw buffer descriptor ring inside the resource heap, between 4 and 64, in whole 2 MiB chunks and within the device's heap size; one draw's descriptors may take at most a quarter of the ring, and a batch is submitted early once it holds half the ring (shader data lives in a data pool that grows instead) | +| `APS5_NO_HEAP_EVICTION` | flag | `core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp` | unset | set: a nearly full GPU heap no longer evicts image slots no draw used for two submissions, so draws take the per-draw path until owners release slots | | `APS5_HOOK_FLUSH_CPU_BLOCKS` | string | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | set to 0 flushes pending images whose every 64 KiB block the CPU wrote, instead of skipping them | | `APS5_HOOK_FULL_SYNC` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | makes the flush hook's recorded-store wait a full Sync under the GpuMutex | | `APS5_HOOK_LOCKED_WAIT` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | keeps the flush hook's pending-write wait under the GpuMutex | @@ -261,6 +270,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_DEFERRED_SETTLES` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | settles watched private host imports as they open instead of through the deferred block-settling path (Windows only) | | `APS5_NO_DEPTH_TRANSFER_PASS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp` | | copies a D32 depth plane into a 32-bit color storage image (one `vkCmdCopyImage` where `APS5_NO_DIRECT_DEPTH_COPY` allows it, else through the staging buffer) instead of reading it into the image in a compute pass (used when `VK_KHR_push_descriptor` is enabled); copies back into the depth plane never use the pass | | `APS5_NO_DIRECT_DEPTH_COPY` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp` | | keeps every copy between a depth plane and a color storage image on the staging buffer instead of one `vkCmdCopyImage` (used when `VK_KHR_maintenance8` is enabled and the storage format is a single component of the aspect's size) | +| `APS5_NO_DEVICE_IMAGE_EXPECT` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp` | | inserts no expected device image keys before a batch, so the device image table serves first uses through its miss ring alone | | `APS5_NO_DISPATCH_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp` | | disables the dispatch cache so every dispatch takes the uncached path | | `APS5_NO_DISPATCH_KEY_HYGIENE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp` | | mixes every shader register into the dispatch key instead of only the registers that affect the dispatch | | `APS5_NO_DISPATCH_LRU` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp` | | clears the whole dispatch cache on overflow instead of evicting the least recently used entry | @@ -525,6 +535,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_VERIFY_DRAW_PIPELINE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | with `APS5_DRAW_PIPELINE` set, runs the full check instead of every 64th reused epoch proof on each thread and prints `[epoch-proofs]` counts | | `APS5_VERIFY_DRAW_RECIPE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCache.cpp` | | re-decodes a draw recipe instead of using the cache and checks the result | | `APS5_VERIFY_DRAW_SNAPSHOTS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | re-reads a reused draw snapshot and counts the ones that differ | +| `APS5_VERIFY_GPU_HEAP` | integer | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/HeapStages.cpp` | | checks heap slots and GPU-read constants, and rebuilds 1 in N GPU-mode stages on the per-draw path and compares them (0 or no number means every stage) | | `APS5_VERIFY_KNOWN_VALUES` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferCopy.cpp` | | serves known values through the hook too and counts mismatches | | `APS5_VERIFY_PROOFS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | runs the full walk beside every own-object refresh proof and aborts on a different decision | | `APS5_VERIFY_RECIPE` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp` | | makes the ordinary prepare path beside a recipe hit and compares the answers | @@ -689,6 +700,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_MIMALLOC` | presence | `core/libs/prx/libkernel/System/src/MimallocRestart.cpp` | | makes the process allocate through mimalloc instead of the system allocator: on Windows the entry stub (`core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp`) loads `mimalloc.dll`, on macOS libkernel restarts the title with `libmimalloc.dylib` in `DYLD_INSERT_LIBRARIES` and on Linux with `libmimalloc.so` in `LD_PRELOAD`; on macOS and Linux both variables stay set after the restart, so processes the title starts inherit them | | `APS5_MIMALLOC_RESTARTED` | presence | `core/libs/prx/libkernel/System/src/MimallocRestart.cpp` | | on macOS and Linux, set by libkernel for the restart that loads mimalloc and removed at start-up, so a restart that did not load it is not repeated | | `APS5_NO_DIRECT_MEMORY_SECTION_POOL` | presence | `core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp` | | on Windows, creates a section for every direct-memory allocation and closes it at release instead of reusing released sections that never held guest data | +| `APS5_NO_HEAP_DEVICE_DESCRIPTORS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp` | | allocates the GPU heap's resource and sampler heaps in host-visible memory instead of device-local host-visible memory (the resource heap stays device-local while it holds the shader-data ring) | | `APS5_NO_JOB_AFFINITY` | presence | `core/libs/prx/libkernel/Pthread/src/Thread.cpp` | | on Windows, disables pinning of job worker threads | | `APS5_NO_SLEEP_WAKE` | presence | `core/libs/prx/libkernel/Time/SleepWake.cpp` | | keeps every sleep of 1 ms or less in full instead of ending it at a GPU batch, batch completion action or flip completion | | `APS5_NO_TIMER_RESOLUTION` | presence | `core/libs/prx/libkernel/Time/Time.cpp` | | keeps the default 15.6 ms scheduler tick instead of raising it to 0.5 ms on Windows | diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index 5c5d7ca6fa3e..b6490c91546c 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -338,6 +338,7 @@ - [Table S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) DISABLE_CUBE_WRAP faults on selection without `VK_EXT_non_seamless_cube_map`: no runtime dimension check, including valid non-cube direct/table T#s. Unselected entries do not fault; with the feature this bit alone is accepted. Table T#/direct S# faults only for cubes needing the missing feature. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. A selected column's range is its record and the bytes of the record before the descriptor. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface has no mapped byte faults when selected instead of becoming a zero texture as a direct T# does; what the console returns for it is not measured. +- A per-dispatch [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) binds every storage element through its float storage view, so a table store through a uint mode (16-bit float and SINT formats written as raw bits) writes zero. The GPU heap gives tables a uint storage column; `agc_driver_image_table_store_gpu_heap` covers it and skips without the heap. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) entry whose T# is a dynamic-resolution variant of another entry (same address, format, tiling and mips, other width or height) borrows that entry's texture: the one the newest storage write used, else the active depth surface, else the largest accessible. Normalized sampling reads the same picture. `image_get_resinfo` (`textureSize`) and texel fetches on a borrower answer with the borrowed texture's extent, not the extent in its own T#, which is not what the console does. The table is not remembered, so it resolves on every use. - [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped from the recompiled code; in the shaders seen they are metadata or constants the caller reads relative to its program counter, and those reads go to the captured guest code, which linking does not change. A caller that jumps through a table among those words, which the control flow analysis reads from the recompiled code, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. - [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block (a conditional branch forward past the call between them is crossed), calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4`, or a 4-dword-aligned slice of an `s_load_dwordx8` or `s_load_dwordx16`, without SOFFSET from a user-data pointer, or from a pointer pair that two `s_mov_b32` of a literal or inline constant set, and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. The descriptor load may be in an earlier block, such as before a per-lane loop around the call, when it is the only write of the descriptor registers that reaches the buffer load on every path back from it, and no `s_call_b64` or statically resolved call lies on those paths; the paths may run through any `s_swappc_b64` that is not resolved statically, the table call itself included, whose callee is assumed to keep the descriptor registers. The chain compares the target each iteration actually loads, so a callee that changes them can only make a later target miss the chain and fault as an unlinked call. Paths follow fall-through, direct branches and the `s_setpc_b64` jumps whose target the control flow analysis resolves. A path that reaches, before a write of the descriptor registers, the program's first instruction, the entry of an in-program function (an `s_call_b64` target, whose callers are not followed), or an instruction that nothing modelled jumps or falls into fails the search. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. Each copy follows its own compares and call, so no branch spans more than one copy. A later call's branch to its chain and its return cross earlier calls' copies through a pair of `s_branch` relays before the first of those copies and after each one, so they don't span more than one copy either. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB, or a call more than 0x7fff dwords before the end of the program and its user-data callees throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. @@ -401,4 +402,6 @@ ### Opt-in draw paths - [Draw finisher](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPipeline.cpp) (`APS5_DRAW_PIPELINE`) - opt-in. The Linux main-image page table it adds matches what Windows already does by default. +- [GPU descriptor heap](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BindlessHeap.cpp) (`APS5_GPU_HEAP`) - opt-in, needs `VK_EXT_descriptor_heap`. A stage whose module keeps a set 0 binding the heap does not map (storage images of registration modules) takes the per-draw path, and so does every other stage of its draw. A stage that needs more image slots than the heap holds also takes the per-draw path; with `APS5_HEAP_SLOT_CAP` far below the default, stages that each fit can still exhaust the heap between their compile and their bind, because the nearly-full margin scales with the cap. Heap stages read guest memory in place and report no BDA fault per use, so the address draw reuse test skips its fault-per-use case with `--heap-stages`. A reused address-based draw whose moved read-only buffer cannot be viewed in place with the alignment offset its build baked into the shader data is built again. - [Multi-submit count overflow and buffer validation](../../../core/libs/prx/libSceAgcDriver/Submit/src/Dcb.cpp) (libSceAgcDriver) - whether the console driver imposes an architectural submission packet limit or allows null buffer pointers with non-zero sizes in multi-submit arrays is unmeasured; AgcDriver::SubmitMany validates count against 32-bit pointer array bounds overflow and rejects null addresses for non-empty command buffers with a runtime exception. +- [Device image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/DeviceImageTable.cpp) (`APS5_GPU_PATH`) - opt-in, takes effect only with `APS5_GPU_HEAP` on a device with VK_EXT_descriptor_heap. A draw whose expected images the table cannot take (a T# the CPU cannot evaluate before the draw is recorded, more than 16 candidates for one use, a full table or a heap past its margin) is still recorded with its device columns: a selected image that is not resolved yet reads zeros in that batch and is resolved from the miss ring for the next. Only later draws of the program take the CPU table map; a dispatch is recorded again at once. Recording the same draw again through the CPU map is not implemented. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 2847a509eb02..a4b8ffd1b91f 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -468,12 +468,16 @@ A first run can stop at one very large compute shader while the GPU driver compi ### Pipeline journal -Besides the recompiled shaders, the shader cache directory holds `pipelines.journal`: every Vulkan graphics pipeline the pipeline store kept and every compute pipeline a run built, with the specialized shader modules they were built from. The next run of the title builds those pipelines on background threads while it loads, so a draw or dispatch the earlier run already made does not compile its pipeline when it is first reached. The journal belongs to the GPU and driver that wrote it, like the driver's pipeline cache file next to it: a run on another GPU or driver version starts it over, and the driver's own pipeline cache decides how fast each pipeline is rebuilt. Two runs sharing the directory take turns on `pipelines.journal.lock`; a run that cannot lock it records nothing. `APS5_NO_PIPELINE_JOURNAL=1` disables recording and replay; `APS5_PIPELINE_JOURNAL_THREADS` sets the replay threads (half the CPU threads by default; `0` records without replaying), `APS5_PIPELINE_JOURNAL_ENTRIES` how many pipelines are replayed (4096, the last recorded) and `APS5_PIPELINE_STORE_ENTRIES` how many graphics pipelines stay resident (by default the store grows from 1024 up to 4096 to hold the replay and the pipelines that keep coming back), and `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024) the file size at which recording stops. Deleting the file loses nothing but the head start. With `APS5_PROFILE_DRAW=1` a `[pipeline-journal]` line reports what was loaded, replayed and recorded. +Besides the recompiled shaders, the shader cache directory holds `pipelines.journal` (`pipelines-heap.journal` with `APS5_GPU_HEAP`, whose pipelines differ): every Vulkan graphics pipeline the pipeline store kept and every compute pipeline a run built, with the specialized shader modules they were built from. The next run of the title builds those pipelines on background threads while it loads, so a draw or dispatch the earlier run already made does not compile its pipeline when it is first reached. The journal belongs to the GPU and driver that wrote it, like the driver's pipeline cache file next to it: a run on another GPU or driver version starts it over, and the driver's own pipeline cache decides how fast each pipeline is rebuilt. Two runs sharing the directory take turns on `pipelines.journal.lock`; a run that cannot lock it records nothing. `APS5_NO_PIPELINE_JOURNAL=1` disables recording and replay; `APS5_PIPELINE_JOURNAL_THREADS` sets the replay threads (half the CPU threads by default; `0` records without replaying), `APS5_PIPELINE_JOURNAL_ENTRIES` how many pipelines are replayed (4096, the last recorded) and `APS5_PIPELINE_STORE_ENTRIES` how many graphics pipelines stay resident (by default the store grows from 1024 up to 4096 to hold the replay and the pipelines that keep coming back), and `APS5_PIPELINE_JOURNAL_MAX_MIB` (1024) the file size at which recording stops. Deleting the file loses nothing but the head start. With `APS5_PROFILE_DRAW=1` a `[pipeline-journal]` line reports what was loaded, replayed and recorded. ### Draw pipeline `APS5_DRAW_PIPELINE=1` (off by default) moves more of each draw off the thread that reads the title's command buffers: a second thread finishes each draw while the next one is read, the thread that records draws skips work whose inputs did not change, and checks that a draw's resources are still current are reused until the title or the GPU writes memory. It is meant with `APS5_PIPELINED_DRAWS=32 APS5_PIPELINE_DISPATCH=1`. The picture does not change; if it does, that is a bug, and a run without the switch shows whether it is the cause. `APS5_VERIFY_DRAW_PIPELINE=1` runs the full checks next to the reused ones, at a cost in speed, and prints `[epoch-proofs]` lines with how many differed (any nonzero count is a bug). `APS5_PROFILE_DRAW=1` finishes every draw on the reading thread, so a run with it does not measure this switch. To see the hand-off without that, set `APS5_PIPELINE_STATS` (any value; `1` also turns on Vulkan pipeline statistics, which cost time): every 10 s a `[finisher]` line counts the draws handed to the second thread and those finished on the reading thread, and `[finisher-held]` lines count the draws held back by why their writes are not known. +### GPU descriptor heap + +`APS5_GPU_HEAP=1` (off by default) binds the textures and samplers of most draws through one table the GPU indexes, instead of building them again for each draw. It needs a GPU driver with `VK_EXT_descriptor_heap`; without it the switch does nothing. Use it with `APS5_DRAW_PIPELINE=1` and the switches listed there. The first run with the switch compiles its own variant of each shader, so expect the same stutter as with an empty shader cache once. Wrong or missing textures or flicker that go away without the switch are bugs. `APS5_VERIFY_GPU_HEAP=1` checks the table against the bound textures, and builds some draws the old way and compares them; it prints `[heap-slots]` and `[gpu-heap]` lines (a slot holding another texture or a differing draw is a bug). + ### Texture cache budget Image and buffer memory allocations retry after releasing unused pooled buffers and reclaimable texture cache entries. Recovery preserves storage images with pending GPU results and resources retained by recorded work. `[bufferpool]` recovery and failure lines report the requested allocation, memory released and retry count. An allocation still fails if recovery cannot free enough memory. When the `VK_EXT_memory_budget` budget of the device-local heap has no room for an image, device buffer or resident heap mirror slab, the texture caches first release reclaimable entries for the allocation and what the heap is over its budget; the allocation goes to host memory only if they release less than that and the budget read again still has no room.