perf(agc): prebuilt fragment libraries that match the draw (push-free offset, depth formats)

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp

@@ -29,6 +29,9 @@ class ShaderPreparationContext;
 
 namespace AgcDriver {
 class VulkanDevice;
+namespace Graphics {
+struct ShaderStages;
+}
 }
 
 namespace AgcDriver::DriverDetail {
@@ -235,6 +238,7 @@ void PrioritizeRegistrationPreparation(PreparedShaders& prepared);
 void AwaitRegistrationPreparation();
 void SubmitLibraryPrebuild(std::function<void()> task);
 void AwaitLibraryPrebuilds();
+void PrebuildStageLibrary(const std::shared_ptr<VulkanDevice>& device, const Graphics::ShaderStages& stages, ShaderRecompiler::ShaderStage stage, std::shared_ptr<const ShaderRecompiler::RecompileResult> result, std::uint32_t pushOffset);
 void CloseRegistrationPreparation();
 unsigned RegistrationPreparationThreads();
 

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp

@@ -1552,10 +1552,12 @@ bool LibraryPrebuildDisabled() {
     return disabled;
 }
 
+}
+
 void PrebuildStageLibrary(const std::shared_ptr<VulkanDevice>& device, const Graphics::ShaderStages& stages, ShaderRecompiler::ShaderStage stage, std::shared_ptr<const ShaderRecompiler::RecompileResult> result, std::uint32_t pushOffset) {
     if (LibraryPrebuildDisabled() || result == nullptr || (stage != ShaderRecompiler::ShaderStage::Vertex && stage != ShaderRecompiler::ShaderStage::Fragment)) return;
     SubmitLibraryPrebuild([device, stages, stage, result = std::move(result), pushOffset] {
-        const std::array<Graphics::CompiledShader, 1> shaders{{{stage, result.get(), pushOffset}}};
+        const std::array<Graphics::CompiledShader, 1> shaders{{{stage, result.get(), result->pushConstants.empty() ? 0u : pushOffset}}};
         try {
             static_cast<void>(device->PrebuildGraphicsLibraries(stages, shaders));
         } catch (const std::exception& error) {
@@ -1564,6 +1566,8 @@ void PrebuildStageLibrary(const std::shared_ptr<VulkanDevice>& device, const Gra
     });
 }
 
+namespace {
+
 std::vector<PreparedShaders::Entry> PrepareRegistered(RegisteredPreparation& plan) {
     auto& request = *plan.request;
     std::vector<PreparedShaders::Entry> entries;

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -4589,7 +4589,16 @@ bool VulkanDevice::PrebuildGraphicsLibraries(const Graphics::ShaderStages& stage
         else group.push_back({ShaderRecompiler::ShaderStage::Fragment, fragmentStub.get(), Graphics::PipelinePushSlotBytes});
     }
     const auto layouts = Graphics::DescribeSetLayouts(context, group);
-    return Graphics::Pipeline::Prebuild(context, state, Graphics::VertexInputLayout{}, layouts.View(), group);
+    const bool built = Graphics::Pipeline::Prebuild(context, state, Graphics::VertexInputLayout{}, layouts.View(), group);
+    for (const auto& [depthFormat, stencilFormat] : Graphics::PipelineLibraryDepthFormats(context.device)) {
+        Graphics::DepthTarget depth{};
+        depth.format = depthFormat;
+        depth.extent = {1, 1};
+        depth.samples = 1;
+        state.depth = depth;
+        Graphics::Pipeline::Prebuild(context, state, Graphics::VertexInputLayout{}, layouts.View(), group);
+    }
+    return built;
 }
 
 std::shared_ptr<const Graphics::AheadDraw> VulkanDevice::PrepareAheadDraw(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> shaders) const {

core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp

@@ -9,6 +9,7 @@
 #include <cstddef>
 #include <cstdint>
 #include <span>
+#include <utility>
 #include <vector>
 
 namespace AgcDriver::Graphics {
@@ -59,6 +60,8 @@ struct PipelineLibraryCounters {
 
 PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device);
 
+std::vector<std::pair<VkFormat, VkFormat>> PipelineLibraryDepthFormats(VkDevice device);
+
 }
 
 #endif

core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp

@@ -11,6 +11,7 @@
 #include <map>
 #include <mutex>
 #include <optional>
+#include <set>
 #include <string_view>
 
 namespace AgcDriver::Graphics {
@@ -45,6 +46,7 @@ struct DeviceLibraries {
     std::map<Key, VkPipelineLayout> layouts;
     std::array<std::map<Key, VkPipeline>, 4> parts;
     PipelineLibraryCounters counters;
+    std::set<std::pair<VkFormat, VkFormat>> depthFormats;
 };
 
 struct LibraryStore {
@@ -252,6 +254,7 @@ VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPip
             }
         }
         ++device.counters.built[part];
+        if (!partsOnly && flags[part] == VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT && (rendering.depthAttachmentFormat != VK_FORMAT_UNDEFINED || rendering.stencilAttachmentFormat != VK_FORMAT_UNDEFINED)) device.depthFormats.emplace(rendering.depthAttachmentFormat, rendering.stencilAttachmentFormat);
         libraries.push_back(handle);
     }
     if (partsOnly) return VK_NULL_HANDLE;
@@ -391,4 +394,12 @@ PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device) {
     return found == store.devices.end() ? PipelineLibraryCounters{} : found->second.counters;
 }
 
+std::vector<std::pair<VkFormat, VkFormat>> PipelineLibraryDepthFormats(VkDevice device) {
+    auto& store = Libraries();
+    std::lock_guard lock(store.mutex);
+    const auto found = store.devices.find(device);
+    if (found == store.devices.end()) return {};
+    return {found->second.depthFormats.begin(), found->second.depthFormats.end()};
+}
+
 }

core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp

@@ -1,8 +1,10 @@
 #include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp"
+#include "DepthFastClearHarness.hpp"
 #include "VulkanTestDevice.hpp"
 #include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp"
 #include "SceShaders.hpp"
 #include "prx/libSceAgcDriver/Execution/include/Driver/Draw/DrawCache.hpp"
+#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp"
 #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp"
 #include "Optimization/ResourceProgram.hpp"
@@ -673,6 +675,75 @@ void CheckMergedHullVertexTables(AgcDriver::VulkanDevice& device) {
     }
 }
 
+void PrebuildStageWithoutPushConstants(AgcDriver::VulkanDevice& device) {
+    using namespace AgcDriver::DriverDetail;
+    if (!device.Target().registrationModules) return;
+    const auto* disabled = std::getenv("APS5_NO_LIBRARY_PREBUILD");
+    if (disabled != nullptr && std::string_view(disabled) == "1") return;
+    auto result = std::make_shared<ShaderRecompiler::RecompileResult>();
+    result->spirv = {0x07230203u, 0x00010000u, 0u, 6u, 0u, 0x00020011u, 1u, 0x0003000eu, 0u, 1u, 0x0005000fu, 4u, 4u, 0x6e69616du, 0u, 0x00030010u, 4u, 7u, 0x00020013u, 2u, 0x00030021u, 3u, 2u, 0x00050036u, 2u, 4u, 0u, 3u, 0x000200f8u, 5u, 0x000100fdu, 0x00010038u};
+    result->variantId = 0x5ca1ab1e0000feedull;
+    result->runtimeAbiVersion = ShaderRecompiler::RuntimeAbi::Version;
+    Require(result->pushConstants.empty(), "the push-constant-free test fragment shader has push constants");
+    const AgcDriver::Graphics::ShaderStages stages{AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, {}, {}};
+    const std::shared_ptr<AgcDriver::VulkanDevice> shared(&device, [](AgcDriver::VulkanDevice*) {});
+    const auto before = AgcDriver::Graphics::PipelineLibraryCountersOf(device.Device());
+    PrebuildStageLibrary(shared, stages, ShaderRecompiler::ShaderStage::Fragment, result, AgcDriver::Graphics::PipelinePushSlotBytes);
+    AwaitLibraryPrebuilds();
+    const auto prebuilt = AgcDriver::Graphics::PipelineLibraryCountersOf(device.Device());
+    Require(prebuilt.built[2] - before.built[2] == 1u, "the registration prebuild of a push-constant-free fragment shader did not build its fragment library");
+    const std::array<AgcDriver::Graphics::CompiledShader, 1> drawShaders{{{ShaderRecompiler::ShaderStage::Fragment, result.get(), 0u}}};
+    Require(device.PrebuildGraphicsLibraries(stages, drawShaders), "the draw-side lookup of a push-constant-free fragment shader was not built");
+    const auto drawn = AgcDriver::Graphics::PipelineLibraryCountersOf(device.Device());
+    Require(drawn.built[2] == prebuilt.built[2], "the draw built a fragment library the registration prebuild of a push-constant-free shader should have provided");
+}
+
+alignas(4096) std::array<float, DepthFastClearHarness::Width * DepthFastClearHarness::Height> PrebuildDepthPlane{};
+alignas(4096) std::array<std::byte, DepthFastClearHarness::Width * DepthFastClearHarness::Height> PrebuildStencilPlane{};
+alignas(256) constexpr std::array<std::uint32_t, 4> PrebuildOtherPixelCode{0x7e0002f0u, 0xf800180fu, 0x00000000u, 0xbf810000u};
+
+void PrebuildCoversRecordedDepthFormats(AgcDriver::VulkanDevice& device) {
+    namespace Harness = DepthFastClearHarness;
+    if (!device.Target().registrationModules) return;
+    const auto* disabled = std::getenv("APS5_NO_LIBRARY_PREBUILD");
+    if (disabled != nullptr && std::string_view(disabled) == "1") return;
+    if (!device.graphicsContext().graphicsPipelineLibrary) return;
+    const auto first = Harness::Compile(device);
+    auto second = first;
+    ShaderRecompiler::ShaderPixelStageInfo pixel{};
+    pixel.wave32 = true;
+    pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY);
+    pixel.posX = true;
+    pixel.posY = true;
+    pixel.targetOutputMode[0] = 9;
+    pixel.targetExportMapping.fill(0xe4u);
+    const std::array<ShaderRecompiler::MemoryRegion, 1> pixelMemory{{{reinterpret_cast<std::uintptr_t>(PrebuildOtherPixelCode.data()), std::as_bytes(std::span(PrebuildOtherPixelCode))}}};
+    ShaderRecompiler::RecompileRequest request{
+        {ShaderRecompiler::ShaderStage::Fragment, reinterpret_cast<std::uintptr_t>(PrebuildOtherPixelCode.data()), PrebuildOtherPixelCode, 0, {}},
+        {32, 0, std::vector<std::uint32_t>(4, 0u), std::nullopt, pixel, std::nullopt, pixelMemory},
+        device.Target(),
+        PixelPushLayout(first.vertexPush, device.Target())
+    };
+    request.useCache = false;
+    second.pixel = ShaderRecompiler::Recompile(request);
+    Require(second.pixel.Identity() != first.pixel.Identity(), "the second depth-format test fragment shader has the identity of the first");
+    const Harness::Surface surface{reinterpret_cast<std::uintptr_t>(PrebuildDepthPlane.data()), reinterpret_cast<std::uintptr_t>(PrebuildStencilPlane.data()), 0, false, VK_FORMAT_D32_SFLOAT_S8_UINT};
+    const auto deviceHandle = device.Device();
+    const auto beforeFirst = AgcDriver::Graphics::PipelineLibraryCountersOf(deviceHandle);
+    Harness::Draw(device, first, surface, {});
+    const auto afterFirst = AgcDriver::Graphics::PipelineLibraryCountersOf(deviceHandle);
+    Require(afterFirst.built[2] - beforeFirst.built[2] == 1u, "the first depth-format draw did not build its fragment library");
+    const auto recorded = AgcDriver::Graphics::PipelineLibraryDepthFormats(deviceHandle);
+    Require(std::ranges::find(recorded, std::pair{VK_FORMAT_D32_SFLOAT_S8_UINT, VK_FORMAT_D32_SFLOAT_S8_UINT}) != recorded.end(), "the draw did not record its depth and stencil attachment formats");
+    const AgcDriver::Graphics::ShaderStages stages{AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt};
+    const std::array<AgcDriver::Graphics::CompiledShader, 1> fragmentOnly{{{ShaderRecompiler::ShaderStage::Fragment, &second.pixel, PixelPushOffset(second.vertexPush, device.Target())}}};
+    Require(device.PrebuildGraphicsLibraries(stages, fragmentOnly), "the prebuild of the second depth-format fragment shader was refused");
+    const auto beforeSecond = AgcDriver::Graphics::PipelineLibraryCountersOf(deviceHandle);
+    Harness::Draw(device, second, surface, {});
+    const auto afterSecond = AgcDriver::Graphics::PipelineLibraryCountersOf(deviceHandle);
+    Require(afterSecond.built[2] == beforeSecond.built[2], "the draw built a fragment library the prebuild should have provided for a depth format draws use");
+}
+
 void NullPixelMatchesRegistration(AgcDriver::VulkanDevice& device) {
     using namespace AgcDriver::DriverDetail;
     Fixture front;
@@ -730,11 +801,13 @@ int main(int argc, char** argv) {
         const auto dump = argc == 2 ? std::filesystem::path(argv[1]) : std::filesystem::path{};
         RetainedPixelMatchesNullProgram(*device);
         NullPixelMatchesRegistration(*device);
+        PrebuildStageWithoutPushConstants(*device);
         Check(*device, AgcDriver::Graphics::ShaderPath::Vertex, dump);
         Check(*device, AgcDriver::Graphics::ShaderPath::Geometry, dump);
         Check(*device, AgcDriver::Graphics::ShaderPath::Tessellation, dump);
         CheckMergedHull(*device);
         CheckMergedHullVertexTables(*device);
+        PrebuildCoversRecordedDepthFormats(*device);
         std::cout << "prepared graphics ABI tests passed\n";
         return 0;
     } catch (const std::exception& error) {

docs/dev/debt/libSceAgcDriver.md

@@ -123,7 +123,7 @@
 - [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): resident graphics pipelines are bounded by count, not bytes: the store grows from 1024 to 4096 entries for a replay that does not fit and for evicted pipelines that keep coming back, and the last `APS5_PIPELINE_JOURNAL_ENTRIES` (4096) records are all replayed, so a pipeline the title never asks for again costs its memory until teardown. `agc_driver_recorder_tests --benchmark-pipeline-store-memory [count]` gives a floor with the triangle shaders: about 59 KiB host and no device memory per pipeline on an RTX 4090, 100 KiB host and 8 KiB device-local on a UHD 770; larger shaders cost more. The file is rewritten to three quarters of `APS5_PIPELINE_JOURNAL_MAX_MIB` when full. A compute pipeline is replayed over the layout `DescribeSetLayouts` derives, checked only under `APS5_VERIFY_SET_LAYOUTS=1`. The header's format number guards the recipe codec and the store key.
 - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline
 - [Fragment shader interlock pipelines](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp) (libSceAgcDriver): a pipeline whose fragment shader declares a fragment shader interlock capability is never built from graphics pipeline libraries, on every driver, so it is compiled whole at the draw with no fast link to bridge the compile. RADV's fast link ([mesa#16653](https://gitlab.freedesktop.org/mesa/mesa/-/work_items/16653), Mesa 26.2.5, RX 7900 GRE) jumps to the PS epilog without waiting for the shader's stores, and the epilog's `done` export ends the GFX11 primitive-ordered section, so the next overlapping wave reads the old value: `agc_driver_pixel_interlock` lost read-modify-writes in 108 of 120 loaded runs under `APS5_NO_PIPELINE_LTO=1`. On an RTX 4090 (NVIDIA 617.42, Windows) the fast link kept the ordering in 240 of 240 loaded runs; other drivers are not measured.
-- [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV.
+- [Prebuilt pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver): each registered vertex or pixel shader in registration form gets its pre-rasterization or fragment-shader library built on one background thread, with a fixed state the draw may not use: triangle list, counter-clockwise front face, no depth clamp, depth bias or conservative rasterization, first provoking vertex, one sample. Libraries are kept until the device is torn down; on Astro's Playroom 1948 of them added 0.44 GB of peak memory on RADV. A fragment-shader library is also prebuilt for each depth and stencil attachment format pair a draw has built one for, so a format is covered only after one draw used it and each recorded pair multiplies the fragment libraries built per shader; the recorded pairs are never pruned and the cost is unmeasured.
 - [`APS5_THREAD_PRIORITY=rt`](../../../core/libs/prx/libSceAgcDriver/Execution/src/ThreadPriority.cpp) (libSceAgcDriver) lowers the process's hard `RLIMIT_RTTIME` when it is above `APS5_THREAD_RTTIME_US` or rtkit's `RTTimeUSecMax`, because [rtkit](https://github.com/heftig/rtkit/blob/c295fa849f52b487be6433e69e08b46251950399/rtkit-daemon.c#L592-L637) (`verify_process_rttime`) reads the hard limit from `/proc/<pid>/limits` and refuses real-time scheduling when it is higher. An unprivileged process cannot raise the hard limit again, so it stays lowered for the process and its threads even when rtkit then refuses the request (burst limit, user or start-time checks) and the thread throws. rtkit's `MaxRealtimePriority` and `RTTimeUSecMax` are read first, so an unreachable rtkit or a priority above its maximum leaves the limit untouched. The lowered limit only matters for threads that later run under a real-time policy
 - On a device without VK_EXT_robustness2's `nullDescriptor`, such as MoltenVK, a null image descriptor in a typed heap is a [placeholder image](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PlaceholderImages.cpp) of that heap's type, 1×1 and cleared to zero, which every null descriptor of the heap shares. Reads give zeros, as `nullDescriptor` allows, but stores and atomics to it are kept, so a later read of a null storage image can return them, and a depth comparison compares against depth 0. A heap whose placeholder the device cannot create, for example 1D or 3D depth, 64-bit atomics without 64-bit image atomics, or multisampled storage without `shaderStorageImageMultisample`, throws when a null descriptor reaches it.
 - [Draw input snapshots](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver): `PrepareDrawBindings` caps read-only input buffer snapshots at 64 MiB (overridable via `APS5_DRAW_SNAPSHOT_MAX_MIB`). Larger inputs (streaming heaps, bindless pools) or draws exceeding the cumulative budget skip the per-draw snapshot and bind in place, avoiding multi-gigabyte CPU copy stalls (such as Ratchet & Clank: Rift Apart PPSA01474 stalling at frame ~380 copying 36 GB every 5 s). If the title rewrites those large ranges on the CPU before GPU completion, the in-place binding observes the modification, as verified by `drawSnapshotCapTests`.