[HOLD] perf(agc): pre-warm graphics pipelines outside GpuMutex during draw preparation

core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp

@@ -303,6 +303,7 @@ class VulkanDevice {
     IndirectDrawSupport DrawIndirectSupport() const;
     void EnqueueDraw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span<const Graphics::CompiledShader> shaders, std::span<const Graphics::GuestMemorySnapshot> snapshots = {});
     bool PrebuildGraphicsLibraries(const Graphics::ShaderStages& stages, std::span<const Graphics::CompiledShader> shaders);
+    void PrewarmDrawPipeline(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> shaders);
 
 private:
     VulkanDevice(const PresentationWindow* window, bool compileOnly);

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp

@@ -354,6 +354,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
                 rejected = std::move(*known);
                 return DrawVerdict::Rejected;
             }
+            localDevice->PrewarmDrawPipeline(graphics, stages);
             lockForDraw();
             noteDrawWriters(stages, submission.queue);
             phaseTiming.Phase(DrawRowVectors);
@@ -380,6 +381,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span<const std::uint32_t> packe
             rejected = std::move(*known);
             return DrawVerdict::Rejected;
         }
+        localDevice->PrewarmDrawPipeline(graphics, stages);
     }
     lockForDraw();
     noteDrawWriters(stages, submission.queue);

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -3213,6 +3213,10 @@ bool VulkanDevice::PrebuildGraphicsLibraries(const Graphics::ShaderStages& stage
     return Graphics::Pipeline::Prebuild(context, state, Graphics::VertexInputLayout{}, layouts.View(), group);
 }
 
+void VulkanDevice::PrewarmDrawPipeline(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> shaders) {
+    Graphics::PrewarmPipeline(graphicsContext(), graphics, shaders);
+}
+
 void VulkanDevice::Draw(const Graphics::State& graphics, const Pm4::DrawParameters& draw, std::span<const Graphics::CompiledShader> shaders, std::span<const Graphics::GuestMemorySnapshot> snapshots, std::shared_ptr<const DrawRecipe>* recipe) {
     for (const auto& shader : shaders) {
         require(shader.program != nullptr, "missing compiled shader");

core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp

@@ -18,6 +18,7 @@ struct HostImport;
 // built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under
 // APS5_NO_DRAW_RECIPE=1).
 void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots = {}, std::shared_ptr<const DrawRecipe>* recipe = nullptr);
+void PrewarmPipeline(const Context& context, const State& state, std::span<const CompiledShader> shaders);
 std::optional<std::string> KnownValidationFailure(const Context& context, std::span<const CompiledShader> shaders, const State& state);
 std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span<const VkImageView> targetViews);
 

core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp

@@ -1837,6 +1837,21 @@ std::optional<std::string> KnownValidationFailure(const Context& context, std::s
     return std::nullopt;
 }
 
+void PrewarmPipeline(const Context& context, const State& state, std::span<const CompiledShader> shaders) {
+    if (shaders.empty() || shaders.front().program == nullptr || context.descriptorCache == nullptr) return;
+    bool memoized = false, hit = false;
+    const auto fragmentOutputs = CachedFragmentOutputs(context, shaders, state, memoized, hit);
+    const auto masked = maskedState(state, fragmentOutputs);
+    const State& pipelineState = masked.has_value() ? *masked : state;
+    const auto& attributes = shaders.front().program->vertexAttributes;
+    const auto vertexInput = BuildVertexInputLayout(context, attributes);
+    const auto described = DescribeSetLayouts(context, shaders);
+    static const bool drawTransitions = std::getenv("APS5_DRAW_TRANSITIONS") != nullptr;
+    static const bool recordDraws = std::getenv("APS5_SYNC_DRAWS") == nullptr;
+    const bool lean = recordDraws && !drawTransitions;
+    CachedPipeline(context, pipelineState, vertexInput, described.View(), shaders, lean ? VK_IMAGE_LAYOUT_GENERAL : VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+}
+
 void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span<const CompiledShader> shaders, std::span<const GuestMemorySnapshot> snapshots, std::shared_ptr<const DrawRecipe>* recipeOut) {
     if (context.recorder != nullptr) context.recorder->BoundKeptBytes();
     PerformanceTimer timing("Graphics.Draw");

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -2566,6 +2566,7 @@ struct MockVulkan {
     std::map<VkPipelineLayout, std::vector<VkPushConstantRange>> pipelineLayoutPushRanges;
     std::map<VkGraphicsPipelineLibraryFlagsEXT, VkPipelineLayout> libraryLayouts;
     std::uint32_t pipelineCreateCount = 0;
+    std::uint32_t graphicsPipelineCreateCount = 0;
     std::vector<std::array<std::uint32_t, 3>> pipelineSpecializations;
     VkPipeline boundPipeline = VK_NULL_HANDLE;
     std::vector<std::byte> lastPushConstants;
@@ -2731,11 +2732,14 @@ VKAPI_ATTR VkResult VKAPI_CALL mockCreatePipelineLayout(VkDevice, const VkPipeli
 }
 
 VKAPI_ATTR VkResult VKAPI_CALL mockCreateGraphicsPipelines(VkDevice, VkPipelineCache, std::uint32_t count, const VkGraphicsPipelineCreateInfo* infos, const VkAllocationCallbacks*, VkPipeline* pipelines) {
-    Require(count == 1 && (infos[0].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0, "mock expects a graphics pipeline library");
-    const auto* library = static_cast<const VkGraphicsPipelineLibraryCreateInfoEXT*>(infos[0].pNext);
-    Require(library != nullptr && library->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, "missing graphics pipeline library description");
-    mock.libraryLayouts[library->flags] = infos[0].layout;
+    Require(count == 1, "mock expects a single graphics pipeline");
+    if ((infos[0].flags & VK_PIPELINE_CREATE_LIBRARY_BIT_KHR) != 0) {
+        const auto* library = static_cast<const VkGraphicsPipelineLibraryCreateInfoEXT*>(infos[0].pNext);
+        Require(library != nullptr && library->sType == VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, "missing graphics pipeline library description");
+        mock.libraryLayouts[library->flags] = infos[0].layout;
+    }
     *pipelines = makeHandle<VkPipeline>();
+    ++mock.graphicsPipelineCreateCount;
     ++mock.live;
     return VK_SUCCESS;
 }
@@ -2744,6 +2748,16 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyPipelineLayout(VkDevice, VkPipelineLayout,
     --mock.live;
 }
 
+VKAPI_ATTR VkResult VKAPI_CALL mockCreateRenderPass(VkDevice, const VkRenderPassCreateInfo*, const VkAllocationCallbacks*, VkRenderPass* pass) {
+    *pass = makeHandle<VkRenderPass>();
+    ++mock.live;
+    return VK_SUCCESS;
+}
+
+VKAPI_ATTR void VKAPI_CALL mockDestroyRenderPass(VkDevice, VkRenderPass, const VkAllocationCallbacks*) {
+    --mock.live;
+}
+
 VKAPI_ATTR VkResult VKAPI_CALL mockCreateShaderModule(VkDevice, const VkShaderModuleCreateInfo*, const VkAllocationCallbacks*, VkShaderModule* module) {
     *module = makeHandle<VkShaderModule>();
     ++mock.live;
@@ -2852,6 +2866,8 @@ PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) {
         {"vkCreateGraphicsPipelines", reinterpret_cast<PFN_vkVoidFunction>(mockCreateGraphicsPipelines)},
         {"vkCreatePipelineLayout", reinterpret_cast<PFN_vkVoidFunction>(mockCreatePipelineLayout)},
         {"vkDestroyPipelineLayout", reinterpret_cast<PFN_vkVoidFunction>(mockDestroyPipelineLayout)},
+        {"vkCreateRenderPass", reinterpret_cast<PFN_vkVoidFunction>(mockCreateRenderPass)},
+        {"vkDestroyRenderPass", reinterpret_cast<PFN_vkVoidFunction>(mockDestroyRenderPass)},
         {"vkCreateShaderModule", reinterpret_cast<PFN_vkVoidFunction>(mockCreateShaderModule)},
         {"vkDestroyShaderModule", reinterpret_cast<PFN_vkVoidFunction>(mockDestroyShaderModule)},
         {"vkCreateComputePipelines", reinterpret_cast<PFN_vkVoidFunction>(mockCreateComputePipelines)},
@@ -4085,6 +4101,37 @@ void rectListTests() {
     }
 }
 
+void prewarmPipelineTests() {
+    using namespace ShaderRecompiler;
+    using namespace AgcDriver::Graphics;
+    mock = MockVulkan{};
+    auto context = mockContext();
+    context.device = makeHandle<VkDevice>();
+    context.limits.maxColorAttachments = 8;
+    context.limits.maxPushConstantsSize = 256;
+    DescriptorCache cache(context);
+    context.descriptorCache = &cache;
+    RecompileResult vertex;
+    vertex.spirv = makeModule({});
+    vertex.variantId = 101;
+    RecompileResult fragment;
+    fragment.spirv = makeModule({.fragment = true});
+    fragment.variantId = 102;
+    const std::array<CompiledShader, 2> shaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}};
+    State state{};
+    state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST;
+    state.hasColorTarget = true;
+    state.color.format = VK_FORMAT_R8G8B8A8_UNORM;
+    state.colors.push_back(state.color);
+    state.blend.colorWriteMask = 15;
+    state.blends.push_back(state.blend);
+    const auto createsBefore = mock.graphicsPipelineCreateCount;
+    PrewarmPipeline(context, state, shaders);
+    Require(mock.graphicsPipelineCreateCount == createsBefore + 1, "PrewarmPipeline did not create a pipeline on miss");
+    PrewarmPipeline(context, state, shaders);
+    Require(mock.graphicsPipelineCreateCount == createsBefore + 1, "PrewarmPipeline recreated an already warmed pipeline");
+}
+
 ShaderRecompiler::RecompileResult recompilePixel(std::initializer_list<std::uint32_t> controls, std::span<const std::uint32_t> code) {
     auto queue = makeState();
     queue.context[0x1b3] = 0x2u;
@@ -5715,6 +5762,7 @@ int main(int argc, char** argv) {
         parallelCompareTests();
         pixelParameterSlotTests();
         rectListTests();
+        prewarmPipelineTests();
         floatControlsModeTests();
         barycentricEmulationTests();
         copiedWriterIndirectOverlapTests();