fix(agc): raise no end-of-pipe event from flips and carry the release's interrupt context

core/libs/prx/libSceAgcDriver/Eq/include/Event.hpp

@@ -4,7 +4,7 @@
 #include <cstdint>
 #include "SceTypes.hpp"
 
-bool AgcDriverDeliverEopInterrupt(std::uint32_t queue);
+void AgcDriverDeliverEopInterrupt(std::uint32_t queue, std::uint32_t context);
 
 extern "C" int APS5_VABI sceAgcDriverAddEqEvent(KernelEqueue eq, int id, void* udata);
 extern "C" int APS5_VABI sceAgcDriverDeleteEqEvent(KernelEqueue eq, int id);

core/libs/prx/libSceAgcDriver/Eq/src/Event.cpp

@@ -33,9 +33,11 @@ int AddEvent(KernelEqueue eq, std::uint32_t queue, std::uintptr_t ident, void* u
     event.event.ident = ident;
     event.event.filter = EvfiltGraphicsCore;
     event.event.flags = EV_ADD | EV_CLEAR;
+    event.event.data = static_cast<intptr_t>(queue);
     event.event.udata = udata;
-    event.filter.triggerFunc = [](KernelEqueueEvent* e, void*) {
-        e->event.data = e->triggered ? e->event.data + 1 : 1;
+    event.filter.triggerFunc = [](KernelEqueueEvent* e, void* data) {
+        if (data == nullptr) throw std::runtime_error("graphics event triggered without its interrupt data");
+        e->event.data = static_cast<intptr_t>(*static_cast<const std::uint64_t*>(data));
         e->triggered = true;
     };
     event.filter.resetFunc = [](KernelEqueueEvent* e) {
@@ -67,20 +69,17 @@ int DeleteEvent(KernelEqueue eq, std::uintptr_t ident, const char* caller) {
 
 }
 
-bool AgcDriverDeliverEopInterrupt(std::uint32_t queue) {
+void AgcDriverDeliverEopInterrupt(std::uint32_t queue, std::uint32_t context) {
     std::vector<Registration> registrations;
     {
         std::lock_guard lock(g_mutex);
         registrations = g_registrations;
     }
-    bool delivered = false;
     for (const auto& registration : registrations) {
-        if (registration.queue == queue) {
-            EqueueTriggerEvent_nid_postfix(registration.eq, registration.ident, EvfiltGraphicsCore, nullptr);
-            delivered = true;
-        }
+        if (registration.queue != queue) continue;
+        std::uint64_t data = static_cast<std::uint64_t>(static_cast<std::uint8_t>(registration.queue)) | (static_cast<std::uint64_t>(context) << 16u);
+        EqueueTriggerEvent_nid_postfix(registration.eq, registration.ident, EvfiltGraphicsCore, &data);
     }
-    return delivered;
 }
 
 extern "C" {

core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp

@@ -17,22 +17,14 @@ uint32_t APS5_VABI sceAgcDriverGetEqContextId(const KernelEvent* ev) {
     if (ev->filter != -14) {
         throw std::runtime_error(std::string(__func__) + ": not a graphics event");
     }
-    if (ev->ident > std::numeric_limits<uint32_t>::max()) {
-        throw std::runtime_error(std::string(__func__) + ": context id overflow");
-    }
-    return static_cast<uint32_t>(ev->ident);
+    return static_cast<uint32_t>(static_cast<std::uint64_t>(ev->data) >> 16u);
 }
 
 int APS5_VABI sceAgcDriverGetEqEventType(const KernelEvent* ev) {
     if (ev == nullptr || reinterpret_cast<std::uintptr_t>(ev) % alignof(KernelEvent) != 0) {
         throw std::runtime_error(std::string(__func__) + ": null or misaligned event");
     }
-    if (ev->filter == -14) {
-        if (ev->ident > static_cast<std::uintptr_t>(std::numeric_limits<int>::max())) {
-            throw std::runtime_error(std::string(__func__) + ": graphics event type overflow");
-        }
-        return static_cast<int>(ev->ident);
-    }
+    if (ev->filter == -14) return static_cast<int>(static_cast<std::uint32_t>(static_cast<std::uint64_t>(ev->data)));
     if (ev->data < std::numeric_limits<int>::min() || ev->data > std::numeric_limits<int>::max()) {
         throw std::runtime_error(std::string(__func__) + ": event type overflow");
     }

core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp

@@ -145,10 +145,13 @@ class Driver {
     static bool copySegment(Submission& submission, const std::uint32_t* guest, std::size_t words, std::size_t& budget);
     static void readRegisterLists(Submission& submission);
     void waitForFlipRoom(const Submission& submission);
-    void holdFlipBehindWorker(const Submission& submission);
+    void holdFlipBehindWorker(const Submission& submission, bool cpuFlip);
     bool awaitsTitle(std::uint64_t awaited) const;
     void noteAwaitingTitle(std::uint32_t queue, std::uint64_t awaited);
     void releaseFlipHold();
+    void finishFlipWork(std::uint64_t workSerial);
+    std::uint64_t submitFlipWork(std::uint32_t queue);
+    void waitFlipWork(std::uint64_t workSerial);
     void reserveOutputs(Submission& submission);
     void executeRewindTail(const Submission& stalled);
     void enqueue(Submission submission);
@@ -159,7 +162,7 @@ class Driver {
     void deferCompletion(const Submission& submission);
     void submitOpenWork(const std::shared_ptr<SubmissionProgress>& progress = {});
     void wakeQueueZeroForCompletions();
-    void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites, const std::shared_ptr<SubmissionProgress>& progress);
+    void completeSubmission(std::uint64_t serial, std::uint64_t received, std::span<const std::uint64_t> labelWrites, const std::shared_ptr<SubmissionProgress>& progress, bool holdsFlip);
     static bool waitFree(const Submission& submission);
     bool queue0Before(std::uint64_t received) const;
     bool waitFreeBefore(std::uint64_t received) const;
@@ -298,11 +301,11 @@ class Driver {
     static std::vector<std::pair<std::uint64_t, std::uint64_t>> drawWriteRanges(const Graphics::State& graphics, std::span<const Graphics::CompiledShader> stages);
     void commitDraw(std::shared_ptr<VulkanDevice> localDevice, std::uint32_t queue, const Graphics::State& graphics, const Pm4::DrawParameters& drawParameters, std::span<const Graphics::CompiledShader> stages, std::span<const Graphics::GuestMemorySnapshot> snapshots, const std::shared_ptr<const DrawRecipe>& recipe, const std::vector<std::shared_ptr<DispatchVariant>>& recipeStages, std::uint64_t drawKey, VulkanDevice::DrawTraceContext trace = {}, std::uint64_t shape = 0);
     bool labelPacketQueues(std::span<const std::uint32_t> packet, std::uint32_t opcode);
-    bool enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue);
+    bool enqueueLabelPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, std::optional<std::uint32_t> interrupt);
     bool enqueueDmaPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state);
     bool enqueueConstantRamDump(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state);
     void commitStore(std::uint32_t queue, std::uint64_t address, std::span<const std::byte> bytes);
-    void commitLabel(std::uint32_t queue, std::uint64_t address, std::span<const std::byte> bytes, bool endOfPipeInterrupt);
+    void commitLabel(std::uint32_t queue, std::uint64_t address, std::span<const std::byte> bytes, std::optional<std::uint32_t> interrupt);
     void submitDueAfterCommit(VulkanDevice* localDevice);
     std::optional<WrittenBuffer> newestWriterLocked(std::uint64_t begin, std::uint64_t end) const;
     std::optional<WrittenBuffer> newestWriter(std::uint64_t begin, std::uint64_t end);
@@ -382,15 +385,15 @@ class Driver {
     void recordQueuedLabelsByTry(std::uint32_t queue, std::atomic<std::uint64_t>& counter);
     static void recordQueuedLabelsFromHook();
     void flushBetweenPackets(std::uint32_t queue, std::uint32_t header, bool labelPacket);
-    bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump);
+    bool preparePacketMemory(const Submission& submission, QueueState& queue, std::span<const std::uint32_t> packet, std::uint32_t header, std::uint32_t opcode, std::optional<std::uint32_t> interrupt, bool& wroteOnGpu, bool& endOfPipeInterrupt, bool& interruptDeferred, bool& drawPacket, bool& sampleDump);
     void dumpSampleCounters(std::uint64_t address);
     template <typename TWork>
     static void timed(double WorkerProfile::*bucket, TWork&& work);
     FrameTiming* frameTiming();
     FrameTiming* includeTimingSubmission(const Submission& submission, bool firstSegment);
     void execute(const Submission& submission);
-    void deliverGuestEopInterrupt(std::uint32_t queue);
-    void deliverFlipEopInterrupt(std::uint32_t queue);
+    void deliverGuestEopInterrupt(std::uint32_t queue, std::uint32_t context);
+    static std::optional<std::uint32_t> releaseInterrupt(std::span<const std::uint32_t> packet, std::uint32_t opcode, bool carriesFlip);
     void markCompleted(std::uint64_t serial);
     static const std::atomic<std::uint64_t>*& workerQueued();
     void reapCompletionLabels();
@@ -416,9 +419,6 @@ class Driver {
     std::atomic<std::uint64_t> flipSerial{0};
     std::atomic<std::uint64_t> flipBatchesUnsignaled{0};
 
-    std::mutex eopSelfSignalMutex;
-    std::unordered_map<std::uint32_t, std::uint32_t> eopSelfSignalFlips;
-
     alignas(64) std::atomic<int> packetsInFlight{0};
     std::atomic<std::uint64_t> packetsDone{0};
 
@@ -501,6 +501,8 @@ class Driver {
     std::atomic<std::uint64_t> queue0Awaited{0};
     std::atomic<std::uint32_t> flipsAhead{0};
     std::atomic<std::uint32_t> flipHolders{0};
+    std::atomic<std::uint32_t> flipWorkOpen{0};
+    std::atomic<std::uint64_t> flipWorkSerial{0};
     std::atomic<bool> queue0AwaitsTitle{false};
 
     std::atomic<std::uint64_t> evidenceReads{0};

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp

@@ -19,13 +19,14 @@
 #include <cstdlib>
 #include <functional>
 #include <shared_mutex>
+#include <stdexcept>
+#include <string>
 #include <utility>
 
 namespace AgcDriver::DriverDetail {
 
 namespace {
 
-constexpr std::uint32_t eopSelfSignalFlipLimit = 4;
 
 struct RingOrder {
     std::uint32_t queue = 0xffffffffu;
@@ -121,22 +122,18 @@ bool& drawnSinceSeal() {
 
 }
 
-void Driver::deliverGuestEopInterrupt(std::uint32_t queue) {
-    if (!AgcDriverDeliverEopInterrupt(queue)) return;
-    std::lock_guard<std::mutex> lock(eopSelfSignalMutex);
-    eopSelfSignalFlips[queue] = 0;
+void Driver::deliverGuestEopInterrupt(std::uint32_t queue, std::uint32_t context) {
+    AgcDriverDeliverEopInterrupt(queue, context);
 }
 
-void Driver::deliverFlipEopInterrupt(std::uint32_t queue) {
-    {
-        std::lock_guard<std::mutex> lock(eopSelfSignalMutex);
-        const auto latched = eopSelfSignalFlips.find(queue);
-        if (latched != eopSelfSignalFlips.end()) {
-            if (++latched->second <= eopSelfSignalFlipLimit) return;
-            eopSelfSignalFlips.erase(latched);
-        }
-    }
-    AgcDriverDeliverEopInterrupt(queue);
+std::optional<std::uint32_t> Driver::releaseInterrupt(std::span<const std::uint32_t> packet, std::uint32_t opcode, bool carriesFlip) {
+    if (opcode != 0x49 || carriesFlip || packet.size() < 3) return std::nullopt;
+    const auto interruptSelect = (packet[2] >> 24u) & 7u;
+    if (interruptSelect == 0 || interruptSelect == 3) return std::nullopt;
+    if (packet.size() < 8) throw std::runtime_error("AGC: an interrupting release has no interrupt context dword");
+    const auto context = packet[7] & 0x0fffffffu;
+    if ((context & 0x08000000u) != 0) throw std::runtime_error("AGC: a release asks for typed interrupt context " + std::to_string(context) + ", which is not delivered as a graphics end-of-pipe event; not implemented");
+    return context;
 }
 
 template <typename TWork>
@@ -162,14 +159,25 @@ void Driver::execute(const Submission& submission) {
     auto* submissionTiming = includeTimingSubmission(submission, true);
     struct FlipHold {
         Driver& driver;
+        std::uint32_t queue;
         bool held;
+        bool working;
         void Release() {
             if (!held) return;
             held = false;
             driver.releaseFlipHold();
         }
-        ~FlipHold() { Release(); }
-    } flipHold{*this, submission.holdsFlip};
+        void Finish() {
+            if (!working) return;
+            const auto workSerial = driver.submitFlipWork(queue);
+            working = false;
+            driver.finishFlipWork(workSerial);
+        }
+        ~FlipHold() {
+            Release();
+            if (working) driver.finishFlipWork(0);
+        }
+    } flipHold{*this, submission.queue, submission.holdsFlip, submission.holdsFlip};
     const bool pipelined = submission.queue == 0 && DrawPipeline::Depth() != 0;
     static const bool orderedWaits = std::getenv("APS5_PIPELINE_DRAIN_LABELS") == nullptr;
     DrawPipeline::Active() = pipelined;
@@ -294,7 +302,7 @@ void Driver::execute(const Submission& submission) {
         if (submission.queue != 0 && Graphics::Recorder::DispatchOverlapEnabled()) noteRingPacket(submission.queue, submission.serial, opcode, header);
         PerformanceTimer timing("Driver.Packet");
         if (!predicated) noteLabelIssue(packet, submission.queue);
-        if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && (enqueueLabelPacket(packet, opcode, submission.queue) || enqueueDmaPacket(packet, opcode, submission.queue, queue) || enqueueConstantRamDump(packet, opcode, submission.queue, queue))) {
+        if (pipelined && !predicated && header != FlipPacketHeader && header != RenderingWaitPacketHeader && !submission.flips.contains(cursor) && (enqueueLabelPacket(packet, opcode, submission.queue, releaseInterrupt(packet, opcode, false)) || enqueueDmaPacket(packet, opcode, submission.queue, queue) || enqueueConstantRamDump(packet, opcode, submission.queue, queue))) {
             traceLabel(packet, submission.queue);
             recent.Record(cursor);
             cursor = nextCursor;
@@ -385,7 +393,7 @@ void Driver::execute(const Submission& submission) {
         if (profilePackets) packetProfile.flushMs += std::chrono::duration<double, std::milli>(packetTimer.start - flushStart).count();
 
         bool wroteOnGpu = false, endOfPipeInterrupt = false, interruptDeferred = false, drawPacket = false, sampleDump = false;
-        const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump);
+        const bool drains = preparePacketMemory(submission, queue, packet, header, opcode, releaseInterrupt(packet, opcode, submission.flips.contains(cursor)), wroteOnGpu, endOfPipeInterrupt, interruptDeferred, drawPacket, sampleDump);
         traceLabel(packet, submission.queue);
         timing.Mark("prepare_memory");
 
@@ -409,12 +417,10 @@ void Driver::execute(const Submission& submission) {
                     Pm4::Execute(packet, queue);
                     if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue);
                 }
-                if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue);
             }
             if (deviceUse.owns_lock()) deviceUse.unlock();
             CheckFailure();
             std::uint64_t batchesAtFlip = 0, unsignaledAtFlip = 0;
-            bool flipInterruptDeferred = false;
             Graphics::StorageTexture::NoteFlip();
             if (!drains) {
 
@@ -424,13 +430,8 @@ void Driver::execute(const Submission& submission) {
                 if (localDevice != nullptr) static_cast<void>(Graphics::StorageTexture::StoreDrawnLinear("flip"));
 
                 if (!recordLabelsForPacket(localDevice.get(), submission.queue) && localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0);
-                if (localDevice != nullptr) {
-                    localDevice->FlipBatches(batchesAtFlip, unsignaledAtFlip);
-                    const auto queueId = submission.queue;
-                    flipInterruptDeferred = localDevice->AfterRecordedWork([this, queueId] { deliverFlipEopInterrupt(queueId); }, false);
-                }
+                if (localDevice != nullptr) localDevice->FlipBatches(batchesAtFlip, unsignaledAtFlip);
             }
-            if (!flipInterruptDeferred) deliverFlipEopInterrupt(submission.queue);
             ++flipsCounted;
             Graphics::NoteHostImportFrame();
             if (batchesAtFlip != 0) flipSerial = batchesAtFlip;
@@ -593,7 +594,7 @@ void Driver::execute(const Submission& submission) {
                     if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue);
                 }
             }
-            if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue);
+            if (endOfPipeInterrupt && !interruptDeferred) deliverGuestEopInterrupt(submission.queue, packet[7] & 0x0fffffffu);
         }
         if (drawPacket || (sampleDump && wroteOnGpu)) Graphics::Recorder::CountRecordedWork();
         timing.Mark(header == RenderingWaitPacketHeader ? "rendering_wait" : opcode == 0x3c || opcode == 0x93 ? "memory_wait" : opcode == 0x15 || opcode == 0x16 ? "dispatch" : drawPacket ? "draw" : "execute");
@@ -608,12 +609,15 @@ void Driver::execute(const Submission& submission) {
     static const bool submitAtEnd = std::getenv("APS5_SUBMIT_AT_END") != nullptr;
     if (pipelined && orderedCompletion && deferredLabels().labels.empty()) {
         if (submission.rewindTail == nullptr) {
+            flipHold.working = false;
             deferCompletion(submission);
             return;
         }
         if ((std::atomic_ref<std::uint32_t>(*const_cast<std::uint32_t*>(submission.rewindTail - 1)).load(std::memory_order_acquire) & 0x80000000u) != 0) {
-            DrawPipeline::Queue0().Enqueue([this, progress = submission.progress] {
-                submitOpenWork(progress);
+            const bool holdsFlip = std::exchange(flipHold.working, false);
+            DrawPipeline::Queue0().Enqueue([this, holdsFlip, progress = submission.progress] {
+                if (holdsFlip) finishFlipWork(submitFlipWork(0));
+                else submitOpenWork(progress);
                 wakeQueueZeroForCompletions();
             }, {});
             endTiming.Finish();
@@ -626,6 +630,7 @@ void Driver::execute(const Submission& submission) {
         auto& costs = submissionCosts(submission.queue);
         if (!submitAtEnd && submission.rewindTail == nullptr && submission.queue == 0 && workerQueued() != nullptr && workerQueued()->load(std::memory_order_acquire) != 0) {
             ++costs.endSkipped;
+            flipHold.Finish();
             return;
         }
         ++costs.endSubmits;
@@ -636,6 +641,7 @@ void Driver::execute(const Submission& submission) {
         recordDeferredLabels(localDevice.get(), submission.queue);
         if (localDevice != nullptr) localDevice->SubmitRecorded(submission.queue == 0);
     }
+    flipHold.Finish();
     endTiming.Finish();
     if (submission.rewindTail != nullptr) {
         if (submission.progress) submitOpenWork(submission.progress);