perf(agc): back the GDS with anonymous shared memory on integrated GPUs and keep its DMA_DATA on the GPU

core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp

@@ -14,6 +14,11 @@ namespace AgcDriver::Pm4 {
 
 constexpr std::size_t GdsBytes = 0x10000;
 std::uint64_t GdsAddress();
+void RequestImportableGds(bool requested);
+bool ImportableGdsRequested();
+bool GdsShared();
+bool GdsImportable();
+void RejectGdsImport();
 
 struct DrawParameters {
     std::uint64_t indexAddress;

core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp

@@ -16,16 +16,40 @@
 #include <cstdlib>
 #include <cstring>
 #include <limits>
+#include <optional>
 
 namespace AgcDriver::DriverDetail {
 
 namespace {
 
-bool gdsWritesPending(std::span<const std::uint32_t> packet) {
+bool gdsWritesPending(std::span<const std::uint32_t> packet, const VulkanDevice* localDevice) {
     const auto address = Pm4::GdsAddress() + packet[4];
     const auto bytes = static_cast<std::size_t>(Pm4::DmaBytes(packet));
     const std::array<std::pair<std::uint64_t, std::uint64_t>, 1> range{{{address, address + bytes}}};
-    return Graphics::Recorder::SnapshotWriteOverlaps(address, bytes) || Graphics::StorageTexture::AnyPendingOverlaps(range) || Graphics::AnyShadowedOverlaps(address, bytes);
+    if (Graphics::Recorder::SnapshotWriteOverlaps(address, bytes) || Graphics::StorageTexture::AnyPendingOverlaps(range) || Graphics::AnyShadowedOverlaps(address, bytes)) return true;
+    if (localDevice == nullptr || !Pm4::GdsImportable()) return false;
+    std::lock_guard gpuLock(GuestMemory::GpuMutex());
+    return localDevice->StoresPendingOver(address, bytes);
+}
+
+std::uint64_t dmaAddress(std::uint32_t low, std::uint32_t high) {
+    return low | (static_cast<std::uint64_t>(high) << 32u);
+}
+
+bool gdsSpan(std::uint64_t offset, std::size_t bytes) {
+    return offset <= Pm4::GdsBytes && bytes <= Pm4::GdsBytes - offset;
+}
+
+std::optional<Pm4::MemoryCopy> decodeCopy(std::span<const std::uint32_t> packet) {
+    if (auto copy = Pm4::DecodeMemoryCopy(packet)) return copy;
+    if (packet.size() != 7 || !Pm4::GdsImportable()) return std::nullopt;
+    const auto source = Pm4::DmaSource(packet);
+    const auto destination = Pm4::DmaDestination(packet);
+    const std::size_t bytes = Pm4::DmaBytes(packet);
+    if (bytes == 0) return std::nullopt;
+    if (source == Pm4::DmaSelectGds && (destination == 0 || destination == 3) && gdsSpan(packet[2], bytes)) return Pm4::MemoryCopy{Pm4::GdsAddress() + packet[2], dmaAddress(packet[4], packet[5]), bytes};
+    if ((source == 0 || source == 3) && destination == Pm4::DmaSelectGds && gdsSpan(packet[4], bytes)) return Pm4::MemoryCopy{dmaAddress(packet[2], packet[3]), Pm4::GdsAddress() + packet[4], bytes};
+    return std::nullopt;
 }
 
 }
@@ -130,7 +154,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
     if (!drainAll && !cpuStores && (opcode == 0x40 || opcode == 0x50 || opcode == 0x83)) {
         constexpr std::size_t gpuStoreLimit = 65536;
         bool drained = true;
-        const auto copy = opcode == 0x50 ? Pm4::DecodeMemoryCopy(packet) : std::nullopt;
+        const auto copy = opcode == 0x50 ? decodeCopy(packet) : std::nullopt;
         const auto copyBehindPendingSource = [&] {
             if (!copy.has_value() || copy->bytes > gpuStoreLimit || !GuestMemory::Accessible(reinterpret_cast<const void*>(copy->source), copy->bytes) || !GuestMemory::Accessible(reinterpret_cast<const void*>(copy->destination), copy->bytes, true)) return false;
             const auto end = copy->source + copy->bytes;
@@ -148,16 +172,16 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue
             if (outcome.path != 1 && outcome.path != 3) throw std::runtime_error("AGC driver: a DMA_DATA copy of up to 64 KiB that GpuCopyPossible accepted took copy path " + std::to_string(outcome.path));
             return true;
         };
-        if (opcode == 0x50 && packet.size() >= 7 && Pm4::DmaDestination(packet) == Pm4::DmaSelectGds) {
-            if (!gdsWritesPending(packet)) {
-                orderedAlready = true;
-                drained = false;
-            }
-        } else if (copyBehindPendingSource()) {
+        const bool gdsDestination = opcode == 0x50 && packet.size() >= 7 && Pm4::DmaDestination(packet) == Pm4::DmaSelectGds;
+        const bool gdsOnGpu = gdsDestination && Pm4::GdsImportable();
+        if ((!gdsDestination || gdsOnGpu) && copyBehindPendingSource()) {
             ++copiesBehindPendingSources;
             wroteOnGpu = true;
             drained = false;
-        } else if (const auto store = Pm4::ResolveStore(packet, queue, gpuStoreLimit)) {
+        } else if (gdsDestination && !gdsWritesPending(packet, device.Load().get())) {
+            orderedAlready = true;
+            drained = false;
+        } else if (const auto store = gdsDestination && !gdsOnGpu ? std::nullopt : Pm4::ResolveStore(packet, queue, gpuStoreLimit)) {
             const auto bytes = store->Bytes();
             if (bytes.empty()) {
 
@@ -313,7 +337,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) {
 bool Driver::enqueueDmaPacket(std::span<const std::uint32_t> packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) {
     if (opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false;
     const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u);
-    if (Pm4::DmaDestination(packet) == Pm4::DmaSelectGds && gdsWritesPending(packet)) return false;
+    if (Pm4::DmaDestination(packet) == Pm4::DmaSelectGds && gdsWritesPending(packet, device.Load().get())) return false;
     constexpr std::size_t immediateLimit = std::size_t{16} << 20u;
     constexpr std::size_t gpuStoreLimit = 65536;
     if (source == 2 && Pm4::DmaBytes(packet) > gpuStoreLimit && Pm4::DmaDestination(packet) != Pm4::DmaSelectGds) {

core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp

@@ -8,6 +8,7 @@
 #include <thread>
 #include <chrono>
 #include <cstdio>
+#include <cstdlib>
 #include <cstring>
 #include <limits>
 #include <memory>
@@ -16,10 +17,55 @@
 
 namespace AgcDriver::Pm4 {
 
+namespace {
+
+std::atomic<bool> importableGdsRequested{false};
+std::atomic<bool> gdsShared{false};
+std::atomic<bool> gdsImportable{false};
+
+}
+
+void RequestImportableGds(bool requested) {
+    importableGdsRequested.store(requested, std::memory_order_release);
+}
+
+bool ImportableGdsRequested() {
+    return importableGdsRequested.load(std::memory_order_acquire);
+}
+
+bool GdsShared() {
+    static_cast<void>(GdsAddress());
+    return gdsShared.load(std::memory_order_acquire);
+}
+
+bool GdsImportable() {
+    static_cast<void>(GdsAddress());
+    return gdsImportable.load(std::memory_order_acquire);
+}
+
+void RejectGdsImport() {
+    gdsImportable.store(false, std::memory_order_release);
+}
+
 std::uint64_t GdsAddress() {
     static const std::uint64_t address = [] {
         constexpr int ReadWrite = 0x3;
         constexpr int PrivateAnonymous = 0x1002;
+        constexpr int Shared = 0x1;
+        constexpr int OpenReadWrite = 0x2;
+        const char* const anonymousSharedMemory = reinterpret_cast<const char*>(1);
+        if (ImportableGdsRequested() && std::getenv("APS5_NO_SHARED_GDS") == nullptr) {
+            if (const int descriptor = shm_open_nid_postfix(anonymousSharedMemory, OpenReadWrite, 0600); descriptor >= 0) {
+                void* shared = ftruncate_nid_postfix(descriptor, GdsBytes) == 0 ? mmap_nid_postfix(nullptr, GdsBytes, ReadWrite, Shared, descriptor, 0) : nullptr;
+                close_nid_postfix(descriptor);
+                if (shared != nullptr && shared != reinterpret_cast<void*>(static_cast<std::uintptr_t>(-1))) {
+                    std::memset(shared, 0, GdsBytes);
+                    gdsShared.store(true, std::memory_order_release);
+                    gdsImportable.store(true, std::memory_order_release);
+                    return reinterpret_cast<std::uint64_t>(shared);
+                }
+            }
+        }
         void* mapped = mmap_nid_postfix(nullptr, GdsBytes, ReadWrite, PrivateAnonymous, -1, 0);
         if (mapped == nullptr || mapped == reinterpret_cast<void*>(static_cast<std::uintptr_t>(-1))) throw std::runtime_error("cannot allocate the global data share");
         return reinterpret_cast<std::uint64_t>(mapped);

core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp

@@ -1636,6 +1636,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
         state->drawInputsInPlace = std::getenv("APS5_NO_DRAW_INPUTS_IN_PLACE") == nullptr && (std::getenv("APS5_DRAW_INPUTS_IN_PLACE") != nullptr || DrawInputsInPlaceUsable(driverProperties.driverID, windows));
         if (!compileOnly) Graphics::SetHostImportWindow(8ull << 20u);
         if (!compileOnly) Graphics::SetHostImportMaxBytes(maintenance3.maxMemoryAllocationSize);
+        if (!compileOnly) Pm4::RequestImportableGds(state->properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU);
 #ifdef _WIN32
         // Imported pages are locked; locking is bounded by the process working-set minimum.
         if (const char* value = std::getenv("APS5_WORKING_SET_MIB"); value != nullptr && !compileOnly) {
@@ -2260,6 +2261,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) :
     Graphics::PrepareImportWatch(graphicsContext());
     state->bufferPool = std::make_shared<Graphics::BufferPool>(graphicsContext());
     state->emptyBuffer = std::make_unique<Graphics::Buffer>(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT);
+    if (!compileOnly && Pm4::GdsImportable() && Graphics::HostImportFor(graphicsContext(), Pm4::GdsAddress(), Pm4::GdsBytes, Graphics::ImportRequester::Map) == nullptr) Pm4::RejectGdsImport();
     state->pipelineCache = std::make_unique<Graphics::PipelineCache>(graphicsContext(), state->properties, window != nullptr || compileOnly);
     state->detiler = std::make_unique<Graphics::TextureDetiler>(graphicsContext());
     state->textureCache = std::make_unique<Graphics::TextureCache>(graphicsContext());

core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gds_dma_fill_tests.cmake

@@ -0,0 +1,8 @@
+if(BUILD_TESTING)
+    add_test(NAME agc_driver_gds_dma_fill_private COMMAND agc_driver_gds_dma_fill_tests)
+    set_tests_properties(agc_driver_gds_dma_fill_private PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_SHARED_GDS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}")
+    if(WIN32)
+        set_tests_properties(agc_driver_gds_dma_fill_private PROPERTIES ENVIRONMENT_MODIFICATION
+            "PATH=path_list_prepend:$<TARGET_FILE_DIR:libc>")
+    endif()
+endif()

core/libs/prx/libSceAgcDriver/tests/execution/GdsDmaFill.cpp

@@ -73,6 +73,13 @@ std::uint64_t Address(const void* pointer) {
     return static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(pointer));
 }
 
+constexpr std::array<std::uint32_t, 6> StoreProgram{
+    0x34060082,
+    0x7e0c02ff, 0x5eed0001u,
+    0xe0702000, 0x80000603,
+    0xbf810000,
+};
+
 std::array<std::uint32_t, 7> FillGds(std::uint32_t value, std::uint32_t offset = GdsOffset, std::uint32_t bytes = FillBytes) {
     return {0xc0055000u, (2u << 29u) | (1u << 20u), value, 0u, offset, 0u, bytes};
 }
@@ -85,6 +92,10 @@ std::array<std::uint32_t, 7> CopyGdsTo(std::uint64_t destination, std::uint32_t
     return {0xc0055000u, 1u << 29u, offset, 0u, static_cast<std::uint32_t>(destination), static_cast<std::uint32_t>(destination >> 32u), bytes};
 }
 
+std::array<std::uint32_t, 7> CopyToGds(std::uint64_t source, std::uint32_t offset, std::uint32_t bytes) {
+    return {0xc0055000u, 1u << 20u, static_cast<std::uint32_t>(source), static_cast<std::uint32_t>(source >> 32u), offset, 0u, bytes};
+}
+
 template <std::size_t Count>
 void Append(std::vector<std::uint32_t>& words, const std::array<std::uint32_t, Count>& packet) {
     words.insert(words.end(), packet.begin(), packet.end());
@@ -136,6 +147,11 @@ int main() {
         const auto device = OpenVulkanTestDevice();
         if (!device) return VulkanTestSkipped;
         using namespace AgcDriver::DriverDetail;
+        const bool sharedMapping = AgcDriver::Pm4::GdsShared();
+        check(sharedMapping == (AgcDriver::Pm4::ImportableGdsRequested() && std::getenv("APS5_NO_SHARED_GDS") == nullptr), sharedMapping ? "the GDS is in shared memory although that was not requested or is switched off" : "the GDS kept its private mapping although shared memory was requested");
+        const bool shared = AgcDriver::Pm4::GdsImportable();
+        check(!shared || sharedMapping, "the GDS counts as importable without a shared mapping");
+        if (sharedMapping && !shared) std::printf("the device refused the host import of the shared GDS: the copy path is checked\n");
         auto* readBack = GuestBlock();
         auto* filled = GuestBlock();
         auto* held = GuestBlock();
@@ -272,6 +288,8 @@ int main() {
             Submit(std::move(words));
             AgcDriverWaitIdle_nid_postfix();
             const auto drainsBefore = storesDrained.load();
+            const auto gpuStoresBefore = storesOnGpu.load() + storesBehindCompletions.load();
+            const auto gpuCopiesBefore = copiesBehindPendingSources.load();
             words.clear();
             AppendDispatch(words, code, output);
             Append(words, FillGds(3000u, CounterOffset, 16));
@@ -288,7 +306,43 @@ int main() {
             check(first[0] == 500u, "the dispatch before the fill read " + std::to_string(first[0]) + ", not the earlier fill's 500");
             check(second[0] == 3000u, "the dispatch after the fill, in the same submission, read " + std::to_string(second[0]) + ", not the fill's 3000");
             check(counters[0] == 3032u && counters[1] == 2968u && counters[2] == 3032u, "the GDS counters after a dispatch, a fill and a dispatch are " + std::to_string(counters[0]) + ", " + std::to_string(counters[1]) + " and " + std::to_string(counters[2]) + ", not 3032, 2968 and 3032: the first dispatch's atomics landed after the fill");
-            check(storesDrained.load() == drainsBefore + 1, "a fill behind a dispatch still using the GDS did not drain the device as main does");
+            if (shared) {
+                check(storesDrained.load() == drainsBefore, "a fill of the shared GDS behind a dispatch still using it drained the device");
+                check(storesOnGpu.load() + storesBehindCompletions.load() == gpuStoresBefore + 1, "a fill of the shared GDS behind a dispatch still using it was not stored on the GPU");
+                check(copiesBehindPendingSources.load() == gpuCopiesBefore + 1, "the copy out of the shared GDS behind the dispatches using it was not recorded on the GPU");
+            } else {
+                check(storesDrained.load() == drainsBefore + 1, "a fill behind a dispatch still using the GDS did not drain the device as main does");
+            }
+        }
+
+        {
+            constexpr std::uint32_t OrderOffset = 0x400;
+            constexpr std::uint32_t OrderValue = 0x5151a1a1u;
+            auto* storeCode = GuestBlock();
+            auto* orderOutput = GuestBlock();
+            auto* orderReadBack = GuestBlock();
+            std::memcpy(storeCode, StoreProgram.data(), sizeof(StoreProgram));
+            const auto drainsBefore = storesDrained.load();
+            const auto gpuStoresBefore = storesOnGpu.load() + storesBehindCompletions.load();
+            const auto gpuCopiesBefore = copiesBehindPendingSources.load();
+            words.clear();
+            AppendDispatch(words, storeCode, orderOutput);
+            Append(words, CopyToGds(Address(orderOutput), OrderOffset, 4));
+            Append(words, FillGds(OrderValue, OrderOffset, 4));
+            Append(words, CopyGdsTo(Address(orderReadBack), OrderOffset, 4));
+            Submit(std::move(words));
+            AgcDriverWaitIdle_nid_postfix();
+            std::array<std::uint32_t, 1> value{};
+            AgcDriver::GuestMemory::Read(Address(orderReadBack), std::as_writable_bytes(std::span(value)), 4);
+            std::array<std::uint32_t, 1> stored{};
+            AgcDriver::GuestMemory::Read(Address(orderOutput), std::as_writable_bytes(std::span(stored)), 4);
+            check(stored[0] == 0x5eed0001u, "the dispatch that writes the copy's source stored " + std::to_string(stored[0]) + ", not 0x5eed0001");
+            check(value[0] == OrderValue, "a GDS fill after a copy into the GDS from the output of a dispatch that does not use the GDS read back as " + std::to_string(value[0]) + ", not the fill's value: the copy landed after the fill");
+            if (shared) {
+                check(copiesBehindPendingSources.load() > gpuCopiesBefore, "the copy into the shared GDS behind the dispatch writing its source was not recorded on the GPU");
+                check(storesOnGpu.load() + storesBehindCompletions.load() > gpuStoresBefore, "the fill of the shared GDS behind the GPU copy into it was not stored on the GPU");
+                check(storesDrained.load() == drainsBefore, "a DMA_DATA into or out of the shared GDS behind a dispatch drained the device");
+            }
         }
 
         LibcRunShutdown_nid_postfix();

docs/dev/ENVIRONMENT.md

@@ -348,6 +348,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti
 | `APS5_NO_SAMPLER_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` |  | creates a sampler per binding instead of caching them |
 | `APS5_NO_SEPARATE_QUEUED_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` |  | lets a queued label enter the recorded table and replace a recorded entry of its dword |
 | `APS5_NO_SHAPE_MEMO` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` |  | looks up every draw's resident targets and pipeline anew instead of repeating the ones an earlier draw of the same register shape used |
+| `APS5_NO_SHARED_GDS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp` |  | keeps the GDS in a private guest mapping, which the GPU cannot import, instead of anonymous shared memory on an integrated GPU with host imports; DMA_DATA into the GDS behind GPU writes to it then drains, and DMA_DATA out of it is read on the CPU |
 | `APS5_NO_SINGLE_PASS_COLLECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp` |  | walks the written pages once without clearing them and, when any were written, a second time clearing them, instead of one cleared pass |
 | `APS5_NO_SITE_SAMPLING` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` |  | disables per-site sampling of write evidence |
 | `APS5_NO_SNAPSHOT_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` |  | skips the live compare that proves a captured snapshot still equals the guest bytes |