diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index 0b9bc1f9f84e..7403a695b27a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -14,6 +14,11 @@ namespace AgcDriver::Pm4 { constexpr std::size_t GdsBytes = 0x10000; std::uint64_t GdsAddress(); +void RequestImportableGds(bool requested); +bool ImportableGdsRequested(); +bool GdsShared(); +bool GdsImportable(); +void RejectGdsImport(); struct DrawParameters { std::uint64_t indexAddress; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index 4705332a2500..887e45d26520 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -16,16 +16,40 @@ #include #include #include +#include namespace AgcDriver::DriverDetail { namespace { -bool gdsWritesPending(std::span packet) { +bool gdsWritesPending(std::span packet, const VulkanDevice* localDevice) { const auto address = Pm4::GdsAddress() + packet[4]; const auto bytes = static_cast(Pm4::DmaBytes(packet)); const std::array, 1> range{{{address, address + bytes}}}; - return Graphics::Recorder::SnapshotWriteOverlaps(address, bytes) || Graphics::StorageTexture::AnyPendingOverlaps(range) || Graphics::AnyShadowedOverlaps(address, bytes); + if (Graphics::Recorder::SnapshotWriteOverlaps(address, bytes) || Graphics::StorageTexture::AnyPendingOverlaps(range) || Graphics::AnyShadowedOverlaps(address, bytes)) return true; + if (localDevice == nullptr || !Pm4::GdsImportable()) return false; + std::lock_guard gpuLock(GuestMemory::GpuMutex()); + return localDevice->StoresPendingOver(address, bytes); +} + +std::uint64_t dmaAddress(std::uint32_t low, std::uint32_t high) { + return low | (static_cast(high) << 32u); +} + +bool gdsSpan(std::uint64_t offset, std::size_t bytes) { + return offset <= Pm4::GdsBytes && bytes <= Pm4::GdsBytes - offset; +} + +std::optional decodeCopy(std::span packet) { + if (auto copy = Pm4::DecodeMemoryCopy(packet)) return copy; + if (packet.size() != 7 || !Pm4::GdsImportable()) return std::nullopt; + const auto source = Pm4::DmaSource(packet); + const auto destination = Pm4::DmaDestination(packet); + const std::size_t bytes = Pm4::DmaBytes(packet); + if (bytes == 0) return std::nullopt; + if (source == Pm4::DmaSelectGds && (destination == 0 || destination == 3) && gdsSpan(packet[2], bytes)) return Pm4::MemoryCopy{Pm4::GdsAddress() + packet[2], dmaAddress(packet[4], packet[5]), bytes}; + if ((source == 0 || source == 3) && destination == Pm4::DmaSelectGds && gdsSpan(packet[4], bytes)) return Pm4::MemoryCopy{dmaAddress(packet[2], packet[3]), Pm4::GdsAddress() + packet[4], bytes}; + return std::nullopt; } } @@ -130,7 +154,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue if (!drainAll && !cpuStores && (opcode == 0x40 || opcode == 0x50 || opcode == 0x83)) { constexpr std::size_t gpuStoreLimit = 65536; bool drained = true; - const auto copy = opcode == 0x50 ? Pm4::DecodeMemoryCopy(packet) : std::nullopt; + const auto copy = opcode == 0x50 ? decodeCopy(packet) : std::nullopt; const auto copyBehindPendingSource = [&] { if (!copy.has_value() || copy->bytes > gpuStoreLimit || !GuestMemory::Accessible(reinterpret_cast(copy->source), copy->bytes) || !GuestMemory::Accessible(reinterpret_cast(copy->destination), copy->bytes, true)) return false; const auto end = copy->source + copy->bytes; @@ -148,16 +172,16 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue if (outcome.path != 1 && outcome.path != 3) throw std::runtime_error("AGC driver: a DMA_DATA copy of up to 64 KiB that GpuCopyPossible accepted took copy path " + std::to_string(outcome.path)); return true; }; - if (opcode == 0x50 && packet.size() >= 7 && Pm4::DmaDestination(packet) == Pm4::DmaSelectGds) { - if (!gdsWritesPending(packet)) { - orderedAlready = true; - drained = false; - } - } else if (copyBehindPendingSource()) { + const bool gdsDestination = opcode == 0x50 && packet.size() >= 7 && Pm4::DmaDestination(packet) == Pm4::DmaSelectGds; + const bool gdsOnGpu = gdsDestination && Pm4::GdsImportable(); + if ((!gdsDestination || gdsOnGpu) && copyBehindPendingSource()) { ++copiesBehindPendingSources; wroteOnGpu = true; drained = false; - } else if (const auto store = Pm4::ResolveStore(packet, queue, gpuStoreLimit)) { + } else if (gdsDestination && !gdsWritesPending(packet, device.Load().get())) { + orderedAlready = true; + drained = false; + } else if (const auto store = gdsDestination && !gdsOnGpu ? std::nullopt : Pm4::ResolveStore(packet, queue, gpuStoreLimit)) { const auto bytes = store->Bytes(); if (bytes.empty()) { @@ -313,7 +337,7 @@ void Driver::dumpSampleCounters(std::uint64_t address) { bool Driver::enqueueDmaPacket(std::span packet, std::uint32_t opcode, std::uint32_t queue, const QueueState& state) { if (opcode != 0x50 || queue != 0 || packet.size() < 7 || !DrawPipeline::Active() || !deferredLabels().labels.empty()) return false; const auto source = ((packet[1] >> 29u) & 3u) | ((packet[6] >> 24u) & 4u) | ((packet[6] >> 25u) & 8u); - if (Pm4::DmaDestination(packet) == Pm4::DmaSelectGds && gdsWritesPending(packet)) return false; + if (Pm4::DmaDestination(packet) == Pm4::DmaSelectGds && gdsWritesPending(packet, device.Load().get())) return false; constexpr std::size_t immediateLimit = std::size_t{16} << 20u; constexpr std::size_t gpuStoreLimit = 65536; if (source == 2 && Pm4::DmaBytes(packet) > gpuStoreLimit && Pm4::DmaDestination(packet) != Pm4::DmaSelectGds) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index e427f319d5ec..9b15e3012dca 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -16,10 +17,55 @@ namespace AgcDriver::Pm4 { +namespace { + +std::atomic importableGdsRequested{false}; +std::atomic gdsShared{false}; +std::atomic gdsImportable{false}; + +} + +void RequestImportableGds(bool requested) { + importableGdsRequested.store(requested, std::memory_order_release); +} + +bool ImportableGdsRequested() { + return importableGdsRequested.load(std::memory_order_acquire); +} + +bool GdsShared() { + static_cast(GdsAddress()); + return gdsShared.load(std::memory_order_acquire); +} + +bool GdsImportable() { + static_cast(GdsAddress()); + return gdsImportable.load(std::memory_order_acquire); +} + +void RejectGdsImport() { + gdsImportable.store(false, std::memory_order_release); +} + std::uint64_t GdsAddress() { static const std::uint64_t address = [] { constexpr int ReadWrite = 0x3; constexpr int PrivateAnonymous = 0x1002; + constexpr int Shared = 0x1; + constexpr int OpenReadWrite = 0x2; + const char* const anonymousSharedMemory = reinterpret_cast(1); + if (ImportableGdsRequested() && std::getenv("APS5_NO_SHARED_GDS") == nullptr) { + if (const int descriptor = shm_open_nid_postfix(anonymousSharedMemory, OpenReadWrite, 0600); descriptor >= 0) { + void* shared = ftruncate_nid_postfix(descriptor, GdsBytes) == 0 ? mmap_nid_postfix(nullptr, GdsBytes, ReadWrite, Shared, descriptor, 0) : nullptr; + close_nid_postfix(descriptor); + if (shared != nullptr && shared != reinterpret_cast(static_cast(-1))) { + std::memset(shared, 0, GdsBytes); + gdsShared.store(true, std::memory_order_release); + gdsImportable.store(true, std::memory_order_release); + return reinterpret_cast(shared); + } + } + } void* mapped = mmap_nid_postfix(nullptr, GdsBytes, ReadWrite, PrivateAnonymous, -1, 0); if (mapped == nullptr || mapped == reinterpret_cast(static_cast(-1))) throw std::runtime_error("cannot allocate the global data share"); return reinterpret_cast(mapped); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 773cd03320ad..8b8c3c943284 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -1636,6 +1636,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->drawInputsInPlace = std::getenv("APS5_NO_DRAW_INPUTS_IN_PLACE") == nullptr && (std::getenv("APS5_DRAW_INPUTS_IN_PLACE") != nullptr || DrawInputsInPlaceUsable(driverProperties.driverID, windows)); if (!compileOnly) Graphics::SetHostImportWindow(8ull << 20u); if (!compileOnly) Graphics::SetHostImportMaxBytes(maintenance3.maxMemoryAllocationSize); + if (!compileOnly) Pm4::RequestImportableGds(state->properties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU); #ifdef _WIN32 // Imported pages are locked; locking is bounded by the process working-set minimum. if (const char* value = std::getenv("APS5_WORKING_SET_MIB"); value != nullptr && !compileOnly) { @@ -2260,6 +2261,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : Graphics::PrepareImportWatch(graphicsContext()); state->bufferPool = std::make_shared(graphicsContext()); state->emptyBuffer = std::make_unique(graphicsContext(), Graphics::EmptyBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + if (!compileOnly && Pm4::GdsImportable() && Graphics::HostImportFor(graphicsContext(), Pm4::GdsAddress(), Pm4::GdsBytes, Graphics::ImportRequester::Map) == nullptr) Pm4::RejectGdsImport(); state->pipelineCache = std::make_unique(graphicsContext(), state->properties, window != nullptr || compileOnly); state->detiler = std::make_unique(graphicsContext()); state->textureCache = std::make_unique(graphicsContext()); diff --git a/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gds_dma_fill_tests.cmake b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gds_dma_fill_tests.cmake new file mode 100644 index 000000000000..7a6d61800ba2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/cmake/agc_driver_gds_dma_fill_tests.cmake @@ -0,0 +1,8 @@ +if(BUILD_TESTING) + add_test(NAME agc_driver_gds_dma_fill_private COMMAND agc_driver_gds_dma_fill_tests) + set_tests_properties(agc_driver_gds_dma_fill_private PROPERTIES TIMEOUT 30 ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_SHARED_GDS=1" SKIP_RETURN_CODE 77 WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + if(WIN32) + set_tests_properties(agc_driver_gds_dma_fill_private PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GdsDmaFill.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GdsDmaFill.cpp index 2acc0815bb89..9d55a5dedcd3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GdsDmaFill.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GdsDmaFill.cpp @@ -73,6 +73,13 @@ std::uint64_t Address(const void* pointer) { return static_cast(reinterpret_cast(pointer)); } +constexpr std::array StoreProgram{ + 0x34060082, + 0x7e0c02ff, 0x5eed0001u, + 0xe0702000, 0x80000603, + 0xbf810000, +}; + std::array FillGds(std::uint32_t value, std::uint32_t offset = GdsOffset, std::uint32_t bytes = FillBytes) { return {0xc0055000u, (2u << 29u) | (1u << 20u), value, 0u, offset, 0u, bytes}; } @@ -85,6 +92,10 @@ std::array CopyGdsTo(std::uint64_t destination, std::uint32_t return {0xc0055000u, 1u << 29u, offset, 0u, static_cast(destination), static_cast(destination >> 32u), bytes}; } +std::array CopyToGds(std::uint64_t source, std::uint32_t offset, std::uint32_t bytes) { + return {0xc0055000u, 1u << 20u, static_cast(source), static_cast(source >> 32u), offset, 0u, bytes}; +} + template void Append(std::vector& words, const std::array& packet) { words.insert(words.end(), packet.begin(), packet.end()); @@ -136,6 +147,11 @@ int main() { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; using namespace AgcDriver::DriverDetail; + const bool sharedMapping = AgcDriver::Pm4::GdsShared(); + check(sharedMapping == (AgcDriver::Pm4::ImportableGdsRequested() && std::getenv("APS5_NO_SHARED_GDS") == nullptr), sharedMapping ? "the GDS is in shared memory although that was not requested or is switched off" : "the GDS kept its private mapping although shared memory was requested"); + const bool shared = AgcDriver::Pm4::GdsImportable(); + check(!shared || sharedMapping, "the GDS counts as importable without a shared mapping"); + if (sharedMapping && !shared) std::printf("the device refused the host import of the shared GDS: the copy path is checked\n"); auto* readBack = GuestBlock(); auto* filled = GuestBlock(); auto* held = GuestBlock(); @@ -272,6 +288,8 @@ int main() { Submit(std::move(words)); AgcDriverWaitIdle_nid_postfix(); const auto drainsBefore = storesDrained.load(); + const auto gpuStoresBefore = storesOnGpu.load() + storesBehindCompletions.load(); + const auto gpuCopiesBefore = copiesBehindPendingSources.load(); words.clear(); AppendDispatch(words, code, output); Append(words, FillGds(3000u, CounterOffset, 16)); @@ -288,7 +306,43 @@ int main() { check(first[0] == 500u, "the dispatch before the fill read " + std::to_string(first[0]) + ", not the earlier fill's 500"); check(second[0] == 3000u, "the dispatch after the fill, in the same submission, read " + std::to_string(second[0]) + ", not the fill's 3000"); check(counters[0] == 3032u && counters[1] == 2968u && counters[2] == 3032u, "the GDS counters after a dispatch, a fill and a dispatch are " + std::to_string(counters[0]) + ", " + std::to_string(counters[1]) + " and " + std::to_string(counters[2]) + ", not 3032, 2968 and 3032: the first dispatch's atomics landed after the fill"); - check(storesDrained.load() == drainsBefore + 1, "a fill behind a dispatch still using the GDS did not drain the device as main does"); + if (shared) { + check(storesDrained.load() == drainsBefore, "a fill of the shared GDS behind a dispatch still using it drained the device"); + check(storesOnGpu.load() + storesBehindCompletions.load() == gpuStoresBefore + 1, "a fill of the shared GDS behind a dispatch still using it was not stored on the GPU"); + check(copiesBehindPendingSources.load() == gpuCopiesBefore + 1, "the copy out of the shared GDS behind the dispatches using it was not recorded on the GPU"); + } else { + check(storesDrained.load() == drainsBefore + 1, "a fill behind a dispatch still using the GDS did not drain the device as main does"); + } + } + + { + constexpr std::uint32_t OrderOffset = 0x400; + constexpr std::uint32_t OrderValue = 0x5151a1a1u; + auto* storeCode = GuestBlock(); + auto* orderOutput = GuestBlock(); + auto* orderReadBack = GuestBlock(); + std::memcpy(storeCode, StoreProgram.data(), sizeof(StoreProgram)); + const auto drainsBefore = storesDrained.load(); + const auto gpuStoresBefore = storesOnGpu.load() + storesBehindCompletions.load(); + const auto gpuCopiesBefore = copiesBehindPendingSources.load(); + words.clear(); + AppendDispatch(words, storeCode, orderOutput); + Append(words, CopyToGds(Address(orderOutput), OrderOffset, 4)); + Append(words, FillGds(OrderValue, OrderOffset, 4)); + Append(words, CopyGdsTo(Address(orderReadBack), OrderOffset, 4)); + Submit(std::move(words)); + AgcDriverWaitIdle_nid_postfix(); + std::array value{}; + AgcDriver::GuestMemory::Read(Address(orderReadBack), std::as_writable_bytes(std::span(value)), 4); + std::array stored{}; + AgcDriver::GuestMemory::Read(Address(orderOutput), std::as_writable_bytes(std::span(stored)), 4); + check(stored[0] == 0x5eed0001u, "the dispatch that writes the copy's source stored " + std::to_string(stored[0]) + ", not 0x5eed0001"); + check(value[0] == OrderValue, "a GDS fill after a copy into the GDS from the output of a dispatch that does not use the GDS read back as " + std::to_string(value[0]) + ", not the fill's value: the copy landed after the fill"); + if (shared) { + check(copiesBehindPendingSources.load() > gpuCopiesBefore, "the copy into the shared GDS behind the dispatch writing its source was not recorded on the GPU"); + check(storesOnGpu.load() + storesBehindCompletions.load() > gpuStoresBefore, "the fill of the shared GDS behind the GPU copy into it was not stored on the GPU"); + check(storesDrained.load() == drainsBefore, "a DMA_DATA into or out of the shared GDS behind a dispatch drained the device"); + } } LibcRunShutdown_nid_postfix(); diff --git a/docs/dev/ENVIRONMENT.md b/docs/dev/ENVIRONMENT.md index f517fb7a3ccd..f7e7f69b59ca 100644 --- a/docs/dev/ENVIRONMENT.md +++ b/docs/dev/ENVIRONMENT.md @@ -348,6 +348,7 @@ The code reads environment names through `getenv`, `IsEnvironmentSet`, `kibSetti | `APS5_NO_SAMPLER_CACHE` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp` | | creates a sampler per binding instead of caching them | | `APS5_NO_SEPARATE_QUEUED_LABELS` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp` | | lets a queued label enter the recorded table and replace a recorded entry of its dword | | `APS5_NO_SHAPE_MEMO` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp` | | looks up every draw's resident targets and pipeline anew instead of repeating the ones an earlier draw of the same register shape used | +| `APS5_NO_SHARED_GDS` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp` | | keeps the GDS in a private guest mapping, which the GPU cannot import, instead of anonymous shared memory on an integrated GPU with host imports; DMA_DATA into the GDS behind GPU writes to it then drains, and DMA_DATA out of it is read on the CPU | | `APS5_NO_SINGLE_PASS_COLLECT` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp` | | walks the written pages once without clearing them and, when any were written, a second time clearing them, instead of one cleared pass | | `APS5_NO_SITE_SAMPLING` | presence | `core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WriteEvidence.cpp` | | disables per-site sampling of write evidence | | `APS5_NO_SNAPSHOT_CHECK` | presence | `core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp` | | skips the live compare that proves a captured snapshot still equals the guest bytes |