diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 27696dd47614..446776d29236 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -1530,6 +1530,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL tests/TextureTiling.cpp tests/GuestTextureResource.cpp tests/GuestSamplerResource.cpp + tests/ImageTableEntryValidation.cpp tests/TextureDetiler.cpp tests/DepthSurfaceReuse.cpp ) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 21a08c936111..e00121881956 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -66,9 +66,14 @@ inline void ReportDeviceLost() { if (DeviceLostHook() && !reported.exchange(true)) DeviceLostHook()(); } +struct VulkanError : std::runtime_error { + VulkanError(VkResult result, const std::string& message) : std::runtime_error(message), result(result) {} + VkResult result; +}; + inline void Check(VkResult result, const char* operation) { if (result == VK_ERROR_DEVICE_LOST) ReportDeviceLost(); - if (result != VK_SUCCESS) throw std::runtime_error(std::string("AGC graphics: ") + operation + ": Vulkan result " + std::to_string(result)); + if (result != VK_SUCCESS) throw VulkanError(result, std::string("AGC graphics: ") + operation + ": Vulkan result " + std::to_string(result)); } inline void Require(bool condition, const char* reason) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index a85a85837466..89251c3374c7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -104,6 +104,7 @@ struct RegisteredPiece { // Whether a readable registered allocation contains [address, address + bytes) right now (one // registry lease): a storage image whose memory was freed or re-registered has nothing to store to. bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes, std::vector* pieces = nullptr); +bool RegisteredReadableOverlaps(std::uint64_t address, std::size_t bytes); std::uint64_t RegisteredReadableEnd(std::uint64_t address); // A persistent device copy of one registered range of the main guest image (which cannot be host diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index 6a5b97e7f28b..fab734a7cc11 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -37,6 +37,10 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); VkFormat HostStorageFormat(const Context& context, VkFormat format); bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys); std::uint64_t SampledTextureMemory(); +void RequireSampledImageWithinDeviceLimits(const Context& context, const GuestTextureResource& descriptor, const SurfaceGeometry& geometry, bool depthCompare); +struct TextureMemoryExhausted : VulkanError { + using VulkanError::VulkanError; +}; bool ClearKeepsDenormals(const Context& context, VkFormat format); bool ClearColorForTexel(VkFormat format, std::uint32_t elementBytes, std::span pattern, VkClearColorValue& clear); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 1b77cf8faf48..4a721f30a729 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -3076,6 +3076,18 @@ bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes, std::vec return true; } +bool RegisteredReadableOverlaps(std::uint64_t address, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const auto end = address + bytes; + const auto next = std::upper_bound(lease.begin(), lease.end(), address, [](std::uint64_t value, const auto& range) { return value < range->address; }); + if (next != lease.begin()) { + const auto& previous = *std::prev(next); + if (address - previous->address < previous->bytes) return true; + } + return next != lease.end() && (*next)->address < end; +} + std::uint64_t RegisteredReadableEnd(std::uint64_t address) { if (address == std::numeric_limits::max()) return 0; const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 3f54d9efc05c..e0ea139c92fc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -4606,9 +4606,13 @@ void ShaderResources::resolveImageTables() { shape = binding.imageShapes.at(element); firstLayer = shape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; if (!firstLayer && !MatchesGuestDimension(shape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the table entry's image shape (shape " + std::to_string(static_cast(shape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); - guestBytes = DescribeSurface(resource).guestBytes; - if (!GuestMemory::Accessible(reinterpret_cast(resource.baseAddress), static_cast(guestBytes)) && GuestMemory::CommittedRanges(resource.baseAddress, static_cast(guestBytes), false).empty()) throw std::runtime_error("AGC graphics: the table entry's surface at 0x" + HexText(resource.baseAddress) + " (0x" + HexText(guestBytes) + " bytes) is not mapped"); + const auto geometry = DescribeSurface(resource); + guestBytes = geometry.guestBytes; + RequireSampledImageWithinDeviceLimits(context, resource, geometry, binding.imageDepthCompare.at(element)); + if (!RegisteredReadableOverlaps(resource.baseAddress, static_cast(guestBytes))) throw std::runtime_error("AGC graphics: the table entry's surface at 0x" + HexText(resource.baseAddress) + " (0x" + HexText(guestBytes) + " bytes) is not mapped in guest memory"); static_cast(ResolveTextureFormat(resource.format)); + } catch (const VulkanError&) { + throw; } catch (const std::runtime_error& error) { table.map[faults + element] = rejectTableElement(table, words, error.what()); if (reusable) fresh.elements.push_back({{}, {}, {}, 0, false, true, error.what()}); @@ -4661,6 +4665,12 @@ void ShaderResources::resolveImageTables() { const VkImageViewType multisampleViewType = shape == ShaderRecompiler::DescriptorImageShape::Image2DMultisampleArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D; try { texture = cachedTexture(context, words, resource, components, guestBytes, depthCompare, multisampleViewType); + } catch (const TextureMemoryExhausted& error) { + table.map[faults + element] = rejectTableElement(table, words, error.what()); + reusable = false; + continue; + } catch (const VulkanError&) { + throw; } catch (const std::runtime_error& error) { table.map[faults + element] = rejectTableElement(table, words, error.what()); if (reusable) fresh.elements.push_back({{}, {}, {}, 0, false, true, error.what()}); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 2beb5439d484..f00ff068be05 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -290,12 +290,100 @@ std::atomic& SampledMemoryCounter() { return bytes; } +template +auto ownTextureMemory(TMake&& make) { + try { + return make(); + } catch (const VulkanError& error) { + if (error.result != VK_ERROR_OUT_OF_HOST_MEMORY && error.result != VK_ERROR_OUT_OF_DEVICE_MEMORY) throw; + throw TextureMemoryExhausted(error.result, error.what()); + } +} + } std::uint64_t SampledTextureMemory() { return SampledMemoryCounter().load(std::memory_order_relaxed); } +namespace { + +const char* dimensionName(TextureDimension dimension) { + switch (dimension) { + case TextureDimension::k1D: return "1D"; + case TextureDimension::k2D: return "2D"; + case TextureDimension::k2DArray: return "2D array"; + case TextureDimension::kCube: return "cube"; + case TextureDimension::k3D: return "3D"; + case TextureDimension::k1DArray: return "1D array"; + case TextureDimension::k2DMsaa: return "2D multisampled"; + } + throw std::runtime_error("AGC graphics: unknown guest texture dimension " + std::to_string(static_cast(dimension))); +} + +std::optional sampledImageFormatLimits(const Context& context, VkFormat format, VkImageType type, VkImageCreateFlags flags) { + static std::mutex mutex; + static std::map, std::optional> limits; + const auto key = std::make_tuple(context.physical, format, type, flags); + { + std::lock_guard lock(mutex); + if (const auto found = limits.find(key); found != limits.end()) return found->second; + } + Require(context.imageFormatProperties != nullptr, "the device has no vkGetPhysicalDeviceImageFormatProperties"); + VkImageFormatProperties properties{}; + const auto result = context.imageFormatProperties(context.physical, format, type, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, flags, &properties); + std::optional supported; + if (result != VK_ERROR_FORMAT_NOT_SUPPORTED) { + Check(result, "vkGetPhysicalDeviceImageFormatProperties sampled texture"); + supported = properties; + } + std::lock_guard lock(mutex); + limits.emplace(key, supported); + return supported; +} + +} + +void RequireSampledImageWithinDeviceLimits(const Context& context, const GuestTextureResource& descriptor, const SurfaceGeometry& geometry, bool depthCompare) { + const auto colorFormat = SampledTextureFormat(context, descriptor.format); + const auto format = !depthCompare ? colorFormat : colorFormat == VK_FORMAT_R32_SFLOAT ? VK_FORMAT_D32_SFLOAT : colorFormat == VK_FORMAT_R16_UNORM ? VK_FORMAT_D16_UNORM : colorFormat; + const auto type = ImageTypeFor(descriptor.dimension); + const bool cube = descriptor.dimension == TextureDimension::kCube; + const VkImageCreateFlags flags = (cube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u) | (SrgbTwinFormat(context, format) != VK_FORMAT_UNDEFINED ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0u); + const VkExtent3D extent{descriptor.width, descriptor.height, geometry.imageDepth}; + const auto supported = sampledImageFormatLimits(context, format, type, flags); + const auto blockWidth = static_cast(BlockWidth(descriptor.format)); + const auto leastBytes = (extent.width + blockWidth - 1u) / blockWidth * ((extent.height + blockWidth - 1u) / blockWidth) * BytesPerElement(descriptor.format) * extent.depth * geometry.imageLayers; + const bool extentFits = supported && extent.width <= supported->maxExtent.width && extent.height <= supported->maxExtent.height && extent.depth <= supported->maxExtent.depth; + const bool layersFit = supported && geometry.imageLayers <= supported->maxArrayLayers; + const bool bytesFit = supported && leastBytes <= supported->maxResourceSize; + std::uint32_t largestHeap = 0; + VkDeviceSize heapBytes = 0; + for (std::uint32_t memoryType = 0; memoryType < std::min(context.memory.memoryTypeCount, VK_MAX_MEMORY_TYPES); ++memoryType) { + const auto heap = context.memory.memoryTypes[memoryType].heapIndex; + if (heap >= VK_MAX_MEMORY_HEAPS || context.memory.memoryHeaps[heap].size <= heapBytes) continue; + largestHeap = heap; + heapBytes = context.memory.memoryHeaps[heap].size; + } + const bool heapFits = leastBytes <= heapBytes; + if (extentFits && layersFit && bytesFit && heapFits) return; + char fields[224]; + std::snprintf(fields, sizeof(fields), "guest texture 0x%llx (%ux%u, %s, depth or last array slice %u, base array %u, format %u, mips %u)", static_cast(descriptor.baseAddress), descriptor.width, descriptor.height, dimensionName(descriptor.dimension), descriptor.depthOrLastArray, descriptor.baseArray, descriptor.format, descriptor.mipCount); + char text[448]; + if (!supported) { + std::snprintf(text, sizeof(text), "AGC graphics: %s needs a sampled VkFormat %d image of type %d with create flags 0x%x, which vkGetPhysicalDeviceImageFormatProperties reports unsupported", fields, static_cast(format), static_cast(type), static_cast(flags)); + } else if (!layersFit) { + std::snprintf(text, sizeof(text), "AGC graphics: %s needs %u array layers, more than the %u of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, geometry.imageLayers, supported->maxArrayLayers, static_cast(format)); + } else if (!extentFits) { + std::snprintf(text, sizeof(text), "AGC graphics: %s needs an extent of %ux%ux%u, more than the %ux%ux%u of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, extent.width, extent.height, extent.depth, supported->maxExtent.width, supported->maxExtent.height, supported->maxExtent.depth, static_cast(format)); + } else if (!bytesFit) { + std::snprintf(text, sizeof(text), "AGC graphics: %s needs at least %llu bytes, more than the maxResourceSize of %llu of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, static_cast(leastBytes), static_cast(supported->maxResourceSize), static_cast(format)); + } else { + std::snprintf(text, sizeof(text), "AGC graphics: %s needs at least %llu bytes, more than the %llu bytes of memory heap %u, the largest a memory type allocates from", fields, static_cast(leastBytes), static_cast(heapBytes), largestHeap); + } + throw std::runtime_error(text); +} + Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span snapshot, bool depthCompare) : context(context) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; @@ -338,7 +426,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | (srgbExpand ? VK_IMAGE_USAGE_TRANSFER_SRC_BIT : 0u); imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE; imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage"); + ownTextureMemory([&] { Check(context.Function("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage"); }); owned = std::make_shared(context, image, VK_NULL_HANDLE); VkMemoryRequirements requirements{}; @@ -347,13 +435,15 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex allocation.allocationSize = requirements.size; allocationBytes = requirements.size; allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture"); - owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture"); - Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); + ownTextureMemory([&] { + owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture"); + Check(context.Function("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory"); + }); const VkImage uploadImage = image; std::shared_ptr uploadOwned; if (srgbExpand) { - auto expanded = CreateExpandImage(context, descriptor, geometry.imageDepth, geometry.imageLayers); + auto expanded = ownTextureMemory([&] { return CreateExpandImage(context, descriptor, geometry.imageDepth, geometry.imageLayers); }); uploadOwned = std::move(owned); owned = std::move(expanded.owned); image = expanded.image; @@ -374,10 +464,10 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex // Entries may also be extents ("3840x2160"), since heap addresses change between runs. dumpWanted = dumpList.find(address) != std::string::npos || dumpList.find(extent) != std::string::npos; } - auto staging = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + auto staging = ownTextureMemory([&] { return std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); }); std::memcpy(staging->Bytes().data(), snapshot.data(), snapshot.size()); - auto tiled = std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); - auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + auto tiled = ownTextureMemory([&] { return std::make_shared(context, static_cast(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); }); + auto linear = ownTextureMemory([&] { return std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); }); if (profile) Profile().allocate += timer.lap(); if (profile) Profile().read += timer.lap(); diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index c0bec7d2f713..d4a36cdee161 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -9131,6 +9131,10 @@ int main(int argc, char** argv) { std::cout << "address-space device teardown releases cached guest allocations\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--image-table-entry-validation") { + RunImageTableEntryValidationTests(); + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--buffer-budget-placement") { bufferBudgetPlacementTests(); return 0; @@ -9367,6 +9371,7 @@ int main(int argc, char** argv) { RunGuestTextureResourceTests(); RunDepthSurfaceReuseTests(); RunGuestSamplerResourceTests(); + RunImageTableEntryValidationTests(); mock = MockVulkan{}; auto textureDetilerContext = mockContext(); textureDetilerContext.limits.minStorageBufferOffsetAlignment = 16; diff --git a/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp b/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp index 28a442b75de4..77a3b89b327e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp @@ -36,6 +36,7 @@ void RunTextureTilingTests(); void RunGuestTextureResourceTests(); void RunDepthSurfaceReuseTests(); void RunGuestSamplerResourceTests(); +void RunImageTableEntryValidationTests(); void RunTextureDetilerTests(const AgcDriver::Graphics::Context& context, const TextureDetilerTestAccess& access); #endif diff --git a/core/libs/prx/libSceAgcDriver/tests/ImageTableEntryValidation.cpp b/core/libs/prx/libSceAgcDriver/tests/ImageTableEntryValidation.cpp new file mode 100644 index 000000000000..7aceba253aa6 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ImageTableEntryValidation.cpp @@ -0,0 +1,861 @@ +#include "GraphicsTests.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "ImageTableAbi.hpp" +#include "Recompiler.hpp" +#include "RuntimeAbi.hpp" +#include "RdnaDecoder/RdnaDescriptorFormat.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; +namespace Abi = ShaderRecompiler::ImageTableAbi; +using Shape = ShaderRecompiler::DescriptorImageShape; + +constexpr std::uint32_t FormatR8G8B8A8Unorm = 56; +constexpr std::uint32_t FormatR16G16B16A16Float = 71; +constexpr std::uint32_t SwModeLinear = 0; +constexpr std::uint32_t SwMode64KBS = 9; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Type2DArray = 13; +constexpr std::uint32_t MaxArrayLayers = 2048; +constexpr std::uint32_t Elements = 6; +constexpr VkDeviceSize KiB = 1024; +constexpr VkDeviceSize MiB = 1024 * KiB; + +struct MockDevice { + std::uintptr_t next = 0x10000; + std::map images; + std::map bufferSizes; + std::map> bufferMemory; + std::map> memories; + std::map bufferWrites; + std::vector missing; + VkResult imageResult = VK_SUCCESS; + VkFormat rejectedImageFormat = VK_FORMAT_UNDEFINED; + VkResult rejectedImageResult = VK_SUCCESS; + VkResult imageFormatResult = VK_SUCCESS; + std::uint32_t imageFormatQueries = 0; + VkExtent3D maxExtent{32768, 32768, 1}; + VkDeviceSize maxResourceSize = VkDeviceSize{1} << 40u; + std::vector bufferCreations; + std::set destroyedImages; + std::map imageMemory; + VkImage memoryImage = VK_NULL_HANDLE; + std::uint32_t imageMemoryWidth = 0; + std::uint32_t imageMemoryAttempts = 0; + std::uint32_t imageMemoryFailures = 0; + VkResult imageMemoryResult = VK_SUCCESS; + VkDeviceSize bufferMemorySize = 0; + VkResult bufferMemoryResult = VK_SUCCESS; + std::uint32_t bufferMemoryFailures = 0; + VkResult submitResult = VK_SUCCESS; + std::array heapBudget{}; + std::array heapUsage{}; +}; + +MockDevice mock; + +template +THandle newHandle() { + return reinterpret_cast(mock.next += 0x10); +} + +VKAPI_ATTR VkResult VKAPI_CALL createImage(VkDevice, const VkImageCreateInfo* info, const VkAllocationCallbacks*, VkImage* image) { + if (mock.imageResult != VK_SUCCESS) return mock.imageResult; + if (info->format == mock.rejectedImageFormat && mock.rejectedImageResult != VK_SUCCESS) return mock.rejectedImageResult; + *image = newHandle(); + mock.images[*image] = *info; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL imageMemoryRequirements(VkDevice, VkImage image, VkMemoryRequirements* requirements) { + *requirements = {65536, 1024, 0x3}; + mock.memoryImage = image; +} + +VKAPI_ATTR VkResult VKAPI_CALL bindImageMemory(VkDevice, VkImage image, VkDeviceMemory memory, VkDeviceSize) { + mock.imageMemory[image] = memory; + mock.memoryImage = VK_NULL_HANDLE; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage image, const VkAllocationCallbacks*) { mock.destroyedImages.insert(image); } + +VKAPI_ATTR VkResult VKAPI_CALL createImageView(VkDevice, const VkImageViewCreateInfo*, const VkAllocationCallbacks*, VkImageView* view) { + *view = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyImageView(VkDevice, VkImageView, const VkAllocationCallbacks*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createBuffer(VkDevice, const VkBufferCreateInfo* info, const VkAllocationCallbacks*, VkBuffer* buffer) { + *buffer = newHandle(); + mock.bufferSizes[*buffer] = info->size; + mock.bufferCreations.push_back(info->size); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL bufferMemoryRequirements(VkDevice, VkBuffer buffer, VkMemoryRequirements* requirements) { + *requirements = {(mock.bufferSizes.at(buffer) + 255u) / 256u * 256u, 256, 0x3}; + mock.memoryImage = VK_NULL_HANDLE; +} + +VKAPI_ATTR VkResult VKAPI_CALL allocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + const bool image = mock.memoryImage != VK_NULL_HANDLE; + if (image) ++mock.imageMemoryAttempts; + if (image && mock.imageMemoryFailures != 0 && (mock.imageMemoryWidth == 0u || mock.images.at(mock.memoryImage).extent.width == mock.imageMemoryWidth)) { + --mock.imageMemoryFailures; + return mock.imageMemoryResult; + } + if (!image && mock.bufferMemorySize != 0 && info->allocationSize == mock.bufferMemorySize) { + ++mock.bufferMemoryFailures; + return mock.bufferMemoryResult; + } + *memory = newHandle(); + mock.memories[*memory].resize(static_cast(info->allocationSize)); + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL bindBufferMemory(VkDevice, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) { + mock.bufferMemory[buffer] = {memory, offset}; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mapMemory(VkDevice, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize, VkMemoryMapFlags, void** data) { + *data = mock.memories.at(memory).data() + offset; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL unmapMemory(VkDevice, VkDeviceMemory) {} + +VKAPI_ATTR void VKAPI_CALL destroyBuffer(VkDevice, VkBuffer buffer, const VkAllocationCallbacks*) { + mock.bufferSizes.erase(buffer); + mock.bufferMemory.erase(buffer); +} + +VKAPI_ATTR void VKAPI_CALL freeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { mock.memories.erase(memory); } + +VKAPI_ATTR VkResult VKAPI_CALL createDescriptorSetLayout(VkDevice, const VkDescriptorSetLayoutCreateInfo*, const VkAllocationCallbacks*, VkDescriptorSetLayout* layout) { + *layout = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyDescriptorSetLayout(VkDevice, VkDescriptorSetLayout, const VkAllocationCallbacks*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createPipelineLayout(VkDevice, const VkPipelineLayoutCreateInfo*, const VkAllocationCallbacks*, VkPipelineLayout* layout) { + *layout = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyPipelineLayout(VkDevice, VkPipelineLayout, const VkAllocationCallbacks*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createShaderModule(VkDevice, const VkShaderModuleCreateInfo*, const VkAllocationCallbacks*, VkShaderModule* module) { + *module = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyShaderModule(VkDevice, VkShaderModule, const VkAllocationCallbacks*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createComputePipelines(VkDevice, VkPipelineCache, std::uint32_t count, const VkComputePipelineCreateInfo*, const VkAllocationCallbacks*, VkPipeline* pipelines) { + for (std::uint32_t index = 0; index < count; ++index) pipelines[index] = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyPipeline(VkDevice, VkPipeline, const VkAllocationCallbacks*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createDescriptorPool(VkDevice, const VkDescriptorPoolCreateInfo*, const VkAllocationCallbacks*, VkDescriptorPool* pool) { + *pool = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyDescriptorPool(VkDevice, VkDescriptorPool, const VkAllocationCallbacks*) {} +VKAPI_ATTR VkResult VKAPI_CALL resetDescriptorPool(VkDevice, VkDescriptorPool, VkDescriptorPoolResetFlags) { return VK_SUCCESS; } + +VKAPI_ATTR VkResult VKAPI_CALL allocateDescriptorSets(VkDevice, const VkDescriptorSetAllocateInfo* info, VkDescriptorSet* sets) { + for (std::uint32_t index = 0; index < info->descriptorSetCount; ++index) sets[index] = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL updateDescriptorSets(VkDevice, std::uint32_t count, const VkWriteDescriptorSet* writes, std::uint32_t, const VkCopyDescriptorSet*) { + for (std::uint32_t index = 0; index < count; ++index) { + if (writes[index].pBufferInfo != nullptr && writes[index].descriptorCount != 0) mock.bufferWrites[writes[index].dstBinding] = writes[index].pBufferInfo[0]; + } +} + +VKAPI_ATTR VkResult VKAPI_CALL allocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo* info, VkCommandBuffer* commands) { + for (std::uint32_t index = 0; index < info->commandBufferCount; ++index) commands[index] = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL freeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t, const VkCommandBuffer*) {} + +VKAPI_ATTR VkResult VKAPI_CALL createFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) { + *fence = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL destroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) {} +VKAPI_ATTR VkResult VKAPI_CALL beginCommandBuffer(VkCommandBuffer, const VkCommandBufferBeginInfo*) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL endCommandBuffer(VkCommandBuffer) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL queueSubmit(VkQueue, std::uint32_t, const VkSubmitInfo*, VkFence) { return mock.submitResult; } +VKAPI_ATTR VkResult VKAPI_CALL waitForFences(VkDevice, std::uint32_t, const VkFence*, VkBool32, std::uint64_t) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL getFenceStatus(VkDevice, VkFence) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL queueWaitIdle(VkQueue) { return VK_SUCCESS; } +VKAPI_ATTR void VKAPI_CALL cmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {} +VKAPI_ATTR void VKAPI_CALL cmdCopyBuffer(VkCommandBuffer, VkBuffer, VkBuffer, std::uint32_t, const VkBufferCopy*) {} +VKAPI_ATTR void VKAPI_CALL cmdCopyBufferToImage(VkCommandBuffer, VkBuffer, VkImage, VkImageLayout, std::uint32_t, const VkBufferImageCopy*) {} +VKAPI_ATTR void VKAPI_CALL cmdBindPipeline(VkCommandBuffer, VkPipelineBindPoint, VkPipeline) {} +VKAPI_ATTR void VKAPI_CALL cmdBindDescriptorSets(VkCommandBuffer, VkPipelineBindPoint, VkPipelineLayout, std::uint32_t, std::uint32_t, const VkDescriptorSet*, std::uint32_t, const std::uint32_t*) {} +VKAPI_ATTR void VKAPI_CALL cmdPushConstants(VkCommandBuffer, VkPipelineLayout, VkShaderStageFlags, std::uint32_t, std::uint32_t, const void*) {} +VKAPI_ATTR void VKAPI_CALL cmdDispatch(VkCommandBuffer, std::uint32_t, std::uint32_t, std::uint32_t) {} + +VKAPI_ATTR void VKAPI_CALL formatProperties(VkPhysicalDevice, VkFormat, VkFormatProperties* properties) { + *properties = {}; + properties->optimalTilingFeatures = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT | VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT | VK_FORMAT_FEATURE_COLOR_ATTACHMENT_BIT | VK_FORMAT_FEATURE_TRANSFER_SRC_BIT | VK_FORMAT_FEATURE_TRANSFER_DST_BIT; +} + +VKAPI_ATTR VkResult VKAPI_CALL imageFormatProperties(VkPhysicalDevice, VkFormat, VkImageType, VkImageTiling, VkImageUsageFlags, VkImageCreateFlags, VkImageFormatProperties* properties) { + ++mock.imageFormatQueries; + *properties = {}; + if (mock.imageFormatResult != VK_SUCCESS) return mock.imageFormatResult; + *properties = {mock.maxExtent, 16, MaxArrayLayers, VK_SAMPLE_COUNT_1_BIT, mock.maxResourceSize}; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL memoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) { + for (auto* next = static_cast(properties->pNext); next != nullptr; next = next->pNext) { + if (next->sType != VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT) continue; + auto& reported = *reinterpret_cast(next); + std::copy(mock.heapBudget.begin(), mock.heapBudget.end(), reported.heapBudget); + std::copy(mock.heapUsage.begin(), mock.heapUsage.end(), reported.heapUsage); + } +} + +PFN_vkVoidFunction VKAPI_CALL deviceProc(VkDevice, const char* name) { + static const std::map table{ + {"vkCreateImage", reinterpret_cast(createImage)}, + {"vkGetImageMemoryRequirements", reinterpret_cast(imageMemoryRequirements)}, + {"vkBindImageMemory", reinterpret_cast(bindImageMemory)}, + {"vkDestroyImage", reinterpret_cast(destroyImage)}, + {"vkCreateImageView", reinterpret_cast(createImageView)}, + {"vkDestroyImageView", reinterpret_cast(destroyImageView)}, + {"vkCreateBuffer", reinterpret_cast(createBuffer)}, + {"vkGetBufferMemoryRequirements", reinterpret_cast(bufferMemoryRequirements)}, + {"vkAllocateMemory", reinterpret_cast(allocateMemory)}, + {"vkBindBufferMemory", reinterpret_cast(bindBufferMemory)}, + {"vkMapMemory", reinterpret_cast(mapMemory)}, + {"vkUnmapMemory", reinterpret_cast(unmapMemory)}, + {"vkDestroyBuffer", reinterpret_cast(destroyBuffer)}, + {"vkFreeMemory", reinterpret_cast(freeMemory)}, + {"vkCreateDescriptorSetLayout", reinterpret_cast(createDescriptorSetLayout)}, + {"vkDestroyDescriptorSetLayout", reinterpret_cast(destroyDescriptorSetLayout)}, + {"vkCreatePipelineLayout", reinterpret_cast(createPipelineLayout)}, + {"vkDestroyPipelineLayout", reinterpret_cast(destroyPipelineLayout)}, + {"vkCreateShaderModule", reinterpret_cast(createShaderModule)}, + {"vkDestroyShaderModule", reinterpret_cast(destroyShaderModule)}, + {"vkCreateComputePipelines", reinterpret_cast(createComputePipelines)}, + {"vkDestroyPipeline", reinterpret_cast(destroyPipeline)}, + {"vkCreateDescriptorPool", reinterpret_cast(createDescriptorPool)}, + {"vkDestroyDescriptorPool", reinterpret_cast(destroyDescriptorPool)}, + {"vkResetDescriptorPool", reinterpret_cast(resetDescriptorPool)}, + {"vkAllocateDescriptorSets", reinterpret_cast(allocateDescriptorSets)}, + {"vkUpdateDescriptorSets", reinterpret_cast(updateDescriptorSets)}, + {"vkAllocateCommandBuffers", reinterpret_cast(allocateCommandBuffers)}, + {"vkFreeCommandBuffers", reinterpret_cast(freeCommandBuffers)}, + {"vkCreateFence", reinterpret_cast(createFence)}, + {"vkDestroyFence", reinterpret_cast(destroyFence)}, + {"vkBeginCommandBuffer", reinterpret_cast(beginCommandBuffer)}, + {"vkEndCommandBuffer", reinterpret_cast(endCommandBuffer)}, + {"vkQueueSubmit", reinterpret_cast(queueSubmit)}, + {"vkWaitForFences", reinterpret_cast(waitForFences)}, + {"vkGetFenceStatus", reinterpret_cast(getFenceStatus)}, + {"vkQueueWaitIdle", reinterpret_cast(queueWaitIdle)}, + {"vkCmdPipelineBarrier", reinterpret_cast(cmdPipelineBarrier)}, + {"vkCmdCopyBuffer", reinterpret_cast(cmdCopyBuffer)}, + {"vkCmdCopyBufferToImage", reinterpret_cast(cmdCopyBufferToImage)}, + {"vkCmdBindPipeline", reinterpret_cast(cmdBindPipeline)}, + {"vkCmdBindDescriptorSets", reinterpret_cast(cmdBindDescriptorSets)}, + {"vkCmdPushConstants", reinterpret_cast(cmdPushConstants)}, + {"vkCmdDispatch", reinterpret_cast(cmdDispatch)}, + }; + const auto it = table.find(name); + if (it != table.end()) return it->second; + mock.missing.emplace_back(name); + return nullptr; +} + +Context mockContext() { + static int deviceTag = 0; + Context context{}; + context.device = reinterpret_cast(&deviceTag); + context.deviceProc = deviceProc; + context.formatProperties = formatProperties; + context.imageFormatProperties = imageFormatProperties; + context.memory.memoryTypeCount = 2; + context.memory.memoryTypes[0] = {VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0}; + context.memory.memoryTypes[1] = {VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT, 1}; + context.memory.memoryHeapCount = 2; + context.memory.memoryHeaps[0] = {std::uint64_t{1} << 32u, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + context.memory.memoryHeaps[1] = {std::uint64_t{1} << 32u, 0}; + context.nullDescriptors = true; + context.imageTableCapacity = Elements; + context.limits.maxMemoryAllocationCount = 4096; + context.limits.nonCoherentAtomSize = 64; + context.limits.minStorageBufferOffsetAlignment = 16; + context.limits.maxStorageBufferRange = 1u << 30u; + context.limits.maxBoundDescriptorSets = 1; + context.limits.maxPerStageDescriptorStorageBuffers = 16; + context.limits.maxDescriptorSetStorageBuffers = 32; + context.limits.maxPerStageResources = 128; + context.limits.maxPerStageDescriptorSampledImages = Elements; + context.limits.maxDescriptorSetSampledImages = Elements; + context.limits.maxImageDimension1D = 32768; + context.limits.maxImageDimension2D = 32768; + context.limits.maxImageDimension3D = 256; + context.limits.maxImageDimensionCube = 32768; + context.limits.maxImageArrayLayers = MaxArrayLayers; + return context; +} + +std::array textureWords(std::uint64_t address, std::uint32_t format, std::uint32_t width, std::uint32_t height, std::uint32_t swizzleMode, std::uint32_t type, std::uint32_t lastSlice) { + std::array words{}; + const auto base = address >> 8u; + words[0] = static_cast(base); + words[1] = static_cast(base >> 32u) | (format << 20u) | (((width - 1u) & 0x3u) << 30u); + words[2] = ((width - 1u) >> 2u) | ((height - 1u) << 14u) | (1u << 31u); + words[3] = 4u | (5u << 3u) | (6u << 6u) | (7u << 9u) | (swizzleMode << 20u) | (type << 28u); + words[4] = lastSlice; + return words; +} + +std::uint64_t addressOf(const void* pointer) { + return static_cast(reinterpret_cast(pointer)); +} + +struct Entry { + std::array words; + Shape shape; +}; + +struct CachedTexturesCleared { + VkDevice device; + explicit CachedTexturesCleared(VkDevice device) : device(device) {} + CachedTexturesCleared(const CachedTexturesCleared&) = delete; + CachedTexturesCleared& operator=(const CachedTexturesCleared&) = delete; + ~CachedTexturesCleared() { ClearCachedTextures(device); } +}; + +struct Block { + std::byte* data; + std::size_t bytes; + explicit Block(std::size_t bytes) : data(static_cast(::operator new(bytes, std::align_val_t{4096}))), bytes(bytes) { std::memset(data, 0, bytes); } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + ~Block() { ::operator delete(data, std::align_val_t{4096}); } +}; + +struct Registration { + void* data; + Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, false); } + Registration(const Registration&) = delete; + Registration& operator=(const Registration&) = delete; + ~Registration() { GuestAllocations::Mutation().Remove(data); } +}; + +constexpr std::size_t GuestBytes = std::size_t{MaxArrayLayers} * 2048u + 65536u; + +std::vector resolveTable(const Context& context, std::span entries) { + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding map{}; + map.descriptorSet = ShaderRecompiler::RuntimeAbi::DescriptorSet; + map.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + map.role = ShaderRecompiler::DescriptorRole::ImageTableMap; + map.binding = static_cast(ShaderRecompiler::RuntimeAbi::Binding::ImageTableMap); + map.count = 1; + map.readOnly = false; + map.guestDescriptor.assign(Abi::HeaderWords + entries.size(), 0u); + map.guestDescriptor[Abi::HeaderVersion] = Abi::Version; + map.guestDescriptor[Abi::HeaderElementCount] = static_cast(entries.size()); + map.guestDescriptor[Abi::HeaderElementFaults] = Abi::HeaderWords; + map.guestDescriptor[Abi::HeaderSamplerFaults] = Abi::HeaderWords + static_cast(entries.size()); + program.bindings.push_back(map); + ShaderRecompiler::DescriptorBinding images{}; + images.descriptorSet = ShaderRecompiler::RuntimeAbi::DescriptorSet; + images.kind = ShaderRecompiler::DescriptorKind::SampledImage; + images.role = ShaderRecompiler::DescriptorRole::ImageTable; + images.binding = ShaderRecompiler::RuntimeAbi::ImageTableBinding; + images.count = static_cast(entries.size()); + for (const auto& entry : entries) { + images.guestDescriptor.insert(images.guestDescriptor.end(), entry.words.begin(), entry.words.end()); + images.imageShapes.push_back(entry.shape); + } + images.imageDepthCompare.assign(entries.size(), false); + images.imageSamplers.assign(entries.size(), 0u); + program.bindings.push_back(std::move(images)); + const CompiledShader shader{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + mock.bufferWrites.clear(); + ShaderResources resources(context, shader, {}, true); + resources.Complete(); + const auto write = mock.bufferWrites.find(map.binding); + Require(write != mock.bufferWrites.end(), "the image table map was not written to its descriptor"); + const auto& [memory, offset] = mock.bufferMemory.at(write->second.buffer); + const auto& bytes = mock.memories.at(memory); + std::vector words(Abi::HeaderWords + entries.size()); + Require(offset + write->second.offset + words.size() * sizeof(std::uint32_t) <= bytes.size(), "the image table map buffer is shorter than its map"); + std::memcpy(words.data(), bytes.data() + offset + write->second.offset, words.size() * sizeof(std::uint32_t)); + return words; +} + +std::string createdImages() { + std::string created; + for (const auto& [image, info] : mock.images) created += " " + std::to_string(info.extent.width) + "x" + std::to_string(info.extent.height) + "x" + std::to_string(info.extent.depth) + "/" + std::to_string(info.arrayLayers) + " layers"; + return created.empty() ? " none" : created; +} + +void requireServed(const std::vector& map, std::uint32_t element, const std::string& what) { + Require(map[Abi::HeaderWords + element] == 0u, what + ": the entry was rejected (fault code " + std::to_string(map[Abi::HeaderWords + element]) + "); images created:" + createdImages() + "; Vulkan functions the mock lacks: " + std::to_string(mock.missing.size())); +} + +void requireRejected(const std::vector& map, std::uint32_t element, const std::string& what) { + Require((map[Abi::HeaderWords + element] & Abi::PoisonFlag) != 0u, what + ": the entry was served instead of faulting when selected; images created:" + createdImages()); +} + +std::array layeredWords(const Block& guest, std::uint32_t layers) { + return textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 256, 256, SwModeLinear, Type2DArray, layers - 1u); +} + +std::size_t imagesWithLayers(std::uint32_t layers) { + return static_cast(std::count_if(mock.images.begin(), mock.images.end(), [&](const auto& image) { return image.second.extent.width == 256 && image.second.arrayLayers == layers; })); +} + +VkDeviceSize largestBufferCreated() { + return mock.bufferCreations.empty() ? 0 : *std::max_element(mock.bufferCreations.begin(), mock.bufferCreations.end()); +} + +void testTableEntries(Context context) { + static int physicalTag = 0; + context.physical = reinterpret_cast(&physicalTag); + const Block guest(GuestBytes), host(4096); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const auto k1 = textureWords(0x200, FormatR16G16B16A16Float, 256, 256, SwMode64KBS, Type2DArray, 7424); + const auto decoded = DecodeTextureResource(k1); + Require(decoded.baseAddress == 0x200 && decoded.width == 256 && decoded.height == 256 && decoded.format == FormatR16G16B16A16Float, "the 256x256 T# at 0x200 decoded to another surface"); + Require(decoded.dimension == TextureDimension::k2DArray && decoded.tileMode == TextureTileMode::kStandard64KB && decoded.depthOrLastArray == 7424 && decoded.mipCount == 1, "the 256x256 T# at 0x200 decoded to other array fields"); + Require(DescribeSurface(decoded).layers == 7425, "the 256x256 T# at 0x200 must describe 7425 layers"); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + {k1, Shape::Image2DArray}, + {textureWords(addressOf(host.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2DArray, MaxArrayLayers), Shape::Image2DArray}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2DArray, MaxArrayLayers - 1u), Shape::Image2DArray}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 16384, 1, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.maxExtent = {8192, 8192, 1}; + const auto map = resolveTable(context, entries); + mock.maxExtent = {32768, 32768, 1}; + requireServed(map, 0, "an 8x8 texture in registered guest memory"); + requireRejected(map, 1, "the 256x256 2D array of 7425 layers at 0x200"); + requireRejected(map, 2, "an 8x8 texture in host memory no guest allocation holds"); + requireRejected(map, 3, "an 8x8 2D array of 2049 layers in registered guest memory"); + requireServed(map, 4, "an 8x8 2D array of 2048 layers in registered guest memory"); + requireRejected(map, 5, "a 16384x1 texture over a format limit of 8192"); + Require(map[Abi::HeaderPoisonCount] == 4u, "the table holds " + std::to_string(map[Abi::HeaderPoisonCount]) + " rejected entries, expected 4"); + Require(mock.images.size() == 2, "the rejected entries created images:" + createdImages()); + for (const auto& [image, info] : mock.images) { + Require(info.extent.width == 8 && info.extent.height == 8 && info.extent.depth == 1 && (info.arrayLayers == 1 || info.arrayLayers == MaxArrayLayers), "an image other than the two served entries was created:" + createdImages()); + } +} + +void testResourceSize(Context context) { + static int physicalTag = 0; + context.physical = reinterpret_cast(&physicalTag); + const Block guest(65536); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 32, 32, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 64, 64, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.maxResourceSize = 8192; + const auto map = resolveTable(context, entries); + mock.maxResourceSize = VkDeviceSize{1} << 40u; + requireServed(map, 0, "a 32x32 RGBA8 texture of 4096 bytes under a maxResourceSize of 8192"); + requireRejected(map, 1, "a 64x64 RGBA8 texture of 16384 bytes over a maxResourceSize of 8192"); + Require(mock.images.size() == 1, "the texture over maxResourceSize created an image:" + createdImages()); +} + +void testUnsupportedImageCombination(Context context) { + static int physicalTag = 0; + context.physical = reinterpret_cast(&physicalTag); + const Block guest(4096); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.imageFormatResult = VK_ERROR_FORMAT_NOT_SUPPORTED; + const auto queriesBefore = mock.imageFormatQueries; + const auto map = resolveTable(context, entries); + mock.imageFormatResult = VK_SUCCESS; + Require(mock.imageFormatQueries > queriesBefore, "the unsupported image combination did not reach the format query"); + requireRejected(map, 0, "an 8x8 registered texture whose exact image combination is unsupported"); + Require(mock.images.empty(), "the unsupported image combination reached vkCreateImage:" + createdImages()); +} + +void testPartlyRegisteredSurface(const Context& context) { + const Block straddling(8192); + const Registration registered(straddling.data + 4096, 4096); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(straddling.data), FormatR8G8B8A8Unorm, 64, 32, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + const auto map = resolveTable(context, entries); + requireServed(map, 0, "a texture whose second page lies in a registered guest allocation"); +} + +void testVulkanErrorPropagates(const Context& context) { + const Block guest(4096); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.imageResult = VK_ERROR_DEVICE_LOST; + std::optional thrown; + std::string message; + try { + static_cast(resolveTable(context, entries)); + } catch (const VulkanError& error) { + thrown = error.result; + message = error.what(); + } + mock.imageResult = VK_SUCCESS; + Require(thrown == VK_ERROR_DEVICE_LOST, "VK_ERROR_DEVICE_LOST from an image table entry's vkCreateImage was turned into a faulting entry instead of reaching the caller"); + Require(message.find("vkCreateImage: Vulkan result -4") != std::string::npos, "the propagated device loss names another call: " + message); +} + +void testHeapSize(Context context) { + static int physicalTag = 0; + context.physical = reinterpret_cast(&physicalTag); + context.memoryProperties2 = memoryProperties2; + const Block guest(MiB); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {layeredWords(guest, 2), Shape::Image2DArray}, + {layeredWords(guest, 3), Shape::Image2DArray}, + }}; + mock.images.clear(); + mock.heapBudget = {64 * MiB, 64 * MiB}; + mock.heapUsage = {64 * MiB - 512 * KiB, 64 * MiB - 256 * KiB}; + auto map = resolveTable(context, entries); + mock.heapBudget = {}; + mock.heapUsage = {}; + requireServed(map, 1, "a 256x256 RGBA8 array of 3 layers (786432 bytes) in heaps of 4 GiB whose VK_EXT_memory_budget budgets leave 524288 and 262144 bytes"); + Require(imagesWithLayers(3) == 1, "the 3-layer entry within the heaps' sizes created no image:" + createdImages()); + context.memory.memoryHeaps[0].size = 512 * KiB; + context.memory.memoryHeaps[1].size = 256 * KiB; + ClearCachedTextures(context.device); + mock.images.clear(); + mock.bufferCreations.clear(); + map = resolveTable(context, entries); + requireServed(map, 0, "a 256x256 RGBA8 array of 2 layers (524288 bytes) in heaps of 524288 and 262144 bytes"); + requireRejected(map, 1, "a 256x256 RGBA8 array of 3 layers (786432 bytes) in heaps of 524288 and 262144 bytes"); + Require(imagesWithLayers(3) == 0, "the 3-layer entry larger than every heap created an image:" + createdImages()); + Require(largestBufferCreated() < 3 * 256 * KiB, "the 3-layer entry larger than every heap reached the texture upload: a buffer of " + std::to_string(largestBufferCreated()) + " bytes was created"); +} + +void testOutOfMemoryEntry(const Context& context) { + const Block guest(2 * MiB); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array imageEntries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 16, 16, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 4, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + for (const auto result : {VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_HOST_MEMORY}) { + const auto name = "Vulkan result " + std::to_string(result); + mock.images.clear(); + mock.destroyedImages.clear(); + mock.imageMemoryResult = result; + mock.imageMemoryWidth = 16u; + mock.imageMemoryFailures = std::numeric_limits::max(); + std::vector map; + std::string thrown; + try { + map = resolveTable(context, imageEntries); + } catch (const VulkanError& error) { + thrown = error.what(); + } + const auto unused = mock.imageMemoryFailures; + mock.imageMemoryFailures = 0; + mock.imageMemoryWidth = 0u; + mock.imageMemoryResult = VK_SUCCESS; + Require(unused < std::numeric_limits::max(), name + " was never returned: the 16x16 entry allocated no image memory"); + Require(thrown.empty(), name + " from the 16x16 entry's image allocation reached the caller instead of faulting that entry: " + thrown); + requireRejected(map, 0, "a 16x16 texture whose image allocation fails with " + name); + requireServed(map, 1, "a 4x8 texture after the entry whose image allocation failed with " + name); + const auto failed = std::find_if(mock.images.begin(), mock.images.end(), [](const auto& image) { return image.second.extent.width == 16; }); + Require(failed != mock.images.end() && mock.destroyedImages.contains(failed->first), "the image of the entry whose allocation failed with " + name + " was not destroyed:" + createdImages()); + } + const std::array bufferEntries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 512, 520, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 16, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.destroyedImages.clear(); + mock.bufferMemorySize = 512 * 520 * 4; + mock.bufferMemoryResult = VK_ERROR_OUT_OF_DEVICE_MEMORY; + mock.bufferMemoryFailures = 0; + std::vector map; + std::string thrown; + try { + map = resolveTable(context, bufferEntries); + } catch (const VulkanError& error) { + thrown = error.what(); + } + const auto failures = mock.bufferMemoryFailures; + mock.bufferMemorySize = 0; + mock.bufferMemoryResult = VK_SUCCESS; + Require(failures != 0, "the 512x520 entry allocated no upload buffer of 1064960 bytes"); + Require(thrown.empty(), "VK_ERROR_OUT_OF_DEVICE_MEMORY from the 512x520 entry's upload buffer allocation reached the caller instead of faulting that entry: " + thrown); + requireRejected(map, 0, "a 512x520 texture whose upload buffer allocation fails with VK_ERROR_OUT_OF_DEVICE_MEMORY after its image was allocated"); + requireServed(map, 1, "an 8x16 texture after the entry whose upload buffer allocation failed"); + const auto failed = std::find_if(mock.images.begin(), mock.images.end(), [](const auto& image) { return image.second.extent.width == 512; }); + Require(failed != mock.images.end() && mock.destroyedImages.contains(failed->first) && mock.imageMemory.contains(failed->first) && !mock.memories.contains(mock.imageMemory.at(failed->first)), "the image of the 512x520 entry whose upload buffer failed, or its memory, was not released:" + createdImages()); +} + +void testImageMemoryRecovery(const Context& context) { + const Block guest(65536); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 32, 16, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.imageMemoryAttempts = 0; + mock.imageMemoryResult = VK_ERROR_OUT_OF_DEVICE_MEMORY; + mock.imageMemoryFailures = 1; + const auto map = resolveTable(context, entries); + const auto unused = mock.imageMemoryFailures; + mock.imageMemoryFailures = 0; + mock.imageMemoryResult = VK_SUCCESS; + Require(unused == 0 && mock.imageMemoryAttempts >= 2u, "the sampled image did not retry its initial device-memory refusal"); + requireServed(map, 0, "a sampled image recovered by the common allocation path"); +} + +void testExpandedImageMemory(const Context& context) { + const Block guest(65536); + const Registration registered(guest.data, guest.bytes); + auto expandedContext = context; + const auto format = ShaderRecompiler::IrBufferFormat::Format8Srgb; + expandedContext.srgbDecodeFormats = ShaderRecompiler::SrgbDecodeBit(format); + expandedContext.srgbCopyFormats = expandedContext.srgbDecodeFormats; + TextureDetiler detiler(expandedContext); + const auto descriptor = DecodeTextureResource(textureWords(addressOf(guest.data), static_cast(format), 8, 8, SwModeLinear, Type2D, 0)); + const auto bytes = DescribeSurface(descriptor).guestBytes; + const auto snapshot = std::span(static_cast(guest.data), static_cast(bytes)); + for (const auto result : {VK_ERROR_OUT_OF_DEVICE_MEMORY, VK_ERROR_OUT_OF_HOST_MEMORY, VK_ERROR_DEVICE_LOST}) { + mock.images.clear(); + mock.rejectedImageFormat = VK_FORMAT_R8G8B8A8_UNORM; + mock.rejectedImageResult = result; + std::optional failure; + bool own = false; + try { + static_cast(Texture(expandedContext, detiler, descriptor, VkComponentMapping{}, snapshot)); + } catch (const TextureMemoryExhausted& error) { + failure = error.result; + own = true; + } catch (const VulkanError& error) { + failure = error.result; + } + mock.rejectedImageFormat = VK_FORMAT_UNDEFINED; + mock.rejectedImageResult = VK_SUCCESS; + Require(mock.images.size() == 1u && mock.images.begin()->second.format == VK_FORMAT_R8_UNORM, "the expanded-image failure did not follow a successful upload-image allocation"); + Require(failure == result && own == (result != VK_ERROR_DEVICE_LOST), "the expanded sampled image classified its allocation failure incorrectly"); + } +} + +void testUploadSubmitOutOfMemoryPropagates(const Context& context) { + const Block guest(4096); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 2, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.submitResult = VK_ERROR_OUT_OF_DEVICE_MEMORY; + std::optional thrown; + std::string message; + try { + static_cast(resolveTable(context, entries)); + } catch (const VulkanError& error) { + thrown = error.result; + message = error.what(); + } + mock.submitResult = VK_SUCCESS; + Require(thrown == VK_ERROR_OUT_OF_DEVICE_MEMORY, "VK_ERROR_OUT_OF_DEVICE_MEMORY from the vkQueueSubmit of an image table entry's upload was turned into a faulting entry instead of reaching the caller"); + Require(message.find("vkQueueSubmit graphics: Vulkan result -2") != std::string::npos, "the propagated out-of-memory error names another call: " + message); +} + +void testDeviceLossFromImageMemoryPropagates(const Context& context) { + const Block guest(4096); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const std::array entries{{ + {textureWords(addressOf(guest.data), FormatR8G8B8A8Unorm, 8, 4, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + mock.imageMemoryResult = VK_ERROR_DEVICE_LOST; + mock.imageMemoryFailures = 1; + std::optional thrown; + std::string message; + try { + static_cast(resolveTable(context, entries)); + } catch (const VulkanError& error) { + thrown = error.result; + message = error.what(); + } + mock.imageMemoryFailures = 0; + mock.imageMemoryResult = VK_SUCCESS; + Require(thrown == VK_ERROR_DEVICE_LOST, "VK_ERROR_DEVICE_LOST from an image table entry's vkAllocateMemory was turned into a faulting entry instead of reaching the caller"); + Require(message.find("vkAllocateMemory texture: ") != std::string::npos, "the propagated device loss names another call: " + message); +} + +struct ArenaBlock { + std::byte* data = nullptr; + std::size_t bytes; + explicit ArenaBlock(std::size_t bytes) : bytes(bytes) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes)); + Require(data != nullptr, "cannot allocate a guest arena block for the image table reuse test"); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); +#endif + } + ArenaBlock(const ArenaBlock&) = delete; + ArenaBlock& operator=(const ArenaBlock&) = delete; + ~ArenaBlock() { +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(data, bytes); +#endif + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } +}; + +void requireResolutions(const ImageTableReuseCounts& before, std::uint64_t reused, std::uint64_t resolved, const std::string& what) { + const auto now = ImageTableReuses(); + Require(now.reused - before.reused == reused && now.resolved - before.resolved == resolved, what + ": " + std::to_string(now.reused - before.reused) + " tables reused and " + std::to_string(now.resolved - before.resolved) + " resolved, expected " + std::to_string(reused) + " and " + std::to_string(resolved)); +} + +void testOutOfMemoryTableResolvedAgain(const Context& context) { + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "guest arena unavailable or not write-watched: reuse of an image table with an entry out of memory not tested\n"; + return; + } + std::exception_ptr failure; + std::thread([&] { + try { + const ArenaBlock guest(65536); + const Registration registered(guest.data, guest.bytes); + const CachedTexturesCleared cleared(context.device); + const auto address = addressOf(guest.data); + const std::array served{{ + {textureWords(address, FormatR8G8B8A8Unorm, 16, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(address, FormatR8G8B8A8Unorm, 4, 4, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + const std::array exhausted{{ + {textureWords(address, FormatR8G8B8A8Unorm, 16, 16, SwModeLinear, Type2D, 0), Shape::Image2D}, + {textureWords(address, FormatR8G8B8A8Unorm, 8, 8, SwModeLinear, Type2D, 0), Shape::Image2D}, + }}; + mock.images.clear(); + AgcDriver::GuestMemory::BumpCollectEpoch(); + auto before = ImageTableReuses(); + requireServed(resolveTable(context, served), 0, "a 16x8 texture in a guest arena block"); + requireServed(resolveTable(context, served), 1, "a 4x4 texture in a guest arena block"); + requireResolutions(before, 1, 1, "a table of two served entries resolved twice in one collect epoch"); + mock.imageMemoryResult = VK_ERROR_OUT_OF_DEVICE_MEMORY; + mock.imageMemoryWidth = 16u; + mock.imageMemoryFailures = std::numeric_limits::max(); + before = ImageTableReuses(); + auto map = resolveTable(context, exhausted); + const auto unused = mock.imageMemoryFailures; + mock.imageMemoryFailures = 0; + mock.imageMemoryWidth = 0u; + mock.imageMemoryResult = VK_SUCCESS; + Require(unused < std::numeric_limits::max(), "the 16x16 entry in the guest arena block allocated no image memory"); + requireRejected(map, 0, "a 16x16 texture in a guest arena block whose image allocation fails with VK_ERROR_OUT_OF_DEVICE_MEMORY"); + requireServed(map, 1, "an 8x8 texture next to the entry whose image allocation failed"); + map = resolveTable(context, exhausted); + requireResolutions(before, 0, 2, "a table whose entry ran out of memory, resolved again in the same collect epoch"); + requireServed(map, 0, "the 16x16 texture once its image allocation succeeds"); + requireServed(map, 1, "the 8x8 texture next to it"); + } catch (...) { + failure = std::current_exception(); + } + }).join(); + if (failure) std::rethrow_exception(failure); +} + +} + +void RunImageTableEntryValidationTests() { + Require(Recorder::Active() == nullptr, "the image table entry validation tests record into their own command batches"); + mock = MockDevice{}; + { + auto context = mockContext(); + TextureDetiler detiler(context); + context.detiler = &detiler; + const CachedTexturesCleared cleared(context.device); + testTableEntries(context); + testResourceSize(context); + testUnsupportedImageCombination(context); + testPartlyRegisteredSurface(context); + testVulkanErrorPropagates(context); + testHeapSize(context); + testOutOfMemoryEntry(context); + testImageMemoryRecovery(context); + testExpandedImageMemory(context); + testUploadSubmitOutOfMemoryPropagates(context); + testDeviceLossFromImageMemoryPropagates(context); + testOutOfMemoryTableResolvedAgain(context); + Require(mock.missing.empty(), "the mock lacks the Vulkan function " + (mock.missing.empty() ? std::string{} : mock.missing.front())); + } + mock = MockDevice{}; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp index 3268ff39d811..8bdc405ecea3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BindlessImages.cpp @@ -2,6 +2,7 @@ #include "Recompiler.hpp" #include "ImageTableAbi.hpp" #include "VulkanTestDevice.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include #include #include @@ -29,6 +30,12 @@ std::array Buffer(const void* pointer, std::uint32_t stride, s } void Run(AgcDriver::VulkanDevice& device) { + struct Registration { + Registration() { GuestAllocations::Mutation().Add(textures.data(), sizeof(textures), true, true, false); } + Registration(const Registration&) = delete; + Registration& operator=(const Registration&) = delete; + ~Registration() { GuestAllocations::Mutation().Remove(textures.data()); } + } registered; textures[0].fill(0x40000000u); textures[1].fill(0x40400000u); const auto threads = std::min(32u, device.Target().subgroupSize); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp index cb8958107b62..3c65a0bbd71f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSint8.cpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include #include #include @@ -187,6 +188,12 @@ alignas(256) const std::array TableCode{ }; void CheckTable(AgcDriver::VulkanDevice& device) { + struct Registration { + Registration() { GuestAllocations::Mutation().Add(TableTextures, sizeof(TableTextures), true, true, false); } + Registration(const Registration&) = delete; + Registration& operator=(const Registration&) = delete; + ~Registration() { GuestAllocations::Mutation().Remove(TableTextures); } + } registered; TableTextures[0].bytes.fill(0x80u); TableTextures[1].bytes.fill(0xffu); for (std::uint32_t entry = 0; entry < 2u; ++entry) Tables.heap[entry] = TextureDescriptor(TableTextures[entry].bytes.data(), Sint8, 4u, 4u, IdentitySwizzle); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp index 7f9ee6befdfb..d8e120422717 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageTable.cpp @@ -293,6 +293,21 @@ constexpr std::array PointClamp{0x92u, (4u * 256u) << 12u, 0u, constexpr std::array LinearWrap{0u, (4u * 256u) << 12u, (1u << 20u) | (1u << 22u), 0u}; constexpr std::array PointWrap{0u, (4u * 256u) << 12u, 1u << 24u, 0u}; +struct GuestRegistration { + void* data; + GuestRegistration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, false); } + GuestRegistration(const GuestRegistration&) = delete; + GuestRegistration& operator=(const GuestRegistration&) = delete; + ~GuestRegistration() { GuestAllocations::Mutation().Remove(data); } +}; + +struct TexelRegistrations { + GuestRegistration floatTexels{FloatTexels.data(), sizeof(FloatTexels)}; + GuestRegistration uintTexels{UintTexels.data(), sizeof(UintTexels)}; + GuestRegistration sintTexels{SintTexels.data(), sizeof(SintTexels)}; + GuestRegistration depthStorage{DepthStorage.data(), sizeof(DepthStorage)}; +}; + std::array Image(const void* texels, std::uint32_t format, std::uint32_t type = Type2D, std::uint32_t swizzleMode = 0u) { const auto address = static_cast(reinterpret_cast(texels)); return {static_cast(address >> 8u), static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Width - 1u) & 3u) << 30u), ((Width - 1u) >> 2u) | ((Height - 1u) << 14u) | (1u << 31u), 0xfacu | (swizzleMode << 20u) | (type << 28u)}; @@ -833,6 +848,7 @@ void RunTests(AgcDriver::VulkanDevice& device) { Keys[0] = 0; const auto before = Run(device, TableCode, 1, Lanes, 64); + const GuestRegistration otherTexels(OtherFloatTexels.data(), sizeof(OtherFloatTexels)); SetRecord(0, PointClamp, Image(OtherFloatTexels.data(), Format32Float)); const auto after = Run(device, TableCode, 1, Lanes, 64); Require(after.variantId == before.variantId && after.artifactId == before.artifactId, "image table: other table contents specialized another module"); @@ -1296,7 +1312,7 @@ void RunPointerStorageWriteTests(AgcDriver::VulkanDevice& device) { void* data; Registration(void* data, std::size_t bytes) : data(data) { GuestAllocations::Mutation().Add(data, bytes, true, true, true); } ~Registration() { GuestAllocations::Mutation().Remove(data); } - } registeredRecords(records.data(), sizeof(records)), registeredOtherTexels(OtherFloatTexels.data(), sizeof(OtherFloatTexels)); + } registeredRecords(records.data(), sizeof(records)), registeredOtherTexels(OtherFloatTexels.data(), sizeof(OtherFloatTexels)), registeredIsolatedTexels(isolatedTexels.data(), sizeof(isolatedTexels)); for (std::uint32_t x = 0; x < isolatedTexels.size(); ++x) isolatedTexels[x] = 1024.0f + static_cast(x); const auto run = [&](std::span code, bool alias, std::uint32_t waveSize = 64u, std::uint32_t selectedLanes = 0u) { records.fill(0u); @@ -1478,22 +1494,28 @@ int main() { #endif const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - RunTests(*device); - RunBorderSwizzleTests(*device); - RunAnisoOverrideTests(*device); - RunReadTests(*device); - RunLoopKeyTests(*device); - RunGuardedTableTests(*device); - RunSelectedRecordTests(*device); - RunModeTests(*device); + { + const TexelRegistrations texels; + RunTests(*device); + RunBorderSwizzleTests(*device); + RunAnisoOverrideTests(*device); + RunReadTests(*device); + RunLoopKeyTests(*device); + RunGuardedTableTests(*device); + RunSelectedRecordTests(*device); + RunModeTests(*device); + } RunStoreTests(*device); RunSelectedStoreTests(*device); - RunPointerTests(*device); - RunPointerRecordTests(*device); - RunPointerChainRecordTests(*device); - RunSplitRecordTests(*device); - RunPointerStorageWriteTests(*device); - RunGraphicsTests(*device); + { + const TexelRegistrations texels; + RunPointerTests(*device); + RunPointerRecordTests(*device); + RunPointerChainRecordTests(*device); + RunSplitRecordTests(*device); + RunPointerStorageWriteTests(*device); + RunGraphicsTests(*device); + } std::puts("image table tests passed"); return 0; } catch (const std::exception& error) { diff --git a/docs/dev/debt/libSceAgcDriver.md b/docs/dev/debt/libSceAgcDriver.md index cbf0a781c3af..b4d5f0e598f8 100644 --- a/docs/dev/debt/libSceAgcDriver.md +++ b/docs/dev/debt/libSceAgcDriver.md @@ -75,7 +75,7 @@ - [Sample masks](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - `PA_SC_AA_MASK_X0Y0_X1Y0` and `PA_SC_AA_MASK_X0Y1_X1Y1` hold one 16-bit sample mask for each pixel of the aligned 2x2 quad ([Mesa gfx10.json](https://gitlab.freedesktop.org/mesa/mesa/-/blob/7b87f8ad226e/src/amd/registers/gfx10.json#L13512)); only the bits of the `MSAA_NUM_SAMPLES` rasterized samples count, and at one sample bit 0 of a pixel's field covers every sample of the target the draw is raised to. A mask equal for the four pixels is the Vulkan `pSampleMask`, the case [radv](https://gitlab.freedesktop.org/mesa/mesa/-/blob/7b87f8ad226e/src/amd/vulkan/radv_cmd_buffer.c#L13633) and [radeonsi](https://gitlab.freedesktop.org/mesa/mesa/-/blob/7b87f8ad226e/src/gallium/drivers/radeonsi/si_state.c#L4218) program; with early fragment tests it needs `earlyFragmentSampleMaskTestBeforeSampleCounting` ([radv](https://gitlab.freedesktop.org/mesa/mesa/-/blob/7b87f8ad226e/src/amd/vulkan/radv_physical_device.c#L2079) reports it). A mask that differs between the pixels is applied in the pixel shader without early fragment tests: a pixel whose mask covers all or none of its samples is demoted to a helper invocation by `gl_FragCoord` parity and killed when the program returns (the Radeon 860M's Windows driver 32.0.13058.8001 writes undefined colour for a demoted pixel whose export leaves its output unwritten), and otherwise the pixel's mask is ANDed into `gl_SampleMask` (with alpha-to-coverage and the mask export) after demoting invocations whose `SampleMask` input it leaves empty. A pixel program with early Z that writes memory under depth, stencil or depth-bounds tests is therefore refused. That XnYm is the pixel at (x mod 2, y mod 2) = (n, m) is read from the field names; that the mask removes samples before the depth and stencil tests and occlusion counting is inferred from radv implementing the Vulkan sample mask test with it; and that it takes no part in the centroid (`AA_MASK_CENTROID_DTMN` 0) is inferred from radv setting that bit only for conservative rasterization. None of it is measured on gfx1013. A mask equal for the four pixels that removes some but not all samples of a pixel is refused with active centroid inputs, since where Vulkan puts the centroid relative to `pSampleMask` is implementation-defined, and any partial mask is refused with conservative rasterization (radv sets `PREZ_AA_MASK_ENABLE` and `POSTZ_AA_MASK_ENABLE` [there](https://gitlab.freedesktop.org/mesa/mesa/-/blob/7b87f8ad226e/src/amd/vulkan/radv_cmd_buffer.c#L13653), which 0x6001 leaves clear). - [Covered-centroid raster state](../../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) accepts PA_SC_AA_CONFIG.COVERED_CENTROID_IS_CENTER only when the pixel program is skipped or SPI_PS_INPUT_ENA and SPI_PS_INPUT_ADDR have no active perspective or linear centroid input. Bit 29 follows [PS5_Mesa GFX10.3 registers](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/registers/gfx103.json); [RADV](https://github.com/mihawk-99/PS5_Mesa/blob/0b2d6d1a61d9bbf89cf8beb88a696144f67c61f8/src/amd/vulkan/radv_cmd_buffer.c) sets it on GFX10.3 and later. Active centroid interpolation and unspecified input registers still throw. Console covered-pixel centroid results were not measured. - [Compute LDS beyond the device's shared-memory limit](../../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords, rounded up to an even count so that 64-bit atomics stay 8-byte aligned, per workgroup (the dword after the LDS is the atomic lock on a device without `shaderBufferInt64Atomics`), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. -- [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. +- [AGC buffer allocations](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) retry host/device out-of-memory once after reclaiming idle pool buffers and fully unused slab blocks. Live buffers and blocks with live slots are retained. Live images and other retained resources have no global memory bound; allocation failure still throws when reclaim cannot satisfy the request. An image table entry whose own texture allocation fails this way faults instead (see the image table entries). - [Host placement after a device-memory refusal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp) (libSceAgcDriver): when the driver refuses a device-local allocation with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool's trim and the texture-cache release, device buffers and images (textures, storage textures, block, repacked, depth, depth plane and stencil copies, render, depth and multisampled targets and attachment proxies) are allocated in a host memory type their requirements allow. A host-placed device buffer leaves the pool on release. A host-placed image keeps that placement until it is destroyed and is never moved back to device memory, so every access to it crosses PCIe for its lifetime. The placements are counted in a rate-limited `[bufferpool]` line with the cumulative MiB, not the resident bytes. The 1x1 denormal clear probe still allocates device memory directly and throws on a refusal. - [Pipeline journal](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineJournal.cpp) (libSceAgcDriver): the journal is append-only and never compacted, so a module a later run no longer builds pipelines from stays in the file until it is deleted, and the whole file is read into memory while the replay runs (its modules reference that buffer until the last replayed pipeline was built). Only the first `APS5_PIPELINE_JOURNAL_ENTRIES` pipelines recorded are replayed (a record is never rewritten, so a pipeline past the bound stays past it), and the pipeline store evicts the least recently used entry past `APS5_PIPELINE_STORE_ENTRIES` (1024 each): a title with more configurations rebuilds the rest at the draw as before. A compute pipeline is replayed over the set layout `DescribeSetLayouts` derives from the result's bindings, which a dispatch that finds it resident checks against its `ShaderResources` layout only under `APS5_VERIFY_SET_LAYOUTS=1`. Recording and replay are best effort: a record whose write fails is lost, an entry whose build fails is skipped, and both are counted in the `[pipeline-journal]` line rather than stopping the run. A replayed pipeline that the title never asks for again costs its build time and its driver memory (its optimized relink is deferred to its first draw, so a run whose first frames bind many replayed pipelines queues their relinks then); nothing ages recipes out of the file. Two runs sharing the file take turns under its lock file, but each only knows the modules it loaded or wrote itself, so a module the other run appended meanwhile may be appended again (the loader keeps the first copy). Records a run appends after another run's load are replayed by that run only after its next start. The file header's format number is the only guard against a change of the recipe codec, `CanonicalPipelineState` or the store key that keeps the structures' sizes (the recompiler's source version does not cover the driver's sources). - [Graphics pipeline libraries](../../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline @@ -167,7 +167,9 @@ - [Table T#s](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) paired with direct S#s fault when a paired border class differs from the decoded image class. Depth-bit views use float borders. Matching float pairs and inactive mismatches do not fault. - [Table S#](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) DISABLE_CUBE_WRAP faults on selection without `VK_EXT_non_seamless_cube_map`: no runtime dimension check, including valid non-cube direct/table T#s. Unselected entries do not fault; with the feature this bit alone is accepted. Table T#/direct S# faults only for cubes needing the missing feature. - An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) whose record range overlaps a buffer or storage image written by its own dispatch faults (`table written by its dispatch`) when any entry is selected, even if the write misses the selected record. A selected column's range is its record and the bytes of the record before the descriptor. -- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface has no mapped byte faults when selected instead of becoming a zero texture as a direct T# does; what the console returns for it is not measured. +- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) T# whose surface overlaps no readable registered guest allocation (`RegisteredReadableOverlaps`: the guest heap, direct and flexible mappings and registered guest images) faults when selected instead of becoming a zero texture as a direct T# does; host memory the guest never mapped (PPSA01342 holds a table entry at 0x200) is not read. A surface that only partly overlaps registered memory is still read from every committed host page it covers. What the console returns for such an entry is not measured. +- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) T# whose sampled image the device does not support for its format, type and flags (`vkGetPhysicalDeviceImageFormatProperties` reports `VK_ERROR_FORMAT_NOT_SUPPORTED`), or that needs more array layers, a larger extent or more bytes than the query allows, or more bytes than the largest memory heap a memory type allocates from (`RequireSampledImageWithinDeviceLimits`; an allocation larger than its heap is invalid usage, `VUID-vkAllocateMemory-pAllocateInfo-01713`), faults when selected, before its guest memory is read or any image is created, also where a view of an existing image would have served it. The console samples such a T# (its `DEPTH` and `BASE_ARRAY` fields name up to 8192 slices and `WIDTH` and `HEIGHT` up to 16384 texels, Mesa `src/amd/registers/gfx10-rsrc.json`); the NVIDIA GeForce RTX 3060 (driver 591.86) allows 2048 array layers. The byte bounds are level 0 of the image, a lower bound of its size. The heap bound is the heap's size, not what its `VK_EXT_memory_budget` budget leaves: in a PPSA01342 run on that GPU a 7520387072-byte texture image was allocated in host heap 1 right after the budget showed no host heap with room for it. A direct T# over the limits still creates the image. The query uses `TRANSFER_DST | SAMPLED`; the sRGB upload's additional `TRANSFER_SRC` usage and its secondary RGBA8 expansion image are not separately checked before creation. +- An [image table](../../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) T# whose snapshot texture runs out of memory faults when selected instead of stopping the draw or dispatch, and the table is resolved again on its next use: `vkCreateImage`, `vkAllocateMemory` or `vkBindImageMemory` of the texture's image, or the allocation of its staging, tiled or linear upload buffer (after the buffer pool's reclaim and retry), returned `VK_ERROR_OUT_OF_HOST_MEMORY` or `VK_ERROR_OUT_OF_DEVICE_MEMORY` (`TextureMemoryExhausted`). The image's own allocation follows the common pool-trim, texture-cache reclaim and allowed-host fallback path; only a final refusal becomes this entry fault. Any other `VulkanError` of the sampled-texture lookup still reaches the draw or dispatch, out-of-memory ones included (a recorder submit, a storage image refresh or flush, a view or copy of an existing image), as does a host allocation failure of the CPU snapshot of the surface (`std::bad_alloc`). The written-entry storage-texture lookup retains its existing `std::runtime_error` catch, which can turn a `VulkanError` into an entry fault. The console has no such failure. In a PPSA01342 run on an NVIDIA GeForce RTX 3060 (driver 591.86) the 7520387072-byte host staging buffer of a table entry decoded as an array of 1793 layers failed with `VK_ERROR_OUT_OF_DEVICE_MEMORY` after the pool reclaimed 1867581668 bytes, and presentation stopped. - [User-data callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is a user-data pair, or a copy of one made with `s_mov_b32` or `s_mov_b64` in the same block, calls the code at the address that user data holds. At dispatch the callee is read from guest memory up to its `s_setpc_b64` on the link pair, one copy of it is appended per call after the caller's last instruction, and the call is rewritten to `s_call_b64` to that copy. The words after the caller's last instruction are dropped from the recompiled code; in the shaders seen they are metadata or constants the caller reads relative to its program counter, and those reads go to the captured guest code, which linking does not change. A caller that jumps through a table among those words, which the control flow analysis reads from the recompiled code, throws. A copy runs at another program counter than the callee, so a callee that reads its program counter, makes a call, ends the program, jumps through another pair, returns before code it branches to, spans more than 64 KiB or is not readable throws. Callees are read again whenever a dispatch misses the dispatch cache; a dispatch that hits it reuses the linked program, so a title that rewrote a callee in place at the same address would keep running the old code. - [Table callees](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a compute shader's `s_swappc_b64` whose source pair is two consecutive dwords of an `s_buffer_load_dword*`, or a copy of them made with `s_mov_b32` or `s_mov_b64` in the same block (a conditional branch forward past the call between them is crossed), calls one of the targets that buffer holds when the load's buffer descriptor comes from an `s_load_dwordx4` without SOFFSET from a user-data pointer, or from a pointer pair that two `s_mov_b32` of a literal or inline constant set, and the load's SOFFSET from an `s_lshl_b32` by 2 to 24 bits. At dispatch the descriptor and the buffer are read from guest memory, and every non-zero 64-bit value at the loaded offset plus a multiple of the shift's stride, below the descriptor's size (stride times num_records, or num_records bytes with stride 0), is read as a user-data callee and appended once per call. The call becomes a branch to a chain that compares the source pair with each target (`s_cmp_lg_u32` with a literal and `s_cbranch_scc1` per dword) and `s_call_b64`s the copy that matches; no match runs the original `s_swappc_b64` as an unlinked call. There is one chain for each value SCC has at the call, and the chain sets SCC back to that value (`s_cmp_eq_u32 0, 0` or `s_cmp_lg_u32 0, 0`) before the callee or the unlinked call runs. The searched range is the descriptor's whole size, which covers the range the load can reach whether it is checked in bytes or in strides; which one the PS5 uses for `s_buffer_load` was not measured. More than 64 distinct targets, a buffer above 64 MiB or a copy out of `s_call_b64` range throws. As with user-data callees, the buffer is read again only when a dispatch misses the dispatch cache, so a target a title adds to the buffer later is an unlinked call when a cached program runs. - [AGC FS table](../../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Lifecycle.cpp) (libSceAgcDriver) - the driver maps 2 MiB at 0xF_E0000000 when it is created and fills the table that libSceAgc keeps at 0xF_E0040000 (six 16-byte V# slots, slot 0 describing eight 16-byte records) the way PS5 12.00 libSceAgcDriver and libSceAgc do, but every record points at one AnyPS5 routine that only returns (`s_setpc_b64` on the link pair). Sony's routines, which order work before a title's `ds_ordered_count`, are not modeled, and the other five slots stay zero.