fix(agc): fault image table entries the device or guest memory cannot hold

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -1530,6 +1530,7 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL
         tests/TextureTiling.cpp
         tests/GuestTextureResource.cpp
         tests/GuestSamplerResource.cpp
+        tests/ImageTableEntryValidation.cpp
         tests/TextureDetiler.cpp
         tests/DepthSurfaceReuse.cpp
 )

core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp

@@ -66,9 +66,14 @@ inline void ReportDeviceLost() {
     if (DeviceLostHook() && !reported.exchange(true)) DeviceLostHook()();
 }
 
+struct VulkanError : std::runtime_error {
+    VulkanError(VkResult result, const std::string& message) : std::runtime_error(message), result(result) {}
+    VkResult result;
+};
+
 inline void Check(VkResult result, const char* operation) {
     if (result == VK_ERROR_DEVICE_LOST) ReportDeviceLost();
-    if (result != VK_SUCCESS) throw std::runtime_error(std::string("AGC graphics: ") + operation + ": Vulkan result " + std::to_string(result));
+    if (result != VK_SUCCESS) throw VulkanError(result, std::string("AGC graphics: ") + operation + ": Vulkan result " + std::to_string(result));
 }
 
 inline void Require(bool condition, const char* reason) {

core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp

@@ -104,6 +104,7 @@ struct RegisteredPiece {
 // Whether a readable registered allocation contains [address, address + bytes) right now (one
 // registry lease): a storage image whose memory was freed or re-registered has nothing to store to.
 bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes, std::vector<RegisteredPiece>* pieces = nullptr);
+bool RegisteredReadableOverlaps(std::uint64_t address, std::size_t bytes);
 std::uint64_t RegisteredReadableEnd(std::uint64_t address);
 
 // A persistent device copy of one registered range of the main guest image (which cannot be host

core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp

@@ -37,6 +37,10 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat);
 VkFormat HostStorageFormat(const Context& context, VkFormat format);
 bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys);
 std::uint64_t SampledTextureMemory();
+void RequireSampledImageWithinDeviceLimits(const Context& context, const GuestTextureResource& descriptor, const SurfaceGeometry& geometry, bool depthCompare);
+struct TextureMemoryExhausted : VulkanError {
+    using VulkanError::VulkanError;
+};
 bool ClearKeepsDenormals(const Context& context, VkFormat format);
 bool ClearColorForTexel(VkFormat format, std::uint32_t elementBytes, std::span<const std::uint32_t, 4> pattern, VkClearColorValue& clear);
 

core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp

@@ -3076,6 +3076,18 @@ bool RegisteredReadableCovers(std::uint64_t address, std::size_t bytes, std::vec
     return true;
 }
 
+bool RegisteredReadableOverlaps(std::uint64_t address, std::size_t bytes) {
+    if (bytes == 0 || bytes > std::numeric_limits<std::uint64_t>::max() - address) return false;
+    const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();
+    const auto end = address + bytes;
+    const auto next = std::upper_bound(lease.begin(), lease.end(), address, [](std::uint64_t value, const auto& range) { return value < range->address; });
+    if (next != lease.begin()) {
+        const auto& previous = *std::prev(next);
+        if (address - previous->address < previous->bytes) return true;
+    }
+    return next != lease.end() && (*next)->address < end;
+}
+
 std::uint64_t RegisteredReadableEnd(std::uint64_t address) {
     if (address == std::numeric_limits<std::uint64_t>::max()) return 0;
     const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix();

core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp

@@ -4606,9 +4606,13 @@ void ShaderResources::resolveImageTables() {
                 shape = binding.imageShapes.at(element);
                 firstLayer = shape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray;
                 if (!firstLayer && !MatchesGuestDimension(shape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the table entry's image shape (shape " + std::to_string(static_cast<int>(shape)) + ", dimension " + std::to_string(static_cast<int>(resource.dimension)) + ")");
-                guestBytes = DescribeSurface(resource).guestBytes;
-                if (!GuestMemory::Accessible(reinterpret_cast<const void*>(resource.baseAddress), static_cast<std::size_t>(guestBytes)) && GuestMemory::CommittedRanges(resource.baseAddress, static_cast<std::size_t>(guestBytes), false).empty()) throw std::runtime_error("AGC graphics: the table entry's surface at 0x" + HexText(resource.baseAddress) + " (0x" + HexText(guestBytes) + " bytes) is not mapped");
+                const auto geometry = DescribeSurface(resource);
+                guestBytes = geometry.guestBytes;
+                RequireSampledImageWithinDeviceLimits(context, resource, geometry, binding.imageDepthCompare.at(element));
+                if (!RegisteredReadableOverlaps(resource.baseAddress, static_cast<std::size_t>(guestBytes))) throw std::runtime_error("AGC graphics: the table entry's surface at 0x" + HexText(resource.baseAddress) + " (0x" + HexText(guestBytes) + " bytes) is not mapped in guest memory");
                 static_cast<void>(ResolveTextureFormat(resource.format));
+            } catch (const VulkanError&) {
+                throw;
             } catch (const std::runtime_error& error) {
                 table.map[faults + element] = rejectTableElement(table, words, error.what());
                 if (reusable) fresh.elements.push_back({{}, {}, {}, 0, false, true, error.what()});
@@ -4661,6 +4665,12 @@ void ShaderResources::resolveImageTables() {
             const VkImageViewType multisampleViewType = shape == ShaderRecompiler::DescriptorImageShape::Image2DMultisampleArray ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D;
             try {
                 texture = cachedTexture(context, words, resource, components, guestBytes, depthCompare, multisampleViewType);
+            } catch (const TextureMemoryExhausted& error) {
+                table.map[faults + element] = rejectTableElement(table, words, error.what());
+                reusable = false;
+                continue;
+            } catch (const VulkanError&) {
+                throw;
             } catch (const std::runtime_error& error) {
                 table.map[faults + element] = rejectTableElement(table, words, error.what());
                 if (reusable) fresh.elements.push_back({{}, {}, {}, 0, false, true, error.what()});

core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp

@@ -290,12 +290,100 @@ std::atomic<std::uint64_t>& SampledMemoryCounter() {
     return bytes;
 }
 
+template<typename TMake>
+auto ownTextureMemory(TMake&& make) {
+    try {
+        return make();
+    } catch (const VulkanError& error) {
+        if (error.result != VK_ERROR_OUT_OF_HOST_MEMORY && error.result != VK_ERROR_OUT_OF_DEVICE_MEMORY) throw;
+        throw TextureMemoryExhausted(error.result, error.what());
+    }
+}
+
 }
 
 std::uint64_t SampledTextureMemory() {
     return SampledMemoryCounter().load(std::memory_order_relaxed);
 }
 
+namespace {
+
+const char* dimensionName(TextureDimension dimension) {
+    switch (dimension) {
+        case TextureDimension::k1D: return "1D";
+        case TextureDimension::k2D: return "2D";
+        case TextureDimension::k2DArray: return "2D array";
+        case TextureDimension::kCube: return "cube";
+        case TextureDimension::k3D: return "3D";
+        case TextureDimension::k1DArray: return "1D array";
+        case TextureDimension::k2DMsaa: return "2D multisampled";
+    }
+    throw std::runtime_error("AGC graphics: unknown guest texture dimension " + std::to_string(static_cast<int>(dimension)));
+}
+
+std::optional<VkImageFormatProperties> sampledImageFormatLimits(const Context& context, VkFormat format, VkImageType type, VkImageCreateFlags flags) {
+    static std::mutex mutex;
+    static std::map<std::tuple<VkPhysicalDevice, VkFormat, VkImageType, VkImageCreateFlags>, std::optional<VkImageFormatProperties>> limits;
+    const auto key = std::make_tuple(context.physical, format, type, flags);
+    {
+        std::lock_guard lock(mutex);
+        if (const auto found = limits.find(key); found != limits.end()) return found->second;
+    }
+    Require(context.imageFormatProperties != nullptr, "the device has no vkGetPhysicalDeviceImageFormatProperties");
+    VkImageFormatProperties properties{};
+    const auto result = context.imageFormatProperties(context.physical, format, type, VK_IMAGE_TILING_OPTIMAL, VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT, flags, &properties);
+    std::optional<VkImageFormatProperties> supported;
+    if (result != VK_ERROR_FORMAT_NOT_SUPPORTED) {
+        Check(result, "vkGetPhysicalDeviceImageFormatProperties sampled texture");
+        supported = properties;
+    }
+    std::lock_guard lock(mutex);
+    limits.emplace(key, supported);
+    return supported;
+}
+
+}
+
+void RequireSampledImageWithinDeviceLimits(const Context& context, const GuestTextureResource& descriptor, const SurfaceGeometry& geometry, bool depthCompare) {
+    const auto colorFormat = SampledTextureFormat(context, descriptor.format);
+    const auto format = !depthCompare ? colorFormat : colorFormat == VK_FORMAT_R32_SFLOAT ? VK_FORMAT_D32_SFLOAT : colorFormat == VK_FORMAT_R16_UNORM ? VK_FORMAT_D16_UNORM : colorFormat;
+    const auto type = ImageTypeFor(descriptor.dimension);
+    const bool cube = descriptor.dimension == TextureDimension::kCube;
+    const VkImageCreateFlags flags = (cube ? VK_IMAGE_CREATE_CUBE_COMPATIBLE_BIT : 0u) | (SrgbTwinFormat(context, format) != VK_FORMAT_UNDEFINED ? VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT : 0u);
+    const VkExtent3D extent{descriptor.width, descriptor.height, geometry.imageDepth};
+    const auto supported = sampledImageFormatLimits(context, format, type, flags);
+    const auto blockWidth = static_cast<std::uint64_t>(BlockWidth(descriptor.format));
+    const auto leastBytes = (extent.width + blockWidth - 1u) / blockWidth * ((extent.height + blockWidth - 1u) / blockWidth) * BytesPerElement(descriptor.format) * extent.depth * geometry.imageLayers;
+    const bool extentFits = supported && extent.width <= supported->maxExtent.width && extent.height <= supported->maxExtent.height && extent.depth <= supported->maxExtent.depth;
+    const bool layersFit = supported && geometry.imageLayers <= supported->maxArrayLayers;
+    const bool bytesFit = supported && leastBytes <= supported->maxResourceSize;
+    std::uint32_t largestHeap = 0;
+    VkDeviceSize heapBytes = 0;
+    for (std::uint32_t memoryType = 0; memoryType < std::min<std::uint32_t>(context.memory.memoryTypeCount, VK_MAX_MEMORY_TYPES); ++memoryType) {
+        const auto heap = context.memory.memoryTypes[memoryType].heapIndex;
+        if (heap >= VK_MAX_MEMORY_HEAPS || context.memory.memoryHeaps[heap].size <= heapBytes) continue;
+        largestHeap = heap;
+        heapBytes = context.memory.memoryHeaps[heap].size;
+    }
+    const bool heapFits = leastBytes <= heapBytes;
+    if (extentFits && layersFit && bytesFit && heapFits) return;
+    char fields[224];
+    std::snprintf(fields, sizeof(fields), "guest texture 0x%llx (%ux%u, %s, depth or last array slice %u, base array %u, format %u, mips %u)", static_cast<unsigned long long>(descriptor.baseAddress), descriptor.width, descriptor.height, dimensionName(descriptor.dimension), descriptor.depthOrLastArray, descriptor.baseArray, descriptor.format, descriptor.mipCount);
+    char text[448];
+    if (!supported) {
+        std::snprintf(text, sizeof(text), "AGC graphics: %s needs a sampled VkFormat %d image of type %d with create flags 0x%x, which vkGetPhysicalDeviceImageFormatProperties reports unsupported", fields, static_cast<int>(format), static_cast<int>(type), static_cast<unsigned>(flags));
+    } else if (!layersFit) {
+        std::snprintf(text, sizeof(text), "AGC graphics: %s needs %u array layers, more than the %u of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, geometry.imageLayers, supported->maxArrayLayers, static_cast<int>(format));
+    } else if (!extentFits) {
+        std::snprintf(text, sizeof(text), "AGC graphics: %s needs an extent of %ux%ux%u, more than the %ux%ux%u of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, extent.width, extent.height, extent.depth, supported->maxExtent.width, supported->maxExtent.height, supported->maxExtent.depth, static_cast<int>(format));
+    } else if (!bytesFit) {
+        std::snprintf(text, sizeof(text), "AGC graphics: %s needs at least %llu bytes, more than the maxResourceSize of %llu of VkFormat %d images (vkGetPhysicalDeviceImageFormatProperties)", fields, static_cast<unsigned long long>(leastBytes), static_cast<unsigned long long>(supported->maxResourceSize), static_cast<int>(format));
+    } else {
+        std::snprintf(text, sizeof(text), "AGC graphics: %s needs at least %llu bytes, more than the %llu bytes of memory heap %u, the largest a memory type allocates from", fields, static_cast<unsigned long long>(leastBytes), static_cast<unsigned long long>(heapBytes), largestHeap);
+    }
+    throw std::runtime_error(text);
+}
+
 Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTextureResource& descriptor, VkComponentMapping components, std::span<const std::byte> snapshot, bool depthCompare) : context(context) {
 
     static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr;
@@ -338,7 +426,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
         imageInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT | (srgbExpand ? VK_IMAGE_USAGE_TRANSFER_SRC_BIT : 0u);
         imageInfo.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
         imageInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
-        Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage");
+        ownTextureMemory([&] { Check(context.Function<PFN_vkCreateImage>("vkCreateImage")(context.device, &imageInfo, nullptr, &image), "vkCreateImage"); });
         owned = std::make_shared<OwnedImage>(context, image, VK_NULL_HANDLE);
 
         VkMemoryRequirements requirements{};
@@ -347,13 +435,15 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
         allocation.allocationSize = requirements.size;
         allocationBytes = requirements.size;
         allocation.memoryTypeIndex = context.DeviceMemoryType(requirements.memoryTypeBits, requirements.size, "texture");
-        owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture");
-        Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory");
+        ownTextureMemory([&] {
+            owned->memory = AllocateImageMemory(context, allocation, requirements.memoryTypeBits, "vkAllocateMemory texture");
+            Check(context.Function<PFN_vkBindImageMemory>("vkBindImageMemory")(context.device, image, owned->memory, 0), "vkBindImageMemory");
+        });
 
         const VkImage uploadImage = image;
         std::shared_ptr<OwnedImage> uploadOwned;
         if (srgbExpand) {
-            auto expanded = CreateExpandImage(context, descriptor, geometry.imageDepth, geometry.imageLayers);
+            auto expanded = ownTextureMemory([&] { return CreateExpandImage(context, descriptor, geometry.imageDepth, geometry.imageLayers); });
             uploadOwned = std::move(owned);
             owned = std::move(expanded.owned);
             image = expanded.image;
@@ -374,10 +464,10 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex
                 // Entries may also be extents ("3840x2160"), since heap addresses change between runs.
                 dumpWanted = dumpList.find(address) != std::string::npos || dumpList.find(extent) != std::string::npos;
             }
-            auto staging = std::make_shared<Buffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+            auto staging = ownTextureMemory([&] { return std::make_shared<Buffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_TRANSFER_SRC_BIT); });
             std::memcpy(staging->Bytes().data(), snapshot.data(), snapshot.size());
-            auto tiled = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT);
-            auto linear = std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT);
+            auto tiled = ownTextureMemory([&] { return std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(guestBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); });
+            auto linear = ownTextureMemory([&] { return std::make_shared<DeviceBuffer>(context, static_cast<std::size_t>(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); });
             if (profile) Profile().allocate += timer.lap();
             if (profile) Profile().read += timer.lap();
 

core/libs/prx/libSceAgcDriver/tests/Graphics.cpp

@@ -9131,6 +9131,10 @@ int main(int argc, char** argv) {
             std::cout << "address-space device teardown releases cached guest allocations\n";
             return 0;
         }
+        if (argc == 2 && std::string_view(argv[1]) == "--image-table-entry-validation") {
+            RunImageTableEntryValidationTests();
+            return 0;
+        }
         if (argc == 2 && std::string_view(argv[1]) == "--buffer-budget-placement") {
             bufferBudgetPlacementTests();
             return 0;
@@ -9367,6 +9371,7 @@ int main(int argc, char** argv) {
         RunGuestTextureResourceTests();
         RunDepthSurfaceReuseTests();
         RunGuestSamplerResourceTests();
+        RunImageTableEntryValidationTests();
         mock = MockVulkan{};
         auto textureDetilerContext = mockContext();
         textureDetilerContext.limits.minStorageBufferOffsetAlignment = 16;

core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp

@@ -36,6 +36,7 @@ void RunTextureTilingTests();
 void RunGuestTextureResourceTests();
 void RunDepthSurfaceReuseTests();
 void RunGuestSamplerResourceTests();
+void RunImageTableEntryValidationTests();
 void RunTextureDetilerTests(const AgcDriver::Graphics::Context& context, const TextureDetilerTestAccess& access);
 
 #endif