Skip to content

fix(agc): place device memory in a host heap only when that heap has room #6553

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
fdfawef324 wants to merge 1 commit into
base: main
Choose a base branch
from
+122 −9
Draft

fix(agc): place device memory in a host heap only when that heap has room #6553

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 26 additions & 9 deletions core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,10 @@ struct Context {
throw std::runtime_error("AGC graphics: required Vulkan memory type is unavailable");
}

static bool HeapFits(const VkPhysicalDeviceMemoryBudgetPropertiesEXT& reported, std::uint32_t heap, VkDeviceSize bytes) {
return reported.heapBudget[heap] != 0 && reported.heapUsage[heap] < reported.heapBudget[heap] && bytes <= reported.heapBudget[heap] - reported.heapUsage[heap];
}

std::uint32_t BufferMemoryType(std::uint32_t mask, VkMemoryPropertyFlags flags, VkDeviceSize bytes) const {
constexpr VkMemoryPropertyFlags coherent = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
auto type = MemoryType(mask, flags);
Expand All @@ -283,13 +287,10 @@ struct Context {
VkPhysicalDeviceMemoryProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2, &reported};
memoryProperties2(physical, &properties);
const auto heap = memory.memoryTypes[type].heapIndex;
const auto fits = [&](std::uint32_t candidate) {
return reported.heapBudget[candidate] != 0 && reported.heapUsage[candidate] < reported.heapBudget[candidate] && bytes <= reported.heapBudget[candidate] - reported.heapUsage[candidate];
};
if (reported.heapBudget[heap] == 0 || fits(heap)) return type;
if (reported.heapBudget[heap] == 0 || HeapFits(reported, heap, bytes)) return type;
for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
const auto& candidate = memory.memoryTypes[i];
if ((mask & (1u << i)) != 0 && (candidate.propertyFlags & flags) == flags && candidate.heapIndex != heap && fits(candidate.heapIndex)) return i;
if ((mask & (1u << i)) != 0 && (candidate.propertyFlags & flags) == flags && candidate.heapIndex != heap && HeapFits(reported, candidate.heapIndex, bytes)) return i;
}
return type;
}
Expand All @@ -304,13 +305,21 @@ struct Context {
const auto heap = memory.memoryTypes[type].heapIndex;
const auto budget = reported.heapBudget[heap];
const auto usage = reported.heapUsage[heap];
if (budget == 0 || (usage < budget && bytes <= budget - usage)) return type;
const auto host = HostMemoryType(mask);
if (!host) return type;
if (budget == 0 || HeapFits(reported, heap, bytes)) return type;
const auto host = HostMemoryTypeWithRoom(mask, reported, bytes);
if (!host) {
if (!HostMemoryType(mask)) return type;
static std::atomic<std::uint64_t> kept{0}, keptBytes{0};
const auto count = kept.fetch_add(1) + 1;
const auto total = keptBytes.fetch_add(bytes) + bytes;
if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget and no allowed host heap has room, %llu bytes kept in device-local memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast<unsigned long long>(usage >> 20u), static_cast<unsigned long long>(budget >> 20u), static_cast<unsigned long long>(bytes), type, static_cast<unsigned long long>(count), static_cast<unsigned long long>(total >> 20u));
return type;
}
const auto hostHeap = memory.memoryTypes[*host].heapIndex;
static std::atomic<std::uint64_t> placements{0}, placedBytes{0};
const auto count = placements.fetch_add(1) + 1;
const auto total = placedBytes.fetch_add(bytes) + bytes;
if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB\n", what, heap, static_cast<unsigned long long>(usage >> 20u), static_cast<unsigned long long>(budget >> 20u), static_cast<unsigned long long>(bytes), *host, static_cast<unsigned long long>(count), static_cast<unsigned long long>(total >> 20u));
if (count <= 5 || (count & (count - 1)) == 0) std::fprintf(stderr, "[gpu] %s: device-local heap %u uses %llu of its %llu MiB budget, %llu bytes placed in host memory type %u; %llu such allocations, %llu MiB; host heap %u uses %llu of its %llu MiB budget\n", what, heap, static_cast<unsigned long long>(usage >> 20u), static_cast<unsigned long long>(budget >> 20u), static_cast<unsigned long long>(bytes), *host, static_cast<unsigned long long>(count), static_cast<unsigned long long>(total >> 20u), hostHeap, static_cast<unsigned long long>(reported.heapUsage[hostHeap] >> 20u), static_cast<unsigned long long>(reported.heapBudget[hostHeap] >> 20u));
return *host;
}

Expand All @@ -321,6 +330,14 @@ struct Context {
return std::nullopt;
}

std::optional<std::uint32_t> HostMemoryTypeWithRoom(std::uint32_t mask, const VkPhysicalDeviceMemoryBudgetPropertiesEXT& reported, VkDeviceSize bytes) const {
for (std::uint32_t i = 0; i < memory.memoryTypeCount; ++i) {
const auto heap = memory.memoryTypes[i].heapIndex;
if ((mask & (1u << i)) != 0 && (memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0 && HeapFits(reported, heap, bytes)) return i;
}
return std::nullopt;
}

std::optional<std::uint32_t> RefusedDeviceMemoryFallback(std::uint32_t mask, std::uint32_t refused) const {
if (refused >= memory.memoryTypeCount || (memory.memoryHeaps[memory.memoryTypes[refused].heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) == 0) return std::nullopt;
return HostMemoryType(mask);
Expand Down
47 changes: 47 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ struct MockDevice {
VkDeviceSize limit = 0;
VkResult pressureResult = VK_ERROR_OUT_OF_DEVICE_MEMORY;
std::uint64_t attempts = 0;
bool traceAttempts = false;
std::vector<std::uint32_t> attemptedTypes;
VkResult mapResult = VK_SUCCESS;
VkResult bindResult = VK_SUCCESS;
unsigned binds = 0;
Expand All @@ -75,6 +77,7 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu

VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) {
++mock.attempts;
if (mock.traceAttempts) mock.attemptedTypes.push_back(info->memoryTypeIndex);
auto result = VK_SUCCESS;
if (!mock.results.empty()) {
result = mock.results.front();
Expand Down Expand Up @@ -170,11 +173,15 @@ Context mockContext() {

VkDeviceSize reportedBudget = 0;
VkDeviceSize reportedUsage = 0;
VkDeviceSize reportedHostBudget = 8000 * MiB;
VkDeviceSize reportedHostUsage = 2000 * MiB;

VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) {
auto* reported = static_cast<VkPhysicalDeviceMemoryBudgetPropertiesEXT*>(properties->pNext);
reported->heapBudget[0] = reportedBudget;
reported->heapUsage[0] = reportedUsage;
reported->heapBudget[1] = reportedHostBudget;
reported->heapUsage[1] = reportedHostUsage;
}

Context budgetContext() {
Expand Down Expand Up @@ -240,6 +247,45 @@ void RefusedDevicePlacement() {
Expect(mock.liveBuffers.empty(), "a failed device buffer left its VkBuffer alive");
}

void BothHeapsOverBudget() {
mock = MockDevice{};
constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
reportedBudget = 1620 * MiB;
reportedUsage = 1600 * MiB;
reportedHostUsage = reportedHostBudget - 32 * MiB;
{
auto context = budgetContext();
DeviceBuffer kept(context, 176 * MiB, usage);
Expect(mock.memoryTypes.at(MemoryOf(kept.Handle())) == 0, "with both heaps over their budgets, a device buffer was placed in a host heap without room instead of device-local memory");
}
{
mock = MockDevice{};
mock.traceAttempts = true;
auto context = budgetContext();
mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY};
DeviceBuffer refused(context, 176 * MiB, usage);
Expect(mock.memoryTypes.at(MemoryOf(refused.Handle())) == 1, "with both heaps over their budgets, a device buffer refused device-local memory was not placed in the host memory type");
Expect(mock.attemptedTypes == std::vector<std::uint32_t>{0, 1}, "with both heaps over their budgets, a device buffer did not try device-local memory before host memory");
}
{
mock = MockDevice{};
mock.traceAttempts = true;
auto context = budgetContext();
VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO};
allocation.allocationSize = 64 * MiB;
allocation.memoryTypeIndex = context.DeviceMemoryType(3, allocation.allocationSize, "test image");
const auto kept = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory test image");
Expect(mock.memoryTypes.at(kept) == 0 && mock.attemptedTypes == std::vector<std::uint32_t>{0}, "with both heaps over their budgets, an image was placed in host memory before the driver refused device-local memory");
mockFreeMemory(VK_NULL_HANDLE, kept, nullptr);
mock.attemptedTypes.clear();
mock.results = {VK_ERROR_OUT_OF_DEVICE_MEMORY};
const auto refused = AllocateImageMemory(context, allocation, 3, "vkAllocateMemory test image");
Expect(mock.memoryTypes.at(refused) == 1 && mock.attemptedTypes == std::vector<std::uint32_t>{0, 1}, "with both heaps over their budgets, a refused image did not try device-local memory before host memory");
mockFreeMemory(VK_NULL_HANDLE, refused, nullptr);
}
reportedHostUsage = 2000 * MiB;
}

void AttachmentReclaim() {
mock = MockDevice{};
auto context = budgetContext();
Expand Down Expand Up @@ -874,6 +920,7 @@ int main(int argc, char** argv) {
ImageCacheReclaim();
HostPlacement();
RefusedDevicePlacement();
BothHeapsOverBudget();
AttachmentReclaim();
} catch (const std::exception& error) {
std::fprintf(stderr, "FAIL: %s\n", error.what());
Expand Down
49 changes: 49 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/DeviceMemoryFallback.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,24 @@ void Expect(bool condition, const std::string& what) {

constexpr std::uint32_t HostType = 0;
constexpr std::uint32_t DeviceType = 1;
constexpr std::uint32_t SecondHostType = 2;
constexpr std::uint64_t MiB = 1ull << 20u;

std::uint64_t reportedBudget = 0;
std::uint64_t reportedUsage = 0;
std::uint64_t reportedHostBudget = 0;
std::uint64_t reportedHostUsage = 0;
std::uint64_t reportedSecondHostBudget = 0;
std::uint64_t reportedSecondHostUsage = 0;

VKAPI_ATTR void VKAPI_CALL mockMemoryProperties2(VkPhysicalDevice, VkPhysicalDeviceMemoryProperties2* properties) {
auto* reported = static_cast<VkPhysicalDeviceMemoryBudgetPropertiesEXT*>(properties->pNext);
reported->heapBudget[0] = reportedBudget;
reported->heapUsage[0] = reportedUsage;
reported->heapBudget[1] = reportedHostBudget;
reported->heapUsage[1] = reportedHostUsage;
reported->heapBudget[2] = reportedSecondHostBudget;
reported->heapUsage[2] = reportedSecondHostUsage;
}

Context mockContext(bool budgetExtension) {
Expand All @@ -53,6 +58,46 @@ void Choose(const char* what, bool budgetExtension, std::uint64_t budget, std::u
Expect(type == expected, std::string(what) + ": chose memory type " + std::to_string(type) + ", expected " + std::to_string(expected));
}

void HostHeapTests() {
const std::uint32_t both = (1u << HostType) | (1u << DeviceType);
reportedHostBudget = 8000 * MiB;
reportedHostUsage = 2000 * MiB;
Choose("host heap has room", true, 4096 * MiB, 4096 * MiB, both, HostType);
reportedHostUsage = 8000 * MiB - 64 * MiB;
Choose("allocation fills the host heap exactly", true, 4096 * MiB, 4096 * MiB, both, HostType);
reportedHostUsage = 8000 * MiB - 32 * MiB;
Choose("host heap nearly full keeps the device-local type", true, 4096 * MiB, 4096 * MiB, both, DeviceType);
reportedHostUsage = 9000 * MiB;
Choose("host heap over its budget keeps the device-local type", true, 4096 * MiB, 4096 * MiB, both, DeviceType);
reportedHostBudget = 0;
reportedHostUsage = 0;
Choose("unreported host budget is not assumed free", true, 4096 * MiB, 4096 * MiB, both, DeviceType);
}

void SecondHostHeapTests() {
auto context = mockContext(true);
context.memory.memoryHeapCount = 3;
context.memory.memoryHeaps[2] = {16384 * MiB, 0};
context.memory.memoryTypeCount = 3;
context.memory.memoryTypes[SecondHostType] = {0, 2};
reportedBudget = 4096 * MiB;
reportedUsage = 4096 * MiB;
reportedHostBudget = 8000 * MiB;
reportedHostUsage = 8000 * MiB - 32 * MiB;
reportedSecondHostBudget = 8000 * MiB;
reportedSecondHostUsage = 2000 * MiB;
const auto all = (1u << HostType) | (1u << DeviceType) | (1u << SecondHostType);
const auto type = context.DeviceMemoryType(all, 64 * MiB, "test");
Expect(type == SecondHostType, "a full first host heap stopped the search for a host heap with room: chose memory type " + std::to_string(type));
const auto masked = context.DeviceMemoryType((1u << HostType) | (1u << DeviceType), 64 * MiB, "test");
Expect(masked == DeviceType, "with the only allowed host heap full, memory type " + std::to_string(masked) + " was chosen instead of the device-local type");
reportedSecondHostUsage = 9000 * MiB;
const auto bothFull = context.DeviceMemoryType(all, 64 * MiB, "test");
Expect(bothFull == DeviceType, "with every host heap full, a host type was chosen: memory type " + std::to_string(bothFull));
reportedSecondHostBudget = 0;
reportedSecondHostUsage = 0;
}

void RefusedFallbackTests() {
const auto context = mockContext(true);
const std::uint32_t both = (1u << HostType) | (1u << DeviceType);
Expand Down Expand Up @@ -108,13 +153,17 @@ void BufferPlacementTests() {
int main() {
try {
const std::uint32_t both = (1u << HostType) | (1u << DeviceType);
reportedHostBudget = 8000 * MiB;
reportedHostUsage = 2000 * MiB;
Choose("no VK_EXT_memory_budget", false, 0, 0, both, DeviceType);
Choose("budget not reported", true, 0, 4000 * MiB, both, DeviceType);
Choose("room in the device-local budget", true, 3000 * MiB, 2000 * MiB, both, DeviceType);
Choose("allocation fills the budget exactly", true, 3000 * MiB, 2936 * MiB, both, DeviceType);
Choose("allocation exceeds the budget", true, 3000 * MiB, 2937 * MiB, both, HostType);
Choose("usage already over the budget", true, 2952 * MiB, 3633 * MiB, both, HostType);
Choose("no host type allowed", true, 3000 * MiB, 3000 * MiB, 1u << DeviceType, DeviceType);
HostHeapTests();
SecondHostHeapTests();
BufferPlacementTests();
RefusedFallbackTests();
} catch (const std::exception& error) {
Expand Down
Loading