diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index ed53a1f55bcf..e5e05107f753 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -486,6 +486,20 @@ if(BUILD_TESTING) add_test(NAME agc_pipeline_link_optimization COMMAND agc_pipeline_link_optimization_tests) endif() +add_test_executable(agc_device_fault_tests EXCLUDE_FROM_ALL tests/DeviceFault.cpp) +target_include_directories(agc_device_fault_tests PRIVATE ${LIBS_INCLUDE_DIR} ${ANYPS5_VULKAN_HEADERS_INCLUDE_DIR}) +set_target_properties(agc_device_fault_tests PROPERTIES CXX_EXTENSIONS OFF) +target_link_libraries(agc_device_fault_tests PRIVATE libc libkernel) +configure_windows_unwind(agc_device_fault_tests) +if(BUILD_TESTING) + set_target_properties(agc_device_fault_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_device_fault COMMAND agc_device_fault_tests) + if(WIN32) + set_tests_properties(agc_device_fault PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_subgroup_clock_tests EXCLUDE_FROM_ALL tests/SubgroupClock.cpp Execution/src/SubgroupClock.cpp diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/DeviceFault.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/DeviceFault.hpp new file mode 100644 index 000000000000..558a25bde7ac --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/DeviceFault.hpp @@ -0,0 +1,79 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DEVICEFAULT_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DEVICEFAULT_HPP + +#ifndef VK_NO_PROTOTYPES +#define VK_NO_PROTOTYPES +#endif +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver { + +inline const char* DeviceFaultAddressKind(VkDeviceFaultAddressTypeEXT type) { + switch (type) { + case VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT: return "none"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT: return "read invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT: return "write invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT: return "execute invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT: return "instruction pointer unknown"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT: return "instruction pointer invalid"; + case VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT: return "instruction pointer fault"; + default: return "unknown"; + } +} + +inline std::string DeviceFaultText(const char (&text)[VK_MAX_DESCRIPTION_SIZE]) { + return std::string(text, strnlen(text, VK_MAX_DESCRIPTION_SIZE)); +} + +inline std::vector DeviceFaultReport(VkDevice device, PFN_vkGetDeviceFaultInfoEXT faultInfo, VkQueue queue, PFN_vkGetQueueCheckpointDataNV checkpointData) { + std::vector lines; + char line[VK_MAX_DESCRIPTION_SIZE + 128]; + if (checkpointData != nullptr && queue != VK_NULL_HANDLE) { + std::uint32_t count = 0; + checkpointData(queue, &count, nullptr); + std::vector points(count, VkCheckpointDataNV{VK_STRUCTURE_TYPE_CHECKPOINT_DATA_NV}); + if (count != 0) checkpointData(queue, &count, points.data()); + points.resize(count); + if (points.empty()) lines.emplace_back("[device-fault] no checkpoint reached"); + for (const auto& point : points) { + std::snprintf(line, sizeof(line), "[device-fault] checkpoint: program 0x%llx at stage 0x%x", static_cast(reinterpret_cast(point.pCheckpointMarker)), static_cast(point.stage)); + lines.emplace_back(line); + } + } + if (faultInfo == nullptr || device == VK_NULL_HANDLE) return lines; + VkDeviceFaultCountsEXT counts{VK_STRUCTURE_TYPE_DEVICE_FAULT_COUNTS_EXT}; + if (faultInfo(device, &counts, nullptr) != VK_SUCCESS) { + lines.emplace_back("[device-fault] vkGetDeviceFaultInfoEXT reported nothing"); + return lines; + } + std::vector addresses(counts.addressInfoCount); + std::vector vendors(counts.vendorInfoCount); + counts.vendorBinarySize = 0; + VkDeviceFaultInfoEXT info{VK_STRUCTURE_TYPE_DEVICE_FAULT_INFO_EXT}; + info.pAddressInfos = addresses.empty() ? nullptr : addresses.data(); + info.pVendorInfos = vendors.empty() ? nullptr : vendors.data(); + const auto result = faultInfo(device, &counts, &info); + addresses.resize(std::min(addresses.size(), counts.addressInfoCount)); + vendors.resize(std::min(vendors.size(), counts.vendorInfoCount)); + std::snprintf(line, sizeof(line), "[device-fault] result %d: %s", static_cast(result), DeviceFaultText(info.description).c_str()); + lines.emplace_back(line); + for (const auto& address : addresses) { + std::snprintf(line, sizeof(line), "[device-fault] %s at 0x%llx (precision 0x%llx)", DeviceFaultAddressKind(address.addressType), static_cast(address.reportedAddress), static_cast(address.addressPrecision)); + lines.emplace_back(line); + } + for (const auto& vendor : vendors) { + std::snprintf(line, sizeof(line), "[device-fault] vendor: %s (code 0x%llx, data 0x%llx)", DeviceFaultText(vendor.description).c_str(), static_cast(vendor.vendorFaultCode), static_cast(vendor.vendorFaultData)); + lines.emplace_back(line); + } + return lines; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 6e644f6c8432..e1d2d3aab037 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -9,6 +9,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include "prx/libSceAgcDriver/Execution/include/DeviceFault.hpp" #include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp" #include "prx/libSceAgcDriver/Execution/include/PipelineLinkOptimization.hpp" #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" @@ -69,6 +70,7 @@ namespace AgcDriver { namespace { void check(VkResult result, const char* operation) { + if (result == VK_ERROR_DEVICE_LOST) Graphics::ReportDeviceLost(); if (result != VK_SUCCESS) { throw std::runtime_error(std::string(operation) + ": Vulkan result " + std::to_string(result)); } @@ -261,6 +263,8 @@ struct VulkanDevice::State { VkPhysicalDeviceProperties properties{}; std::string driverInfo; PFN_vkGetCalibratedTimestampsKHR calibratedTimestamps = nullptr; + PFN_vkCmdSetCheckpointNV checkpointMarker = nullptr; + bool deviceLostHook = false; VkTimeDomainKHR hostTimeDomain = VK_TIME_DOMAIN_DEVICE_KHR; VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; std::vector capabilities{1}; @@ -647,6 +651,10 @@ struct VulkanDevice::State { ~State() { contextReady = false; + if (deviceLostHook && Graphics::DeviceLostOwner() == this) { + Graphics::DeviceLostHook() = nullptr; + Graphics::DeviceLostOwner() = nullptr; + } if (compileOnly && device != VK_NULL_HANDLE) { reinterpret_cast(deviceProc(device, "vkDestroyDevice"))(device, nullptr); device = VK_NULL_HANDLE; @@ -1631,6 +1639,27 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : pipelineFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &pipelineFeatures; } + VkPhysicalDeviceFaultFeaturesEXT faultFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT}; + if (hasExtension(VK_EXT_DEVICE_FAULT_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &faultFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + } + const bool deviceFault = faultFeatures.deviceFault == VK_TRUE; + if (deviceFault) { + faultFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FAULT_FEATURES_EXT}; + faultFeatures.deviceFault = VK_TRUE; + deviceExtensions.push_back(VK_EXT_DEVICE_FAULT_EXTENSION_NAME); + deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); + deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); + faultFeatures.pNext = byteFeatures.pNext; + byteFeatures.pNext = &faultFeatures; + } + const bool checkpoints = std::getenv("APS5_GPU_CHECKPOINTS") != nullptr && hasExtension(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME); + if (checkpoints) { + deviceExtensions.push_back(VK_NV_DEVICE_DIAGNOSTIC_CHECKPOINTS_EXTENSION_NAME); + deviceInfo.enabledExtensionCount = static_cast(deviceExtensions.size()); + deviceInfo.ppEnabledExtensionNames = deviceExtensions.data(); + } bdaFeatures.pNext = &byteFeatures; VkPhysicalDeviceRobustness2FeaturesEXT robustness2{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; if (hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { @@ -1673,6 +1702,16 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window, bool compileOnly) : state->DeviceFunction("vkGetDeviceQueue")(state->device, family, 0, &state->queue); state->queueFamily = family; if (state->sparseFamily != VK_QUEUE_FAMILY_IGNORED) state->DeviceFunction("vkGetDeviceQueue")(state->device, state->sparseFamily, 0, &state->sparseQueue); + if (checkpoints) state->checkpointMarker = state->DeviceFunction("vkCmdSetCheckpointNV"); + if (deviceFault || checkpoints) { + const auto faultInfo = deviceFault ? state->DeviceFunction("vkGetDeviceFaultInfoEXT") : nullptr; + const auto checkpointData = checkpoints ? state->DeviceFunction("vkGetQueueCheckpointDataNV") : nullptr; + Graphics::DeviceLostHook() = [device = state->device, queue = state->queue, faultInfo, checkpointData] { + for (const auto& line : DeviceFaultReport(device, faultInfo, queue, checkpointData)) std::fprintf(stderr, "%s\n", line.c_str()); + }; + Graphics::DeviceLostOwner() = state.get(); + state->deviceLostHook = true; + } APS5_LOG_OUT("Vulkan device ready device=%p queue=%p family=%u", reinterpret_cast(state->device), reinterpret_cast(state->queue), family); VkCommandPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; @@ -4070,6 +4109,7 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes, record.pushBytes->data()); } const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId); + if (state->checkpointMarker != nullptr) state->checkpointMarker(commands, reinterpret_cast(static_cast(record.programAddress))); if (argumentImport != nullptr) { context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); } else { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 5f907e3404e9..6a04bd12329c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,23 @@ inline void Require(bool condition, const std::string& reason) { if (!condition) throw std::runtime_error("AGC graphics: " + reason); } +inline std::function& DeviceLostHook() { + static std::function hook; + return hook; +} + +inline const void*& DeviceLostOwner() { + static const void* owner = nullptr; + return owner; +} + +inline void ReportDeviceLost() { + static std::atomic reported{false}; + if (DeviceLostHook() && !reported.exchange(true)) DeviceLostHook()(); +} + inline void Check(VkResult result, const char* operation) { + if (result == VK_ERROR_DEVICE_LOST) ReportDeviceLost(); if (result != VK_SUCCESS) throw std::runtime_error(std::string("AGC graphics: ") + operation + ": Vulkan result " + std::to_string(result)); } diff --git a/core/libs/prx/libSceAgcDriver/tests/DeviceFault.cpp b/core/libs/prx/libSceAgcDriver/tests/DeviceFault.cpp new file mode 100644 index 000000000000..67475a41de2d --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DeviceFault.cpp @@ -0,0 +1,124 @@ +#include "prx/libSceAgcDriver/Execution/include/DeviceFault.hpp" +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +int faultCalls = 0; +bool faultFails = false; + +VKAPI_ATTR VkResult VKAPI_CALL MockFaultInfo(VkDevice, VkDeviceFaultCountsEXT* counts, VkDeviceFaultInfoEXT* info) { + ++faultCalls; + if (faultFails) return VK_ERROR_DEVICE_LOST; + if (info == nullptr) { + counts->addressInfoCount = 2; + counts->vendorInfoCount = 1; + counts->vendorBinarySize = 4096; + return VK_SUCCESS; + } + Require(counts->vendorBinarySize == 0, "the vendor binary was requested"); + Require(counts->addressInfoCount == 2 && info->pAddressInfos != nullptr, "the address array was not passed"); + Require(counts->vendorInfoCount == 1 && info->pVendorInfos != nullptr, "the vendor array was not passed"); + std::memset(info->description, 'd', sizeof(info->description)); + std::memcpy(info->description, "page fault", 10); + info->description[10] = '\0'; + info->pAddressInfos[0] = {VK_DEVICE_FAULT_ADDRESS_TYPE_READ_INVALID_EXT, 0x1234000ull, 0xfffull}; + info->pAddressInfos[1] = {VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_FAULT_EXT, 0xabcdull, 0x3ull}; + std::memset(info->pVendorInfos[0].description, 'v', sizeof(info->pVendorInfos[0].description)); + info->pVendorInfos[0].vendorFaultCode = 0x77ull; + info->pVendorInfos[0].vendorFaultData = 0x88ull; + return VK_SUCCESS; +} + +std::uint32_t checkpointCount = 0; + +VKAPI_ATTR void VKAPI_CALL MockCheckpointData(VkQueue, std::uint32_t* count, VkCheckpointDataNV* data) { + if (data == nullptr) { + *count = checkpointCount; + return; + } + Require(*count == checkpointCount, "the checkpoint array has the wrong size"); + for (std::uint32_t i = 0; i < *count; ++i) { + Require(data[i].sType == VK_STRUCTURE_TYPE_CHECKPOINT_DATA_NV, "checkpoint entries were not initialised"); + data[i].stage = i == 0 ? VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT : VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT; + data[i].pCheckpointMarker = reinterpret_cast(static_cast(0x10000ull + i * 0x100ull)); + } +} + +VkDevice FakeDevice() { + return reinterpret_cast(static_cast(0x1000)); +} + +VkQueue FakeQueue() { + return reinterpret_cast(static_cast(0x2000)); +} + +void FaultInfo() { + faultCalls = 0; + faultFails = false; + const auto lines = AgcDriver::DeviceFaultReport(FakeDevice(), MockFaultInfo, VK_NULL_HANDLE, nullptr); + Require(faultCalls == 2, "the fault info was not queried for counts and then for data"); + Require(lines.size() == 4, "the report does not have a result, two addresses and a vendor line"); + Require(lines[0] == "[device-fault] result 0: page fault", "the description line changed"); + Require(lines[1] == "[device-fault] read invalid at 0x1234000 (precision 0xfff)", "the read fault line changed"); + Require(lines[2] == "[device-fault] instruction pointer fault at 0xabcd (precision 0x3)", "the instruction pointer line changed"); + Require(lines[3] == "[device-fault] vendor: " + std::string(VK_MAX_DESCRIPTION_SIZE, 'v') + " (code 0x77, data 0x88)", "an unterminated vendor description was not bounded"); +} + +void FaultInfoFails() { + faultCalls = 0; + faultFails = true; + const auto lines = AgcDriver::DeviceFaultReport(FakeDevice(), MockFaultInfo, VK_NULL_HANDLE, nullptr); + Require(faultCalls == 1, "a failed count query was followed by a data query"); + Require(lines.size() == 1 && lines[0] == "[device-fault] vkGetDeviceFaultInfoEXT reported nothing", "a failed query was not reported"); +} + +void Checkpoints() { + checkpointCount = 2; + auto lines = AgcDriver::DeviceFaultReport(VK_NULL_HANDLE, nullptr, FakeQueue(), MockCheckpointData); + Require(lines.size() == 2, "each checkpoint gets one line"); + Require(lines[0] == "[device-fault] checkpoint: program 0x10000 at stage 0x1", "the first checkpoint line changed"); + Require(lines[1] == "[device-fault] checkpoint: program 0x10100 at stage 0x2000", "the second checkpoint line changed"); + checkpointCount = 0; + lines = AgcDriver::DeviceFaultReport(VK_NULL_HANDLE, nullptr, FakeQueue(), MockCheckpointData); + Require(lines.size() == 1 && lines[0] == "[device-fault] no checkpoint reached", "an empty checkpoint list was not reported"); +} + +void Nothing() { + faultCalls = 0; + Require(AgcDriver::DeviceFaultReport(FakeDevice(), nullptr, FakeQueue(), nullptr).empty(), "a device without either extension reported something"); + Require(faultCalls == 0, "the fault info was queried without the extension"); +} + +void AddressKinds() { + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(VK_DEVICE_FAULT_ADDRESS_TYPE_NONE_EXT), "none") == 0, "none"); + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(VK_DEVICE_FAULT_ADDRESS_TYPE_WRITE_INVALID_EXT), "write invalid") == 0, "write invalid"); + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(VK_DEVICE_FAULT_ADDRESS_TYPE_EXECUTE_INVALID_EXT), "execute invalid") == 0, "execute invalid"); + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_UNKNOWN_EXT), "instruction pointer unknown") == 0, "instruction pointer unknown"); + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(VK_DEVICE_FAULT_ADDRESS_TYPE_INSTRUCTION_POINTER_INVALID_EXT), "instruction pointer invalid") == 0, "instruction pointer invalid"); + Require(std::strcmp(AgcDriver::DeviceFaultAddressKind(static_cast(99)), "unknown") == 0, "an unknown kind"); +} + +} + +int main() { + try { + FaultInfo(); + FaultInfoFails(); + Checkpoints(); + Nothing(); + AddressKinds(); + } catch (const std::exception& error) { + std::fprintf(stderr, "agc_device_fault: %s\n", error.what()); + return 1; + } + std::puts("agc_device_fault: passed"); + return 0; +} diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index ba5c74087343..060a7a72eca2 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -122,6 +122,10 @@ Each invocation creates one pipeline. With no `--cache-in`, its application pipe Set `APS5_PIPELINE_STATS=1` to capture and print driver statistics for each newly created graphics or compute pipeline. This requires `VK_KHR_pipeline_executable_properties` and `pipelineExecutableInfo`; an unsupported device fails with an error. Statistic names and units are driver-specific. Capturing statistics can increase pipeline compilation cost. The setting is disabled by default. +## Device loss reports + +On a device that supports `VK_EXT_device_fault`, the first `VK_ERROR_DEVICE_LOST` prints the driver's fault description, the faulting addresses and the vendor fault codes as `[device-fault]` lines. Set `APS5_GPU_CHECKPOINTS=1` to also record a `VK_NV_device_diagnostic_checkpoints` marker with the program address before each compute dispatch; the report then lists the last markers the GPU reached. Checkpoints are disabled by default and ignored on devices without the extension. + ## Worker sampler Set `APS5_SAMPLE_WORKER=` to sample queue worker 0 every millisecond (in practice every timer tick). Every 20 s and at exit it writes ``, one `module offset self inclusive` line per sampled address, and `.collapsed.txt`, the worker's whole stacks in the collapsed format that speedscope.app and flamegraph.pl read: one `frame;...;frame count` line per distinct stack, root first, each frame as `module+0xRVA`. A walk that ends before the thread's root starts with `[truncated]`. The `#` lines give the samples, the walk depth (64 frames), the truncated walks, the mean and longest time the worker was suspended, and the size and PE timestamp of each module, to check the binary a file is symbolized against. A frame with `.pdata` is stepped by `RtlVirtualUnwind`; any other frame by its DWARF record in `.ehfram`, the unwind data the libraries are built with. Those steps read only inside the worker's stack, and a module is indexed after the worker resumes, the first time a walk meets it.