diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 6ab1458817cb..7fad973a6c7f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -7,6 +7,9 @@ #include "Optimization/DescriptorBindingBuilder.hpp" #include "ShaderCacheDirectory.hpp" #include "ControlFlow/RequestSerializer.hpp" +#include "Optimization/RequestMemoryView.hpp" +#include "Optimization/SrtWalker.hpp" +#include "IntermediateRepresentation/IrBlock.hpp" #include #include #include @@ -18,7 +21,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -247,6 +253,7 @@ CompiledVariant sampleVariant() { info.info.usesDma = true; info.info.usesFaultBuffer = true; info.info.rasterPrimitive = true; + info.info.layerExported = true; info.bindings.pushDataStartDword = 2; info.bindings.memoryOffsetDword = 1; info.bindings.memoryOffsetCount = 5; @@ -1082,6 +1089,743 @@ void verifySpecializationLiveness() { require(SpecializeSpirv(specialized) == specialized, "dead computation removal is not stable"); } +constexpr std::uint64_t PlanSrtAddress = 0x7f0000010000ull; +constexpr std::uint64_t PlanTableAddress = 0x7f0000020000ull; +constexpr std::uint64_t PlanMaterialAddress = 0x7f0000030000ull; +constexpr std::uint64_t PlanPayloadAddress = 0x7f0000040000ull; +constexpr std::uint64_t PlanPointerAddress = 0x7f0000048000ull; +constexpr std::uint64_t PlanOutputAddress = 0x7f0000050000ull; +constexpr std::uint64_t PlanTextureAddress = 0x7f0000060000ull; +constexpr std::uint64_t PlanUnmappedAddress = 0x7f00000f0000ull; + +struct PlanFixture { + std::string name; + ShaderStage stage = ShaderStage::Compute; + std::uint64_t codeAddress = 0; + std::uint32_t waveSize = 64; + std::vector code; + std::vector userData; + std::vector>> regions; + std::optional pixel; + std::vector memory; + + RecompileRequest Request(bool useCache) { + static constexpr std::array capabilities{spv::CapabilityShader, spv::CapabilityImageGatherExtended, spv::CapabilitySampledImageArrayDynamicIndexing, spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, spv::CapabilityClipDistance}; + static constexpr std::array extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"}; + memory.clear(); + for (const auto& [address, words] : regions) memory.push_back({address, std::as_bytes(std::span(words))}); + RecompileRequest request{}; + request.shader = {stage, codeAddress, code, 0, {}}; + request.context.waveSize = waveSize; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + if (stage == ShaderStage::Compute) request.context.compute = ShaderComputeStageInfo{{waveSize, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.pixel = pixel; + request.context.memory = memory; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.bdaAbiVersion = 1u; + request.layout.pushConstantSizeBytes = 128; + request.useCache = useCache; + return request; + } +}; + +std::vector imageWords(std::uint64_t base) { + return {static_cast(base >> 8u), static_cast((base >> 40u) & 0xffu) | (56u << 20u) | (3u << 30u), 3u << 14u, 0xfacu | (9u << 28u), 0u, 0u, 0u, 0u}; +} + +std::vector bufferWords(std::uint64_t base, std::uint32_t stride, std::uint32_t records, std::uint32_t format = 0xfacu) { + return {static_cast(base), static_cast((base >> 32u) & 0xffffu) | (stride << 16u), records, format}; +} + +void place(std::vector& words, std::size_t offset, const std::vector& values) { + std::copy(values.begin(), values.end(), words.begin() + static_cast(offset)); +} + +std::vector planFixtures() { + std::vector fixtures; + const std::vector srtPointer{static_cast(PlanSrtAddress), static_cast(PlanSrtAddress >> 32u)}; + const auto add = [&](std::string name, std::uint32_t waveSize, std::vector code, std::vector userData, std::vector>> regions) { + PlanFixture fixture; + fixture.name = std::move(name); + fixture.codeAddress = 0x100000u + static_cast(fixtures.size()) * 0x1000u; + fixture.waveSize = waveSize; + fixture.code = std::move(code); + fixture.userData = std::move(userData); + fixture.regions = std::move(regions); + fixtures.push_back(std::move(fixture)); + }; + add("user data buffer", 64, {0xe0700000u, 0x80000000u, 0xbf810000u}, bufferWords(PlanOutputAddress, 0u, 0x40u, 0x00027facu), {}); + + std::vector guarded(32, 0u); + place(guarded, 0, imageWords(PlanTextureAddress)); + place(guarded, 16, bufferWords(PlanOutputAddress, 0u, 16u)); + guarded[20] = 1u; + add("guarded image", 32, {0xf4000400u, 0xfa000050u, 0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xbf068010u, 0xbf850006u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, guarded}}); + add("guarded image stores", 32, {0xf4000400u, 0xfa000050u, 0xbf8cc07fu, 0xbf068010u, 0xbf850006u, 0xf40c0100u, 0xfa000000u, 0xbf8cc07fu, 0xf0200f08u, 0x00010004u, 0xbf820005u, 0xf40c0100u, 0xfa000000u, 0xbf8cc07fu, 0xf0200f10u, 0x00010004u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, guarded}}); + + std::vector heap(32, 0u); + place(heap, 0, imageWords(PlanTextureAddress)); + place(heap, 8, imageWords(PlanTextureAddress + 0x100u)); + place(heap, 24, imageWords(PlanTextureAddress)); + const std::vector materials{0u, 1u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 3u, 0u, 0u}; + std::vector tables(16, 0u); + place(tables, 0, bufferWords(PlanTableAddress, 32u, 4u)); + place(tables, 8, bufferWords(PlanMaterialAddress, 16u, 3u)); + place(tables, 12, bufferWords(PlanOutputAddress, 0u, 16u)); + add("bindless material table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x93109010u, 0xf4200406u, 0x20000004u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); + add("bindless whole table", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xf09c0f08u, 0x00450000u, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); + add("bindless sample in a loop latch", 64, {0xf4080100u, 0xfa000000u, 0xf4080200u, 0xfa000010u, 0xf4080300u, 0xfa000020u, 0xf4080700u, 0xfa000030u, 0x7e200500u, 0x8f108510u, 0xf42c0502u, 0x20000000u, 0xbea80380u, 0xf09c0f08u, 0x00450000u, 0x80288128u, 0xbf0a8428u, 0xbf85fffbu, 0xe0700000u, 0x80070000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, tables}, {PlanTableAddress, heap}, {PlanMaterialAddress, materials}}); + + std::vector phis(32, 0u); + place(phis, 0, imageWords(PlanTextureAddress)); + place(phis, 12, {0x49u, 0u, 0x00500000u, 0u}); + place(phis, 16, bufferWords(PlanOutputAddress, 0u, 16u)); + phis[20] = 1u; + place(phis, 24, imageWords(PlanTextureAddress + 0x100u)); + add("descriptor phi sampler", 32, {0xf40c0200u, 0xfa000000u, 0xf4000400u, 0xfa000050u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf4080500u, 0xfa000020u, 0xbf820002u, 0xf4080500u, 0xfa000030u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, phis}}); + add("descriptor phi image", 32, {0xf4000400u, 0xfa000050u, 0xf4080500u, 0xfa000020u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, phis}}); + add("descriptor phi loop", 32, {0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbe910380u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf068011u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xbf8cc07fu, 0x80118111u, 0xbf0a8211u, 0xbf85fff0u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, phis}}); + add("descriptor phi in sequence", 32, {0xf4000400u, 0xfa000050u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xbf068010u, 0xbf850003u, 0xf4080500u, 0xfa000020u, 0xbf820002u, 0xf4080500u, 0xfa000030u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, phis}}); + + std::vector root(16, 0u); + place(root, 0, bufferWords(PlanOutputAddress, 0u, 256u, 0x30027facu)); + root[4] = 1u; + root[6] = static_cast(PlanPayloadAddress); + root[7] = static_cast(PlanPayloadAddress >> 32u); + const std::vector payload{0x3f800000u}; + add("guarded nested read", 64, {0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, root}, {PlanPayloadAddress, payload}}); + auto chained = root; + chained[6] = static_cast(PlanPointerAddress); + chained[7] = static_cast(PlanPointerAddress >> 32u); + add("guarded pointer chain", 64, {0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, 0xbf8cc07fu, 0xbf068008u, 0xbf850006u, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xf4000307u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, chained}, {PlanPayloadAddress, payload}, {PlanPointerAddress, {static_cast(PlanPayloadAddress), static_cast(PlanPayloadAddress >> 32u)}}}); + add("descriptor through a pointer", 64, {0xf4040280u, 0xfa000018u, 0xbf8cc07fu, 0xf4080105u, 0xfa000000u, 0xbf8cc07fu, 0x34040082u, 0xe0701000u, 0x80010002u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, chained}, {PlanPointerAddress, bufferWords(PlanOutputAddress, 0u, 256u, 0x30027facu)}}); + add("descriptor invariant phi", 32, {0xf4000400u, 0xfa000050u, 0xf4080500u, 0xfa000020u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000000u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}, srtPointer, {{PlanSrtAddress, phis}}); + + PlanFixture fragment; + fragment.name = "fragment parameters"; + fragment.stage = ShaderStage::Fragment; + fragment.codeAddress = 0x100000u + static_cast(fixtures.size()) * 0x1000u; + fragment.code = {0xc8100000u, 0xc8110001u, 0xc8140402u, 0xc8150403u, 0xf800180fu, 0x05040504u, 0xbf810000u}; + ShaderPixelStageInfo pixel{}; + pixel.interpolatorCount = 2u; + pixel.interpolatorSettings[1] = 1u; + pixel.wave32 = true; + pixel.inputAddr = PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::LinearCenter); + pixel.hasPerspectiveCenterVgpr = true; + pixel.noPerspective = true; + pixel.targetOutputMode[0] = 9u; + pixel.targetExportMapping[0] = 0xe4u; + fragment.pixel = pixel; + fixtures.push_back(std::move(fragment)); + return fixtures; +} + +IrResourcePlan handBuiltPlan() { + IrResourcePlan plan; + std::uint32_t ids = 0; + const auto make = [&](IrOpcode opcode, IrType type) -> IrValue& { + plan.valueStorage.push_back(std::make_unique(opcode, type, ids++)); + return *plan.valueStorage.back(); + }; + const auto constant = [&](std::uint32_t value) -> IrValue& { + auto& immediate = make(IrOpcode::Void, IrType::U32); + immediate.SetImmediateU32(value); + return immediate; + }; + auto& resource = make(IrOpcode::GetSrtResource, IrType::SrtResource); + const auto readConst = [&](std::uint32_t slot) -> IrValue& { + auto& read = make(IrOpcode::ReadConst, IrType::U32); + read.AddArgument(&resource); + read.AddArgument(&constant(slot)); + return read; + }; + const auto userData = [&](std::uint32_t index) -> IrValue& { + auto& reg = make(IrOpcode::Void, IrType::ScalarReg); + reg.SetRegister({RegisterBank::Scalar, index}); + auto& read = make(IrOpcode::GetUserData, IrType::U32); + read.AddArgument(®); + return read; + }; + auto& handle = make(IrOpcode::CompositeConstructU64, IrType::U64); + handle.AddArgument(&userData(0)); + handle.AddArgument(&userData(1)); + auto& pointer = make(IrOpcode::LoadAddressU32, IrType::U32); + pointer.AddArgument(&handle); + pointer.AddArgument(&constant(8)); + plan.blockStorage.push_back(std::make_unique(3u)); + plan.blockStorage.push_back(std::make_unique(7u)); + auto& phi = make(IrOpcode::Phi, IrType::U32); + phi.AddPhiOperand(plan.blockStorage[0].get(), &readConst(0)); + phi.AddPhiOperand(plan.blockStorage[1].get(), &constant(0)); + auto& condition = make(IrOpcode::Void, IrType::Bool); + condition.SetImmediateU64(1u); + condition.SetFlags(std::uint64_t{0x1234u}); + plan.stage = IrShaderStage::Compute; + plan.shaderHash = 0x123456789abcull; + plan.userDataBase = 2u; + plan.userDataCount = 4u; + plan.srgbDecodeFormats = 5u; + MemoryInfo memory; + memory.kind = ResourceKind::Image; + memory.resource = 1u; + memory.dmask = 0xfu; + memory.imageDimension = RdnaImageDimension::Dim2D; + memory.gpuDescriptor = true; + memory.flushDenormals = true; + MemoryInfo every; + every.kind = ResourceKind::Buffer; + every.resource = 2u; + every.sampler = 3u; + every.offset = 0x14u; + every.secondaryOffset = 0x28u; + every.dmask = 0x5u; + every.dataDwords = 3u; + every.dataBits = 16u; + every.componentIndex = 1u; + every.componentCount = 2u; + every.dataFormat = 10u; + every.numberFormat = 7u; + every.imageSampleFlags = 0x21u; + every.imageDimension = RdnaImageDimension::Dim3D; + every.imageAddressComponents = 3u; + every.addressIsFull = true; + every.dataSigned = true; + every.typed = true; + every.formatted = true; + every.d16 = true; + every.imageHasMip = true; + every.imageR128 = true; + every.imagePacked = true; + every.imageByElements = 2u; + every.idxen = true; + every.offen = true; + every.planningOnly = true; + every.coherent = true; + every.gpuDescriptor = true; + every.flushDenormals = true; + plan.memoryInfo = {memory, every, MemoryInfo{}}; + DescriptorSource buffer; + buffer.dwordCount = 4u; + buffer.dwords = {&readConst(1), &readConst(2), &phi, &pointer}; + DescriptorSource table; + table.dwordCount = 8u; + table.dwords[0] = &readConst(3); + table.indirectImage = DescriptorSource::IndirectImage{3u, 2u, 16u, 4u, 2u, 32u, true}; + DescriptorSource heap; + heap.dwordCount = 4u; + heap.dwords[1] = &readConst(6); + DescriptorSource material; + material.dwordCount = 4u; + material.dwords[2] = &readConst(7); + auto& wide = make(IrOpcode::Void, IrType::U64); + wide.SetImmediateU64(0x8000000000000001ull); + material.dwords[3] = &wide; + plan.descriptorSources = {buffer, table, heap, material}; + plan.controlFlow = {{&condition, {1u, 2u}, {0u}, {}}, {nullptr, {2u}, {3u}, {0u}}, {nullptr, {}, {}, {}}}; + plan.materializationSources = {0u, 2u}; + plan.srtReads = {{&readConst(4), 2u}, {&pointer, 1u}}; + plan.guardedSrtSlots = {1u}; + plan.srtGuardOffset = 6u; + plan.cleanFlatSlots = {0u, 1u}; + plan.pureFlatSlots = {1u, 0u}; + plan.requiresSpecializationMemory = true; + plan.srtPlanComplete = true; + plan.resourceTrackingComplete = true; + plan.info.buffers = {BufferResource{}, BufferResource{2u, 0x40u, 0x100u, 0u, true, true, true, true, true, 0x7u, true, 4u}}; + ImageResource image{}; + image.source = 1u; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2D; + plan.info.images = {image}; + ResourceMaterializer::PrepareImageModes(plan.info); + plan.uniformFill.fill = {UniformFillKind::Buffer, 1u, {4u, 8u, 12u}, 2u, 0x3f800000u}; + auto& fillValue = readConst(5); + plan.uniformFill.values = {&fillValue, &fillValue, nullptr, nullptr}; + return plan; +} + +std::vector encodedPlan(const IrResourcePlan& plan) { + std::vector bytes; + require(ShaderDiskCache::EncodePlan(plan, bytes), "a self-contained plan does not encode"); + return bytes; +} + +bool samePlan(const IrResourcePlan& left, const IrResourcePlan& right) { + if (left.valueStorage.size() != right.valueStorage.size() || left.blockStorage.size() != right.blockStorage.size()) return false; + std::map leftValues; + std::map rightValues; + std::map leftBlocks; + std::map rightBlocks; + for (std::size_t index = 0; index < left.valueStorage.size(); ++index) { + leftValues.emplace(left.valueStorage[index].get(), index); + rightValues.emplace(right.valueStorage[index].get(), index); + } + for (std::size_t index = 0; index < left.blockStorage.size(); ++index) { + leftBlocks.emplace(left.blockStorage[index].get(), index); + rightBlocks.emplace(right.blockStorage[index].get(), index); + } + const auto sameValue = [&](const IrValue* a, const IrValue* b) { + if (a == nullptr || b == nullptr) return a == b; + const auto foundA = leftValues.find(a); + const auto foundB = rightValues.find(b); + return foundA != leftValues.end() && foundB != rightValues.end() && foundA->second == foundB->second; + }; + const auto sameBlock = [&](const IrBlock* a, const IrBlock* b) { + const auto foundA = leftBlocks.find(a); + const auto foundB = rightBlocks.find(b); + return foundA != leftBlocks.end() && foundB != rightBlocks.end() && foundA->second == foundB->second; + }; + for (std::size_t index = 0; index < left.blockStorage.size(); ++index) { + if (left.blockStorage[index]->Id() != right.blockStorage[index]->Id()) return false; + } + for (std::size_t index = 0; index < left.valueStorage.size(); ++index) { + const auto& a = *left.valueStorage[index]; + const auto& b = *right.valueStorage[index]; + if (a.Opcode() != b.Opcode() || a.Type() != b.Type() || a.Id() != b.Id() || a.Flags() != b.Flags() || !(a.Register() == b.Register()) || a.HasImmediate() != b.HasImmediate() || (a.HasImmediate() && a.ImmediateU64() != b.ImmediateU64())) return false; + if (a.ArgumentCount() != b.ArgumentCount() || a.IsPhi() != b.IsPhi() || a.PhiBlockCount() != b.PhiBlockCount()) return false; + for (std::size_t operand = 0; operand < a.ArgumentCount(); ++operand) { + if (!sameValue(a.Argument(operand), b.Argument(operand))) return false; + } + for (std::size_t operand = 0; operand < a.PhiBlockCount(); ++operand) { + if (!sameBlock(a.PhiBlock(operand), b.PhiBlock(operand))) return false; + } + } + if (left.stage != right.stage || left.shaderHash != right.shaderHash || left.userDataBase != right.userDataBase || left.userDataCount != right.userDataCount || left.srgbDecodeFormats != right.srgbDecodeFormats) return false; + if (left.memoryInfo != right.memoryInfo || left.descriptorSources.size() != right.descriptorSources.size() || left.controlFlow.size() != right.controlFlow.size() || left.srtReads.size() != right.srtReads.size()) return false; + for (std::size_t index = 0; index < left.descriptorSources.size(); ++index) { + const auto& a = left.descriptorSources[index]; + const auto& b = right.descriptorSources[index]; + if (a.dwordCount != b.dwordCount || a.indirectImage != b.indirectImage) return false; + for (std::size_t dword = 0; dword < a.dwords.size(); ++dword) { + if (!sameValue(a.dwords[dword], b.dwords[dword])) return false; + } + } + for (std::size_t index = 0; index < left.controlFlow.size(); ++index) { + const auto& a = left.controlFlow[index]; + const auto& b = right.controlFlow[index]; + if (!sameValue(a.condition, b.condition) || a.successors != b.successors || a.sources != b.sources || a.images != b.images) return false; + } + for (std::size_t index = 0; index < left.srtReads.size(); ++index) { + if (!sameValue(left.srtReads[index].value, right.srtReads[index].value) || left.srtReads[index].flatOffset != right.srtReads[index].flatOffset) return false; + } + for (std::size_t index = 0; index < left.uniformFill.values.size(); ++index) { + if (!sameValue(left.uniformFill.values[index], right.uniformFill.values[index])) return false; + } + return left.materializationSources == right.materializationSources && left.guardedSrtSlots == right.guardedSrtSlots && left.srtGuardOffset == right.srtGuardOffset && left.cleanFlatSlots == right.cleanFlatSlots && left.pureFlatSlots == right.pureFlatSlots && + left.requiresSpecializationMemory == right.requiresSpecializationMemory && left.srtPlanComplete == right.srtPlanComplete && left.resourceTrackingComplete == right.resourceTrackingComplete && left.info == right.info && left.uniformFill.fill == right.uniformFill.fill; +} + +struct MaterializedPlan { + bool ok = false; + std::string error; + ResourceSnapshot snapshot; + SrtReadTrace trace; + std::size_t leaf = 0; +}; + +MaterializedPlan materializePlan(const IrResourcePlan& plan, SrtRuntime runtime) { + MaterializedPlan outcome; + runtime.readTrace = &outcome.trace; + try { + ResourceMaterializer{}.Materialize(plan, runtime, outcome.snapshot); + outcome.ok = true; + } catch (const std::exception& error) { + outcome.error = error.what(); + } + outcome.leaf = static_cast(std::find_if(plan.valueStorage.begin(), plan.valueStorage.end(), [&](const auto& value) { return value.get() == outcome.trace.leaf; }) - plan.valueStorage.begin()); + return outcome; +} + +bool sameMaterialization(const MaterializedPlan& left, const MaterializedPlan& right) { + const auto& a = left.snapshot; + const auto& b = right.snapshot; + return left.ok == right.ok && left.error == right.error && a.buffers == b.buffers && a.images == b.images && a.samplers == b.samplers && a.flattenedSrt == b.flattenedSrt && a.userData == b.userData && a.uniformFill == b.uniformFill && a.srtPoison == b.srtPoison && a.nullRootReads == b.nullRootReads && + left.leaf == right.leaf && left.trace.leafSlot == right.trace.leafSlot && left.trace.leaves == right.trace.leaves && left.trace.otherReads == right.trace.otherReads; +} + +void mutateWords(std::vector& words, std::mt19937& random) { + static constexpr std::array addresses{PlanSrtAddress, PlanTableAddress, PlanMaterialAddress, PlanPayloadAddress, PlanPointerAddress, PlanOutputAddress, PlanTextureAddress, PlanUnmappedAddress}; + for (auto& word : words) { + switch (random() % 16u) { + case 0: word = static_cast(random()); break; + case 1: word = 0u; break; + case 2: { + const auto address = addresses[random() % addresses.size()] + (random() % 8u) * 4u; + word = random() % 2u == 0u ? static_cast(address) : static_cast(address >> 32u); + break; + } + case 3: word ^= 1u << (random() % 32u); break; + default: break; + } + } +} + +std::size_t verifyRoundTripRuntimes(const IrResourcePlan& plan, const IrResourcePlan& decoded, const PlanFixture& fixture, std::uint32_t seed, const std::string& what) { + std::mt19937 random(seed); + std::size_t succeeded = 0; + for (std::uint32_t run = 0; run < 1000u; ++run) { + auto userData = fixture.userData; + auto regions = fixture.regions; + if (userData.size() < plan.userDataCount) userData.resize(plan.userDataCount, 0u); + if (run != 0u) { + mutateWords(userData, random); + for (auto& region : regions) mutateWords(region.second, random); + } + std::vector memory; + for (const auto& [address, words] : regions) memory.push_back({address, std::as_bytes(std::span(words))}); + RequestMemoryView view(memory); + const auto runtime = view.MakeRuntime(userData, fixture.codeAddress); + const auto original = materializePlan(plan, runtime); + const auto loaded = materializePlan(decoded, runtime); + require(sameMaterialization(original, loaded), what + ": runtime " + std::to_string(run) + " materialized differently from the decoded plan (" + original.error + " / " + loaded.error + ")"); + if (original.ok) ++succeeded; + } + return succeeded; +} + +void verifyPlanEntryDamage(const std::vector& key, const IrResourcePlan& plan, const std::string& what) { + const auto file = ShaderDiskCache::EncodePlanEntry(key, plan); + require(!file.empty(), what + ": the plan entry was not encoded"); + IrResourcePlan decoded; + require(ShaderDiskCache::DecodePlanEntry(file, key, decoded) == ShaderDiskCache::LoadStatus::Loaded && encodedPlan(decoded) == encodedPlan(plan), what + ": the plan entry does not decode to the plan"); + for (std::size_t size = 0; size < file.size(); size += size < 512 ? 1 : 37) { + IrResourcePlan partial; + require(ShaderDiskCache::DecodePlanEntry(std::span(file).first(size), key, partial) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry truncated to " + std::to_string(size) + " bytes is not rejected"); + } + for (std::size_t offset = 0; offset < file.size(); offset += offset < 512 ? 1 : 29) { + for (const auto bit : {std::byte{0x01}, std::byte{0x80}}) { + auto damaged = file; + damaged[offset] ^= bit; + IrResourcePlan partial; + require(ShaderDiskCache::DecodePlanEntry(damaged, key, partial) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry with byte " + std::to_string(offset) + " flipped is not rejected"); + } + } + auto longer = file; + longer.push_back(std::byte{0}); + require(ShaderDiskCache::DecodePlanEntry(longer, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry with a trailing byte is not rejected"); + auto otherKey = key; + otherKey.back() ^= std::byte{1}; + require(ShaderDiskCache::DecodePlanEntry(file, otherKey, decoded) == ShaderDiskCache::LoadStatus::KeyMismatch, what + ": a plan entry for another key loads"); + auto previousFormat = file; + previousFormat[4] = static_cast(ShaderDiskCache::FormatVersion - 1u); + require(ShaderDiskCache::DecodePlanEntry(previousFormat, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of the previous format loads"); + auto otherSource = file; + otherSource[8] ^= std::byte{1}; + require(ShaderDiskCache::DecodePlanEntry(otherSource, key, decoded) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry of another source version loads"); + CompiledVariant variant; + require(ShaderDiskCache::DecodeEntry(file, key, variant) == ShaderDiskCache::LoadStatus::Rejected, what + ": a plan entry decodes as a compiled variant"); +} + +void verifyPlanPayloadDamage(const IrResourcePlan& plan, const std::string& what) { + const auto bytes = encodedPlan(plan); + for (std::size_t size = 0; size < bytes.size(); ++size) { + IrResourcePlan partial; + require(!ShaderDiskCache::DecodePlan(std::span(bytes).first(size), partial), what + ": a plan truncated to " + std::to_string(size) + " bytes decodes"); + } + std::size_t rejected = 0; + for (std::size_t offset = 0; offset < bytes.size(); ++offset) { + for (const auto bit : {std::byte{0x01}, std::byte{0x40}}) { + auto damaged = bytes; + damaged[offset] ^= bit; + IrResourcePlan decoded; + if (!ShaderDiskCache::DecodePlan(damaged, decoded)) { + ++rejected; + continue; + } + std::vector again; + require(ShaderDiskCache::EncodePlan(decoded, again), what + ": a plan accepted with byte " + std::to_string(offset) + " flipped is not self-contained"); + } + } + require(rejected != 0u, what + ": no damaged plan was rejected"); +} + +void verifyMalformedPlans() { + const auto rejects = [](const std::string& what, const std::function& damage) { + auto plan = handBuiltPlan(); + damage(plan); + std::vector bytes; + require(ShaderDiskCache::EncodePlan(plan, bytes), "malformed plan: " + what + " did not encode"); + IrResourcePlan decoded; + require(!ShaderDiskCache::DecodePlan(bytes, decoded), "malformed plan: " + what + " decodes"); + }; + rejects("a branch with one successor", [](IrResourcePlan& plan) { plan.controlFlow[0].successors.pop_back(); }); + rejects("a successor past the blocks", [](IrResourcePlan& plan) { plan.controlFlow[1].successors[0] = 3u; }); + rejects("a block source past the sources", [](IrResourcePlan& plan) { plan.controlFlow[0].sources[0] = 4u; }); + rejects("a block image past the images", [](IrResourcePlan& plan) { plan.controlFlow[1].images[0] = 1u; }); + rejects("a materialized source past the sources", [](IrResourcePlan& plan) { plan.materializationSources.push_back(4u); }); + rejects("a buffer past the sources", [](IrResourcePlan& plan) { plan.info.buffers[0].source = 4u; }); + rejects("an image past the sources", [](IrResourcePlan& plan) { plan.info.images[0].source = 5u; }); + rejects("a descriptor of nine dwords", [](IrResourcePlan& plan) { plan.descriptorSources[1].dwordCount = 9u; }); + rejects("a uniform fill of five words", [](IrResourcePlan& plan) { plan.uniformFill.fill.words = 5u; }); + rejects("an image table heap past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].indirectImage->heapSource = 4u; }); + rejects("an image table material past the sources", [](IrResourcePlan& plan) { plan.descriptorSources[1].indirectImage->materialSource = 4u; }); + auto foreign = handBuiltPlan(); + IrValue stray(IrOpcode::Void, IrType::U32, 99u); + foreign.srtReads[0].value = &stray; + std::vector bytes; + require(!ShaderDiskCache::EncodePlan(foreign, bytes), "a plan reading a value it does not own was encoded"); + auto unblocked = handBuiltPlan(); + auto& phi = *unblocked.valueStorage.emplace_back(std::make_unique(IrOpcode::Phi, IrType::U32, 100u)); + phi.AddArgument(unblocked.valueStorage[0].get()); + unblocked.srtReads[0].value = φ + bytes.clear(); + require(!ShaderDiskCache::EncodePlan(unblocked, bytes), "a phi without predecessor blocks was encoded"); + auto emptied = handBuiltPlan(); + emptied.valueStorage.push_back(nullptr); + bytes.clear(); + require(!ShaderDiskCache::EncodePlan(emptied, bytes), "a plan with an empty value slot was encoded"); +} + +void verifyPlanRoundTrip() { + const auto handBuilt = handBuiltPlan(); + const auto handBytes = encodedPlan(handBuilt); + IrResourcePlan handDecoded; + require(ShaderDiskCache::DecodePlan(handBytes, handDecoded), "the hand-built plan does not decode"); + require(encodedPlan(handDecoded) == handBytes && handDecoded.valueStorage.size() == handBuilt.valueStorage.size() && handDecoded.blockStorage.size() == handBuilt.blockStorage.size(), "the hand-built plan changed in a round trip"); + const auto& phi = *handDecoded.descriptorSources[0].dwords[2]; + require(phi.IsPhi() && phi.PhiBlockCount() == 2u && phi.PhiBlock(0)->Id() == 3u && phi.PhiBlock(1)->Id() == 7u && phi.Argument(1)->ImmediateU32() == 0u, "the decoded phi lost its blocks or operands"); + require(handDecoded.uniformFill.values[0] == handDecoded.uniformFill.values[1] && handDecoded.uniformFill.values[0] != nullptr && handDecoded.uniformFill.values[2] == nullptr, "the decoded uniform fill lost its shared or null values"); + require(samePlan(handBuilt, handDecoded), "the decoded hand-built plan differs from the plan in a field or a value"); + PlanFixture handFixture; + handFixture.userData = {static_cast(PlanSrtAddress), static_cast(PlanSrtAddress >> 32u), 0u, 0u}; + std::vector handSrt(16, 0u); + place(handSrt, 0, bufferWords(PlanOutputAddress, 0u, 16u)); + handFixture.regions = {{PlanSrtAddress, handSrt}, {PlanPayloadAddress, {0x3f800000u}}}; + static_cast(verifyRoundTripRuntimes(handBuilt, handDecoded, handFixture, 7u, "hand-built plan")); + verifyPlanPayloadDamage(handBuilt, "hand-built plan"); + verifyMalformedPlans(); + + auto fixtures = planFixtures(); + std::vector> plans; + std::size_t index = 0; + for (auto& fixture : fixtures) { + auto request = fixture.Request(false); + const auto plan = GetResourcePlan(request); + plans.push_back(plan); + const auto bytes = encodedPlan(*plan); + IrResourcePlan decoded; + require(ShaderDiskCache::DecodePlan(bytes, decoded), fixture.name + ": the plan does not decode"); + require(encodedPlan(decoded) == bytes, fixture.name + ": the plan changed in a round trip"); + require(samePlan(*plan, decoded), fixture.name + ": the decoded plan differs from the plan in a field or a value"); + require(decoded.valueStorage.size() == plan->valueStorage.size() && decoded.blockStorage.size() == plan->blockStorage.size(), fixture.name + ": the decoded plan holds another graph"); + const auto succeeded = verifyRoundTripRuntimes(*plan, decoded, fixture, 0x5eedu + static_cast(index), fixture.name); + require(succeeded != 0u, fixture.name + ": no runtime materialized"); + std::vector key; + ShaderDiskCache::BuildPlanKey(request, 32u, key); + if (index % 4u == 0u) verifyPlanEntryDamage(key, *plan, fixture.name); + std::cout << "plan round trip: " << fixture.name << ", " << plan->valueStorage.size() << " values, " << ShaderDiskCache::EncodePlanEntry(key, *plan).size() << " bytes per entry, " << succeeded << " of 1000 runtimes materialized\n"; + ++index; + } + const auto covers = [&](const std::function& predicate) { + return std::any_of(plans.begin(), plans.end(), [&](const std::shared_ptr& plan) { return predicate(*plan); }); + }; + require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.descriptorSources.begin(), plan.descriptorSources.end(), [](const DescriptorSource& source) { return source.indirectImage.has_value(); }); }), "no fixture plans an indirect image table"); + require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.controlFlow.begin(), plan.controlFlow.end(), [](const ResourceBlock& block) { return block.condition != nullptr; }); }), "no fixture plans a control-flow condition"); + require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.valueStorage.begin(), plan.valueStorage.end(), [](const auto& value) { return value->IsPhi(); }); }), "no fixture plans a phi"); + require(covers([](const IrResourcePlan& plan) { return !plan.guardedSrtSlots.empty(); }), "no fixture plans a guarded SRT read"); + require(covers([](const IrResourcePlan& plan) { return std::any_of(plan.pureFlatSlots.begin(), plan.pureFlatSlots.end(), [](std::uint8_t pure) { return pure != 0u; }); }), "no fixture plans a pure flat slot"); +} + +void verifyPlanKey() { + auto fixtures = planFixtures(); + auto& fixture = fixtures[1]; + const auto planKey = [&](const std::function& change, std::uint32_t hostSubgroupSize = 32u) { + auto request = fixture.Request(true); + change(request); + std::vector key; + ShaderDiskCache::BuildPlanKey(request, hostSubgroupSize, key); + return key; + }; + const auto key = planKey([](RecompileRequest&) {}); + require(planKey([](RecompileRequest&) {}) == key, "the plan key is not deterministic"); + std::vector variantKey; + ShaderDiskCache::BuildKey(fixture.Request(true), 32u, variantKey); + require(variantKey != key && ShaderDiskCache::EntryName(variantKey) != ShaderDiskCache::EntryName(key), "the plan and the compiled variant share an entry"); + require(planKey([](RecompileRequest& request) { request.layout = {1u, 4u, 16u, 64u}; }) == key, "the binding layout changed the plan key"); + require(planKey([](RecompileRequest& request) { request.shader.codeAddress += 0x1000u; request.context.memory = {}; }) == key, "the code address or the memory changed the plan key"); + auto userData = fixture.userData; + userData[0] ^= 0x100u; + require(planKey([&](RecompileRequest& request) { request.context.userData = userData; }) == key, "user data values changed the plan key"); + const auto original = fixture.code; + for (std::size_t word = 0; word < original.size(); ++word) { + for (std::uint32_t bit = 0; bit < 32u; ++bit) { + fixture.code = original; + fixture.code[word] ^= 1u << bit; + require(planKey([](RecompileRequest&) {}) != key, "the plan key ignores code word " + std::to_string(word) + " bit " + std::to_string(bit)); + } + } + fixture.code = original; + fixture.code.push_back(0xbf810000u); + require(planKey([](RecompileRequest&) {}) != key, "the plan key ignores an appended code word"); + fixture.code = original; + require(planKey([](RecompileRequest&) {}, 64u) != key, "the plan key ignores the host subgroup size"); + require(planKey([](RecompileRequest& request) { request.target.subgroupSize = 64u; }) != key, "the plan key ignores the target"); + require(planKey([](RecompileRequest& request) { request.target.srgbDecodeFormats = 2u; }) != key, "the plan key ignores the sRGB formats decoded in the shader"); + require(planKey([](RecompileRequest& request) { request.context.waveSize = 64u; request.context.compute->numThreads[0] = 64u; }) != key, "the plan key ignores the wave size"); + require(planKey([](RecompileRequest& request) { request.context.floatMode = ShaderFloatMode{0xc0u, true, false, false}; }) != key, "the plan key ignores the float mode"); +} + +void appendWords(std::vector& out, std::span words) { + const auto count = static_cast(words.size()); + const auto countBytes = std::as_bytes(std::span(&count, 1)); + out.insert(out.end(), countBytes.begin(), countBytes.end()); + const auto bytes = std::as_bytes(words); + out.insert(out.end(), bytes.begin(), bytes.end()); +} + +void appendSnapshot(std::vector& out, const ResourceCapture& capture) { + const auto& snapshot = capture.snapshot; + for (const auto* values : {&snapshot.buffers, &snapshot.images, &snapshot.samplers}) { + std::vector words; + for (const auto& value : *values) { + words.push_back(value.dwordCount); + words.insert(words.end(), value.dwords.begin(), value.dwords.end()); + } + appendWords(out, words); + } + appendWords(out, snapshot.flattenedSrt); + appendWords(out, snapshot.userData); + const auto& fill = snapshot.uniformFill; + appendWords(out, std::vector{static_cast(fill.kind), fill.resource, fill.groupStride[0], fill.groupStride[1], fill.groupStride[2], fill.words, fill.value, snapshot.nullRootReads}); + std::vector poison; + for (const auto& entry : snapshot.srtPoison) poison.insert(poison.end(), {entry.slot, entry.pc, static_cast(entry.address), static_cast(entry.address >> 32u)}); + appendWords(out, poison); + std::vector leaves{capture.readTrace.leafSlot}; + for (const auto& [slot, address] : capture.readTrace.leaves) leaves.insert(leaves.end(), {slot, static_cast(address), static_cast(address >> 32u)}); + appendWords(out, leaves); + std::vector reads; + for (const auto address : capture.readTrace.otherReads) reads.insert(reads.end(), {static_cast(address), static_cast(address >> 32u)}); + appendWords(out, reads); +} + +std::vector> preparedOutputs(std::vector& fixtures, std::vector>* plans = nullptr) { + std::vector> outputs; + for (auto& fixture : fixtures) { + const auto request = fixture.Request(true); + const auto handle = PrepareShader(request); + RequestMemoryView view(request.context.memory); + const auto capture = CaptureResources(request, view.MakeRuntime(request.context.userData, request.shader.codeAddress), *handle); + const auto result = MaterializeShader(request, *capture, *handle); + std::vector bytes; + appendWords(bytes, GetPreparedArtifact(*handle).spirv.Words()); + ShaderDiskCache::EncodeResult(*result, bytes); + appendSnapshot(bytes, *capture); + require(ShaderDiskCache::EncodePlan(*capture->plan, bytes), fixture.name + ": the captured plan does not encode"); + if (plans != nullptr) plans->push_back(capture->plan); + outputs.push_back(std::move(bytes)); + } + return outputs; +} + +std::filesystem::path planOutputPath(std::size_t index) { + return ShaderCacheDirectory() / "plan-outputs" / ("fixture-" + std::to_string(index) + ".bin"); +} + +std::filesystem::path planEntryPath(PlanFixture& fixture) { + std::vector key; + ShaderDiskCache::BuildPlanKey(fixture.Request(true), 32u, key); + return ShaderDiskCache::EntryDirectory() / ShaderDiskCache::EntryName(key); +} + +int runPlanLoadingProcess(std::string_view mode) { + if (mode == "--load-plan-switch") setEnvironment("APS5_TRACE_BINDLESS", "1"); + if (mode == "--load-plan-off") setEnvironment("APS5_NO_PERF_DISK_PLANS", "1"); + auto fixtures = planFixtures(); + const auto count = fixtures.size(); + std::vector> plans; + const auto outputs = preparedOutputs(fixtures, &plans); + ShaderDiskCache::Flush(); + const auto totals = ShaderDiskCache::Totals(); + const auto builds = FrontEndBuilds(); + const auto counts = " (front end builds " + std::to_string(builds) + ", plan hits " + std::to_string(totals.planHits) + ", plan misses " + std::to_string(totals.planMisses) + ", plan load failures " + std::to_string(totals.planLoadFailures) + ", variant hits " + std::to_string(totals.hits) + ", variant misses " + std::to_string(totals.misses) + ")"; + if (mode == "--load-plan-switch") { + require(builds == count && totals.planHits == 0u && totals.planMisses == count && totals.misses == count, "another switch value reused stored plans or variants" + counts); + std::cout << "a process with another switch value rebuilt all " << count << " plans\n"; + return 0; + } + if (mode == "--load-plan") { + require(builds == 0u && totals.planHits == count && totals.planMisses == 0u && totals.planLoadFailures == 0u && totals.hits == count && totals.misses == 0u, "the second process ran the front end" + counts); + } else if (mode == "--load-plan-off") { + require(builds == count && totals.planHits == 0u && totals.planMisses == 0u && totals.planWrites == 0u && totals.hits == count && totals.misses == 0u, "APS5_NO_PERF_DISK_PLANS=1 used the stored plans or missed the stored variants" + counts); + } else { + require(builds == 1u && totals.planHits == count - 1u && totals.planLoadFailures == 1u && totals.planWrites == 1u && totals.hits == count && totals.misses == 0u, "the damaged plan was not rejected and rebuilt alone" + counts); + } + for (std::size_t index = 0; index < count; ++index) { + std::vector expected; + require(ReadWholeFile(planOutputPath(index), expected), "cannot read the first process's output for " + fixtures[index].name); + require(outputs[index] == expected, fixtures[index].name + ": the SPIR-V, bindings, push constants, snapshot or plan differ from the first process"); + } + if (mode == "--load-plan") { + auto changed = planFixtures(); + auto& fixture = changed.front(); + fixture.code.insert(fixture.code.end() - 1, 0xbf800000u); + const auto request = fixture.Request(true); + static_cast(PrepareShader(request)); + const auto after = ShaderDiskCache::Totals(); + require(FrontEndBuilds() == 1u && after.planHits == count && after.planMisses == 1u, "a changed code word reused a stored plan"); + std::cout << "second process prepared " << count << " shaders from stored plans without running the front end\n"; + } else if (mode == "--load-plan-off") { + std::cout << "APS5_NO_PERF_DISK_PLANS=1 ran the front end for all " << count << " shaders with the same output\n"; + } else { + std::cout << "a damaged plan entry was rejected, rebuilt and stored again with the same output\n"; + } + for (std::size_t index = 0; index < count; ++index) { + const auto fresh = GetResourcePlan(fixtures[index].Request(false)); + require(samePlan(*plans[index], *fresh), fixtures[index].name + ": the plan this process used differs from the plan its front end builds"); + } + ShaderDiskCache::Flush(); + return 0; +} + +void verifyPlanAcrossProcesses(const char* self) { + auto fixtures = planFixtures(); + const auto builds = FrontEndBuilds(); + const auto before = ShaderDiskCache::Totals(); + const auto outputs = preparedOutputs(fixtures); + ShaderDiskCache::Flush(); + const auto after = ShaderDiskCache::Totals(); + require(FrontEndBuilds() - builds == fixtures.size() && after.planMisses - before.planMisses == fixtures.size() && after.planWrites - before.planWrites == fixtures.size(), "the first process did not build and store every plan"); + std::filesystem::create_directories(planOutputPath(0).parent_path()); + for (std::size_t index = 0; index < outputs.size(); ++index) require(WriteFileAtomically(planOutputPath(index), outputs[index]), "cannot write the first process's output"); + auto refused = fixtures[1]; + refused.codeAddress = 0x200000u; + refused.code = {0xf40c0200u, 0xfa000000u, 0xf4000400u, 0xfa000050u, 0xbf8cc07fu, 0xbf068010u, 0xbf850003u, 0xf4080500u, 0xfa000020u, 0xbf820002u, 0xf4080500u, 0x20000000u, 0xf4080600u, 0xfa000040u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; + bool threw = false; + try { + static_cast(PrepareShader(refused.Request(true))); + } catch (const std::runtime_error&) { + threw = true; + } + ShaderDiskCache::Flush(); + require(threw && ShaderDiskCache::Totals().planWrites == after.planWrites && !std::filesystem::exists(planEntryPath(refused)), "a plan the front end refused was stored"); + std::uintmax_t planBytes = 0; + for (auto& fixture : fixtures) planBytes += std::filesystem::file_size(planEntryPath(fixture)); + std::cout << fixtures.size() << " plan entries, " << planBytes / fixtures.size() << " bytes per entry on average\n"; + const auto run = [&](const char* mode) { + const std::string command = "\"" + std::string(self) + "\" " + mode; + require(std::system(command.c_str()) == 0, std::string("the process ") + mode + " failed"); + }; + run("--load-plan"); + run("--load-plan-off"); + std::vector entry; + const auto damaged = planEntryPath(fixtures.front()); + require(ReadWholeFile(damaged, entry), "cannot read a stored plan entry"); + entry[entry.size() / 2] ^= std::byte{0x20}; + require(WriteFileAtomically(damaged, entry), "cannot damage a stored plan entry"); + run("--load-plan-corrupt"); + std::vector rewritten; + std::vector key; + ShaderDiskCache::BuildPlanKey(fixtures.front().Request(true), 32u, key); + IrResourcePlan plan; + require(ReadWholeFile(damaged, rewritten) && ShaderDiskCache::DecodePlanEntry(rewritten, key, plan) == ShaderDiskCache::LoadStatus::Loaded, "the rebuilt plan was not stored again"); + run("--load-plan-switch"); +} + void verifyDefaultDirectory(const char* self) { setEnvironment("ANYPS5_SHADER_CACHE_DIR", ""); setEnvironment("ANYPS5_NO_SHADER_CACHE", "1"); @@ -1099,6 +1843,7 @@ int main(int argc, char** argv) { try { if (argc == 2 && std::string_view(argv[1]) == "--load") return runLoadingProcess(); if (argc == 2 && std::string_view(argv[1]) == "--no-failure-memo") return runWithoutFailureMemo(); + if (argc == 2 && std::string_view(argv[1]).starts_with("--load-plan")) return runPlanLoadingProcess(argv[1]); const auto directory = std::filesystem::temp_directory_path() / ("aps5-shader-disk-cache-test-" + std::to_string(std::random_device{}())); std::filesystem::remove_all(directory); verifyDefaultDirectory(argv[0]); @@ -1110,8 +1855,11 @@ int main(int argc, char** argv) { verifyArtifactStorageIsolation(); verifyKeySensitivity(); verifyFloatModeSerialization(); + verifyPlanRoundTrip(); + verifyPlanKey(); verifyStore(); verifyAcrossProcesses(argv[0]); + verifyPlanAcrossProcesses(argv[0]); verifyEmissionFailureMemo(); verifyFailureMemoSwitch(argv[0]); verifyInvocationIsolation(); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index 909f3cea1352..2e4660e269f2 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -20,6 +20,7 @@ class ShaderPreparationContext { [[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request); [[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparationContext* preparation); +[[nodiscard]] std::uint64_t FrontEndBuilds(); [[nodiscard]] std::shared_ptr GetResourcePlan(const RecompileRequest& request); struct SourceEntry; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index aacd8989dd68..46f382ce1880 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -117,6 +117,8 @@ std::uint32_t DeviceMemoryLdsBytes(const RecompileRequest& request) { return bytes + 4u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; } +std::atomic frontEndBuilds{0}; + } struct PreparedControlFlow { @@ -156,11 +158,16 @@ std::shared_ptr ShaderPreparationContext::AcquireFron return prepared; } +std::uint64_t FrontEndBuilds() { + return frontEndBuilds.load(std::memory_order_relaxed); +} + IrProgram PrepareResourceProgram(const RecompileRequest& request) { return PrepareResourceProgram(request, nullptr); } IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparationContext* preparation) { + frontEndBuilds.fetch_add(1, std::memory_order_relaxed); ShaderPreparationContext local; const auto frontend = (preparation != nullptr ? *preparation : local).AcquireFrontend(request); const auto stageKind = toShaderStageKind(request.shader.stage); @@ -357,6 +364,14 @@ bool FailureMemo() { return memo; } +bool diskPlans() { + static const bool enabled = [] { + const char* dumpIr = std::getenv("APS5_DUMP_IR"); + return std::getenv("APS5_NO_PERF_DISK_PLANS") == nullptr && (dumpIr == nullptr || *dumpIr == '\0'); + }(); + return enabled && ShaderDiskCache::Enabled() && !DebugProbeActive(); +} + std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr, bool reclaim = true) { static PrimedSharedMutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. @@ -400,13 +415,26 @@ std::shared_ptr getSource(const RecompileRequest& request, ShaderPr std::lock_guard lock(source->mutex); if (source->plan == nullptr) { if (FailureMemo() && source->planFailure) std::rethrow_exception(source->planFailure); - try { - ResourceProgram resource(request, preparation, reclaim); - source->plan = std::move(resource.plan); - source->program = std::move(resource.program); - } catch (...) { - if (FailureMemo()) source->planFailure = std::current_exception(); - throw; + const bool disk = diskPlans(); + std::vector planKey; + if (disk) { + ShaderDiskCache::BuildPlanKey(request, HostSubgroupSize(request), planKey); + IrResourcePlan loaded; + if (ShaderDiskCache::LoadPlan(planKey, loaded)) { + loaded.shaderHash = request.shader.codeAddress; + source->plan = std::make_shared(std::move(loaded)); + } + } + if (source->plan == nullptr) { + try { + ResourceProgram resource(request, preparation, reclaim); + source->plan = std::move(resource.plan); + source->program = std::move(resource.program); + } catch (...) { + if (FailureMemo()) source->planFailure = std::current_exception(); + throw; + } + if (disk) ShaderDiskCache::StorePlan(std::move(planKey), source->plan); } } } diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index f25dc30969f0..8d934a1716fe 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -31,6 +32,8 @@ #include #include #include +#include +#include namespace ShaderRecompiler { @@ -94,10 +97,56 @@ static_assert(sizeof(IrDescriptorBinding) == 32, "IrDescriptorBinding changed: u static_assert(sizeof(BindingAllocationResult) == 152, "BindingAllocationResult changed: update the allocation encoder"); static_assert(sizeof(CompiledBindingLayout) == 80, "CompiledBindingLayout changed: update the allocation encoder"); static_assert(sizeof(BindingLayout) == 16, "BindingLayout changed: update BuildKey"); +static_assert(sizeof(IrResourcePlan) == 584, "IrResourcePlan changed: update EncodePlan and DecodePlan"); +static_assert(sizeof(IrValue) == 152, "IrValue changed: update the plan value encoder"); +static_assert(sizeof(GuestRegister) == 8, "GuestRegister changed: update the plan value encoder"); +static_assert(sizeof(MemoryInfo) == 80, "MemoryInfo changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource) == 104, "DescriptorSource changed: update the plan encoder"); +static_assert(sizeof(DescriptorSource::IndirectImage) == 28, "IndirectImage changed: update the plan encoder"); +static_assert(sizeof(ResourceBlock) == 80, "ResourceBlock changed: update the plan encoder"); +static_assert(sizeof(SrtRead) == 16, "SrtRead changed: update the plan encoder"); +static_assert(sizeof(UniformFill) == 28, "UniformFill changed: update the plan encoder"); +static_assert(sizeof(UniformFillPlan) == 64, "UniformFillPlan changed: update the plan encoder"); #endif namespace { +struct AnyField { + template + operator TField() const; +}; + +template +constexpr bool bracedWith(std::index_sequence) { + return requires { TStruct{(static_cast(Index), AnyField{})...}; }; +} + +template +constexpr std::size_t fieldCount() { + if constexpr (bracedWith(std::make_index_sequence{})) { + return fieldCount(); + } else { + return Count; + } +} + +static_assert(fieldCount() == 23, "ShaderInfo changed: update the info encoder"); +static_assert(fieldCount() == 12, "BufferResource changed: update the info encoder"); +static_assert(fieldCount() == 37, "ImageResource changed: update the info encoder"); +static_assert(fieldCount() == 7, "SamplerResource changed: update the info encoder"); +static_assert(fieldCount() == 3, "SampledResourcePair changed: update the info encoder"); +static_assert(fieldCount() == 5, "StageInput changed: update the info encoder"); +static_assert(fieldCount() == 4, "StageOutput changed: update the info encoder"); +static_assert(fieldCount() == 21, "IrResourcePlan changed: update EncodePlan and DecodePlan"); +static_assert(fieldCount() == 2, "GuestRegister changed: update the plan value encoder"); +static_assert(fieldCount() == 30, "MemoryInfo changed: update the plan encoder"); +static_assert(fieldCount() == 3, "DescriptorSource changed: update the plan encoder"); +static_assert(fieldCount() == 7, "IndirectImage changed: update the plan encoder"); +static_assert(fieldCount() == 4, "ResourceBlock changed: update the plan encoder"); +static_assert(fieldCount() == 2, "SrtRead changed: update the plan encoder"); +static_assert(fieldCount() == 5, "UniformFill changed: update the plan encoder"); +static_assert(fieldCount() == 2, "UniformFillPlan changed: update the plan encoder"); + constexpr std::uint32_t FileMagic = 0x43535041u; struct FileHeader { @@ -138,6 +187,14 @@ class Writer { for (const bool flag : flags) Value(flag); } + void Compact(std::uint64_t value) { + while (value >= 0x80u) { + Raw(static_cast(value | 0x80u)); + value >>= 7u; + } + Raw(static_cast(value)); + } + void Text(const std::string& text) { Value(text.size()); const auto bytes = std::as_bytes(std::span(text.data(), text.size())); @@ -167,6 +224,22 @@ class Reader { [[nodiscard]] bool Ok() const { return ok; } [[nodiscard]] bool Done() const { return ok && position == bytes.size(); } + [[nodiscard]] std::size_t Remaining() const { return ok ? bytes.size() - position : 0; } + void Fail() { ok = false; } + + [[nodiscard]] std::uint64_t Compact() { + std::uint64_t value = 0; + for (std::uint32_t shift = 0; shift < 64u; shift += 7u) { + const auto byte = Raw(); + if (!ok) return 0; + value |= static_cast(byte & 0x7fu) << shift; + if ((byte & 0x80u) != 0u) continue; + if ((byte == 0u && shift != 0u) || (shift == 63u && byte > 1u)) ok = false; + return ok ? value : 0; + } + ok = false; + return 0; + } template void Value(TValue& value) requires (std::is_integral_v || std::is_enum_v) { @@ -452,15 +525,7 @@ void decodeLayout(Reader& reader, IrBindingLayout& layout) { }); } -void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { - writer.Value(compiled.stage); - writer.Value(compiled.shaderHash); - writer.Value(compiled.waveSize); - writer.Value(compiled.userDataBase); - writer.Value(compiled.userDataCount); - writer.Value(compiled.scratchDwords); - writer.Value(compiled.paramExportMask); - const auto& info = compiled.info; +void encodeShaderInfo(Writer& writer, const ShaderInfo& info) { writer.Value(info.scratchDwords); writer.Value(info.sharedMemoryBytes); writer.List(info.buffers, [](Writer& out, const BufferResource& buffer) { @@ -558,18 +623,9 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { writer.Value(info.dispatchThreadLimit); writer.Value(info.rasterPrimitive); writer.Value(info.layerExported); - encodeLayout(writer, compiled.bindings); } -void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { - reader.Value(compiled.stage); - reader.Value(compiled.shaderHash); - reader.Value(compiled.waveSize); - reader.Value(compiled.userDataBase); - reader.Value(compiled.userDataCount); - reader.Value(compiled.scratchDwords); - reader.Value(compiled.paramExportMask); - auto& info = compiled.info; +void decodeShaderInfo(Reader& reader, ShaderInfo& info) { reader.Value(info.scratchDwords); reader.Value(info.sharedMemoryBytes); reader.List(info.buffers, 27, [](Reader& in, BufferResource& buffer) { @@ -669,6 +725,29 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { reader.Value(info.dispatchThreadLimit); reader.Value(info.rasterPrimitive); reader.Value(info.layerExported); +} + +void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { + writer.Value(compiled.stage); + writer.Value(compiled.shaderHash); + writer.Value(compiled.waveSize); + writer.Value(compiled.userDataBase); + writer.Value(compiled.userDataCount); + writer.Value(compiled.scratchDwords); + writer.Value(compiled.paramExportMask); + encodeShaderInfo(writer, compiled.info); + encodeLayout(writer, compiled.bindings); +} + +void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { + reader.Value(compiled.stage); + reader.Value(compiled.shaderHash); + reader.Value(compiled.waveSize); + reader.Value(compiled.userDataBase); + reader.Value(compiled.userDataCount); + reader.Value(compiled.scratchDwords); + reader.Value(compiled.paramExportMask); + decodeShaderInfo(reader, compiled.info); decodeLayout(reader, compiled.bindings); } @@ -684,12 +763,328 @@ void decodeAllocation(Reader& reader, CompiledBindingLayout& allocation) { reader.Value(allocation.pushConstantSizeBytes); } +constexpr std::uint32_t PlanEntryKind = 0x4e414c50u; +constexpr std::uint32_t NoPlanIndex = std::numeric_limits::max(); + +void encodeMemoryInfo(Writer& out, const MemoryInfo& memory) { + out.Value(memory.kind); + out.Value(memory.resource); + out.Value(memory.sampler); + out.Value(memory.offset); + out.Value(memory.secondaryOffset); + out.Value(memory.dmask); + out.Value(memory.dataDwords); + out.Value(memory.dataBits); + out.Value(memory.componentIndex); + out.Value(memory.componentCount); + out.Value(memory.dataFormat); + out.Value(memory.numberFormat); + out.Value(memory.imageSampleFlags); + out.Value(memory.imageDimension); + out.Value(memory.imageAddressComponents); + out.Value(memory.addressIsFull); + out.Value(memory.dataSigned); + out.Value(memory.typed); + out.Value(memory.formatted); + out.Value(memory.d16); + out.Value(memory.imageHasMip); + out.Value(memory.imageR128); + out.Value(memory.imagePacked); + out.Value(memory.imageByElements); + out.Value(memory.idxen); + out.Value(memory.offen); + out.Value(memory.planningOnly); + out.Value(memory.coherent); + out.Value(memory.gpuDescriptor); + out.Value(memory.flushDenormals); +} + +void decodeMemoryInfo(Reader& in, MemoryInfo& memory) { + in.Value(memory.kind); + in.Value(memory.resource); + in.Value(memory.sampler); + in.Value(memory.offset); + in.Value(memory.secondaryOffset); + in.Value(memory.dmask); + in.Value(memory.dataDwords); + in.Value(memory.dataBits); + in.Value(memory.componentIndex); + in.Value(memory.componentCount); + in.Value(memory.dataFormat); + in.Value(memory.numberFormat); + in.Value(memory.imageSampleFlags); + in.Value(memory.imageDimension); + in.Value(memory.imageAddressComponents); + in.Value(memory.addressIsFull); + in.Value(memory.dataSigned); + in.Value(memory.typed); + in.Value(memory.formatted); + in.Value(memory.d16); + in.Value(memory.imageHasMip); + in.Value(memory.imageR128); + in.Value(memory.imagePacked); + in.Value(memory.imageByElements); + in.Value(memory.idxen); + in.Value(memory.offen); + in.Value(memory.planningOnly); + in.Value(memory.coherent); + in.Value(memory.gpuDescriptor); + in.Value(memory.flushDenormals); +} + +bool encodePlan(Writer& writer, const IrResourcePlan& plan) { + if (std::ranges::any_of(plan.valueStorage, [](const auto& value) { return value == nullptr; }) || std::ranges::any_of(plan.blockStorage, [](const auto& block) { return block == nullptr; })) return false; + std::unordered_map values; + std::unordered_map blocks; + values.reserve(plan.valueStorage.size()); + blocks.reserve(plan.blockStorage.size()); + for (std::size_t index = 0; index < plan.valueStorage.size(); ++index) values.emplace(plan.valueStorage[index].get(), static_cast(index)); + for (std::size_t index = 0; index < plan.blockStorage.size(); ++index) blocks.emplace(plan.blockStorage[index].get(), static_cast(index)); + bool owned = values.size() == plan.valueStorage.size() && blocks.size() == plan.blockStorage.size(); + std::uint32_t current = 0; + const auto valueIndex = [&](const IrValue* value) { + if (value == nullptr) return NoPlanIndex; + const auto found = values.find(value); + if (found == values.end()) { + owned = false; + return NoPlanIndex; + } + return found->second; + }; + writer.Value(plan.stage); + writer.Value(plan.shaderHash); + writer.Value(plan.userDataBase); + writer.Value(plan.userDataCount); + writer.Value(plan.srgbDecodeFormats); + writer.List(plan.blockStorage, [](Writer& out, const std::unique_ptr& block) { out.Value(block->Id()); }); + writer.List(plan.valueStorage, [&](Writer& out, const std::unique_ptr& value) { + const auto bank = static_cast(value->Register().bank); + if (bank > 0xfu) owned = false; + out.Compact(static_cast(value->Opcode())); + out.Compact(static_cast(value->Type())); + out.Compact(value->Id()); + out.Compact(value->Flags()); + out.Compact((static_cast(value->Register().index) << 4u) | (bank & 0xfu)); + out.Compact((static_cast(value->ArgumentCount()) << 1u) | (value->HasImmediate() ? 1u : 0u)); + if (value->HasImmediate()) out.Compact(value->ImmediateU64()); + for (const auto* argument : value->Arguments()) { + const auto index = valueIndex(argument); + const auto delta = static_cast(index) - static_cast(current); + out.Compact(argument == nullptr ? 0u : ((static_cast(delta) << 1u) ^ static_cast(delta >> 63)) + 1u); + } + if (value->IsPhi() && value->PhiBlockCount() != value->ArgumentCount()) owned = false; + for (std::size_t operand = 0; value->IsPhi() && operand < value->ArgumentCount(); ++operand) { + const auto found = operand < value->PhiBlockCount() ? blocks.find(value->PhiBlock(operand)) : blocks.end(); + if (found == blocks.end()) owned = false; + out.Compact(found != blocks.end() ? found->second : 0u); + } + ++current; + }); + writer.List(plan.memoryInfo, encodeMemoryInfo); + writer.List(plan.descriptorSources, [&](Writer& out, const DescriptorSource& source) { + for (const auto* dword : source.dwords) out.Value(valueIndex(dword)); + out.Value(source.dwordCount); + const auto table = source.indirectImage.value_or(DescriptorSource::IndirectImage{}); + out.Value(source.indirectImage.has_value()); + out.Value(table.materialSource); + out.Value(table.heapSource); + out.Value(table.selectorStride); + out.Value(table.selectorOffset); + out.Value(table.keyArg); + out.Value(table.entryOffset); + out.Value(table.hasMaterial); + }); + writer.List(plan.controlFlow, [&](Writer& out, const ResourceBlock& block) { + out.Value(valueIndex(block.condition)); + out.Values(std::span(block.successors)); + out.Values(std::span(block.sources)); + out.Values(std::span(block.images)); + }); + writer.Values(std::span(plan.materializationSources)); + writer.List(plan.srtReads, [&](Writer& out, const SrtRead& read) { + out.Value(valueIndex(read.value)); + out.Value(read.flatOffset); + }); + writer.Values(std::span(plan.guardedSrtSlots)); + writer.Value(plan.srtGuardOffset); + writer.Values(std::span(plan.cleanFlatSlots)); + writer.Values(std::span(plan.pureFlatSlots)); + writer.Value(plan.requiresSpecializationMemory); + writer.Value(plan.srtPlanComplete); + writer.Value(plan.resourceTrackingComplete); + encodeShaderInfo(writer, plan.info); + const auto& fill = plan.uniformFill.fill; + writer.Value(fill.kind); + writer.Value(fill.resource); + for (const auto stride : fill.groupStride) writer.Value(stride); + writer.Value(fill.words); + writer.Value(fill.value); + for (const auto* value : plan.uniformFill.values) writer.Value(valueIndex(value)); + return owned; +} + +struct PlanValueRecord { + IrOpcode opcode = IrOpcode::Identity; + IrType type = IrType::Void; + std::uint32_t id = 0; + std::uint64_t flags = 0; + bool hasImmediate = false; + std::uint64_t immediate = 0; + GuestRegister registerValue{}; + std::vector arguments; + std::vector phiBlocks; +}; + +bool decodePlan(Reader& reader, IrResourcePlan& plan) { + reader.Value(plan.stage); + reader.Value(plan.shaderHash); + reader.Value(plan.userDataBase); + reader.Value(plan.userDataCount); + reader.Value(plan.srgbDecodeFormats); + std::vector blockIds; + reader.Values(blockIds); + std::vector records; + std::uint32_t current = 0; + reader.List(records, 6, [&](Reader& in, PlanValueRecord& record) { + constexpr std::uint64_t Word = std::numeric_limits::max(); + const auto opcode = in.Compact(); + const auto type = in.Compact(); + const auto id = in.Compact(); + record.flags = in.Compact(); + const auto registerValue = in.Compact(); + const auto shape = in.Compact(); + const auto arguments = shape >> 1u; + if (opcode >= static_cast(IrOpcode::Count) || type > Word || id > Word || (registerValue >> 4u) > Word || arguments > in.Remaining()) { + in.Fail(); + return; + } + record.opcode = static_cast(opcode); + record.type = static_cast(type); + record.id = static_cast(id); + record.registerValue = {static_cast(registerValue & 0xfu), static_cast(registerValue >> 4u)}; + record.hasImmediate = (shape & 1u) != 0u; + if (record.hasImmediate) record.immediate = in.Compact(); + record.arguments.resize(static_cast(arguments)); + for (auto& argument : record.arguments) { + const auto code = in.Compact(); + if (code == 0u) { + argument = NoPlanIndex; + continue; + } + const auto zigzag = code - 1u; + const auto delta = static_cast(zigzag >> 1u) ^ -static_cast(zigzag & 1u); + const auto index = static_cast(current) + delta; + if (index < 0 || index >= static_cast(records.size())) { + in.Fail(); + return; + } + argument = static_cast(index); + } + if (record.opcode == IrOpcode::Phi) { + record.phiBlocks.resize(record.arguments.size()); + for (auto& block : record.phiBlocks) { + const auto index = in.Compact(); + block = index < blockIds.size() ? static_cast(index) : NoPlanIndex; + if (block == NoPlanIndex) in.Fail(); + } + } + ++current; + }); + if (!reader.Ok()) return false; + plan.blockStorage.reserve(blockIds.size()); + for (const auto id : blockIds) plan.blockStorage.push_back(std::make_unique(id)); + plan.valueStorage.reserve(records.size()); + for (const auto& record : records) { + auto value = std::make_unique(record.opcode, record.type, record.id); + value->SetFlags(record.flags); + if (record.hasImmediate) value->SetImmediateU64(record.immediate); + value->SetRegister(record.registerValue); + plan.valueStorage.push_back(std::move(value)); + } + bool valid = true; + const auto valueAt = [&](std::uint32_t index) -> IrValue* { + if (index == NoPlanIndex) return nullptr; + if (index >= plan.valueStorage.size()) { + valid = false; + return nullptr; + } + return plan.valueStorage[index].get(); + }; + for (std::size_t index = 0; index < records.size(); ++index) { + const auto& record = records[index]; + auto& value = *plan.valueStorage[index]; + for (std::size_t operand = 0; operand < record.arguments.size(); ++operand) { + auto* argument = valueAt(record.arguments[operand]); + if (value.IsPhi()) { + value.AddPhiOperand(plan.blockStorage[record.phiBlocks[operand]].get(), argument); + } else { + value.AddArgument(argument); + } + } + } + reader.List(plan.memoryInfo, 78, decodeMemoryInfo); + reader.List(plan.descriptorSources, 62, [&](Reader& in, DescriptorSource& source) { + for (auto& dword : source.dwords) dword = valueAt(in.Get()); + in.Value(source.dwordCount); + const bool indirect = in.Get(); + DescriptorSource::IndirectImage table; + in.Value(table.materialSource); + in.Value(table.heapSource); + in.Value(table.selectorStride); + in.Value(table.selectorOffset); + in.Value(table.keyArg); + in.Value(table.entryOffset); + in.Value(table.hasMaterial); + source.indirectImage = indirect ? std::optional(table) : std::nullopt; + }); + reader.List(plan.controlFlow, 28, [&](Reader& in, ResourceBlock& block) { + block.condition = valueAt(in.Get()); + in.Values(block.successors); + in.Values(block.sources); + in.Values(block.images); + }); + reader.Values(plan.materializationSources); + reader.List(plan.srtReads, 8, [&](Reader& in, SrtRead& read) { + read.value = valueAt(in.Get()); + in.Value(read.flatOffset); + }); + reader.Values(plan.guardedSrtSlots); + reader.Value(plan.srtGuardOffset); + reader.Values(plan.cleanFlatSlots); + reader.Values(plan.pureFlatSlots); + reader.Value(plan.requiresSpecializationMemory); + reader.Value(plan.srtPlanComplete); + reader.Value(plan.resourceTrackingComplete); + decodeShaderInfo(reader, plan.info); + auto& fill = plan.uniformFill.fill; + reader.Value(fill.kind); + reader.Value(fill.resource); + for (auto& stride : fill.groupStride) reader.Value(stride); + reader.Value(fill.words); + reader.Value(fill.value); + for (auto& value : plan.uniformFill.values) value = valueAt(reader.Get()); + if (!valid || !reader.Done() || fill.words > plan.uniformFill.values.size()) return false; + const auto knownSource = [&](std::uint32_t source) { return source < plan.descriptorSources.size(); }; + if (!std::ranges::all_of(plan.materializationSources, knownSource)) return false; + if (!std::ranges::all_of(plan.descriptorSources, [&](const DescriptorSource& source) { return source.dwordCount <= source.dwords.size() && (!source.indirectImage.has_value() || (knownSource(source.indirectImage->heapSource) && knownSource(source.indirectImage->materialSource))); })) return false; + if (!std::ranges::all_of(plan.info.buffers, [&](const BufferResource& buffer) { return knownSource(buffer.source); })) return false; + if (!std::ranges::all_of(plan.info.images, [&](const ImageResource& image) { return knownSource(image.source); })) return false; + if (!std::ranges::all_of(plan.info.samplers, [&](const SamplerResource& sampler) { return knownSource(sampler.source); })) return false; + return std::ranges::all_of(plan.controlFlow, [&](const ResourceBlock& block) { + return (block.condition == nullptr || block.successors.size() == 2u) && + std::ranges::all_of(block.successors, [&](std::uint32_t successor) { return successor < plan.controlFlow.size(); }) && + std::ranges::all_of(block.sources, knownSource) && + std::ranges::all_of(block.images, [&](std::uint32_t image) { return image < plan.info.images.size(); }); + }); +} + constexpr std::string_view NeutralSwitches[] = { "APS5_PROFILE_DRAW", "APS5_DUMP_IR", "APS5_NO_CODE_HASH_KEY", "APS5_NO_FAILURE_MEMO", "APS5_NO_PERF_FRONTEND_PAIR", + "APS5_NO_PERF_DISK_PLANS", "APS5_NO_PERF_DOM_INTERSECTION", "APS5_NO_PERF_ITERATION_SELECTION", "APS5_NO_RESULT_MEMO", @@ -717,6 +1112,30 @@ std::string hex(std::uint64_t value) { return text; } +std::vector wrapEntry(std::span key, std::span payload) { + const FileHeader header{FileMagic, FormatVersion, SourceVersion(), key.size(), payload.size(), HashBytes(key), HashBytes(payload)}; + std::vector file(sizeof(header) + key.size() + payload.size()); + std::memcpy(file.data(), &header, sizeof(header)); + std::memcpy(file.data() + sizeof(header), key.data(), key.size()); + if (!payload.empty()) std::memcpy(file.data() + sizeof(header) + key.size(), payload.data(), payload.size()); + return file; +} + +LoadStatus unwrapEntry(std::span file, std::span key, std::span& payload) { + if (file.size() < sizeof(FileHeader)) return LoadStatus::Rejected; + FileHeader header; + std::memcpy(&header, file.data(), sizeof(header)); + if (header.magic != FileMagic || header.format != FormatVersion || header.sourceVersion != SourceVersion()) return LoadStatus::Rejected; + const auto body = file.size() - sizeof(header); + if (header.keyBytes > body || header.payloadBytes != body - header.keyBytes) return LoadStatus::Rejected; + const auto storedKey = file.subspan(sizeof(header), static_cast(header.keyBytes)); + payload = file.subspan(sizeof(header) + static_cast(header.keyBytes)); + if (HashBytes(storedKey) != header.keyHash) return LoadStatus::Rejected; + if (storedKey.size() != key.size() || !std::equal(storedKey.begin(), storedKey.end(), key.begin())) return LoadStatus::KeyMismatch; + if (HashBytes(payload) != header.payloadHash) return LoadStatus::Rejected; + return LoadStatus::Loaded; +} + class DiskStore { public: DiskStore() { @@ -731,26 +1150,71 @@ class DiskStore { [[nodiscard]] const std::filesystem::path& Directory() const { return directory; } bool Load(std::span key, CompiledVariant& variant) { + return load(key, [&](std::span file) { return DecodeEntry(file, key, variant); }, hits, misses, loadFailures); + } + + bool LoadPlan(std::span key, IrResourcePlan& plan) { + return load(key, [&](std::span file) { return DecodePlanEntry(file, key, plan); }, planHits, planMisses, planLoadFailures); + } + + void Store(std::vector key, std::shared_ptr variant) { + enqueue({std::move(key), std::move(variant), nullptr}); + } + + void StorePlan(std::vector key, std::shared_ptr plan) { + enqueue({std::move(key), nullptr, std::move(plan)}); + } + + void Flush() { + std::unique_lock lock(mutex); + idle.wait(lock, [&] { return jobs.empty() && !writing; }); + } + + Counters Totals() const { + Counters counters; + counters.hits = hits.load(std::memory_order_relaxed); + counters.misses = misses.load(std::memory_order_relaxed); + counters.writes = writes.load(std::memory_order_relaxed); + counters.loadFailures = loadFailures.load(std::memory_order_relaxed); + counters.writeFailures = writeFailures.load(std::memory_order_relaxed); + counters.bytesRead = bytesRead.load(std::memory_order_relaxed); + counters.bytesWritten = bytesWritten.load(std::memory_order_relaxed); + counters.planHits = planHits.load(std::memory_order_relaxed); + counters.planMisses = planMisses.load(std::memory_order_relaxed); + counters.planWrites = planWrites.load(std::memory_order_relaxed); + counters.planLoadFailures = planLoadFailures.load(std::memory_order_relaxed); + return counters; + } + +private: + struct Job { + std::vector key; + std::shared_ptr variant; + std::shared_ptr plan; + }; + + template + bool load(std::span key, TDecode&& decode, std::atomic& hitCount, std::atomic& missCount, std::atomic& failureCount) { const auto started = std::chrono::steady_clock::now(); const auto name = EntryName(key); thread_local std::vector* fileSlot = nullptr; auto& file = ThreadOwned(fileSlot); bool loaded = false; if (!ReadWholeFile(directory / name, file)) { - misses.fetch_add(1, std::memory_order_relaxed); + missCount.fetch_add(1, std::memory_order_relaxed); } else { - switch (DecodeEntry(file, key, variant)) { + switch (decode(std::span(file))) { case LoadStatus::Loaded: - hits.fetch_add(1, std::memory_order_relaxed); + hitCount.fetch_add(1, std::memory_order_relaxed); bytesRead.fetch_add(file.size(), std::memory_order_relaxed); loaded = true; break; case LoadStatus::Absent: case LoadStatus::KeyMismatch: - misses.fetch_add(1, std::memory_order_relaxed); + missCount.fetch_add(1, std::memory_order_relaxed); break; case LoadStatus::Rejected: { - const auto failures = loadFailures.fetch_add(1, std::memory_order_relaxed); + const auto failures = failureCount.fetch_add(1, std::memory_order_relaxed); if (failures < 8) std::fprintf(stderr, "[shader-disk-cache] rejected %s (%zu bytes): truncated, corrupt or another format; recompiling\n", name.c_str(), file.size()); break; } @@ -762,37 +1226,14 @@ class DiskStore { return loaded; } - void Store(std::vector key, std::shared_ptr variant) { + void enqueue(Job job) { { std::lock_guard lock(mutex); - jobs.push_back({std::move(key), std::move(variant)}); + jobs.push_back(std::move(job)); } wake.notify_one(); } - void Flush() { - std::unique_lock lock(mutex); - idle.wait(lock, [&] { return jobs.empty() && !writing; }); - } - - Counters Totals() const { - Counters counters; - counters.hits = hits.load(std::memory_order_relaxed); - counters.misses = misses.load(std::memory_order_relaxed); - counters.writes = writes.load(std::memory_order_relaxed); - counters.loadFailures = loadFailures.load(std::memory_order_relaxed); - counters.writeFailures = writeFailures.load(std::memory_order_relaxed); - counters.bytesRead = bytesRead.load(std::memory_order_relaxed); - counters.bytesWritten = bytesWritten.load(std::memory_order_relaxed); - return counters; - } - -private: - struct Job { - std::vector key; - std::shared_ptr variant; - }; - void run() { std::fprintf(stderr, "[shader-disk-cache] %s (source version %s, format %u)\n", directory.string().c_str(), hex(SourceVersion()).c_str(), FormatVersion); housekeeping(); @@ -817,9 +1258,9 @@ class DiskStore { } void write(const Job& job) { - const auto file = EncodeEntry(job.key, *job.variant); - if (WriteFileAtomically(directory / EntryName(job.key), file)) { - writes.fetch_add(1, std::memory_order_relaxed); + const auto file = job.plan != nullptr ? EncodePlanEntry(job.key, *job.plan) : EncodeEntry(job.key, *job.variant); + if (!file.empty() && WriteFileAtomically(directory / EntryName(job.key), file)) { + (job.plan != nullptr ? planWrites : writes).fetch_add(1, std::memory_order_relaxed); bytesWritten.fetch_add(file.size(), std::memory_order_relaxed); } else { const auto failures = writeFailures.fetch_add(1, std::memory_order_relaxed); @@ -869,11 +1310,13 @@ class DiskStore { const auto totals = Totals(); const auto loadNs = loadNanoseconds.load(std::memory_order_relaxed); const auto delta = [](std::uint64_t after, std::uint64_t before) { return static_cast(after - before); }; - if (totals.hits == reported.hits && totals.misses == reported.misses && totals.writes == reported.writes && totals.loadFailures == reported.loadFailures && totals.writeFailures == reported.writeFailures) return; - std::fprintf(stderr, "[shader-disk-cache] (10 s): %llu hits, %llu misses, %llu writes (%.1f MiB), %llu load failures, %llu write failures; %.1f MiB read in %.1f ms (totals: %llu hits, %llu misses, %llu writes)\n", + if (totals.hits == reported.hits && totals.misses == reported.misses && totals.writes == reported.writes && totals.loadFailures == reported.loadFailures && totals.writeFailures == reported.writeFailures && totals.planHits == reported.planHits && totals.planMisses == reported.planMisses && totals.planWrites == reported.planWrites && totals.planLoadFailures == reported.planLoadFailures) return; + std::fprintf(stderr, "[shader-disk-cache] (10 s): %llu hits, %llu misses, %llu writes (%.1f MiB), %llu load failures, %llu write failures; resource plans: %llu hits, %llu misses, %llu writes, %llu load failures; %.1f MiB read in %.1f ms (totals: %llu hits, %llu misses, %llu writes; plans %llu hits, %llu misses)\n", delta(totals.hits, reported.hits), delta(totals.misses, reported.misses), delta(totals.writes, reported.writes), static_cast(totals.bytesWritten - reported.bytesWritten) / (1024.0 * 1024.0), - delta(totals.loadFailures, reported.loadFailures), delta(totals.writeFailures, reported.writeFailures), static_cast(totals.bytesRead - reported.bytesRead) / (1024.0 * 1024.0), static_cast(loadNs - reportedLoadNs) / 1e6, - static_cast(totals.hits), static_cast(totals.misses), static_cast(totals.writes)); + delta(totals.loadFailures, reported.loadFailures), delta(totals.writeFailures, reported.writeFailures), + delta(totals.planHits, reported.planHits), delta(totals.planMisses, reported.planMisses), delta(totals.planWrites, reported.planWrites), delta(totals.planLoadFailures, reported.planLoadFailures), + static_cast(totals.bytesRead - reported.bytesRead) / (1024.0 * 1024.0), static_cast(loadNs - reportedLoadNs) / 1e6, + static_cast(totals.hits), static_cast(totals.misses), static_cast(totals.writes), static_cast(totals.planHits), static_cast(totals.planMisses)); reported = totals; reportedLoadNs = loadNs; } @@ -886,6 +1329,7 @@ class DiskStore { std::deque jobs; bool writing = false; std::atomic hits{0}, misses{0}, writes{0}, loadFailures{0}, writeFailures{0}, bytesRead{0}, bytesWritten{0}, loadNanoseconds{0}; + std::atomic planHits{0}, planMisses{0}, planWrites{0}, planLoadFailures{0}; std::atomic lastReport{0}; std::mutex reportMutex; Counters reported; @@ -923,6 +1367,23 @@ void BuildKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, s key.insert(key.end(), switches.begin(), switches.end()); } +void BuildPlanKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, std::vector& key) { + key.clear(); + Writer writer(key); + writer.Value(FileMagic); + writer.Value(FormatVersion); + writer.Value(SourceVersion()); + writer.Value(PlanEntryKind); + thread_local std::vector* memoryKeySlot = nullptr; + auto& memoryKey = ThreadOwned(memoryKeySlot); + RecompileCacheKey::Build(request, memoryKey); + writer.Values(std::span(memoryKey)); + writer.Values(request.shader.code); + writer.Value(hostSubgroupSize); + const auto& switches = switchKey(); + key.insert(key.end(), switches.begin(), switches.end()); +} + std::string EntryName(std::span key) { return hex(HashBytes(key, 0x5eed0001ull)) + hex(HashBytes(key, 0x5eed0002ull)) + ".bin"; } @@ -950,26 +1411,13 @@ std::vector EncodeEntry(std::span key, const Compile encodeArtifact(writer, variant.artifact); encodeInfo(writer, variant.info); encodeAllocation(writer, variant.bindings); - const FileHeader header{FileMagic, FormatVersion, SourceVersion(), key.size(), payload.size(), HashBytes(key), HashBytes(payload)}; - std::vector file(sizeof(header) + key.size() + payload.size()); - std::memcpy(file.data(), &header, sizeof(header)); - std::memcpy(file.data() + sizeof(header), key.data(), key.size()); - std::memcpy(file.data() + sizeof(header) + key.size(), payload.data(), payload.size()); - return file; + return wrapEntry(key, payload); } LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant) { - if (file.size() < sizeof(FileHeader)) return LoadStatus::Rejected; - FileHeader header; - std::memcpy(&header, file.data(), sizeof(header)); - if (header.magic != FileMagic || header.format != FormatVersion || header.sourceVersion != SourceVersion()) return LoadStatus::Rejected; - const auto body = file.size() - sizeof(header); - if (header.keyBytes > body || header.payloadBytes != body - header.keyBytes) return LoadStatus::Rejected; - const auto storedKey = file.subspan(sizeof(header), static_cast(header.keyBytes)); - const auto payload = file.subspan(sizeof(header) + static_cast(header.keyBytes)); - if (HashBytes(storedKey) != header.keyHash) return LoadStatus::Rejected; - if (storedKey.size() != key.size() || !std::equal(storedKey.begin(), storedKey.end(), key.begin())) return LoadStatus::KeyMismatch; - if (HashBytes(payload) != header.payloadHash) return LoadStatus::Rejected; + std::span payload; + const auto status = unwrapEntry(file, key, payload); + if (status != LoadStatus::Loaded) return status; Reader reader(payload); CompiledVariant decoded; decodeArtifact(reader, decoded.artifact); @@ -982,6 +1430,32 @@ LoadStatus DecodeEntry(std::span file, std::span& out) { + Writer writer(out); + return encodePlan(writer, plan); +} + +bool DecodePlan(std::span bytes, IrResourcePlan& plan) { + Reader reader(bytes); + IrResourcePlan decoded; + if (!decodePlan(reader, decoded)) return false; + plan = std::move(decoded); + return true; +} + +std::vector EncodePlanEntry(std::span key, const IrResourcePlan& plan) { + std::vector payload; + if (!EncodePlan(plan, payload)) return {}; + return wrapEntry(key, payload); +} + +LoadStatus DecodePlanEntry(std::span file, std::span key, IrResourcePlan& plan) { + std::span payload; + const auto status = unwrapEntry(file, key, payload); + if (status != LoadStatus::Loaded) return status; + return DecodePlan(payload, plan) ? LoadStatus::Loaded : LoadStatus::Rejected; +} + bool Enabled() { return store().Enabled(); } @@ -1000,6 +1474,16 @@ void Store(std::vector key, std::shared_ptr va if (instance.Enabled()) instance.Store(std::move(key), std::move(variant)); } +bool LoadPlan(std::span key, IrResourcePlan& plan) { + auto& instance = store(); + return instance.Enabled() && instance.LoadPlan(key, plan); +} + +void StorePlan(std::vector key, std::shared_ptr plan) { + auto& instance = store(); + if (instance.Enabled() && plan != nullptr) instance.StorePlan(std::move(key), std::move(plan)); +} + void Flush() { auto& instance = store(); if (instance.Enabled()) instance.Flush(); diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index 81bbd87bebe6..530ccae9c46a 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -2,6 +2,7 @@ #define CORE_SHADER_RECOMPILER_SHADERDISKCACHE_HPP #include "CompiledVariant.hpp" +#include "IntermediateRepresentation/IrMetadata/ResourcePlan.hpp" #include #include #include @@ -12,7 +13,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 29; +inline constexpr std::uint32_t FormatVersion = 30; enum class LoadStatus { Loaded, @@ -29,6 +30,10 @@ struct Counters { std::uint64_t writeFailures = 0; std::uint64_t bytesRead = 0; std::uint64_t bytesWritten = 0; + std::uint64_t planHits = 0; + std::uint64_t planMisses = 0; + std::uint64_t planWrites = 0; + std::uint64_t planLoadFailures = 0; }; [[nodiscard]] std::uint64_t SourceVersion(); @@ -43,11 +48,19 @@ void EncodeResult(const RecompileResult& result, std::vector& out); [[nodiscard]] std::vector EncodeEntry(std::span key, const CompiledVariant& variant); [[nodiscard]] LoadStatus DecodeEntry(std::span file, std::span key, CompiledVariant& variant); +void BuildPlanKey(const RecompileRequest& request, std::uint32_t hostSubgroupSize, std::vector& key); +[[nodiscard]] bool EncodePlan(const IrResourcePlan& plan, std::vector& out); +[[nodiscard]] bool DecodePlan(std::span bytes, IrResourcePlan& plan); +[[nodiscard]] std::vector EncodePlanEntry(std::span key, const IrResourcePlan& plan); +[[nodiscard]] LoadStatus DecodePlanEntry(std::span file, std::span key, IrResourcePlan& plan); + [[nodiscard]] bool Enabled(); [[nodiscard]] std::filesystem::path EntryDirectory(); [[nodiscard]] bool Load(std::span key, CompiledVariant& variant); void Store(std::vector key, std::shared_ptr variant); +[[nodiscard]] bool LoadPlan(std::span key, IrResourcePlan& plan); +void StorePlan(std::vector key, std::shared_ptr plan); void Flush(); [[nodiscard]] Counters Totals();