Skip to content

perf(shader): probe 32-bit BDA stores and share the write fallback #1282

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Swastik-51 wants to merge 5 commits into from
Closed

perf(shader): probe 32-bit BDA stores and share the write fallback #1282

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
92c461e
perf(recompiler): emit BDA byte stores through one looping helper
popcoder565 Oct 7, 2026
4ead270
perf(recompiler): probe the written-page slots of a BDA write in a loop
popcoder565 Oct 7, 2026
082705e
refactor(recompiler): keep the BDA byte write helper beside the store…
popcoder565 Oct 7, 2026
b3b6398
perf(recompiler): load the written-page slot before the CAS
TheAggie Oct 7, 2026
90d6d56
perf(shader): probe 32-bit BDA stores and use the looping write fallback
Swastik-51 Oct 7, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1606,6 +1606,73 @@ void verifyBdaReadFallbackFunctions() {
}
}

void verifyBdaWriteFallbackFunctions() {
using namespace ShaderRecompiler;
const std::array<std::uint32_t, 3> capabilities{spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess};
const std::array<std::string_view, 2> extensions{"SPV_KHR_physical_storage_buffer", "SPV_KHR_8bit_storage"};
alignas(256) static std::array<std::uint32_t, 32> output{};
const auto outputAddress = static_cast<std::uint64_t>(reinterpret_cast<std::uintptr_t>(output.data()));
const std::array<std::uint32_t, 8> userData{0x10000000u, 0u, 0u, 0u, static_cast<std::uint32_t>(outputAddress), static_cast<std::uint32_t>((outputAddress >> 32u) & 0xffffu), 128u, 0x01016facu};
const auto compile = [&](std::uint32_t stores, bool coherent) {
std::vector<std::uint32_t> code{0x7e020200u, 0x7e040201u};
for (std::uint32_t store = 0; store < stores; ++store) {
code.push_back(0xdc708000u | (coherent ? 0x10000u : 0u) | (4u * store + 4u));
code.push_back(0x00000101u);
}
code.push_back(0xbf810000u);
RecompileRequest request{};
request.shader = {ShaderStage::Compute, 0x50000u, code, 0, {}};
request.context.waveSize = 32;
request.context.userDataBaseRegister = 0;
request.context.userData = userData;
request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u};
request.target.vulkanVersion = 0x00401000u;
request.target.spirvVersion = 0x00010300u;
request.target.subgroupSize = 32;
request.target.bdaAbiVersion = BdaAbi::Version;
request.target.supportedCapabilities = capabilities;
request.target.supportedExtensions = extensions;
request.target.fragmentShaderBarycentricEnabled = false;
request.layout.pushConstantSizeBytes = 128;
request.useCache = false;
return Recompile(request).spirv;
};
for (const bool coherent : {false, true}) {
const auto one = compile(1u, coherent);
const auto five = compile(5u, coherent);
std::map<std::uint32_t, std::string> names;
std::map<std::string, std::uint32_t> definitions;
const auto writer = std::string("write_bda_bytes") + (coherent ? "_coherent" : "");
std::string function;
std::size_t mainLookups = 0;
std::array<std::size_t, 2> writerStores{};
for (std::size_t cursor = 5; cursor < five.size();) {
const auto length = five[cursor] >> 16u;
require(length != 0 && length <= five.size() - cursor, "BDA write functions: truncated SPIR-V instruction");
const auto op = five[cursor] & 0xffffu;
if (op == spv::OpName) names[five[cursor + 1]] = reinterpret_cast<const char*>(&five[cursor + 2]);
if (op == spv::OpFunction) {
function = names[five[cursor + 2]];
if (function == "note_bda_write") {
++definitions[function];
}
if (function.starts_with("write_bda_bytes")) {
require((five[cursor + 3] & spv::FunctionControlDontInlineMask) != 0u, "BDA write functions: a byte write function may be inlined");
++definitions[function];
}
}
if (op == spv::OpFunctionCall && function == "main" && names[five[cursor + 3]] == "get_bda_write_pointer") ++mainLookups;
if (op == spv::OpStore && function == writer && length > 3u) ++writerStores[(five[cursor + 3] & spv::MemoryAccessVolatileMask) != 0u];
cursor += length;
}
require(definitions["note_bda_write"] == 1u && definitions[writer] == 1u, "BDA write functions: the note and byte write functions are not defined");
for (const auto& [name, count] : definitions) require(count == 1u, "BDA write functions: a function is defined twice");
require(mainLookups == 0u, "BDA write functions: a write site looks up its bytes inline");
require(writerStores[coherent] == 1u && writerStores[!coherent] == 0u, "BDA write functions: the byte stores do not keep the access's coherence");
require(five.size() - one.size() < 4u * 300u, "BDA write functions: a write site takes 300 SPIR-V words or more");
}
}

void verifyFunctionLdsBound() {
using namespace ShaderRecompiler;
const auto build = [](const auto& body) {
Expand Down Expand Up @@ -1776,6 +1843,7 @@ int main(int argc, char** argv) {
verifyWaveUniformValues();
verifyTwoLaneUniformValues();
verifyBdaReadFallbackFunctions();
verifyBdaWriteFallbackFunctions();
verifyFunctionLdsBound();
#if ANYPS5_ENABLE_SPIRV_TOOLS
const std::vector<std::uint32_t> minimalSpirv{
Expand Down
2 changes: 2 additions & 0 deletions core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition);
std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits);
void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u);
void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits);
void DefineBdaByteWriteFunctions(SpirvEmitterState& state);
bool BdaByteWritesForced();
std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function<std::uint32_t(std::uint32_t)>& operation);
// Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one
// table lookup for the whole span; the per-byte lookups of EmitBdaRead remain the fallback (and
Expand Down
2 changes: 2 additions & 0 deletions core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,9 @@ struct SpirvEmitterState {
std::uint32_t bdaProbeFunction = 0;
std::uint32_t bdaWritePointerFunction = 0;
std::uint32_t bdaAtomicPointerFunction = 0;
std::uint32_t bdaWriteProbeFunction = 0;
std::uint32_t bdaNoteWriteFunction = 0;
std::array<std::uint32_t, 2> bdaByteWriteFunctions {};
std::uint32_t bdaFaultFunction = 0;
std::array<std::array<std::uint32_t, 2>, 2> bdaDwordReadFunctions {};
std::uint32_t bdaStopValue = 0;
Expand Down
46 changes: 35 additions & 11 deletions core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,22 +130,44 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) {
EmitLabel(state, state.module.AllocateId());
const auto done = state.module.AllocateId();
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), done, spv::StorageClassFunction);
const auto counter = state.module.AllocateId();
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction);
state.module.AddFunction(spv::OpStore, done, ConstantBool(state, false));
state.module.AddFunction(spv::OpStore, counter, constant(0u));
const auto page = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(1));
const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, page, constant(0x9e3779b1u)), constant(32u - static_cast<std::uint32_t>(std::countr_zero(BdaAbi::WrittenPageSlots))));
const auto scope = constant(spv::ScopeDevice);
const auto relaxed = constant(spv::MemorySemanticsMaskNone);
for (std::uint32_t probe = 0; probe < BdaAbi::WrittenPageProbes; probe++) {
const auto pending = state.module.AllocateId();
state.module.AddFunction(spv::OpLoad, boolean, pending, done);
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, pending), [&] {
const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, constant(probe)), constant(BdaAbi::WrittenPageSlots - 1u)));
const auto previous = state.module.AllocateId();
state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u));
const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page));
state.module.AddFunction(spv::OpStore, done, taken);
});
}
const auto header = state.module.AllocateId();
const auto body = state.module.AllocateId();
const auto continuation = state.module.AllocateId();
const auto merge = state.module.AllocateId();
state.module.AddFunction(spv::OpBranch, header);
EmitLabel(state, header);
const auto probe = state.module.AllocateId();
state.module.AddFunction(spv::OpLoad, u32, probe, counter);
const auto found = state.module.AllocateId();
state.module.AddFunction(spv::OpLoad, boolean, found, done);
const auto searching = binary(spv::OpLogicalAnd, boolean, Unary(state, spv::OpLogicalNot, boolean, found), binary(spv::OpULessThan, boolean, probe, constant(BdaAbi::WrittenPageProbes)));
state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone);
state.module.AddFunction(spv::OpBranchConditional, searching, body, merge);
EmitLabel(state, body);
const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u)));
const auto current = state.module.AllocateId();
state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed);
const auto held = binary(spv::OpIEqual, boolean, current, page);
state.module.AddFunction(spv::OpStore, done, held);
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, held), [&] {
const auto previous = state.module.AllocateId();
state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u));
const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page));
state.module.AddFunction(spv::OpStore, done, taken);
});
state.module.AddFunction(spv::OpBranch, continuation);
EmitLabel(state, continuation);
state.module.AddFunction(spv::OpStore, counter, binary(spv::OpIAdd, u32, probe, constant(1u)));
state.module.AddFunction(spv::OpBranch, header);
EmitLabel(state, merge);
const auto noted = state.module.AllocateId();
state.module.AddFunction(spv::OpLoad, boolean, noted, done);
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, noted), [&] {
Expand All @@ -165,7 +187,9 @@ void DefineGetBdaPointer(SpirvEmitterState& state) {
if (state.program.Info().bdaWrites) {
state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write);
state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write);
if (!BdaByteWritesForced()) state.bdaWriteProbeFunction = DefineBdaLookup(state, "probe_bda_write_pointer", false, BdaAbi::Write);
state.bdaNoteWriteFunction = DefineBdaNoteWrite(state);
DefineBdaByteWriteFunctions(state);
}
}

Expand Down
97 changes: 84 additions & 13 deletions core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -226,30 +226,101 @@ void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t
});
}

void DefineBdaByteWriteFunctions(SpirvEmitterState& state) {
const auto u32 = TypeU32(state);
const auto u64 = TypeScalarU64(state);
const auto voidType = state.module.Type(spv::OpTypeVoid);
const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u);
const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType);
const auto type = state.module.Type(spv::OpTypeFunction, voidType, u64, u32, u32, u32);
const auto& memory = state.program.Resources().memoryInfo;
const bool coherentAccesses = std::any_of(memory.begin(), memory.end(), [](const MemoryInfo& info) { return info.coherent; });
for (const bool coherent : {false, true}) {
if (coherent && !coherentAccesses) continue;
const auto function = state.module.AllocateId();
state.module.AddName(function, std::string("write_bda_bytes") + (coherent ? "_coherent" : ""));
state.module.AddFunction(spv::OpFunction, voidType, function, spv::FunctionControlDontInlineMask, type);
const auto address = state.module.AllocateId();
const auto value = state.module.AllocateId();
const auto bytes = state.module.AllocateId();
const auto instruction = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionParameter, u64, address);
state.module.AddFunction(spv::OpFunctionParameter, u32, value);
state.module.AddFunction(spv::OpFunctionParameter, u32, bytes);
state.module.AddFunction(spv::OpFunctionParameter, u32, instruction);
EmitLabel(state, state.module.AllocateId());
const auto counter = state.module.AllocateId();
state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction);
state.module.AddFunction(spv::OpStore, counter, ConstantU32(state, 0u));
const auto header = state.module.AllocateId();
const auto body = state.module.AllocateId();
const auto continuation = state.module.AllocateId();
const auto merge = state.module.AllocateId();
state.module.AddFunction(spv::OpBranch, header);
EmitLabel(state, header);
const auto byte = state.module.AllocateId();
state.module.AddFunction(spv::OpLoad, u32, byte, counter);
const auto pending = Binary(state, spv::OpULessThan, TypeBool(state), byte, bytes);
state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone);
state.module.AddFunction(spv::OpBranchConditional, pending, body, merge);
EmitLabel(state, body);
const auto guest = Binary(state, spv::OpIAdd, u64, address, Unary(state, spv::OpUConvert, u64, byte));
const auto shifted = Binary(state, spv::OpShiftRightLogical, u32, value, Binary(state, spv::OpShiftLeftLogical, u32, byte, ConstantU32(state, 3u)));
EmitBdaStoreAt(state, guest, 1u, bytePointer, Unary(state, spv::OpUConvert, byteType, shifted), instruction, BdaAccessMask(coherent));
state.module.AddFunction(spv::OpBranch, continuation);
EmitLabel(state, continuation);
state.module.AddFunction(spv::OpStore, counter, Binary(state, spv::OpIAdd, u32, byte, ConstantU32(state, 1u)));
state.module.AddFunction(spv::OpBranch, header);
EmitLabel(state, merge);
state.module.AddFunction(spv::OpReturn);
state.module.AddFunction(spv::OpFunctionEnd);
state.bdaByteWriteFunctions[coherent] = function;
}
}

bool BdaByteWritesForced() {
static const bool forced = std::getenv("APS5_BDA_BYTE_WRITES") != nullptr;
return forced;
}
void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) {
auto& state = ctx.state;
if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA store width");
if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing");
const auto instruction = ConstantU32(state, inst.Flags<MemoryFlags>().pc);
const auto accessMask = BdaAccessMask(state, inst);
const auto storeBytes = [&] {
const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u);
const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType);
for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) {
const auto guest = byte == 0u ? address : Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte));
const auto shifted = byte == 0u ? value : Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, ConstantU32(state, byte * 8u));
EmitBdaStoreAt(state, guest, 1u, bytePointer, Unary(state, spv::OpUConvert, byteType, shifted), instruction, accessMask);
}
const auto function = state.bdaByteWriteFunctions[BdaCoherent(state, inst)];
if (function == 0) ctx.Fail(inst, "BDA byte write function is missing");
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), function, address, value, ConstantU32(state, bits / 8u), instruction);
};
if (bits != 32u) {
storeBytes();
return;
}
const auto unaligned = BdaAddressUnaligned(state, address);
EmitIfCondition(state, unaligned, storeBytes);
EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] {
EmitBdaStoreAt(state, address, 4u, TypePhysicalU32Pointer(state), value, instruction, accessMask);
});
const auto accessMask = BdaAccessMask(state, inst);
if (state.bdaWriteProbeFunction == 0) {
storeBytes();
return;
}
const auto physical = state.module.AllocateId();
state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaWriteProbeFunction, address, ConstantU32(state, 4u), instruction);
const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u));
const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), physical, BdaConstant(state, 3u)), BdaConstant(state, 0u));
const auto wide = Binary(state, spv::OpLogicalAnd, TypeBool(state), mapped, aligned);
const auto wideLabel = state.module.AllocateId();
const auto byteLabel = state.module.AllocateId();
const auto merge = state.module.AllocateId();
state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone);
state.module.AddFunction(spv::OpBranchConditional, wide, wideLabel, byteLabel);
EmitLabel(state, wideLabel);
const auto pointer = state.module.AllocateId();
state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, physical);
state.module.AddFunction(spv::OpStore, pointer, value, accessMask, 4u);
state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, address);
state.module.AddFunction(spv::OpBranch, merge);
EmitLabel(state, byteLabel);
storeBytes();
state.module.AddFunction(spv::OpBranch, merge);
EmitLabel(state, merge);
}

std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function<std::uint32_t(std::uint32_t)>& operation) {
Expand Down