From 438978e2a4dcfffbbfdd2a26023f1969520f13a6 Mon Sep 17 00:00:00 2001 From: Mehran Date: Fri, 16 Oct 2026 13:22:13 +0300 Subject: [PATCH] perf(shader): keep a binding's element flags inline --- .../libSceAgcDriver/Graphics/src/Pipeline.cpp | 2 +- .../Graphics/src/ShaderResources.cpp | 4 +- .../prx/libSceAgcDriver/tests/Graphics.cpp | 2 +- .../prx/libSceAgcDriver/tests/Recorder.cpp | 2 +- .../libSceAgcDriver/tests/ShaderDiskCache.cpp | 35 ++++ core/shader/recompiler/Recompiler.hpp | 150 ++++++++++++++++-- core/shader/recompiler/ShaderDiskCache.cpp | 10 +- 7 files changed, 182 insertions(+), 23 deletions(-) diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 470b77f6f3df..b7288b27860a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -220,7 +220,7 @@ void ValidateViewport(const Context& context, const VkViewport& viewport) { } bool ShadersWriteMemory(std::span shaders) { - const auto any = [](const std::vector& written, std::uint32_t count) { return written.size() < count || std::ranges::any_of(written, [](bool element) { return element; }); }; + const auto any = [](const ShaderRecompiler::ElementFlags& written, std::uint32_t count) { return written.size() < count || std::ranges::any_of(written, [](bool element) { return element; }); }; for (const auto& shader : shaders) { if (shader.program == nullptr) continue; for (const auto& binding : shader.program->bindings) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index 231d471add73..9067f1d26814 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -1828,7 +1828,7 @@ struct ReusedTable { std::uint64_t pendingSerial = 0; std::vector words; std::vector shapes; - std::vector depthCompare; + ShaderRecompiler::ElementFlags depthCompare; std::vector elements; std::vector queries; std::vector> ranges; @@ -2644,7 +2644,7 @@ void VisitContentKey(const CompiledShader& shader, bool dataWords, bool movableB visit(static_cast(program.PipelineVariantId())); visit(static_cast(program.PipelineVariantId() >> 32u)); visit(static_cast(program.bindings.size())); - const auto packBits = [&](const std::vector& bits) { + const auto packBits = [&](const ShaderRecompiler::ElementFlags& bits) { visit(static_cast(bits.size())); std::uint32_t word = 0; for (std::size_t i = 0; i < bits.size(); ++i) { diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index 39a2b17eb853..ba11b4a130c3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1947,7 +1947,7 @@ void TuningFieldTests() { } void ShadersWriteMemoryTests() { - const auto binding = [](ShaderRecompiler::DescriptorKind kind, ShaderRecompiler::DescriptorRole role, std::vector written) { + const auto binding = [](ShaderRecompiler::DescriptorKind kind, ShaderRecompiler::DescriptorRole role, ShaderRecompiler::ElementFlags written) { ShaderRecompiler::DescriptorBinding result{}; result.kind = kind; result.role = role; diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index cf83ce9e23f6..408f484787f6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -8321,7 +8321,7 @@ void imageMemoTests(const Device& device, Recorder& recorder) { Require(replaced != other && sample(second) == replaced && sample(first) == remade, "the cache does not answer with the replacements"); const auto firstWords = linearTextureWords(reinterpret_cast(first), width, height, 56); const auto secondWords = linearTextureWords(reinterpret_cast(second), width, height, 56); - const auto bindElements = [&](std::initializer_list> elements, std::vector depthCompare) { + const auto bindElements = [&](std::initializer_list> elements, ShaderRecompiler::ElementFlags depthCompare) { program.bindings[0].count = static_cast(elements.size()); program.bindings[0].guestDescriptor.clear(); for (const auto& words : elements) program.bindings[0].guestDescriptor.insert(program.bindings[0].guestDescriptor.end(), words.begin(), words.end()); diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 44299ca46d7e..c208bb8b490b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -431,6 +431,40 @@ void verifyResultRoundTrip() { require(!ShaderDiskCache::DecodeResult(bytes, decoded), "an incompatible runtime ABI result decodes"); } +void verifyElementFlags() { + ElementFlags flags; + std::vector expected; + for (std::size_t index = 0; index < 70; ++index) { + flags.push_back(index % 3 == 0); + expected.push_back(index % 3 == 0); + require(flags == expected, "element flags differ from a vector after " + std::to_string(index + 1) + " entries"); + } + const auto copy = flags; + require(copy == flags && copy.size() == 70 && copy.at(69) && !copy.at(68), "a copy of spilled element flags differs"); + require(std::count(flags.begin(), flags.end(), true) == 24, "spilled element flags do not iterate"); + auto writable = flags; + writable[69] = false; + writable.back().flip(); + writable[0] = writable[1]; + require(writable.size() == 70 && writable.back() && !writable[0] && writable[3], "a write through a spilled flag was lost"); + writable.resize(3); + require(writable == std::vector{false, false, false}, "shrinking spilled element flags kept the wrong entries"); + writable.resize(5, true); + writable[1] = true; + writable.back().flip(); + require(writable == std::vector{false, true, false, true, false}, "a write through an inline flag was lost"); + auto result = sampleResult(); + result.bindings[1].bufferWritten = flags; + std::vector bytes; + ShaderDiskCache::EncodeResult(result, bytes); + RecompileResult decoded; + require(ShaderDiskCache::DecodeResult(bytes, decoded) && decoded.bindings[1].bufferWritten == expected, "spilled element flags do not round trip"); + flags.assign(3, true); + require(flags == std::vector{true, true, true}, "assigned element flags kept spilled entries"); + flags.clear(); + require(flags.empty() && flags == ElementFlags{}, "cleared element flags are not empty"); +} + void verifyEntryRoundTrip() { SampleRequest sample; const auto key = sample.Key(); @@ -2501,6 +2535,7 @@ int main(int argc, char** argv) { setEnvironment("ANYPS5_SHADER_CACHE_DIR", directory.string()); verifyAtomicReplacement(directory); verifyResultRoundTrip(); + verifyElementFlags(); verifyEntryRoundTrip(); verifyPrebuiltPack(directory / "packs"); verifyArtifactStorageIsolation(); diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 01a51aef33f1..6605559416a0 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -10,10 +10,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -475,6 +477,128 @@ enum class DescriptorRole { ImageStorageTable }; +class ElementFlags { +public: + class reference { + public: + reference(ElementFlags& flags, std::size_t index) : flags(flags), index(index) {} + operator bool() const { return static_cast(flags)[index]; } + reference& operator=(bool flag) { + flags.set(index, flag); + return *this; + } + reference& operator=(const reference& other) { return *this = static_cast(other); } + void flip() { flags.set(index, !static_cast(*this)); } + + private: + ElementFlags& flags; + std::size_t index; + }; + + class const_iterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = bool; + using difference_type = std::ptrdiff_t; + using pointer = void; + using reference = bool; + const_iterator() = default; + const_iterator(const ElementFlags* flags, std::size_t index) : flags(flags), index(index) {} + bool operator*() const { return (*flags)[index]; } + const_iterator& operator++() { + ++index; + return *this; + } + const_iterator operator++(int) { + auto previous = *this; + ++index; + return previous; + } + bool operator==(const const_iterator& other) const { return index == other.index; } + bool operator!=(const const_iterator& other) const { return index != other.index; } + + private: + const ElementFlags* flags = nullptr; + std::size_t index = 0; + }; + + ElementFlags() = default; + ElementFlags(std::initializer_list flags) { + for (const bool flag : flags) push_back(flag); + } + std::size_t size() const { return count; } + bool empty() const { return count == 0; } + bool operator[](std::size_t index) const { return count <= InlineFlags ? ((bits >> index) & 1u) != 0 : spill[index]; } + reference operator[](std::size_t index) { return {*this, index}; } + bool back() const { return (*this)[count - 1]; } + reference back() { return {*this, count - 1}; } + bool at(std::size_t index) const { + if (index >= count) throw std::out_of_range("ElementFlags::at"); + return (*this)[index]; + } + void push_back(bool flag) { + if (count < InlineFlags) { + if (flag) bits |= std::uint64_t{1} << count; + ++count; + return; + } + if (count == InlineFlags) { + for (std::size_t index = 0; index < InlineFlags; ++index) spill.push_back(((bits >> index) & 1u) != 0); + } + spill.push_back(flag); + ++count; + } + void clear() { + bits = 0; + count = 0; + spill.clear(); + } + void assign(std::size_t size, bool flag) { + clear(); + for (std::size_t index = 0; index < size; ++index) push_back(flag); + } + void resize(std::size_t size, bool flag = false) { + if (size < count) { + ElementFlags kept; + for (std::size_t index = 0; index < size; ++index) kept.push_back((*this)[index]); + *this = std::move(kept); + return; + } + while (count < size) push_back(flag); + } + const_iterator begin() const { return {this, 0}; } + const_iterator end() const { return {this, count}; } + bool operator==(const ElementFlags& other) const { + if (count != other.count) return false; + for (std::size_t index = 0; index < count; ++index) { + if ((*this)[index] != other[index]) return false; + } + return true; + } + bool operator==(const std::vector& other) const { + if (count != other.size()) return false; + for (std::size_t index = 0; index < count; ++index) { + if ((*this)[index] != other[index]) return false; + } + return true; + } + +private: + void set(std::size_t index, bool flag) { + if (count > InlineFlags) { + spill[index] = flag; + return; + } + const auto bit = std::uint64_t{1} << index; + bits = flag ? bits | bit : bits & ~bit; + } + + static constexpr std::size_t InlineFlags = 64; + std::uint64_t bits = 0; + std::size_t count = 0; + std::vector spill; +}; + struct DescriptorBinding { DescriptorKind kind; DescriptorRole role; @@ -484,30 +608,30 @@ struct DescriptorBinding { std::vector guestDescriptor; bool readOnly = false; std::optional imageShape; - std::vector samplerDepthCompare; + ElementFlags samplerDepthCompare; // Guest image elements the shader stores to (or updates atomically); the others are only read. - std::vector imageWritten; - std::vector imageDepthCompare; - std::vector imageAtomic; - std::vector imageAtomic64; + ElementFlags imageWritten; + ElementFlags imageDepthCompare; + ElementFlags imageAtomic; + ElementFlags imageAtomic64; std::vector imageShapes; // Guest buffer elements the shader updates atomically (one entry per element of a GuestBuffers // binding, empty otherwise). An atomic on a host-imported range is a serialized PCIe round trip // (~0.4-0.5 us each on NVIDIA), so a driver may keep these elements in device-local memory. - std::vector bufferAtomic; + ElementFlags bufferAtomic; // Guest buffer elements the shader may store to through this V# (any store or atomic in the // program, whatever its offset), one entry per element of a GuestBuffers binding, empty // otherwise. A false entry is proved: every access of that element is a load. A driver may then // skip the write-back and the pending-write note for the element; an element beyond the vector // (a producer that does not fill it) must be treated as written. - std::vector bufferWritten; - std::vector samplerUnnormalized; - std::vector samplerIntegerBorder; - std::vector samplerCoordinatesIgnored; - std::vector imageUnnormalized; - std::vector imageUnnormalizedBaseLevel; + ElementFlags bufferWritten; + ElementFlags samplerUnnormalized; + ElementFlags samplerIntegerBorder; + ElementFlags samplerCoordinatesIgnored; + ElementFlags imageUnnormalized; + ElementFlags imageUnnormalizedBaseLevel; std::vector imageSamplers; - std::vector bufferRead; + ElementFlags bufferRead; std::vector bufferExtent; }; diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 481c5d232bd1..b20c44870ca4 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -88,7 +88,7 @@ namespace ShaderRecompiler::ShaderDiskCache { static_assert(sizeof(CompiledShaderArtifact) == 280, "CompiledShaderArtifact changed: update the artifact encoder"); static_assert(sizeof(ShaderInvocation) == 216, "ShaderInvocation changed: update the invocation encoder"); static_assert(sizeof(RecompileResult) == 544, "RecompileResult changed: update EncodeResult and DecodeResult"); -static_assert(sizeof(DescriptorBinding) == 656, "DescriptorBinding changed: update the binding encoder"); +static_assert(sizeof(DescriptorBinding) == 864, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 16, "FragmentParameter changed: update the parameter encoder"); @@ -258,7 +258,7 @@ class Writer { out.insert(out.end(), bytes.begin(), bytes.end()); } - void Flags(const std::vector& flags) { + void Flags(const ElementFlags& flags) { Value(flags.size()); for (const bool flag : flags) Value(flag); } @@ -359,10 +359,10 @@ class Reader { return SharedSpirv(std::move(words)); } - void Flags(std::vector& flags) { + void Flags(ElementFlags& flags) { const auto count = Count(1); - flags.assign(count, false); - for (std::size_t i = 0; i < count; ++i) flags[i] = Get(); + flags.clear(); + for (std::size_t i = 0; i < count; ++i) flags.push_back(Get()); } void Text(std::string& text) {