perf(agc): store whole words of 1- and 2-byte texels in the detile shader

core/libs/prx/libSceAgcDriver/CMakeLists.txt

@@ -56,6 +56,28 @@ add_custom_command(
         DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
         VERBATIM
 )
+set(agcTextureDetileWordsShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileWords.spv)
+set(agcTextureDetileWordsShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileWords_spv.h)
+add_custom_command(
+        OUTPUT "${agcTextureDetileWordsShaderHeader}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+        COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DWORD_GROUPS=1 -o "${agcTextureDetileWordsShaderSpv}" "${agcTextureDetileShaderSource}"
+        COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileWordsShaderSpv} -DOUTPUT=${agcTextureDetileWordsShaderHeader}
+        -DSYMBOL=TEXTURE_DETILE_WORDS_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+        DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
+        VERBATIM
+)
+set(agcTextureDetileImageWordsShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImageWords.spv)
+set(agcTextureDetileImageWordsShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImageWords_spv.h)
+add_custom_command(
+        OUTPUT "${agcTextureDetileImageWordsShaderHeader}"
+        COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}"
+        COMMAND "$<TARGET_FILE:glslang-standalone>" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -DWORD_GROUPS=1 -o "${agcTextureDetileImageWordsShaderSpv}" "${agcTextureDetileShaderSource}"
+        COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageWordsShaderSpv} -DOUTPUT=${agcTextureDetileImageWordsShaderHeader}
+        -DSYMBOL=TEXTURE_DETILE_IMAGE_WORDS_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake"
+        DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone
+        VERBATIM
+)
 
 set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp)
 foreach(agcReadbackKind IN ITEMS Uint Depth)
@@ -373,6 +395,8 @@ set(agcDriverSources
         ${agcTextureDetileImageShaderHeader}
         ${agcTextureRepackShaderHeader}
         ${agcDispatchArgumentsShaderHeader}
+        ${agcTextureDetileWordsShaderHeader}
+        ${agcTextureDetileImageWordsShaderHeader}
         ${agcSampleCounterShaderHeader}
         ${agcSampleDumpsShaderHeader}
         ${agcPresentationShaderHeaders}

core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp

@@ -92,6 +92,8 @@ namespace AgcDriver::Graphics {
         VkShaderModule imageModule = VK_NULL_HANDLE;
         VkShaderModule repackModule = VK_NULL_HANDLE;
         VkPipeline repackPipeline = VK_NULL_HANDLE;
+        VkShaderModule imageWordModule = VK_NULL_HANDLE;
+        VkShaderModule wordModule = VK_NULL_HANDLE;
         DescriptorRing imageDescriptorSets;
         VkDescriptorSetLayout readbackDescriptorLayout = VK_NULL_HANDLE;
         VkPipelineLayout readbackPipelineLayout = VK_NULL_HANDLE;

core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp

@@ -207,34 +207,27 @@ void copyElement(uint src, uint dst) {
 }
 #endif
 
-void main() {
-    uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin);
-    if (p.x >= params.width || p.y >= params.height) {
-        return;
-    }
-
+bool locate(uvec2 p, out uint src, out uint dst, out uint ranged) {
     uint linearOffset = p.y * params.pitchBytes + p.x * ELEMENT_BYTES;
 
     if (TILE_FAMILY == 0u) {
-        // kLinear: guest and host layouts match; a straight copy at the
-        // same pitch, no swizzle.
+        ranged = linearOffset;
         if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) {
-            return;
+            return false;
         }
 #ifdef LINEAR_IMAGE
-        if (RETILE != 0u) {
-            loadElement(p, params.dstBase + linearOffset - params.tiledBase);
-        } else {
-            storeElement(params.srcBase + linearOffset - params.tiledBase, p);
-        }
+        src = params.srcBase + linearOffset - params.tiledBase;
+        dst = params.dstBase + linearOffset - params.tiledBase;
 #else
         if (RETILE != 0u) {
-            copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase);
+            src = params.srcBase + linearOffset - params.linearBase;
+            dst = params.dstBase + linearOffset - params.tiledBase;
         } else {
-            copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase);
+            src = params.srcBase + linearOffset - params.tiledBase;
+            dst = params.dstBase + linearOffset - params.linearBase;
         }
 #endif
-        return;
+        return true;
     }
 
     uvec2 swizzle = p;
@@ -251,24 +244,106 @@ void main() {
         uint split = equationBit(SPLIT0, p) | (equationBit(SPLIT1, p) << 1) | (equationBit(SPLIT2, p) << 2) | (equationBit(SPLIT3, p) << 3);
         tiledOffset += split * params.splitStride;
     }
+    ranged = tiledOffset;
     if (tiledOffset < params.rangeBegin || tiledOffset >= params.rangeEnd) {
-        return;
+        return false;
     }
     tiledOffset -= params.tiledBase;
     linearOffset -= params.linearBase;
 
 #ifdef LINEAR_IMAGE
+    src = params.srcBase + tiledOffset;
+    dst = params.dstBase + tiledOffset;
+#else
     if (RETILE != 0u) {
-        loadElement(p, params.dstBase + tiledOffset);
+        src = params.srcBase + linearOffset;
+        dst = params.dstBase + tiledOffset;
     } else {
-        storeElement(params.srcBase + tiledOffset, p);
+        src = params.srcBase + tiledOffset;
+        dst = params.dstBase + linearOffset;
     }
-#else
+#endif
+    return true;
+}
+
+void moveElement(uvec2 p) {
+    uint src;
+    uint dst;
+    uint ranged;
+    if (!locate(p, src, dst, ranged)) {
+        return;
+    }
+#ifdef LINEAR_IMAGE
     if (RETILE != 0u) {
-        copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset);
+        loadElement(p, dst);
     } else {
-        copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset);
+        storeElement(src, p);
     }
+#else
+    copyElement(src, dst);
 #endif
 }
 
+#ifdef WORD_GROUPS
+shared uint groupWords[32];
+shared uint groupLanes[32];
+shared uint groupDestinations[64];
+
+void main() {
+    uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin);
+    bool inside = p.x < params.width && p.y < params.height;
+    uint lane = gl_LocalInvocationID.y * gl_WorkGroupSize.x + gl_LocalInvocationID.x;
+    uint count = 4u / ELEMENT_BYTES;
+    uint slot = lane / count;
+    uint index = lane % count;
+    uint src = 0u;
+    uint dst = 0u;
+    uint ranged = 0u;
+    bool valid = inside && locate(p, src, dst, ranged);
+    if (index == 0u) {
+        groupWords[slot] = 0u;
+        groupLanes[slot] = 0u;
+    }
+    groupDestinations[lane] = dst;
+    barrier();
+    if (valid) {
+        uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu;
+#ifdef LINEAR_IMAGE
+        uint value = imageLoad(linearImage, ivec2(p)).x & mask;
+#else
+        uint value = (inputBuffer.data[src >> 2] >> ((src & 3u) * 8u)) & mask;
+#endif
+        atomicOr(groupWords[slot], value << (index * ELEMENT_BYTES * 8u));
+        atomicOr(groupLanes[slot], 1u << index);
+    }
+    barrier();
+    uint first = groupDestinations[slot * count];
+    bool whole = groupLanes[slot] == (1u << count) - 1u && (first & 3u) == 0u;
+    for (uint i = 1u; i < count; ++i) {
+        whole = whole && groupDestinations[slot * count + i] == first + i * ELEMENT_BYTES;
+    }
+    if (whole) {
+        if (index == 0u) {
+#ifdef LINEAR_IMAGE
+            tiledBuffer.data[first >> 2] = groupWords[slot];
+#else
+            outputBuffer.data[first >> 2] = groupWords[slot];
+#endif
+        }
+    } else if (valid) {
+#ifdef LINEAR_IMAGE
+        loadElement(p, dst);
+#else
+        copyElement(src, dst);
+#endif
+    }
+}
+#else
+void main() {
+    uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin);
+    if (p.x >= params.width || p.y >= params.height) {
+        return;
+    }
+    moveElement(p);
+}
+#endif

core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp

@@ -6,6 +6,8 @@
 #include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h"
 #include "prx/libSceAgcDriver/Graphics/shaders/TextureRepack_spv.h"
+#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImageWords_spv.h"
+#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileWords_spv.h"
 #include <algorithm>
 #include <array>
 #include <bit>
@@ -122,6 +124,8 @@ void TextureDetiler::release() noexcept {
     if (imageModule) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, imageModule, nullptr);
     if (repackPipeline) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, repackPipeline, nullptr);
     if (repackModule) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, repackModule, nullptr);
+    if (imageWordModule) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, imageWordModule, nullptr);
+    if (wordModule) context.Function<PFN_vkDestroyShaderModule>("vkDestroyShaderModule")(context.device, wordModule, nullptr);
     if (imagePipelineLayout) context.Function<PFN_vkDestroyPipelineLayout>("vkDestroyPipelineLayout")(context.device, imagePipelineLayout, nullptr);
     if (imageDescriptorLayout) context.Function<PFN_vkDestroyDescriptorSetLayout>("vkDestroyDescriptorSetLayout")(context.device, imageDescriptorLayout, nullptr);
     for (const auto& entry : pipelines) context.Function<PFN_vkDestroyPipeline>("vkDestroyPipeline")(context.device, entry.second, nullptr);
@@ -176,8 +180,12 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem
         values[2] = 2u;
         std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4);
     }
-    if (image && imageModule == VK_NULL_HANDLE) {
-        imageModule = CreateShaderModule(context, TEXTURE_DETILE_IMAGE_SPV, sizeof(TEXTURE_DETILE_IMAGE_SPV), "vkCreateShaderModule");
+    const bool words = elementBytes < 4u && (image ? retile && values[2] != 2u : !retile || values[2] != 2u);
+    auto& shader = image ? (words ? imageWordModule : imageModule) : (words ? wordModule : module);
+    if (shader == VK_NULL_HANDLE) {
+        if (!image) shader = CreateShaderModule(context, TEXTURE_DETILE_WORDS_SPV, sizeof(TEXTURE_DETILE_WORDS_SPV), "vkCreateShaderModule detile words");
+        else if (words) shader = CreateShaderModule(context, TEXTURE_DETILE_IMAGE_WORDS_SPV, sizeof(TEXTURE_DETILE_IMAGE_WORDS_SPV), "vkCreateShaderModule detile image words");
+        else shader = CreateShaderModule(context, TEXTURE_DETILE_IMAGE_SPV, sizeof(TEXTURE_DETILE_IMAGE_SPV), "vkCreateShaderModule");
     }
     std::array<VkSpecializationMapEntry, 26> entries{};
     for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4};
@@ -188,7 +196,7 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem
     specialization.pData = values.data();
     VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO};
     stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
-    stage.module = image ? imageModule : module;
+    stage.module = shader;
     stage.pName = "main";
     stage.pSpecializationInfo = &specialization;
     VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};