fix(shader): gather a 3D image's image_gather4_lz from one slice as measured

core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp

@@ -59,6 +59,16 @@ inline constexpr std::uint32_t ReferenceZero = 3u;
 [[nodiscard]] inline std::uint32_t Reference(std::uint32_t state) { return (state >> ReferenceShift) & 0x3u; }
 }
 
+namespace VolumeGather {
+inline constexpr std::uint32_t ClampXShift = 0u;
+inline constexpr std::uint32_t ClampYShift = 3u;
+inline constexpr std::uint32_t ClampZShift = 6u;
+inline constexpr std::uint32_t BorderShift = 9u;
+inline constexpr std::uint32_t BorderTransparentBlack = 0u;
+inline constexpr std::uint32_t BorderOpaqueBlack = 1u;
+inline constexpr std::uint32_t BorderOpaqueWhite = 2u;
+}
+
 inline constexpr std::uint32_t NoTable = std::numeric_limits<std::uint32_t>::max();
 
 enum class TableOperation : std::uint8_t { Supported, Unsupported };
@@ -106,6 +116,7 @@ struct ImageResource {
     bool layerRows = false;
     std::uint32_t maxArrayLayers = 0u;
     bool optionalDescriptor = false;
+    bool volumeGather = false;
     std::uint32_t byElements = 0;
     std::uint32_t byComponents = 0;
     IrBufferFormat packedFormat = IrBufferFormat::Invalid;
@@ -164,6 +175,7 @@ struct SampledResourcePair {
     std::uint32_t image = 0;
     std::uint32_t sampler = 0;
     std::uint32_t firstUsePc = 0;
+    bool volumeGather = false;
 
     bool operator==(const SampledResourcePair& other) const = default;
 };

core/shader/recompiler/Optimization/include/Optimization/ResourceMaterializer.hpp

@@ -74,6 +74,7 @@ class LayerRowSamplesRequired : public std::runtime_error {
 class ResourceMaterializer {
 public:
     static std::uint32_t EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index);
+    static std::uint32_t VolumeGatherState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index);
     void ApplyStaticInterface(IrProgram& program, bool nativeSampleOffsets = true) const;
     static std::vector<ImageResource> RuntimeImageModes(const ImageResource& image);
     static RuntimeImageModeMatch TryRuntimeImageMode(const ImageResource& image, const DescriptorValue& descriptor, std::span<const ImageResource> modes);

core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp

@@ -402,6 +402,12 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l
     std::vector<std::uint32_t> compareStates(info.images.size());
     for (std::uint32_t index = 0; index < info.images.size(); ++index) {
         const bool convertedSample = !info.images[index].srgbDecodeCompatible && index < info.runtimeImageModes.size() && std::ranges::any_of(info.runtimeImageModes.at(index), [](const ImageResource& mode) { return mode.conversionFormat == IrBufferFormat::Format11_11_10UNorm; });
+        if (info.images[index].volumeGather) {
+            if (info.images[index].depthCompare || convertedSample) fail("a 3D gather of a compared or converted image is not implemented");
+            compareStates[index] = ResourceMaterializer::VolumeGatherState(info, snapshot, index);
+            if (!registrationModules) plan.specialization.push_back({PipelineSpecialization::VolumeGatherBase + index, compareStates[index]});
+            continue;
+        }
         if (!info.images[index].depthCompare && !convertedSample) continue;
         const auto compare = compareStates[index] = ResourceMaterializer::EmulatedCompareState(info, snapshot, index);
         const auto first = PipelineSpecialization::CompareBase + index * PipelineSpecialization::CompareWords;

core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp

@@ -2379,6 +2379,35 @@ std::uint32_t ResourceMaterializer::RuntimeImageMode(const ImageResource& image,
     return *match.mode;
 }
 
+std::uint32_t ResourceMaterializer::VolumeGatherState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) {
+    const auto& image = info.images[index];
+    const auto& descriptor = snapshot.images[index];
+    if (image.table != NoTable) throw std::runtime_error("a 3D gather of an image table entry is not implemented");
+    if (descriptor.dwordCount != 8u) throw std::runtime_error("a 3D gather of an image without an 8-dword descriptor is not implemented");
+    if (nullImageDescriptor(image, descriptor)) return 0u;
+    std::optional<std::uint32_t> samplerState;
+    for (const auto& pair : info.sampledPairs) {
+        if (pair.image != index || !pair.volumeGather) continue;
+        if (pair.sampler >= info.samplers.size() || info.samplers[pair.sampler].table != NoTable) throw std::runtime_error("a 3D gather through a sampler table is not implemented");
+        if (pair.sampler >= snapshot.samplers.size() || snapshot.samplers[pair.sampler].dwordCount != 4u) throw std::runtime_error("a 3D gather has no sampler descriptor");
+        const auto& words = snapshot.samplers[pair.sampler].dwords;
+        if (((words[0] >> 15u) & 0x1u) != 0u) throw std::runtime_error("a 3D gather with unnormalized coordinates is not implemented");
+        const auto clampX = words[0] & 0x7u;
+        const auto clampY = (words[0] >> 3u) & 0x7u;
+        const auto clampZ = (words[0] >> 6u) & 0x7u;
+        const auto border = (words[3] >> 30u) & 0x3u;
+        const bool bordered = clampX >= 4u || clampY >= 4u || clampZ >= 6u;
+        if (bordered && border == 3u) throw std::runtime_error("a 3D gather with a border color table is not implemented");
+        if (bordered && (image.numericClass == IrTextureNumericClass::Uint || image.numericClass == IrTextureNumericClass::Sint)) throw std::runtime_error("a 3D gather of an integer texture with border addressing is not implemented");
+        if (bordered && descriptorImageSwizzle(descriptor) != ShaderImageIdentitySwizzle) throw std::runtime_error("a 3D gather of a swizzled texture with border addressing is not implemented");
+        const auto state = (clampX << VolumeGather::ClampXShift) | (clampY << VolumeGather::ClampYShift) | (clampZ << VolumeGather::ClampZShift) | ((bordered ? border : 0u) << VolumeGather::BorderShift);
+        if (samplerState.has_value() && *samplerState != state) throw std::runtime_error("a 3D gather through samplers that disagree is not implemented");
+        samplerState = state;
+    }
+    if (!samplerState.has_value()) throw std::runtime_error("a 3D gather has no paired sampler");
+    return *samplerState;
+}
+
 std::uint32_t ResourceMaterializer::EmulatedCompareState(const ShaderInfo& info, const ResourceSnapshot& snapshot, std::uint32_t index) {
     return emulatedCompareState(info, snapshot, index);
 }

core/shader/recompiler/Optimization/src/ResourceTracker.cpp

@@ -2313,6 +2313,7 @@ class Tracker {
         const bool flatVolumeLoad = op == IrOpcode::ImageRead && memory.imageSampleFlags == 0u;
         const bool flatVolumeSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero;
         const bool flatVolumeQuery = op == IrOpcode::ImageQueryDimensions;
+        image.volumeGather = image.volumeGather || (op == IrOpcode::ImageGatherRaw && image.dimension == RdnaImageDimension::Dim3D);
         image.flatVolumeCompatible = image.flatVolumeCompatible && (flatVolumeLoad || flatVolumeSample || flatVolumeQuery) && !memory.imagePacked && memory.imageByElements == 0u;
         const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u;
         const auto lineSampleFlags = memory.imageSampleFlags & ~RdnaImageSampleFlagCompare;
@@ -2350,17 +2351,18 @@ class Tracker {
         return static_cast<std::uint32_t>(m_info.samplers.size() - 1);
     }
 
-    void AddSampledPair(std::uint32_t image, std::uint32_t sampler, std::uint32_t pc) {
+    void AddSampledPair(std::uint32_t image, std::uint32_t sampler, std::uint32_t pc, bool volumeGather) {
         for (auto& pair : m_info.sampledPairs) {
             if (pair.image == image && pair.sampler == sampler) {
                 pair.firstUsePc = std::min(pair.firstUsePc, pc);
+                pair.volumeGather = pair.volumeGather || volumeGather;
                 return;
             }
         }
         if (m_info.sampledPairs.size() >= ShaderInfo::MaxSampledPairs) {
             fail("sampled image/sampler pair limit exceeded");
         }
-        m_info.sampledPairs.push_back({image, sampler, pc});
+        m_info.sampledPairs.push_back({image, sampler, pc, volumeGather});
     }
 
     void AddHandlePatch(IrValue* handle, std::uint32_t resource) {
@@ -2489,7 +2491,7 @@ class Tracker {
                 fail("sampler resource limit exceeded");
             }
             AddHandlePatch(samplerHandle, sampler);
-            AddSampledPair(resource, sampler, flags.pc);
+            AddSampledPair(resource, sampler, flags.pc, op == IrOpcode::ImageGatherRaw && m_info.images[resource].dimension == RdnaImageDimension::Dim3D);
         }
         AddMemoryPatch(flags.index, resource, sampler, imageInfo.needsSampler);
     }

core/shader/recompiler/PipelineSpecialization.hpp

@@ -50,6 +50,7 @@ inline constexpr std::uint32_t UnnormalizedBaseLevelBase = 57408u;
 inline constexpr std::uint32_t FmaskFormatBase = 57472u;
 inline constexpr std::uint32_t MirrorUnnormalizedBase = 57536u;
 inline constexpr std::uint32_t MinLodBase = 57600u;
+inline constexpr std::uint32_t VolumeGatherBase = 57664u;
 
 inline constexpr std::uint32_t DescriptorIndex(std::uint32_t binding, std::uint32_t element) {
     return DescriptorIndexBase + binding * DescriptorIndexStride + element;

core/shader/recompiler/ShaderDiskCache.cpp

@@ -99,7 +99,7 @@ static_assert(sizeof(ShaderInfo) == 232, "ShaderInfo changed: update the info en
 static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder");
 static_assert(sizeof(ImageResource) == 108, "ImageResource changed: update the info encoder");
 static_assert(sizeof(SamplerResource) == 24, "SamplerResource changed: update the info encoder");
-static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder");
+static_assert(sizeof(SampledResourcePair) == 16, "SampledResourcePair changed: update the info encoder");
 static_assert(sizeof(StageInput) == 56, "StageInput changed: update the info encoder");
 static_assert(sizeof(StageOutput) == 48, "StageOutput changed: update the info encoder");
 static_assert(sizeof(IrBindingLayout) == 80, "IrBindingLayout changed: update the layout encoder");
@@ -143,9 +143,9 @@ constexpr std::size_t fieldCount() {
 
 static_assert(fieldCount<ShaderInfo>() == 24, "ShaderInfo changed: update the info encoder");
 static_assert(fieldCount<BufferResource>() == 12, "BufferResource changed: update the info encoder");
-static_assert(fieldCount<ImageResource>() == 49, "ImageResource changed: update the info encoder");
+static_assert(fieldCount<ImageResource>() == 50, "ImageResource changed: update the info encoder");
 static_assert(fieldCount<SamplerResource>() == 8, "SamplerResource changed: update the info encoder");
-static_assert(fieldCount<SampledResourcePair>() == 3, "SampledResourcePair changed: update the info encoder");
+static_assert(fieldCount<SampledResourcePair>() == 4, "SampledResourcePair changed: update the info encoder");
 static_assert(fieldCount<StageInput>() == 5, "StageInput changed: update the info encoder");
 static_assert(fieldCount<StageOutput>() == 4, "StageOutput changed: update the info encoder");
 static_assert(fieldCount<IrResourcePlan>() == 25, "IrResourcePlan changed: update EncodePlan and DecodePlan");
@@ -819,6 +819,7 @@ void encodeShaderInfo(Writer& writer, const ShaderInfo& info) {
         out.Value(image.layerRows);
         out.Value(image.maxArrayLayers);
         out.Value(image.optionalDescriptor);
+        out.Value(image.volumeGather);
         out.Value(image.byElements);
         out.Value(image.byComponents);
         out.Value(image.packedFormat);
@@ -842,6 +843,7 @@ void encodeShaderInfo(Writer& writer, const ShaderInfo& info) {
         out.Value(pair.image);
         out.Value(pair.sampler);
         out.Value(pair.firstUsePc);
+        out.Value(pair.volumeGather);
     });
     writer.List(info.inputs, [](Writer& out, const StageInput& input) {
         out.Value(input.kind);
@@ -933,6 +935,7 @@ void decodeShaderInfo(Reader& reader, ShaderInfo& info) {
         in.Value(image.layerRows);
         in.Value(image.maxArrayLayers);
         in.Value(image.optionalDescriptor);
+        in.Value(image.volumeGather);
         in.Value(image.byElements);
         in.Value(image.byComponents);
         in.Value(image.packedFormat);
@@ -954,10 +957,11 @@ void decodeShaderInfo(Reader& reader, ShaderInfo& info) {
         in.Value(sampler.foldTexelOffsetsOnly);
         in.Value(sampler.table);
     });
-    reader.List(info.sampledPairs, 12, [](Reader& in, SampledResourcePair& pair) {
+    reader.List(info.sampledPairs, 13, [](Reader& in, SampledResourcePair& pair) {
         in.Value(pair.image);
         in.Value(pair.sampler);
         in.Value(pair.firstUsePc);
+        in.Value(pair.volumeGather);
     });
     reader.List(info.inputs, 21, [](Reader& in, StageInput& input) {
         in.Value(input.kind);