ControlFlowInstructions.cpp

#include "Translation/ControlFlowInstructions.hpp"
#include "Translation/TranslationContext.hpp"
#include <array>
#include <cstdint>
#include <cstdlib>
#include <stdexcept>
#include <string>

namespace ShaderRecompiler {

void TranslateControlFlowInstruction(IrBuilder& builder, const RdnaInstruction& instruction, const ControlFlowGraph& cfg) {
    throw std::runtime_error("TranslateControlFlowInstruction not implemented");
}

void TranslationContext::sSubvectorLoop(const RdnaInstruction& inst, bool begin) {
    if (program.WaveSize() != 64u) {
        throw std::runtime_error(std::string(begin ? "s_subvector_loop_begin" : "s_subvector_loop_end") + " in a wave" + std::to_string(program.WaveSize()) + " shader is not measured, only wave64 subvector loops are supported");
    }
    if (inst.destination.kind != RdnaOperandKind::ScalarRegister) {
        throw std::runtime_error(std::string(begin ? "s_subvector_loop_begin" : "s_subvector_loop_end") + " with a destination that is not an SGPR is not measured");
    }
    const IrU32 execLo(ir.GetExecLo());
    const IrU32 execHi(ir.GetExecHi());
    const IrU32 counter = readRawU32(inst.destination);
    const IrU32 zero(ir.Constant(0u));
    IrU32 newExecLo = execLo;
    IrU32 newExecHi = execHi;
    IrU32 newCounter = counter;
    if (begin) {
        const IrU1 lowZero(ir.IEqual(execLo.Value(), zero.Value()));
        const IrU1 execZero(ir.IEqual(ir.Emit(IrOpcode::BitwiseOr32, IrType::U32, {&execLo.Value(), &execHi.Value()}), zero.Value()));
        const IrU32 entered(ir.Select(lowZero.Value(), execLo.Value(), execHi.Value()));
        newCounter = IrU32(ir.Select(execZero.Value(), counter.Value(), entered.Value()));
        newExecHi = IrU32(ir.Select(ir.LogicalOr(execZero.Value(), lowZero.Value()), execHi.Value(), zero.Value()));
        instructionBranchCondition = execZero;
    } else {
        const IrU1 highSet(ir.INotEqual(execHi.Value(), zero.Value()));
        const IrU1 counterZero(ir.IEqual(counter.Value(), zero.Value()));
        const IrU1 stay(ir.LogicalOr(highSet.Value(), counterZero.Value()));
        newExecLo = IrU32(ir.Select(highSet.Value(), counter.Value(), ir.Select(counterZero.Value(), execLo.Value(), zero.Value())));
        newExecHi = IrU32(ir.Select(stay.Value(), execHi.Value(), counter.Value()));
        newCounter = IrU32(ir.Select(stay.Value(), counter.Value(), execLo.Value()));
        instructionBranchCondition = IrU1(ir.LogicalNot(stay.Value()));
    }
    writeRawU32(inst.destination, newCounter);
    ir.SetExecLo(newExecLo.Value());
    ir.SetExecHi(newExecHi.Value());
    ir.SetExec(threadBit({newExecLo, newExecHi}).Value());
}

// s_*_saveexec reads its source before it writes the old exec into the destination, so
// `s_and_saveexec_b64 vcc, vcc` (the Bink 2 decoder kernels gate every sparse coefficient load with
// it) masks exec by the old vcc. Reading the source after the destination write turned that form into
// exec = exec & exec. APS5_SAVEEXEC_WRITE_FIRST=1 restores the old order.
bool SaveexecWritesDestinationFirst() {
    static const bool writeFirst = std::getenv("APS5_SAVEEXEC_WRITE_FIRST") != nullptr;
    return writeFirst;
}

void TranslationContext::sSaveexec(const RdnaInstruction& inst, IrOpcode operation, bool negateExec, bool negateSource, bool write64, bool negateResult, bool writeResult) {
    // Callers name the lane-wise operation; the exec mask words themselves combine bitwise.
    const IrOpcode laneOperation = operation;
    if (operation == IrOpcode::LogicalAnd) operation = IrOpcode::BitwiseAnd32;
    else if (operation == IrOpcode::LogicalOr) operation = IrOpcode::BitwiseOr32;
    else if (operation == IrOpcode::LogicalXor) operation = IrOpcode::BitwiseXor32;
    const bool writeFirst = SaveexecWritesDestinationFirst() && !writeResult;
    const bool execSource = sourceAt(inst, 0u).kind == RdnaOperandKind::ExecLo;
    if (write64) {
        const std::array<IrU32, 2> oldExec{IrU32(ir.GetExecLo()), IrU32(ir.GetExecHi())};
        const std::array<IrU32, 2> savedExec{hostExecWord(0u), hostExecWord(1u)};
        if (writeFirst) writeU32Pair(inst.destination, savedExec);
        const std::array<IrU32, 2> source = execSource ? oldExec : readU32Pair(sourceAt(inst, 0u));
        if (!writeFirst && !writeResult) writeU32Pair(inst.destination, savedExec);
        IrValue& lowExecOperand = negateExec ? ir.BitwiseNot(oldExec[0].Value()) : oldExec[0].Value();
        IrValue& lowSourceOperand = negateSource ? ir.BitwiseNot(source[0].Value()) : source[0].Value();
        IrValue& highExecOperand = negateExec ? ir.BitwiseNot(oldExec[1].Value()) : oldExec[1].Value();
        IrValue& highSourceOperand = negateSource ? ir.BitwiseNot(source[1].Value()) : source[1].Value();
        IrValue& lowCombined = ir.Emit(operation, IrType::U32, {&lowExecOperand, &lowSourceOperand});
        IrValue& highCombined = ir.Emit(operation, IrType::U32, {&highExecOperand, &highSourceOperand});
        const IrU32 newExecLo(negateResult ? ir.BitwiseNot(lowCombined) : lowCombined);
        const IrU32 newExecHi(negateResult ? ir.BitwiseNot(highCombined) : highCombined);
        ir.SetExecLo(newExecLo.Value());
        ir.SetExecHi(newExecHi.Value());
        if (writeResult) writeU32Pair(inst.destination, {hostExecWord(0u), hostExecWord(1u)});
        const IrU1 nonZero(ir.LogicalOr(ir.INotEqual(newExecLo.Value(), ir.Constant(0u)), ir.INotEqual(newExecHi.Value(), ir.Constant(0u))));
        ir.SetScc(nonZero.Value());
        ir.SetExec(threadBit({newExecLo, newExecHi}).Value());
        return;
    }
    const IrU32 oldExec(ir.GetExecLo());
    const IrU1 oldLane(ir.GetExec());
    const IrU32 savedExec = hostExecWord(0u);
    const bool lanes = program.WaveSize() == 32u && (execSource || maskLanesExact(sourceAt(inst, 0u)));
    const auto writeSaved = [&](IrU32 word, IrU1 lane) {
        writeRawU32(inst.destination, word);
        if (lanes && inst.destination.kind == RdnaOperandKind::ScalarRegister) {
            const auto reg = static_cast<ScalarReg>(inst.destination.reg);
            ir.SetThreadBitScalarReg(reg, lane.Value());
            ir.SetScalarMaskTag(reg, ir.ConstantBool(true));
        }
    };
    if (writeFirst) writeSaved(savedExec, oldLane);
    const IrU32 source = execSource ? oldExec : readU32(sourceAt(inst, 0u));
    IrU1 sourceLane;
    if (lanes) sourceLane = execSource ? oldLane : readMask(sourceAt(inst, 0u));
    if (!writeFirst && !writeResult) writeSaved(savedExec, oldLane);
    IrValue& execOperand = negateExec ? ir.BitwiseNot(oldExec.Value()) : oldExec.Value();
    IrValue& sourceOperand = negateSource ? ir.BitwiseNot(source.Value()) : source.Value();
    IrValue& combined = ir.Emit(operation, IrType::U32, {&execOperand, &sourceOperand});
    const IrU32 newExec(negateResult ? ir.BitwiseNot(combined) : combined);
    ir.SetExecLo(newExec.Value());
    IrU1 newLane;
    if (lanes) {
        IrValue& execLane = negateExec ? ir.LogicalNot(oldLane.Value()) : oldLane.Value();
        IrValue& sourceLaneOperand = negateSource ? ir.LogicalNot(sourceLane.Value()) : sourceLane.Value();
        IrValue& combinedLane = ir.Emit(laneOperation, IrType::U1, {&execLane, &sourceLaneOperand});
        newLane = IrU1(negateResult ? ir.LogicalNot(combinedLane) : combinedLane);
    } else {
        newLane = threadBit({newExec, IrU32(ir.GetExecHi())});
    }
    if (writeResult) writeSaved(hostExecWord(0u), newLane);
    const IrU1 nonZero(ir.INotEqual(newExec.Value(), ir.Constant(0u)));
    ir.SetScc(nonZero.Value());
    ir.SetExec(newLane.Value());
}

void TranslationContext::addU32(const RdnaInstruction& inst, bool vector, bool useCarryIn) {
    const IrU32 lhs = readU32(sourceAt(inst, 0u));
    const IrU32 rhs = readU32(sourceAt(inst, 1u));
    IrValue& firstAdd = ir.Emit(IrOpcode::IAddCarry32, IrType::U32x2, {&lhs.Value(), &rhs.Value()});
    const IrU32 sum(ir.Emit(IrOpcode::CompositeExtractU32x2, IrType::U32, {&firstAdd, &ir.Constant(0u)}));
    const IrU32 firstCarry(ir.Emit(IrOpcode::CompositeExtractU32x2, IrType::U32, {&firstAdd, &ir.Constant(1u)}));
    if (!useCarryIn) {
        const IrU1 carryOut(ir.INotEqual(firstCarry.Value(), ir.Constant(0u)));
        writeRawU32(inst.destination, inst.destination.clamp ? IrU32(ir.Select(carryOut.Value(), ir.Constant(0xffffffffu), sum.Value())) : sum);
        if (vector) {
            writeMask(inst.destination2, carryOut);
            return;
        }
        ir.SetScc(carryOut.Value());
        return;
    }
    const IrU1 carryIn = !vector ? IrU1(ir.GetScc()) : inst.sourceCount >= 3u ? readMask(sourceAt(inst, 2u)) : IrU1(ir.GetVcc());
    const IrU32 carryInU32(ir.Select(carryIn.Value(), ir.Constant(1u), ir.Constant(0u)));
    IrValue& secondAdd = ir.Emit(IrOpcode::IAddCarry32, IrType::U32x2, {&sum.Value(), &carryInU32.Value()});
    const IrU32 result(ir.Emit(IrOpcode::CompositeExtractU32x2, IrType::U32, {&secondAdd, &ir.Constant(0u)}));
    const IrU32 secondCarry(ir.Emit(IrOpcode::CompositeExtractU32x2, IrType::U32, {&secondAdd, &ir.Constant(1u)}));
    const IrU1 carryOut(ir.LogicalOr(ir.INotEqual(firstCarry.Value(), ir.Constant(0u)), ir.INotEqual(secondCarry.Value(), ir.Constant(0u))));
    writeRawU32(inst.destination, inst.destination.clamp ? IrU32(ir.Select(carryOut.Value(), ir.Constant(0xffffffffu), result.Value())) : result);
    if (vector) {
        writeMask(inst.destination2, carryOut);
        return;
    }
    ir.SetScc(carryOut.Value());
}

void TranslationContext::subU32(const RdnaInstruction& inst, bool vector, bool reverse) {
    const IrU32 first = readU32(sourceAt(inst, 0u));
    const IrU32 second = readU32(sourceAt(inst, 1u));
    const IrU32& lhs = reverse ? second : first;
    const IrU32& rhs = reverse ? first : second;
    const IrU32 result(ir.ISub(lhs.Value(), rhs.Value()));
    const IrU1 borrow(ir.ULessThan(lhs.Value(), rhs.Value()));
    writeRawU32(inst.destination, result);
    if (vector) {
        writeMask(inst.destination2, borrow);
        return;
    }
    ir.SetScc(borrow.Value());
}

void TranslationContext::subbU32(const RdnaInstruction& inst, bool vector, bool reverse) {
    const IrU32 first = readU32(sourceAt(inst, 0u));
    const IrU32 second = readU32(sourceAt(inst, 1u));
    const IrU32& lhs = reverse ? second : first;
    const IrU32& rhs = reverse ? first : second;
    const IrU1 borrowIn = !vector ? IrU1(ir.GetScc()) : inst.sourceCount >= 3u ? readMask(sourceAt(inst, 2u)) : IrU1(ir.GetVcc());
    const IrU32 borrowInU32(ir.Select(borrowIn.Value(), ir.Constant(1u), ir.Constant(0u)));
    const IrU32 partial(ir.ISub(lhs.Value(), rhs.Value()));
    const IrU1 firstBorrow(ir.ULessThan(lhs.Value(), rhs.Value()));
    const IrU32 result(ir.ISub(partial.Value(), borrowInU32.Value()));
    const IrU1 secondBorrow(ir.ULessThan(partial.Value(), borrowInU32.Value()));
    const IrU1 borrowOut(ir.LogicalOr(firstBorrow.Value(), secondBorrow.Value()));
    writeRawU32(inst.destination, inst.destination.clamp ? IrU32(ir.Select(borrowOut.Value(), ir.Constant(0u), result.Value())) : result);
    if (vector) {
        writeMask(inst.destination2, borrowOut);
        return;
    }
    ir.SetScc(borrowOut.Value());
}

bool TranslationContext::vAddSubNcU32(const RdnaInstruction& inst, bool subtract, bool reverse) {
    const IrU32 first = readU32(sourceAt(inst, 0u));
    const IrU32 second = readU32(sourceAt(inst, 1u));
    const IrU32& lhs = reverse ? second : first;
    const IrU32& rhs = reverse ? first : second;
    IrU32 result(subtract ? ir.ISub(lhs.Value(), rhs.Value()) : ir.IAdd(lhs.Value(), rhs.Value()));
    if (inst.destination.clamp) {
        const IrU1 carry(subtract ? ir.ULessThan(lhs.Value(), rhs.Value()) : ir.ULessThan(result.Value(), lhs.Value()));
        result = IrU32(ir.Select(carry.Value(), ir.Constant(subtract ? 0u : 0xffffffffu), result.Value()));
    }
    writeOperand(inst.destination, &result.Value());
    return true;
}

void TranslationContext::sAbsdiffI32(const RdnaInstruction& inst) {
    const IrU32 lhs = readU32(sourceAt(inst, 0u));
    const IrU32 rhs = readU32(sourceAt(inst, 1u));
    const IrU32 difference(ir.ISub(lhs.Value(), rhs.Value()));
    const IrU32 result(ir.Emit(IrOpcode::IAbs32, IrType::U32, {&difference.Value()}));
    ir.SetScc(ir.INotEqual(result.Value(), ir.Constant(0u)));
    writeRawU32(inst.destination, result);
}

void TranslationContext::sAddSubI32(const RdnaInstruction& inst, bool subtract) {
    const IrU32 lhs = readU32(sourceAt(inst, 0u));
    const IrU32 rhs = readU32(sourceAt(inst, 1u));
    const IrU32 result(subtract ? ir.ISub(lhs.Value(), rhs.Value()) : ir.IAdd(lhs.Value(), rhs.Value()));
    const IrU32 signCheck(subtract ? ir.BitwiseAnd(ir.BitwiseXor(lhs.Value(), rhs.Value()), ir.BitwiseXor(lhs.Value(), result.Value()))
                                    : ir.BitwiseAnd(ir.BitwiseXor(lhs.Value(), result.Value()), ir.BitwiseXor(rhs.Value(), result.Value())));
    ir.SetScc(ir.INotEqual(ir.BitwiseAnd(signCheck.Value(), ir.Constant(0x80000000u)), ir.Constant(0u)));
    writeRawU32(inst.destination, result);
}

void TranslationContext::sLshlAddU32(const RdnaInstruction& inst, std::uint32_t shiftAmount) {
    const IrU32 lhs = readU32(sourceAt(inst, 0u));
    const IrU32 rhs = readU32(sourceAt(inst, 1u));
    const IrU32 shifted(ir.ShiftLeftLogical(lhs.Value(), ir.Constant(shiftAmount)));
    const IrU32 result(ir.IAdd(shifted.Value(), rhs.Value()));
    const IrU1 shiftedOut(ir.INotEqual(ir.ShiftRightLogical(lhs.Value(), ir.Constant(32u - shiftAmount)), ir.Constant(0u)));
    ir.SetScc(ir.LogicalOr(shiftedOut.Value(), ir.ULessThan(result.Value(), shifted.Value())));
    writeRawU32(inst.destination, result);
}

void TranslationContext::scalarMinMax32(const RdnaInstruction& inst, IrOpcode valueOpcode, IrOpcode compareOpcode) {
    const IrU32 lhs = readU32(sourceAt(inst, 0u));
    const IrU32 rhs = readU32(sourceAt(inst, 1u));
    const IrU32 result(ir.Emit(valueOpcode, IrType::U32, {&lhs.Value(), &rhs.Value()}));
    const IrU1 selected(ir.Emit(compareOpcode, IrType::U1, {&lhs.Value(), &rhs.Value()}));
    ir.SetScc(selected.Value());
    writeRawU32(inst.destination, result);
}

void TranslationContext::emitControlNop() {
    (void)ir.Emit(IrOpcode::ControlNop, IrType::Void, {});
}

void TranslationContext::emitWaitcnt() {
    (void)ir.Emit(IrOpcode::Waitcnt, IrType::Void, {});
}

void TranslationContext::sBarrier() {
    (void)ir.Emit(IrOpcode::Barrier, IrType::Void, {});
}

void TranslationContext::sSendmsg(const RdnaInstruction& inst) {
    constexpr std::uint32_t GsAllocReq = 9u;
    if (program.Resources().stage == IrShaderStage::Mesh && (inst.rawWords[0] & 0xfu) == GsAllocReq) {
        (void)ir.Emit(IrOpcode::MeshAllocate, IrType::Void, {&ir.GetM0()});
        return;
    }
    (void)ir.Emit(IrOpcode::Sendmsg, IrType::Void, {});
}

void TranslationContext::sTtracedata() {
    (void)ir.Emit(IrOpcode::TtraceData, IrType::Void, {});
}

void TranslationContext::sInstPrefetch() {
    (void)ir.Emit(IrOpcode::InstPrefetch, IrType::Void, {});
}

void TranslationContext::sGetpcB64(const RdnaInstruction& inst) {
    IrValue& base = ir.Emit(IrOpcode::GetShaderBase, IrType::U64, {});
    const IrU64 pc(ir.Emit(IrOpcode::IAdd64, IrType::U64, {&base, &ir.ConstantU64(static_cast<std::uint64_t>(currentProgramCounter) + 4u)}));
    writeU32Pair(inst.destination, extractU64(pc));
}

void TranslationContext::sSwappcB64(const RdnaInstruction& inst) {
    IrValue& base = ir.Emit(IrOpcode::GetShaderBase, IrType::U64, {});
    const IrU64 link(ir.Emit(IrOpcode::IAdd64, IrType::U64, {&base, &ir.ConstantU64(static_cast<std::uint64_t>(currentProgramCounter) + 4u)}));
    writeU32Pair(inst.destination, extractU64(link));
}

void TranslationContext::TranslateUnlinkedCall(const RdnaInstruction& inst) {
    const std::array<IrU32, 2> target = readU32Pair(inst.source0);
    (void)ir.Emit(IrOpcode::UnlinkedCall, IrType::Void, {&target[0].Value(), &target[1].Value(), &ir.Constant(inst.programCounter)});
    program.Info().usesFaultBuffer = true;
    sSwappcB64(inst);
}

void TranslationContext::sCselectB32(const RdnaInstruction& inst) {
    const IrU32 trueValue = readU32(sourceAt(inst, 0u));
    const IrU32 falseValue = readU32(sourceAt(inst, 1u));
    const IrU32 result(ir.Select(ir.GetScc(), trueValue.Value(), falseValue.Value()));
    writeRawU32(inst.destination, result);
}

void TranslationContext::scalarSelect64(const RdnaInstruction& inst, const RdnaOperand& falseSource) {
    const std::array<IrU32, 2> trueValue = readU32Pair(sourceAt(inst, 0u));
    const std::array<IrU32, 2> falseValue = readU32Pair(falseSource);
    IrValue& condition = ir.GetScc();
    const IrU32 low(ir.Select(condition, trueValue[0].Value(), falseValue[0].Value()));
    const IrU32 high(ir.Select(condition, trueValue[1].Value(), falseValue[1].Value()));
    writeU32Pair(inst.destination, {low, high});
}

void TranslationContext::scalarSelectMask64(const RdnaInstruction& inst) {
    if (inst.destination.kind != RdnaOperandKind::ScalarRegister) {
        scalarSelect64(inst, sourceAt(inst, 1u));
        return;
    }
    const RdnaOperand& trueSource = sourceAt(inst, 0u);
    const RdnaOperand& falseSource = sourceAt(inst, 1u);
    IrValue& valid = ir.LogicalAnd(readMaskValid(trueSource).Value(), readMaskValid(falseSource).Value());
    IrValue& bit = ir.Emit(IrOpcode::SelectU1, IrType::U1, {&ir.GetScc(), &readMask(trueSource).Value(), &readMask(falseSource).Value()});
    scalarSelect64(inst, falseSource);
    const auto dst = static_cast<ScalarReg>(inst.destination.reg);
    ir.SetThreadBitScalarReg(dst, bit);
    ir.SetScalarMaskTag(dst, valid);
}

void TranslationContext::movB32(const RdnaInstruction& inst, bool applyFloatModifiers) {
    if (applyFloatModifiers) {
        IrF32 value(*readOperand(sourceAt(inst, 0u), IrType::F32));
        value = applyF32ResultModifiers(inst.destination, value);
        writeOperand(inst.destination, &value.Value());
        return;
    }
    IrValue* value = readOperand(sourceAt(inst, 0u), IrType::U32);
    writeOperand(inst.destination, value);
}

void TranslationContext::sMovB64(const RdnaInstruction& inst) {
    const std::array<IrU32, 2> value = readU32Pair(sourceAt(inst, 0u));
    writeU32Pair(inst.destination, value);
}

void TranslationContext::sWqm(const RdnaInstruction& inst, bool wide) {
    if (wide) {
        const std::array<IrU32, 2> source = readU32Pair(sourceAt(inst, 0u));
        const IrU64 wide64(ir.ConstructU64(source[0].Value(), source[1].Value()));
        const IrU64 result(ir.Emit(IrOpcode::WqmU64, IrType::U64, {&wide64.Value()}));
        const std::array<IrU32, 2> words = extractU64(result);
        writeU32Pair(inst.destination, words);
        ir.SetScc(ir.LogicalOr(ir.INotEqual(words[0].Value(), ir.Constant(0u)), ir.INotEqual(words[1].Value(), ir.Constant(0u))));
        return;
    }
    const IrU32 source = readU32(sourceAt(inst, 0u));
    const IrU64 wide64(ir.ConstructU64(source.Value(), ir.Constant(0u)));
    const IrU64 result(ir.Emit(IrOpcode::WqmU64, IrType::U64, {&wide64.Value()}));
    const IrU32 low = extractU64(result)[0];
    writeRawU32(inst.destination, low);
    ir.SetScc(ir.INotEqual(low.Value(), ir.Constant(0u)));
}

struct RelativeWindowAccess {
    IrValue* index = nullptr;
    IrValue* inRange = nullptr;
};

RelativeWindowAccess relativeWindowAccess(IrBuilder& ir, const IrProgram& program, std::uint32_t reg, IrValue& offset, const char* name, std::uint32_t vectorLimit) {
    const auto& metadata = program.Metadata();
    if (metadata.vectorWindowSize == 0u) {
        if (reg >= vectorLimit) throw std::runtime_error(std::string(name) + " addresses a register outside the relative-addressing span");
        return {nullptr, &ir.ULessThan(offset, ir.Constant(vectorLimit - reg))};
    }
    if (metadata.vectorWindowSize == 0u || reg < metadata.vectorWindowBase) throw std::runtime_error(std::string(name) + " addresses a register outside the relative-addressing window");
    const auto first = reg - metadata.vectorWindowBase;
    IrValue& index = ir.Emit(IrOpcode::UMin32, IrType::U32, {&ir.IAdd(ir.Constant(first), offset), &ir.Constant(metadata.vectorWindowSize - 1u)});
    return {&index, &ir.ULessThan(offset, ir.Constant(metadata.vectorWindowSize - first))};
}

IrU32 TranslationContext::readRelativeVector(const RdnaOperand& base, IrValue& offset, const char* name) {
    if (base.kind != RdnaOperandKind::VectorRegister) {
        throw std::runtime_error(std::string(name) + " source is not a vector register");
    }
    const auto access = relativeWindowAccess(ir, program, base.reg, offset, name, currentVectorLimit);
    if (access.index == nullptr) {
        IrU32 result(ir.GetVectorReg(static_cast<VectorReg>(0)));
        for (std::uint32_t reg = base.reg; reg < currentVectorLimit; ++reg) {
            IrValue& hit = ir.IEqual(offset, ir.Constant(reg - base.reg));
            result = IrU32(ir.Select(hit, ir.GetVectorReg(static_cast<VectorReg>(reg)), result.Value()));
        }
        return applyBitSourceModifiers(base, result);
    }
    IrValue& loaded = ir.Emit(IrOpcode::LoadVectorWindow, IrType::U32, {access.index});
    return applyBitSourceModifiers(base, IrU32(ir.Select(*access.inRange, loaded, ir.GetVectorReg(static_cast<VectorReg>(0)))));
}

void TranslationContext::writeRelativeVector(const RdnaOperand& base, IrValue& offset, IrU32 value, const char* name, IrValue* enable) {
    const RdnaOperand& destination = base;
    if (destination.kind != RdnaOperandKind::VectorRegister) {
        throw std::runtime_error(std::string(name) + " destination is not a vector register");
    }
    const auto access = relativeWindowAccess(ir, program, destination.reg, offset, name, currentVectorLimit);
    if (access.index == nullptr) {
        for (std::uint32_t reg = destination.reg; reg < currentVectorLimit; ++reg) {
            IrValue& previous = ir.GetVectorReg(static_cast<VectorReg>(reg));
            IrValue* merged = nullptr;
            if (destination.dpp) {
                const DppMoveFlags flags{destination.dpp8 ? DppMoveFlags::Lanes8 | destination.dppCtrl : destination.dppCtrl, static_cast<std::uint8_t>(destination.dppRowMask), static_cast<std::uint8_t>(destination.dppBankMask), destination.dppFetchInactive, destination.dppBoundCtrl};
                merged = &ir.Emit(IrOpcode::DppUpdateU32, IrType::U32, {&value.Value(), &previous, &ir.GetExec()}, flags);
            } else merged = &ir.Select(ir.GetExec(), value.Value(), previous);
            IrValue* written = &ir.IEqual(offset, ir.Constant(reg - destination.reg));
            if (enable != nullptr) written = &ir.LogicalAnd(*written, *enable);
            ir.SetVectorReg(static_cast<VectorReg>(reg), ir.Select(*written, *merged, previous));
        }
        return;
    }
    IrValue& previous = ir.Emit(IrOpcode::LoadVectorWindow, IrType::U32, {access.index});
    IrValue* merged = nullptr;
    if (destination.dpp) {
        const DppMoveFlags flags{destination.dpp8 ? DppMoveFlags::Lanes8 | destination.dppCtrl : destination.dppCtrl, static_cast<std::uint8_t>(destination.dppRowMask), static_cast<std::uint8_t>(destination.dppBankMask), destination.dppFetchInactive, destination.dppBoundCtrl};
        merged = &ir.Emit(IrOpcode::DppUpdateU32, IrType::U32, {&value.Value(), &previous, &ir.GetExec()}, flags);
    } else {
        merged = &ir.Select(ir.GetExec(), value.Value(), previous);
    }
    IrValue* written = access.inRange;
    if (enable != nullptr) written = &ir.LogicalAnd(*written, *enable);
    (void)ir.Emit(IrOpcode::StoreVectorWindow, IrType::Void, {access.index, &ir.Select(*written, *merged, previous)});
}

void TranslationContext::vMovrelsB32(const RdnaInstruction& inst) {
    writeRawU32(inst.destination, readRelativeVector(sourceAt(inst, 0u), ir.GetM0(), "v_movrels_b32"));
}

void TranslationContext::vMovreldB32(const RdnaInstruction& inst) {
    const IrU32 value = readU32(sourceAt(inst, 0u));
    writeRelativeVector(inst.destination, ir.GetM0(), value, "v_movreld_b32");
}

void TranslationContext::vMovrelsdB32(const RdnaInstruction& inst, bool split, bool swap) {
    const char* name = swap ? "v_swaprel_b32" : split ? "v_movrelsd_2_b32" : "v_movrelsd_b32";
    IrValue& m0 = ir.GetM0();
    IrValue& sourceOffset = split ? ir.BitwiseAnd(m0, ir.Constant(0x3ffu)) : m0;
    IrValue& destinationOffset = split ? ir.BitwiseAnd(ir.ShiftRightLogical(m0, ir.Constant(16u)), ir.Constant(0x3ffu)) : m0;
    const IrU32 value = readRelativeVector(sourceAt(inst, 0u), sourceOffset, name);
    if (!swap) {
        writeRelativeVector(inst.destination, destinationOffset, value, name);
        return;
    }
    const IrU32 previous = readRelativeVector(plainOperand(inst.destination), destinationOffset, name);
    IrValue& bothInRange = ir.LogicalAnd(*relativeWindowAccess(ir, program, sourceAt(inst, 0u).reg, sourceOffset, name, currentVectorLimit).inRange, *relativeWindowAccess(ir, program, inst.destination.reg, destinationOffset, name, currentVectorLimit).inRange);
    writeRelativeVector(inst.destination, destinationOffset, value, name, &bothInRange);
    writeRelativeVector(sourceAt(inst, 0u), sourceOffset, previous, name, &bothInRange);
}

void TranslationContext::vSwapB32(const RdnaInstruction& inst) {
    const RdnaOperand& source = sourceAt(inst, 0u);
    if (source.kind != RdnaOperandKind::VectorRegister) {
        throw std::runtime_error("v_swap_b32 source is not a vector register");
    }
    const IrU32 value = readU32(source);
    const IrU32 previous = readU32(plainOperand(inst.destination));
    writeRawU32(inst.destination, value);
    writeRawU32(plainOperand(source), previous);
}

void TranslationContext::vReadfirstlaneB32(const RdnaInstruction& inst) {
    const IrU32 value = readU32(sourceAt(inst, 0u));
    const IrU32 result(ir.Emit(IrOpcode::ReadFirstLane, IrType::U32, {&value.Value(), &ir.GetExec()}));
    writeRawU32(inst.destination, result);
}

void TranslationContext::vReadlaneB32(const RdnaInstruction& inst) {
    const IrU32 value = readU32(sourceAt(inst, 0u));
    const IrU32 lane = readU32(sourceAt(inst, 1u));
    const IrU32 result(ir.Emit(IrOpcode::ReadLane, IrType::U32, {&value.Value(), &lane.Value()}));
    writeRawU32(inst.destination, result);
}

void TranslationContext::vWritelaneB32(const RdnaInstruction& inst) {
    const IrU32 value = readU32(sourceAt(inst, 0u));
    const IrU32 lane = readU32(sourceAt(inst, 1u));
    const IrU32 previous = readRawU32(plainOperand(inst.destination));
    const IrU32 result(ir.Emit(IrOpcode::WriteLane, IrType::U32, {&value.Value(), &lane.Value(), &previous.Value()}));
    ir.SetVectorReg(static_cast<VectorReg>(inst.destination.reg), result.Value());
}

void TranslationContext::vPermlane16B32(const RdnaInstruction& inst, bool x16) {
    const IrU32 value = readU32(sourceAt(inst, 0u));
    const IrU32 selectLow = readU32(sourceAt(inst, 1u));
    const IrU32 selectHigh = readU32(sourceAt(inst, 2u));
    const IrU32 previous = readRawU32(plainOperand(inst.destination));
    const PermlaneFlags flags{x16, inst.destination.opSel, inst.destination.opSelHi};
    const IrU32 result(ir.Emit(IrOpcode::Permlane16U32, IrType::U32, {&value.Value(), &selectLow.Value(), &selectHigh.Value(), &ir.GetExec(), &previous.Value()}, flags));
    writeRawU32(inst.destination, result);
}

void TranslateControlFlowInstruction(TranslationContext& context, const RdnaInstruction& instruction) {
    throw std::runtime_error("TranslateControlFlowInstruction not implemented");
}

}