diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 09e307e124316..a6ca48f9da927 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -544,6 +544,11 @@ if(onnxruntime_USE_SVE) message(WARNING "onnxruntime_USE_SVE was set but compiler does not support SVE. It will be disabled.") set(onnxruntime_USE_SVE OFF) endif() + elseif(WIN32 AND onnxruntime_target_platform STREQUAL "ARM64") + # The SVE elementwise kernels ship as portable machine-code assembly + # (hex-encoded instruction words assembled by armasm64), so no compiler + # SVE support is needed; the kernels are runtime-gated on HasArmSve(). + message(STATUS "SVE enabled via portable machine-code kernels.") else() message(WARNING "onnxruntime_USE_SVE was set but it is not supported on this platform. It will be disabled.") set(onnxruntime_USE_SVE OFF) diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 319c897b8349c..95a659118e17b 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -170,6 +170,21 @@ function(setup_mlas_source_for_windows) if (onnxruntime_USE_KLEIDIAI) setup_kleidiai() endif() + + if (onnxruntime_USE_SVE) + # Portable machine-code SVE elementwise kernels: a plain C++ dispatch + # layer plus hex-encoded assembly. kai_asm_macros.h resolves to + # armasm64 directives (AREA/PROC/DCD) during the cl.exe preprocessing + # step applied to mlas_platform_preprocess_srcs below, so no SVE + # toolchain support is required. + target_sources(onnxruntime_mlas PRIVATE + ${MLAS_SRC_DIR}/sve/mlasi_sve.h + ${MLAS_SRC_DIR}/sve/elementwise_sve_dispatch.cpp + ) + list(APPEND mlas_platform_preprocess_srcs ${MLAS_SRC_DIR}/aarch64/elementwise_sve_asm.S) + list(APPEND mlas_private_compile_definitions MLAS_USE_SVE) + set(mlas_private_compile_definitions ${mlas_private_compile_definitions} PARENT_SCOPE) + endif() else() target_sources(onnxruntime_mlas PRIVATE ${MLAS_SRC_DIR}/qgemm_kernel_neon.cpp @@ -552,11 +567,20 @@ else() # Conditionally add the SVE implementation if compiler supports it if (onnxruntime_USE_SVE) list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/mlasi_sve.h) - list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/elementwise_sve.cpp) - list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/elementwise_sve_fp16.cpp) - list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/mlas_sve_fp16.h) - set_source_files_properties(${MLAS_SRC_DIR}/sve/elementwise_sve.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+sve+fp16 ") - set_source_files_properties(${MLAS_SRC_DIR}/sve/elementwise_sve_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+sve+fp16 ") + list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/elementwise_sve_dispatch.cpp) + # The portable machine-code variant is the production default; set + # to OFF to build the SVE intrinsics reference implementation (the + # regeneration source for elementwise_sve_asm.S). + set(onnxruntime_SVE_ELEMENTWISE_ASM ON) + if (onnxruntime_SVE_ELEMENTWISE_ASM) + # Portable machine-code variant (same symbols as the intrinsics TUs). + list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/aarch64/elementwise_sve_asm.S) + else() + list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/elementwise_sve.cpp) + list(APPEND mlas_platform_srcs ${MLAS_SRC_DIR}/sve/elementwise_sve_fp16.cpp) + set_source_files_properties(${MLAS_SRC_DIR}/sve/elementwise_sve.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+sve+fp16 -DMLAS_SVE_SUMEXP_FEXPA=1 ") + set_source_files_properties(${MLAS_SRC_DIR}/sve/elementwise_sve_fp16.cpp PROPERTIES COMPILE_FLAGS " -march=armv8.2-a+sve+fp16 ") + endif() list(APPEND mlas_private_compile_definitions MLAS_USE_SVE) endif() diff --git a/onnxruntime/core/common/cpuid_info.cc b/onnxruntime/core/common/cpuid_info.cc index ec5c1386e8336..17a74dbfe0ed2 100644 --- a/onnxruntime/core/common/cpuid_info.cc +++ b/onnxruntime/core/common/cpuid_info.cc @@ -317,6 +317,12 @@ void CPUIDInfo::ArmWindowsInit() { has_arm_sme2_ = cpuinfo_has_arm_sme2(); } #endif // defined(CPUINFO_SUPPORTED) + +#if defined(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) + // Available from Windows 11 24H2 SDKs; older SDKs lack the constant, in + // which case SVE stays disabled at runtime. + has_arm_sve_ = IsProcessorFeaturePresent(PF_ARM_SVE_INSTRUCTIONS_AVAILABLE) != 0; +#endif } #elif defined(__APPLE__) // ^ defined(_WIN32) diff --git a/onnxruntime/core/mlas/lib/aarch64/elementwise_sve_asm.S b/onnxruntime/core/mlas/lib/aarch64/elementwise_sve_asm.S new file mode 100644 index 0000000000000..d035638506822 --- /dev/null +++ b/onnxruntime/core/mlas/lib/aarch64/elementwise_sve_asm.S @@ -0,0 +1,738 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + elementwise_sve_asm.S + +Abstract: + + Portable machine-code variant of the SVE elementwise_sve kernels, in the style + of Arm's KleidiAI library: every instruction is emitted as a raw + instruction word (GAS ".inst" / armasm64 "DCD" via KAI_ASM_INST), so the + file assembles with toolchains that have no SVE support, and the bytes + are identical on every platform. + + GENERATED FILE - DO NOT EDIT BY HAND. + + Generated from the SVE intrinsics translation unit(s) (sve/elementwise_sve.cpp, sve/elementwise_sve_fp16.cpp), which + remain the reference implementation and regeneration source (script: + sve/gen_sve_asm.py). The functions here export the same extern "C" + symbols; the build links exactly one of the two implementations. The + code is fully position-independent (the generator verifies there are no + relocations, calls, or literal pools). + +--*/ + +#include "kai_asm_macros.h" + + KAI_ASM_CODE(elementwise_sve) + + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveErfKernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveErfKernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveErfKernelImpl) + KAI_ASM_INST(0x910003eb) // mov x11, sp + KAI_ASM_INST(0x0423e3ec) // cntb x12, all, mul #4 + KAI_ASM_INST(0x9101018c) // add x12, x12, #0x40 + KAI_ASM_INST(0xf1403d9f) // cmp x12, #0xf, lsl #12 + KAI_ASM_INST(0x540000ab) // b.lt 24 <.SVLPEND0> + KAI_ASM_INST(0xd1403fff) // sub sp, sp, #0xf, lsl #12 + KAI_ASM_INST(0xf90003ff) // str xzr, [sp] + KAI_ASM_INST(0xd1403d8c) // sub x12, x12, #0xf, lsl #12 + KAI_ASM_INST(0x17fffffb) // b c <.SVLPSPL0> + KAI_ASM_INST(0xcb2c63ff) // sub sp, sp, x12 + KAI_ASM_INST(0x6d0027e8) // stp d8, d9, [sp] + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8544c069) // ld1rw {z9.s}, p0/z, [x3, #16] + KAI_ASM_INST(0x8545c068) // ld1rw {z8.s}, p0/z, [x3, #20] + KAI_ASM_INST(0x8546c07f) // ld1rw {z31.s}, p0/z, [x3, #24] + KAI_ASM_INST(0x8547c07e) // ld1rw {z30.s}, p0/z, [x3, #28] + KAI_ASM_INST(0x8549c07d) // ld1rw {z29.s}, p0/z, [x3, #36] + KAI_ASM_INST(0x854ac07c) // ld1rw {z28.s}, p0/z, [x3, #40] + KAI_ASM_INST(0x854bc07b) // ld1rw {z27.s}, p0/z, [x3, #44] + KAI_ASM_INST(0x854cc07a) // ld1rw {z26.s}, p0/z, [x3, #48] + KAI_ASM_INST(0x854dc079) // ld1rw {z25.s}, p0/z, [x3, #52] + KAI_ASM_INST(0x854ec078) // ld1rw {z24.s}, p0/z, [x3, #56] + KAI_ASM_INST(0x854fc077) // ld1rw {z23.s}, p0/z, [x3, #60] + KAI_ASM_INST(0x8553c076) // ld1rw {z22.s}, p0/z, [x3, #76] + KAI_ASM_INST(0x8554c075) // ld1rw {z21.s}, p0/z, [x3, #80] + KAI_ASM_INST(0x8555c074) // ld1rw {z20.s}, p0/z, [x3, #84] + KAI_ASM_INST(0x8556c073) // ld1rw {z19.s}, p0/z, [x3, #88] + KAI_ASM_INST(0x855ec065) // ld1rw {z5.s}, p0/z, [x3, #120] + KAI_ASM_INST(0x8557c072) // ld1rw {z18.s}, p0/z, [x3, #92] + KAI_ASM_INST(0x8558c071) // ld1rw {z17.s}, p0/z, [x3, #96] + KAI_ASM_INST(0x8559c070) // ld1rw {z16.s}, p0/z, [x3, #100] + KAI_ASM_INST(0x6d012fea) // stp d10, d11, [sp, #16] + KAI_ASM_INST(0x8542c06b) // ld1rw {z11.s}, p0/z, [x3, #8] + KAI_ASM_INST(0x8543c06a) // ld1rw {z10.s}, p0/z, [x3, #12] + KAI_ASM_INST(0x910103e5) // add x5, sp, #0x40 + KAI_ASM_INST(0x6d0237ec) // stp d12, d13, [sp, #32] + KAI_ASM_INST(0x8541c06d) // ld1rw {z13.s}, p0/z, [x3, #4] + KAI_ASM_INST(0x8551c06c) // ld1rw {z12.s}, p0/z, [x3, #68] + KAI_ASM_INST(0x6d033fee) // stp d14, d15, [sp, #48] + KAI_ASM_INST(0x8540c06e) // ld1rw {z14.s}, p0/z, [x3] + KAI_ASM_INST(0x855ac067) // ld1rw {z7.s}, p0/z, [x3, #104] + KAI_ASM_INST(0x855cc060) // ld1rw {z0.s}, p0/z, [x3, #112] + KAI_ASM_INST(0xe58040a0) // str z0, [x5] + KAI_ASM_INST(0x855dc060) // ld1rw {z0.s}, p0/z, [x3, #116] + KAI_ASM_INST(0xe58044a0) // str z0, [x5, #1, mul vl] + KAI_ASM_INST(0x855fc060) // ld1rw {z0.s}, p0/z, [x3, #124] + KAI_ASM_INST(0xe58048a0) // str z0, [x5, #2, mul vl] + KAI_ASM_INST(0x855bc066) // ld1rw {z6.s}, p0/z, [x3, #108] + KAI_ASM_INST(0x8550c064) // ld1rw {z4.s}, p0/z, [x3, #64] + KAI_ASM_INST(0xb4000762) // cbz x2, 1b0 <.SVLPEND0+0x18c> + KAI_ASM_INST(0xd2800004) // mov x4, #0x0 + KAI_ASM_INST(0x04a0e3e3) // cntw x3 + KAI_ASM_INST(0xe5804ca6) // str z6, [x5, #3, mul vl] + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25a21c80) // whilelo p0.s, x4, x2 + KAI_ASM_INST(0xa5444003) // ld1w {z3.s}, p0/z, [x0, x4, lsl #2] + KAI_ASM_INST(0x04e43061) // bic z1.d, z3.d, z4.d + KAI_ASM_INST(0x658781c1) // fmin z1.s, p0/m, z1.s, z14.s + KAI_ASM_INST(0x0420bf80) // movprfx z0, z28 + KAI_ASM_INST(0x65a103a0) // fmla z0.s, p0/m, z29.s, z1.s + KAI_ASM_INST(0x65bb8020) // fmad z0.s, p0/m, z1.s, z27.s + KAI_ASM_INST(0x65ba8020) // fmad z0.s, p0/m, z1.s, z26.s + KAI_ASM_INST(0x65b98020) // fmad z0.s, p0/m, z1.s, z25.s + KAI_ASM_INST(0x65b88020) // fmad z0.s, p0/m, z1.s, z24.s + KAI_ASM_INST(0x65b78020) // fmad z0.s, p0/m, z1.s, z23.s + KAI_ASM_INST(0x65a18020) // fmad z0.s, p0/m, z1.s, z1.s + KAI_ASM_INST(0x04a43000) // eor z0.d, z0.d, z4.d + KAI_ASM_INST(0x658682c0) // fmax z0.s, p0/m, z0.s, z22.s + KAI_ASM_INST(0x0420bcaf) // movprfx z15, z5 + KAI_ASM_INST(0x65b5000f) // fmla z15.s, p0/m, z0.s, z21.s + KAI_ASM_INST(0x658180af) // fsub z15.s, p0/m, z15.s, z5.s + KAI_ASM_INST(0x65b401e0) // fmla z0.s, p0/m, z15.s, z20.s + KAI_ASM_INST(0x910103e5) // add x5, sp, #0x40 + KAI_ASM_INST(0x0420bde2) // movprfx z2, z15 + KAI_ASM_INST(0x659ca1e2) // fcvtzs z2.s, p0/m, z15.s + KAI_ASM_INST(0x85804ca6) // ldr z6, [x5, #3, mul vl] + KAI_ASM_INST(0x65a0826f) // fmad z15.s, p0/m, z19.s, z0.s + KAI_ASM_INST(0x858048a0) // ldr z0, [x5, #2, mul vl] + KAI_ASM_INST(0x04a00042) // add z2.s, z2.s, z0.s + KAI_ASM_INST(0x0420be20) // movprfx z0, z17 + KAI_ASM_INST(0x65af0240) // fmla z0.s, p0/m, z18.s, z15.s + KAI_ASM_INST(0x04779c42) // lsl z2.s, z2.s, #23 + KAI_ASM_INST(0x65b081e0) // fmad z0.s, p0/m, z15.s, z16.s + KAI_ASM_INST(0x65a781e0) // fmad z0.s, p0/m, z15.s, z7.s + KAI_ASM_INST(0x65a681e0) // fmad z0.s, p0/m, z15.s, z6.s + KAI_ASM_INST(0x858040a6) // ldr z6, [x5] + KAI_ASM_INST(0x658d4031) // fcmgt p1.s, p0/z, z1.s, z13.s + KAI_ASM_INST(0x65a681e0) // fmad z0.s, p0/m, z15.s, z6.s + KAI_ASM_INST(0x04233083) // and z3.d, z4.d, z3.d + KAI_ASM_INST(0x858044a6) // ldr z6, [x5, #1, mul vl] + KAI_ASM_INST(0x65a681e0) // fmad z0.s, p0/m, z15.s, z6.s + KAI_ASM_INST(0x0420bc2f) // movprfx z15, z1 + KAI_ASM_INST(0x6582802f) // fmul z15.s, p0/m, z15.s, z1.s + KAI_ASM_INST(0x65828040) // fmul z0.s, p0/m, z0.s, z2.s + KAI_ASM_INST(0x0420bd42) // movprfx z2, z10 + KAI_ASM_INST(0x65af0162) // fmla z2.s, p0/m, z11.s, z15.s + KAI_ASM_INST(0x65838180) // fsubr z0.s, p0/m, z0.s, z12.s + KAI_ASM_INST(0x65a981e2) // fmad z2.s, p0/m, z15.s, z9.s + KAI_ASM_INST(0x65a881e2) // fmad z2.s, p0/m, z15.s, z8.s + KAI_ASM_INST(0x65bf81e2) // fmad z2.s, p0/m, z15.s, z31.s + KAI_ASM_INST(0x65be81e2) // fmad z2.s, p0/m, z15.s, z30.s + KAI_ASM_INST(0x65a10041) // fmla z1.s, p0/m, z2.s, z1.s + KAI_ASM_INST(0x05a1c400) // sel z0.s, p1, z0.s, z1.s + KAI_ASM_INST(0x04633000) // orr z0.d, z0.d, z3.d + KAI_ASM_INST(0xe5444020) // st1w {z0.s}, p0, [x1, x4, lsl #2] + KAI_ASM_INST(0x8b030084) // add x4, x4, x3 + KAI_ASM_INST(0xeb04005f) // cmp x2, x4 + KAI_ASM_INST(0x54fff968) // b.hi d8 <.SVLPEND0+0xb4> + KAI_ASM_INST(0x6d4027e8) // ldp d8, d9, [sp] + KAI_ASM_INST(0x6d412fea) // ldp d10, d11, [sp, #16] + KAI_ASM_INST(0x6d4237ec) // ldp d12, d13, [sp, #32] + KAI_ASM_INST(0x6d433fee) // ldp d14, d15, [sp, #48] + KAI_ASM_INST(0x043f509f) // addvl sp, sp, #4 + KAI_ASM_INST(0x910103ff) // add sp, sp, #0x40 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveErfKernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveLogisticKernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveLogisticKernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveLogisticKernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8540c064) // ld1rw {z4.s}, p0/z, [x3] + KAI_ASM_INST(0x8541c065) // ld1rw {z5.s}, p0/z, [x3, #4] + KAI_ASM_INST(0x8542c066) // ld1rw {z6.s}, p0/z, [x3, #8] + KAI_ASM_INST(0x8543c067) // ld1rw {z7.s}, p0/z, [x3, #12] + KAI_ASM_INST(0x8544c070) // ld1rw {z16.s}, p0/z, [x3, #16] + KAI_ASM_INST(0x8545c071) // ld1rw {z17.s}, p0/z, [x3, #20] + KAI_ASM_INST(0x8546c072) // ld1rw {z18.s}, p0/z, [x3, #24] + KAI_ASM_INST(0x8547c073) // ld1rw {z19.s}, p0/z, [x3, #28] + KAI_ASM_INST(0x8548c074) // ld1rw {z20.s}, p0/z, [x3, #32] + KAI_ASM_INST(0x8549c075) // ld1rw {z21.s}, p0/z, [x3, #36] + KAI_ASM_INST(0x854ac076) // ld1rw {z22.s}, p0/z, [x3, #40] + KAI_ASM_INST(0x854bc077) // ld1rw {z23.s}, p0/z, [x3, #44] + KAI_ASM_INST(0x854cc078) // ld1rw {z24.s}, p0/z, [x3, #48] + KAI_ASM_INST(0x854dc079) // ld1rw {z25.s}, p0/z, [x3, #52] + KAI_ASM_INST(0xb40003e2) // cbz x2, b8 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x04a0e3e4) // cntw x4 + KAI_ASM_INST(0x25b8c01b) // mov z27.s, #0 + KAI_ASM_INST(0x25b9ce1a) // fmov z26.s, #1.000000000000000000e+00 + KAI_ASM_INST(0x25a21c60) // whilelo p0.s, x3, x2 + KAI_ASM_INST(0xa5434003) // ld1w {z3.s}, p0/z, [x0, x3, lsl #2] + KAI_ASM_INST(0x65868083) // fmax z3.s, p0/m, z3.s, z4.s + KAI_ASM_INST(0x658780a3) // fmin z3.s, p0/m, z3.s, z5.s + KAI_ASM_INST(0x0420bc62) // movprfx z2, z3 + KAI_ASM_INST(0x65828062) // fmul z2.s, p0/m, z2.s, z3.s + KAI_ASM_INST(0x0420bce0) // movprfx z0, z7 + KAI_ASM_INST(0x65a60040) // fmla z0.s, p0/m, z2.s, z6.s + KAI_ASM_INST(0x0420be81) // movprfx z1, z20 + KAI_ASM_INST(0x65b30041) // fmla z1.s, p0/m, z2.s, z19.s + KAI_ASM_INST(0x65b08040) // fmad z0.s, p0/m, z2.s, z16.s + KAI_ASM_INST(0x65b58041) // fmad z1.s, p0/m, z2.s, z21.s + KAI_ASM_INST(0x65b18040) // fmad z0.s, p0/m, z2.s, z17.s + KAI_ASM_INST(0x65b68041) // fmad z1.s, p0/m, z2.s, z22.s + KAI_ASM_INST(0x65b28040) // fmad z0.s, p0/m, z2.s, z18.s + KAI_ASM_INST(0x65b78041) // fmad z1.s, p0/m, z2.s, z23.s + KAI_ASM_INST(0x65828060) // fmul z0.s, p0/m, z0.s, z3.s + KAI_ASM_INST(0x65b88041) // fmad z1.s, p0/m, z2.s, z24.s + KAI_ASM_INST(0x658d8020) // fdiv z0.s, p0/m, z0.s, z1.s + KAI_ASM_INST(0x65808320) // fadd z0.s, p0/m, z0.s, z25.s + KAI_ASM_INST(0x65868360) // fmax z0.s, p0/m, z0.s, z27.s + KAI_ASM_INST(0x65878340) // fmin z0.s, p0/m, z0.s, z26.s + KAI_ASM_INST(0xe5434020) // st1w {z0.s}, p0, [x1, x3, lsl #2] + KAI_ASM_INST(0x8b040063) // add x3, x3, x4 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffce8) // b.hi 50 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveLogisticKernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveTanhKernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveTanhKernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveTanhKernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8540c064) // ld1rw {z4.s}, p0/z, [x3] + KAI_ASM_INST(0x8541c065) // ld1rw {z5.s}, p0/z, [x3, #4] + KAI_ASM_INST(0x8542c066) // ld1rw {z6.s}, p0/z, [x3, #8] + KAI_ASM_INST(0x8543c067) // ld1rw {z7.s}, p0/z, [x3, #12] + KAI_ASM_INST(0x8544c070) // ld1rw {z16.s}, p0/z, [x3, #16] + KAI_ASM_INST(0x8545c071) // ld1rw {z17.s}, p0/z, [x3, #20] + KAI_ASM_INST(0x8546c072) // ld1rw {z18.s}, p0/z, [x3, #24] + KAI_ASM_INST(0x8547c073) // ld1rw {z19.s}, p0/z, [x3, #28] + KAI_ASM_INST(0x8548c074) // ld1rw {z20.s}, p0/z, [x3, #32] + KAI_ASM_INST(0x8549c075) // ld1rw {z21.s}, p0/z, [x3, #36] + KAI_ASM_INST(0x854ac076) // ld1rw {z22.s}, p0/z, [x3, #40] + KAI_ASM_INST(0x854bc077) // ld1rw {z23.s}, p0/z, [x3, #44] + KAI_ASM_INST(0x854cc078) // ld1rw {z24.s}, p0/z, [x3, #48] + KAI_ASM_INST(0xb4000362) // cbz x2, a4 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x04a0e3e4) // cntw x4 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25a21c60) // whilelo p0.s, x3, x2 + KAI_ASM_INST(0xa5434002) // ld1w {z2.s}, p0/z, [x0, x3, lsl #2] + KAI_ASM_INST(0x65868082) // fmax z2.s, p0/m, z2.s, z4.s + KAI_ASM_INST(0x658780a2) // fmin z2.s, p0/m, z2.s, z5.s + KAI_ASM_INST(0x0420bc41) // movprfx z1, z2 + KAI_ASM_INST(0x65828041) // fmul z1.s, p0/m, z1.s, z2.s + KAI_ASM_INST(0x0420bce0) // movprfx z0, z7 + KAI_ASM_INST(0x65a60020) // fmla z0.s, p0/m, z1.s, z6.s + KAI_ASM_INST(0x0420bec3) // movprfx z3, z22 + KAI_ASM_INST(0x65b50023) // fmla z3.s, p0/m, z1.s, z21.s + KAI_ASM_INST(0x65b08020) // fmad z0.s, p0/m, z1.s, z16.s + KAI_ASM_INST(0x65b78023) // fmad z3.s, p0/m, z1.s, z23.s + KAI_ASM_INST(0x65b18020) // fmad z0.s, p0/m, z1.s, z17.s + KAI_ASM_INST(0x65b88023) // fmad z3.s, p0/m, z1.s, z24.s + KAI_ASM_INST(0x65b28020) // fmad z0.s, p0/m, z1.s, z18.s + KAI_ASM_INST(0x65b38020) // fmad z0.s, p0/m, z1.s, z19.s + KAI_ASM_INST(0x65b48020) // fmad z0.s, p0/m, z1.s, z20.s + KAI_ASM_INST(0x65828040) // fmul z0.s, p0/m, z0.s, z2.s + KAI_ASM_INST(0x658d8060) // fdiv z0.s, p0/m, z0.s, z3.s + KAI_ASM_INST(0xe5434020) // st1w {z0.s}, p0, [x1, x3, lsl #2] + KAI_ASM_INST(0x8b040063) // add x3, x3, x4 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffd48) // b.hi 48 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveTanhKernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveComputeExpF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveComputeExpF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveComputeExpF32KernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8540c066) // ld1rw {z6.s}, p0/z, [x3] + KAI_ASM_INST(0x8541c067) // ld1rw {z7.s}, p0/z, [x3, #4] + KAI_ASM_INST(0x8542c070) // ld1rw {z16.s}, p0/z, [x3, #8] + KAI_ASM_INST(0x8543c071) // ld1rw {z17.s}, p0/z, [x3, #12] + KAI_ASM_INST(0x8544c072) // ld1rw {z18.s}, p0/z, [x3, #16] + KAI_ASM_INST(0x8545c064) // ld1rw {z4.s}, p0/z, [x3, #20] + KAI_ASM_INST(0x8546c073) // ld1rw {z19.s}, p0/z, [x3, #24] + KAI_ASM_INST(0x8547c074) // ld1rw {z20.s}, p0/z, [x3, #28] + KAI_ASM_INST(0x8548c075) // ld1rw {z21.s}, p0/z, [x3, #32] + KAI_ASM_INST(0x8549c076) // ld1rw {z22.s}, p0/z, [x3, #36] + KAI_ASM_INST(0x854ac077) // ld1rw {z23.s}, p0/z, [x3, #40] + KAI_ASM_INST(0x854bc078) // ld1rw {z24.s}, p0/z, [x3, #44] + KAI_ASM_INST(0xb40003e2) // cbz x2, b0 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x04a0e3e4) // cntw x4 + KAI_ASM_INST(0x25a21c60) // whilelo p0.s, x3, x2 + KAI_ASM_INST(0xa5434002) // ld1w {z2.s}, p0/z, [x0, x3, lsl #2] + KAI_ASM_INST(0x0420bc80) // movprfx z0, z4 + KAI_ASM_INST(0x65b30040) // fmla z0.s, p0/m, z2.s, z19.s + KAI_ASM_INST(0x0420bc01) // movprfx z1, z0 + KAI_ASM_INST(0x65818081) // fsub z1.s, p0/m, z1.s, z4.s + KAI_ASM_INST(0x0420bc43) // movprfx z3, z2 + KAI_ASM_INST(0x65b40023) // fmla z3.s, p0/m, z1.s, z20.s + KAI_ASM_INST(0x65824312) // fcmgt p2.s, p0/z, z24.s, z2.s + KAI_ASM_INST(0x65a382a1) // fmad z1.s, p0/m, z21.s, z3.s + KAI_ASM_INST(0x65974051) // fcmgt p1.s, p0/z, z2.s, z23.s + KAI_ASM_INST(0x0420bc23) // movprfx z3, z1 + KAI_ASM_INST(0x65828023) // fmul z3.s, p0/m, z3.s, z1.s + KAI_ASM_INST(0x0420bce5) // movprfx z5, z7 + KAI_ASM_INST(0x65a10205) // fmla z5.s, p0/m, z16.s, z1.s + KAI_ASM_INST(0x0420be22) // movprfx z2, z17 + KAI_ASM_INST(0x65a10242) // fmla z2.s, p0/m, z18.s, z1.s + KAI_ASM_INST(0x04779c00) // lsl z0.s, z0.s, #23 + KAI_ASM_INST(0x658280c1) // fmul z1.s, p0/m, z1.s, z6.s + KAI_ASM_INST(0x65a58062) // fmad z2.s, p0/m, z3.s, z5.s + KAI_ASM_INST(0x65a30041) // fmla z1.s, p0/m, z2.s, z3.s + KAI_ASM_INST(0x65a00020) // fmla z0.s, p0/m, z1.s, z0.s + KAI_ASM_INST(0x05924000) // mov z0.s, p2/m, #0 + KAI_ASM_INST(0x05a0c6c0) // mov z0.s, p1/m, z22.s + KAI_ASM_INST(0xe5434020) // st1w {z0.s}, p0, [x1, x3, lsl #2] + KAI_ASM_INST(0x8b040063) // add x3, x3, x4 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffca8) // b.hi 40 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveComputeExpF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveComputeSumExpF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveComputeSumExpF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveComputeSumExpF32KernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x854ec0b1) // ld1rw {z17.s}, p0/z, [x5, #56] + KAI_ASM_INST(0x8546c0b2) // ld1rw {z18.s}, p0/z, [x5, #24] + KAI_ASM_INST(0x8547c0b3) // ld1rw {z19.s}, p0/z, [x5, #28] + KAI_ASM_INST(0x8548c0b4) // ld1rw {z20.s}, p0/z, [x5, #32] + KAI_ASM_INST(0x854cc0a7) // ld1rw {z7.s}, p0/z, [x5, #48] + KAI_ASM_INST(0x854dc0b5) // ld1rw {z21.s}, p0/z, [x5, #52] + KAI_ASM_INST(0x8540c076) // ld1rw {z22.s}, p0/z, [x3] + KAI_ASM_INST(0xb4000602) // cbz x2, e0 + KAI_ASM_INST(0x04a0e3e4) // cntw x4 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x0460e3e5) // cnth x5 + KAI_ASM_INST(0x25b8c010) // mov z16.s, #0 + KAI_ASM_INST(0x04703200) // mov z0.d, z16.d + KAI_ASM_INST(0x25a21c61) // whilelo p1.s, x3, x2 + KAI_ASM_INST(0x25a21c80) // whilelo p0.s, x4, x2 + KAI_ASM_INST(0xa5434404) // ld1w {z4.s}, p1/z, [x0, x3, lsl #2] + KAI_ASM_INST(0xa5444003) // ld1w {z3.s}, p0/z, [x0, x4, lsl #2] + KAI_ASM_INST(0x658086c4) // fadd z4.s, p1/m, z4.s, z22.s + KAI_ASM_INST(0x658082c3) // fadd z3.s, p0/m, z3.s, z22.s + KAI_ASM_INST(0x65868624) // fmax z4.s, p1/m, z4.s, z17.s + KAI_ASM_INST(0x65868223) // fmax z3.s, p0/m, z3.s, z17.s + KAI_ASM_INST(0x0420bce2) // movprfx z2, z7 + KAI_ASM_INST(0x65b20482) // fmla z2.s, p1/m, z4.s, z18.s + KAI_ASM_INST(0x0420bce1) // movprfx z1, z7 + KAI_ASM_INST(0x65b20061) // fmla z1.s, p0/m, z3.s, z18.s + KAI_ASM_INST(0x04a0b846) // fexpa z6.s, z2.s + KAI_ASM_INST(0x04a0b825) // fexpa z5.s, z1.s + KAI_ASM_INST(0x658184e2) // fsub z2.s, p1/m, z2.s, z7.s + KAI_ASM_INST(0x658180e1) // fsub z1.s, p0/m, z1.s, z7.s + KAI_ASM_INST(0x65b30444) // fmla z4.s, p1/m, z2.s, z19.s + KAI_ASM_INST(0x65b30023) // fmla z3.s, p0/m, z1.s, z19.s + KAI_ASM_INST(0x65a48682) // fmad z2.s, p1/m, z20.s, z4.s + KAI_ASM_INST(0x65a38281) // fmad z1.s, p0/m, z20.s, z3.s + KAI_ASM_INST(0x0420bc44) // movprfx z4, z2 + KAI_ASM_INST(0x65828444) // fmul z4.s, p1/m, z4.s, z2.s + KAI_ASM_INST(0x0420bc43) // movprfx z3, z2 + KAI_ASM_INST(0x65b50483) // fmla z3.s, p1/m, z4.s, z21.s + KAI_ASM_INST(0x0420bc22) // movprfx z2, z1 + KAI_ASM_INST(0x65828022) // fmul z2.s, p0/m, z2.s, z1.s + KAI_ASM_INST(0x65a304c6) // fmla z6.s, p1/m, z6.s, z3.s + KAI_ASM_INST(0x65b50041) // fmla z1.s, p0/m, z2.s, z21.s + KAI_ASM_INST(0x65a100a5) // fmla z5.s, p0/m, z5.s, z1.s + KAI_ASM_INST(0xb4000061) // cbz x1, b8 + KAI_ASM_INST(0xe5434426) // st1w {z6.s}, p1, [x1, x3, lsl #2] + KAI_ASM_INST(0xe5444025) // st1w {z5.s}, p0, [x1, x4, lsl #2] + KAI_ASM_INST(0x8b050063) // add x3, x3, x5 + KAI_ASM_INST(0x8b050084) // add x4, x4, x5 + KAI_ASM_INST(0x658084c0) // fadd z0.s, p1/m, z0.s, z6.s + KAI_ASM_INST(0x658080b0) // fadd z16.s, p0/m, z16.s, z5.s + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffb68) // b.hi 38 + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x65900000) // fadd z0.s, z0.s, z16.s + KAI_ASM_INST(0x65802000) // faddv s0, p0, z0.s + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_INST(0x25b8c010) // mov z16.s, #0 + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x04703200) // mov z0.d, z16.d + KAI_ASM_INST(0x65900000) // fadd z0.s, z0.s, z16.s + KAI_ASM_INST(0x65802000) // faddv s0, p0, z0.s + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveComputeSumExpF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveReduceMaximumF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveReduceMaximumF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveReduceMaximumF32KernelImpl) + KAI_ASM_INST(0x0420e3e2) // cntb x2 + KAI_ASM_INST(0x05242000) // mov z0.s, s0 + KAI_ASM_INST(0xeb02003f) // cmp x1, x2 + KAI_ASM_INST(0x540002c3) // b.cc 64 + KAI_ASM_INST(0x0423e3e3) // cntb x3, all, mul #4 + KAI_ASM_INST(0x04603002) // mov z2.d, z0.d + KAI_ASM_INST(0x04603001) // mov z1.d, z0.d + KAI_ASM_INST(0x04603003) // mov z3.d, z0.d + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0xa540a005) // ld1w {z5.s}, p0/z, [x0] + KAI_ASM_INST(0xa541a004) // ld1w {z4.s}, p0/z, [x0, #1, mul vl] + KAI_ASM_INST(0xcb020021) // sub x1, x1, x2 + KAI_ASM_INST(0x658680a0) // fmax z0.s, p0/m, z0.s, z5.s + KAI_ASM_INST(0x65868083) // fmax z3.s, p0/m, z3.s, z4.s + KAI_ASM_INST(0xa542a005) // ld1w {z5.s}, p0/z, [x0, #2, mul vl] + KAI_ASM_INST(0xa543a004) // ld1w {z4.s}, p0/z, [x0, #3, mul vl] + KAI_ASM_INST(0x658680a1) // fmax z1.s, p0/m, z1.s, z5.s + KAI_ASM_INST(0x8b030000) // add x0, x0, x3 + KAI_ASM_INST(0x65868082) // fmax z2.s, p0/m, z2.s, z4.s + KAI_ASM_INST(0xeb02003f) // cmp x1, x2 + KAI_ASM_INST(0x54fffea2) // b.cs 28 + KAI_ASM_INST(0x65868060) // fmax z0.s, p0/m, z0.s, z3.s + KAI_ASM_INST(0x65868041) // fmax z1.s, p0/m, z1.s, z2.s + KAI_ASM_INST(0x65868020) // fmax z0.s, p0/m, z0.s, z1.s + KAI_ASM_INST(0xb4000121) // cbz x1, 88 + KAI_ASM_INST(0xd2800002) // mov x2, #0x0 + KAI_ASM_INST(0x04a0e3e3) // cntw x3 + KAI_ASM_INST(0x25a11c40) // whilelo p0.s, x2, x1 + KAI_ASM_INST(0xa5424001) // ld1w {z1.s}, p0/z, [x0, x2, lsl #2] + KAI_ASM_INST(0x8b030042) // add x2, x2, x3 + KAI_ASM_INST(0x65868020) // fmax z0.s, p0/m, z0.s, z1.s + KAI_ASM_INST(0xeb02003f) // cmp x1, x2 + KAI_ASM_INST(0x54ffff68) // b.hi 70 + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x65862000) // fmaxv s0, p0, z0.s + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveReduceMaximumF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveReduceMinimumMaximumF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveReduceMinimumMaximumF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveReduceMinimumMaximumF32KernelImpl) + KAI_ASM_INST(0x0420e3e5) // cntb x5 + KAI_ASM_INST(0xeb05007f) // cmp x3, x5 + KAI_ASM_INST(0x54000663) // b.cc d4 + KAI_ASM_INST(0x0423e3e6) // cntb x6, all, mul #4 + KAI_ASM_INST(0x05c043c4) // mov z4.s, #0xff7fffff + KAI_ASM_INST(0x12b01004) // mov w4, #0x7f7fffff + KAI_ASM_INST(0x04643085) // mov z5.d, z4.d + KAI_ASM_INST(0x05a03880) // mov z0.s, w4 + KAI_ASM_INST(0x04643087) // mov z7.d, z4.d + KAI_ASM_INST(0x04603010) // mov z16.d, z0.d + KAI_ASM_INST(0x04603006) // mov z6.d, z0.d + KAI_ASM_INST(0x04603011) // mov z17.d, z0.d + KAI_ASM_INST(0x04643081) // mov z1.d, z4.d + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0xa540a003) // ld1w {z3.s}, p0/z, [x0] + KAI_ASM_INST(0xa541a002) // ld1w {z2.s}, p0/z, [x0, #1, mul vl] + KAI_ASM_INST(0xcb050063) // sub x3, x3, x5 + KAI_ASM_INST(0x65878060) // fmin z0.s, p0/m, z0.s, z3.s + KAI_ASM_INST(0x65868061) // fmax z1.s, p0/m, z1.s, z3.s + KAI_ASM_INST(0x65878051) // fmin z17.s, p0/m, z17.s, z2.s + KAI_ASM_INST(0xa542a003) // ld1w {z3.s}, p0/z, [x0, #2, mul vl] + KAI_ASM_INST(0x65868047) // fmax z7.s, p0/m, z7.s, z2.s + KAI_ASM_INST(0x65878066) // fmin z6.s, p0/m, z6.s, z3.s + KAI_ASM_INST(0xa543a002) // ld1w {z2.s}, p0/z, [x0, #3, mul vl] + KAI_ASM_INST(0x65868065) // fmax z5.s, p0/m, z5.s, z3.s + KAI_ASM_INST(0x8b060000) // add x0, x0, x6 + KAI_ASM_INST(0x65878050) // fmin z16.s, p0/m, z16.s, z2.s + KAI_ASM_INST(0x65868044) // fmax z4.s, p0/m, z4.s, z2.s + KAI_ASM_INST(0xeb05007f) // cmp x3, x5 + KAI_ASM_INST(0x54fffe22) // b.cs 38 + KAI_ASM_INST(0x65878220) // fmin z0.s, p0/m, z0.s, z17.s + KAI_ASM_INST(0x65878206) // fmin z6.s, p0/m, z6.s, z16.s + KAI_ASM_INST(0x658680e1) // fmax z1.s, p0/m, z1.s, z7.s + KAI_ASM_INST(0x65868085) // fmax z5.s, p0/m, z5.s, z4.s + KAI_ASM_INST(0x658780c0) // fmin z0.s, p0/m, z0.s, z6.s + KAI_ASM_INST(0x658680a1) // fmax z1.s, p0/m, z1.s, z5.s + KAI_ASM_INST(0xb4000163) // cbz x3, bc + KAI_ASM_INST(0xd2800004) // mov x4, #0x0 + KAI_ASM_INST(0x04a0e3e5) // cntw x5 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25a31c80) // whilelo p0.s, x4, x3 + KAI_ASM_INST(0xa5444002) // ld1w {z2.s}, p0/z, [x0, x4, lsl #2] + KAI_ASM_INST(0x8b050084) // add x4, x4, x5 + KAI_ASM_INST(0x65878040) // fmin z0.s, p0/m, z0.s, z2.s + KAI_ASM_INST(0x65868041) // fmax z1.s, p0/m, z1.s, z2.s + KAI_ASM_INST(0xeb04007f) // cmp x3, x4 + KAI_ASM_INST(0x54ffff48) // b.hi a0 + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x65872000) // fminv s0, p0, z0.s + KAI_ASM_INST(0xbd000020) // str s0, [x1] + KAI_ASM_INST(0x65862021) // fmaxv s1, p0, z1.s + KAI_ASM_INST(0xbd000041) // str s1, [x2] + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_INST(0x12b01004) // mov w4, #0x7f7fffff + KAI_ASM_INST(0x05c043c1) // mov z1.s, #0xff7fffff + KAI_ASM_INST(0x05a03880) // mov z0.s, w4 + KAI_ASM_INST(0x17ffffec) // b 90 + KAI_ASM_FUNCTION_END(MlasSveReduceMinimumMaximumF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveComputeSoftmaxOutputF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveComputeSoftmaxOutputF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveComputeSoftmaxOutputF32KernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8540c041) // ld1rw {z1.s}, p0/z, [x2] + KAI_ASM_INST(0xb4000161) // cbz x1, 34 + KAI_ASM_INST(0xd2800002) // mov x2, #0x0 + KAI_ASM_INST(0x04a0e3e3) // cntw x3 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25a11c40) // whilelo p0.s, x2, x1 + KAI_ASM_INST(0xa5424000) // ld1w {z0.s}, p0/z, [x0, x2, lsl #2] + KAI_ASM_INST(0x65828020) // fmul z0.s, p0/m, z0.s, z1.s + KAI_ASM_INST(0xe5424000) // st1w {z0.s}, p0, [x0, x2, lsl #2] + KAI_ASM_INST(0x8b030042) // add x2, x2, x3 + KAI_ASM_INST(0xeb02003f) // cmp x1, x2 + KAI_ASM_INST(0x54ffff48) // b.hi 18 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveComputeSoftmaxOutputF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveComputeLogSoftmaxOutputF32KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveComputeLogSoftmaxOutputF32KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveComputeLogSoftmaxOutputF32KernelImpl) + KAI_ASM_INST(0x2518e3e0) // ptrue p0.b + KAI_ASM_INST(0x8540c061) // ld1rw {z1.s}, p0/z, [x3] + KAI_ASM_INST(0x8541c062) // ld1rw {z2.s}, p0/z, [x3, #4] + KAI_ASM_INST(0xb4000162) // cbz x2, 38 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x04a0e3e4) // cntw x4 + KAI_ASM_INST(0x25a21c60) // whilelo p0.s, x3, x2 + KAI_ASM_INST(0xa5434000) // ld1w {z0.s}, p0/z, [x0, x3, lsl #2] + KAI_ASM_INST(0x65808020) // fadd z0.s, p0/m, z0.s, z1.s + KAI_ASM_INST(0x65818040) // fsub z0.s, p0/m, z0.s, z2.s + KAI_ASM_INST(0xe5434020) // st1w {z0.s}, p0, [x1, x3, lsl #2] + KAI_ASM_INST(0x8b040063) // add x3, x3, x4 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54ffff28) // b.hi 18 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveComputeLogSoftmaxOutputF32KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveTanhFP16KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveTanhFP16KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveTanhFP16KernelImpl) + KAI_ASM_INST(0xb4000502) // cbz x2, a0 + KAI_ASM_INST(0x4f078730) // movi v16.8h, #0xf9 + KAI_ASM_INST(0x12879ee4) // mov w4, #0xffffc308 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x05603891) // mov z17.h, w4 + KAI_ASM_INST(0x0460e3e6) // cnth x6 + KAI_ASM_INST(0x52822704) // mov w4, #0x1138 + KAI_ASM_INST(0x52886105) // mov w5, #0x4308 + KAI_ASM_INST(0x05603887) // mov z7.h, w4 + KAI_ASM_INST(0x5283a064) // mov w4, #0x1d03 + KAI_ASM_INST(0x056038b2) // mov z18.h, w5 + KAI_ASM_INST(0x05603884) // mov z4.h, w4 + KAI_ASM_INST(0x5280f8a5) // mov w5, #0x7c5 + KAI_ASM_INST(0x528314a4) // mov w4, #0x18a5 + KAI_ASM_INST(0x05302210) // mov z16.q, q16 + KAI_ASM_INST(0x056038a6) // mov z6.h, w5 + KAI_ASM_INST(0x2578c034) // mov z20.h, #1 + KAI_ASM_INST(0x05603885) // mov z5.h, w4 + KAI_ASM_INST(0x2578c293) // mov z19.h, #20 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25621c60) // whilelo p0.h, x3, x2 + KAI_ASM_INST(0xa4a34001) // ld1h {z1.h}, p0/z, [x0, x3, lsl #1] + KAI_ASM_INST(0x65478241) // fmin z1.h, p0/m, z1.h, z18.h + KAI_ASM_INST(0x65468221) // fmax z1.h, p0/m, z1.h, z17.h + KAI_ASM_INST(0x0420bc22) // movprfx z2, z1 + KAI_ASM_INST(0x65428022) // fmul z2.h, p0/m, z2.h, z1.h + KAI_ASM_INST(0x0420be00) // movprfx z0, z16 + KAI_ASM_INST(0x65620280) // fmla z0.h, p0/m, z20.h, z2.h + KAI_ASM_INST(0x0420bcc3) // movprfx z3, z6 + KAI_ASM_INST(0x65620263) // fmla z3.h, p0/m, z19.h, z2.h + KAI_ASM_INST(0x65678040) // fmad z0.h, p0/m, z2.h, z7.h + KAI_ASM_INST(0x65658043) // fmad z3.h, p0/m, z2.h, z5.h + KAI_ASM_INST(0x65648040) // fmad z0.h, p0/m, z2.h, z4.h + KAI_ASM_INST(0x65648062) // fmad z2.h, p0/m, z3.h, z4.h + KAI_ASM_INST(0x65428020) // fmul z0.h, p0/m, z0.h, z1.h + KAI_ASM_INST(0x654d8040) // fdiv z0.h, p0/m, z0.h, z2.h + KAI_ASM_INST(0xe4a34020) // st1h {z0.h}, p0, [x1, x3, lsl #1] + KAI_ASM_INST(0x8b060063) // add x3, x3, x6 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffda8) // b.hi 50 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveTanhFP16KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveErfFP16KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveErfFP16KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveErfFP16KernelImpl) + KAI_ASM_INST(0xb4000b22) // cbz x2, 164 + KAI_ASM_INST(0x5286a7e6) // mov w6, #0x353f + KAI_ASM_INST(0x52868045) // mov w5, #0x3402 + KAI_ASM_INST(0x056038d7) // mov z23.h, w6 + KAI_ASM_INST(0x056038b6) // mov z22.h, w5 + KAI_ASM_INST(0x12896c66) // mov w6, #0xffffb49c + KAI_ASM_INST(0x5287b485) // mov w5, #0x3da4 + KAI_ASM_INST(0x056038d5) // mov z21.h, w6 + KAI_ASM_INST(0x056038b4) // mov z20.h, w5 + KAI_ASM_INST(0x128849c6) // mov w6, #0xffffbdb1 + KAI_ASM_INST(0x52878705) // mov w5, #0x3c38 + KAI_ASM_INST(0x056038d3) // mov z19.h, w6 + KAI_ASM_INST(0x056038b2) // mov z18.h, w5 + KAI_ASM_INST(0x12893846) // mov w6, #0xffffb63d + KAI_ASM_INST(0x5287aa45) // mov w5, #0x3d52 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x0460e3e4) // cnth x4 + KAI_ASM_INST(0x056038d1) // mov z17.h, w6 + KAI_ASM_INST(0x056038b0) // mov z16.h, w5 + KAI_ASM_INST(0x5284ebe6) // mov w6, #0x275f + KAI_ASM_INST(0x52875925) // mov w5, #0x3ac9 + KAI_ASM_INST(0x056038c7) // mov z7.h, w6 + KAI_ASM_INST(0x056038a6) // mov z6.h, w5 + KAI_ASM_INST(0x5287ab46) // mov w6, #0x3d5a + KAI_ASM_INST(0x5286ef25) // mov w5, #0x3779 + KAI_ASM_INST(0x056038c5) // mov z5.h, w6 + KAI_ASM_INST(0x056038a4) // mov z4.h, w5 + KAI_ASM_INST(0x2579de19) // fmov z25.h, #-1.000000000000000000e+00 + KAI_ASM_INST(0x2579ce03) // fmov z3.h, #1.000000000000000000e+00 + KAI_ASM_INST(0x2579c202) // fmov z2.h, #4.000000000000000000e+00 + KAI_ASM_INST(0x2579c318) // fmov z24.h, #6.000000000000000000e+00 + KAI_ASM_INST(0xfc1f0fe8) // str d8, [sp, #-16]! + KAI_ASM_INST(0x25621c60) // whilelo p0.h, x3, x2 + KAI_ASM_INST(0xa4a34000) // ld1h {z0.h}, p0/z, [x0, x3, lsl #1] + KAI_ASM_INST(0x0420bc1e) // movprfx z30, z0 + KAI_ASM_INST(0x045ca01e) // fabs z30.h, p0/m, z0.h + KAI_ASM_INST(0x65512001) // fcmlt p1.h, p0/z, z0.h, #0.0 + KAI_ASM_INST(0x0420bfc1) // movprfx z1, z30 + KAI_ASM_INST(0x65478041) // fmin z1.h, p0/m, z1.h, z2.h + KAI_ASM_INST(0x0420bc7d) // movprfx z29, z3 + KAI_ASM_INST(0x656102fd) // fmla z29.h, p0/m, z23.h, z1.h + KAI_ASM_INST(0x654e33a0) // frecpe z0.h, z29.h + KAI_ASM_INST(0x65401ba8) // frecps z8.h, z29.h, z0.h + KAI_ASM_INST(0x65428100) // fmul z0.h, p0/m, z0.h, z8.h + KAI_ASM_INST(0x65401bbd) // frecps z29.h, z29.h, z0.h + KAI_ASM_INST(0x65428021) // fmul z1.h, p0/m, z1.h, z1.h + KAI_ASM_INST(0x654283a0) // fmul z0.h, p0/m, z0.h, z29.h + KAI_ASM_INST(0x65478301) // fmin z1.h, p0/m, z1.h, z24.h + KAI_ASM_INST(0x0420bc1d) // movprfx z29, z0 + KAI_ASM_INST(0x6542801d) // fmul z29.h, p0/m, z29.h, z0.h + KAI_ASM_INST(0x0420bc3b) // movprfx z27, z1 + KAI_ASM_INST(0x6542803b) // fmul z27.h, p0/m, z27.h, z1.h + KAI_ASM_INST(0x0420bec8) // movprfx z8, z22 + KAI_ASM_INST(0x65428008) // fmul z8.h, p0/m, z8.h, z0.h + KAI_ASM_INST(0x657d02a8) // fmla z8.h, p0/m, z21.h, z29.h + KAI_ASM_INST(0x0420bc7a) // movprfx z26, z3 + KAI_ASM_INST(0x0551779a) // mov z26.h, p1/m, #-17408 + KAI_ASM_INST(0x0420bfbf) // movprfx z31, z29 + KAI_ASM_INST(0x6542801f) // fmul z31.h, p0/m, z31.h, z0.h + KAI_ASM_INST(0x0420be1c) // movprfx z28, z16 + KAI_ASM_INST(0x6561023c) // fmla z28.h, p0/m, z17.h, z1.h + KAI_ASM_INST(0x0420bffd) // movprfx z29, z31 + KAI_ASM_INST(0x6542801d) // fmul z29.h, p0/m, z29.h, z0.h + KAI_ASM_INST(0x657b00fc) // fmla z28.h, p0/m, z7.h, z27.h + KAI_ASM_INST(0x6568829f) // fmad z31.h, p0/m, z20.h, z8.h + KAI_ASM_INST(0x654283a0) // fmul z0.h, p0/m, z0.h, z29.h + KAI_ASM_INST(0x656580c1) // fmad z1.h, p0/m, z6.h, z5.h + KAI_ASM_INST(0x657f827d) // fmad z29.h, p0/m, z19.h, z31.h + KAI_ASM_INST(0x657b0081) // fmla z1.h, p0/m, z4.h, z27.h + KAI_ASM_INST(0x657d8240) // fmad z0.h, p0/m, z18.h, z29.h + KAI_ASM_INST(0x654e303b) // frecpe z27.h, z1.h + KAI_ASM_INST(0x655e4051) // fcmgt p1.h, p0/z, z2.h, z30.h + KAI_ASM_INST(0x655b1828) // frecps z8.h, z1.h, z27.h + KAI_ASM_INST(0x6542811b) // fmul z27.h, p0/m, z27.h, z8.h + KAI_ASM_INST(0x655b1821) // frecps z1.h, z1.h, z27.h + KAI_ASM_INST(0x6542803b) // fmul z27.h, p0/m, z27.h, z1.h + KAI_ASM_INST(0x6542837c) // fmul z28.h, p0/m, z28.h, z27.h + KAI_ASM_INST(0x65428380) // fmul z0.h, p0/m, z0.h, z28.h + KAI_ASM_INST(0x655b8020) // fsubr z0.h, p0/m, z0.h, #1.0 + KAI_ASM_INST(0x65428340) // fmul z0.h, p0/m, z0.h, z26.h + KAI_ASM_INST(0x655f8020) // fmin z0.h, p0/m, z0.h, #1.0 + KAI_ASM_INST(0x65468320) // fmax z0.h, p0/m, z0.h, z25.h + KAI_ASM_INST(0x057ac400) // sel z0.h, p1, z0.h, z26.h + KAI_ASM_INST(0xe4a34020) // st1h {z0.h}, p0, [x1, x3, lsl #1] + KAI_ASM_INST(0x8b040063) // add x3, x3, x4 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fff948) // b.hi 80 + KAI_ASM_INST(0xfc4107e8) // ldr d8, [sp], #16 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveErfFP16KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveGeluTanhArgFP16KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveGeluTanhArgFP16KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveGeluTanhArgFP16KernelImpl) + KAI_ASM_INST(0xb40002c2) // cbz x2, 58 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x0460e3e6) // cnth x6 + KAI_ASM_INST(0x52851225) // mov w5, #0x2891 + KAI_ASM_INST(0x52874c44) // mov w4, #0x3a62 + KAI_ASM_INST(0x056038a3) // mov z3.h, w5 + KAI_ASM_INST(0x05603882) // mov z2.h, w4 + KAI_ASM_INST(0x2579c285) // fmov z5.h, #5.000000000000000000e+00 + KAI_ASM_INST(0x2579d284) // fmov z4.h, #-5.000000000000000000e+00 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25621c60) // whilelo p0.h, x3, x2 + KAI_ASM_INST(0xa4a34000) // ld1h {z0.h}, p0/z, [x0, x3, lsl #1] + KAI_ASM_INST(0x0420bc01) // movprfx z1, z0 + KAI_ASM_INST(0x65428001) // fmul z1.h, p0/m, z1.h, z0.h + KAI_ASM_INST(0x65628061) // fmad z1.h, p0/m, z3.h, z2.h + KAI_ASM_INST(0x65428020) // fmul z0.h, p0/m, z0.h, z1.h + KAI_ASM_INST(0x654780a0) // fmin z0.h, p0/m, z0.h, z5.h + KAI_ASM_INST(0x65468080) // fmax z0.h, p0/m, z0.h, z4.h + KAI_ASM_INST(0xe4a34020) // st1h {z0.h}, p0, [x1, x3, lsl #1] + KAI_ASM_INST(0x8b060063) // add x3, x3, x6 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54fffea8) // b.hi 28 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveGeluTanhArgFP16KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveGeluScaleFP16KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveGeluScaleFP16KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveGeluScaleFP16KernelImpl) + KAI_ASM_INST(0xb40001a2) // cbz x2, 34 + KAI_ASM_INST(0xd2800003) // mov x3, #0x0 + KAI_ASM_INST(0x0460e3e5) // cnth x5 + KAI_ASM_INST(0x52873504) // mov w4, #0x39a8 + KAI_ASM_INST(0x05603881) // mov z1.h, w4 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25621c60) // whilelo p0.h, x3, x2 + KAI_ASM_INST(0xa4a34000) // ld1h {z0.h}, p0/z, [x0, x3, lsl #1] + KAI_ASM_INST(0x65428020) // fmul z0.h, p0/m, z0.h, z1.h + KAI_ASM_INST(0xe4a34020) // st1h {z0.h}, p0, [x1, x3, lsl #1] + KAI_ASM_INST(0x8b050063) // add x3, x3, x5 + KAI_ASM_INST(0xeb03005f) // cmp x2, x3 + KAI_ASM_INST(0x54ffff48) // b.hi 18 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveGeluScaleFP16KernelImpl) + + KAI_ASM_ALIGN + KAI_ASM_GLOBAL(MlasSveGeluCombineFP16KernelImpl) + KAI_ASM_FUNCTION_TYPE(MlasSveGeluCombineFP16KernelImpl) +KAI_ASM_FUNCTION_LABEL(MlasSveGeluCombineFP16KernelImpl) + KAI_ASM_INST(0xb4000203) // cbz x3, 40 + KAI_ASM_INST(0xd2800004) // mov x4, #0x0 + KAI_ASM_INST(0x0460e3e5) // cnth x5 + KAI_ASM_INST(0x2579ce03) // fmov z3.h, #1.000000000000000000e+00 + KAI_ASM_INST(0x2579cc02) // fmov z2.h, #5.000000000000000000e-01 + KAI_ASM_INST(0xd503201f) // nop + KAI_ASM_INST(0x25631c80) // whilelo p0.h, x4, x3 + KAI_ASM_INST(0xa4a44000) // ld1h {z0.h}, p0/z, [x0, x4, lsl #1] + KAI_ASM_INST(0xa4a44021) // ld1h {z1.h}, p0/z, [x1, x4, lsl #1] + KAI_ASM_INST(0x65408061) // fadd z1.h, p0/m, z1.h, z3.h + KAI_ASM_INST(0x65428020) // fmul z0.h, p0/m, z0.h, z1.h + KAI_ASM_INST(0x65428040) // fmul z0.h, p0/m, z0.h, z2.h + KAI_ASM_INST(0xe4a44040) // st1h {z0.h}, p0, [x2, x4, lsl #1] + KAI_ASM_INST(0x8b050084) // add x4, x4, x5 + KAI_ASM_INST(0xeb04007f) // cmp x3, x4 + KAI_ASM_INST(0x54fffee8) // b.hi 18 + KAI_ASM_INST(0xd65f03c0) // ret + KAI_ASM_FUNCTION_END(MlasSveGeluCombineFP16KernelImpl) + + KAI_ASM_END diff --git a/onnxruntime/core/mlas/lib/aarch64/kai_asm_macros.h b/onnxruntime/core/mlas/lib/aarch64/kai_asm_macros.h new file mode 100644 index 0000000000000..8136b335798a9 --- /dev/null +++ b/onnxruntime/core/mlas/lib/aarch64/kai_asm_macros.h @@ -0,0 +1,63 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + kai_asm_macros.h + +Abstract: + + Portable assembly macros for AArch64 kernels that must assemble under + both the GNU assembler (Linux, macOS) and Microsoft armasm64 (Windows). + + The macro set and its names are adopted verbatim from Arm's KleidiAI + library (kai/kai_common_sve_asm.S, SPDX-License-Identifier: Apache-2.0), + which inlines this block at the top of each of its portable *_asm.S + files; keeping the same names lets kernels move between the two trees + unchanged. This header exists so ONNX Runtime kernels can share a single + copy. + + Instructions that a given assembler may not accept as mnemonics (all SVE + instructions, in the case of armasm64) are emitted as raw instruction + words via KAI_ASM_INST, which expands to ".inst" under GAS and "DCD" + under armasm64 — the same bytes either way. + +--*/ + +#if defined(_MSC_VER) + + #define KAI_ASM_GLOBAL(name) GLOBAL name + #define KAI_ASM_FUNCTION_TYPE(name) + #define KAI_ASM_FUNCTION_LABEL(name) name PROC + #define KAI_ASM_FUNCTION_END(name) ENDP + + #define KAI_ASM_CODE(name) AREA name, CODE, READONLY + #define KAI_ASM_ALIGN + #define KAI_ASM_LABEL(name) name + #define KAI_ASM_INST(hex) DCD hex + #define KAI_ASM_END END + +#else + + #if defined(__APPLE__) + #define KAI_ASM_GLOBAL(name) .globl _##name + #define KAI_ASM_FUNCTION_TYPE(name) + #define KAI_ASM_FUNCTION_LABEL(name) _##name: + #define KAI_ASM_FUNCTION_END(name) + #else + #define KAI_ASM_GLOBAL(name) .global name + #define KAI_ASM_FUNCTION_TYPE(name) .type name, %function + #define KAI_ASM_FUNCTION_LABEL(name) name: + #define KAI_ASM_FUNCTION_END(name) .size name, .-name + #endif + + #define KAI_ASM_CODE(name) .text + #define KAI_ASM_ALIGN .p2align 4,,11 + #define KAI_ASM_LABEL(name) name: + #define KAI_ASM_INST(hex) .inst hex + #define KAI_ASM_END + +#endif diff --git a/onnxruntime/core/mlas/lib/compute.cpp b/onnxruntime/core/mlas/lib/compute.cpp index 013247ce1806c..973c00661d10d 100644 --- a/onnxruntime/core/mlas/lib/compute.cpp +++ b/onnxruntime/core/mlas/lib/compute.cpp @@ -277,7 +277,7 @@ Return Value: --*/ { -#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64) +#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64) || defined(MLAS_USE_SVE) GetMlasPlatform().ComputeExpF32Kernel(Input, Output, N); #else MlasComputeExpF32Kernel(Input, Output, N); diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index 7e8c5e931b0ef..cfbe80a608126 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -1640,6 +1640,9 @@ struct MLAS_PLATFORM { MLAS_COMPUTE_SUMEXP_FLOAT_KERNEL* ComputeSumExpF32Kernel; MLAS_COMPUTE_LOGSOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeLogSoftmaxOutputF32Kernel; MLAS_COMPUTE_SOFTMAX_OUTPUT_FLOAT_KERNEL* ComputeSoftmaxOutputF32Kernel; + MLAS_COMPUTE_UNARY_FLOAT_KERNEL* ComputeExpF32Kernel; + MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL* ReduceMinimumMaximumF32Kernel; + MLAS_COMPUTE_UNARY_FLOAT_KERNEL* TanhKernelRoutine; #endif #if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64) // Hoisted under combined guard so future "shared between AMD64 and RISCV64" @@ -1647,8 +1650,6 @@ struct MLAS_PLATFORM { // assigns these fields independently in platform.cpp. MLAS_COMPUTE_UNARY_FLOAT_KERNEL* GeluErfKernelRoutine; MLAS_COMPUTE_UNARY_FLOAT_KERNEL* SiluKernelRoutine; - MLAS_COMPUTE_UNARY_FLOAT_KERNEL* TanhKernelRoutine; - MLAS_COMPUTE_UNARY_FLOAT_KERNEL* ComputeExpF32Kernel; #endif #if defined(MLAS_TARGET_RISCV64) && defined(MLAS_USE_RVV) @@ -1683,7 +1684,6 @@ MLAS_COMPUTE_TANH_FP16_KERNEL* TanhFP16KernelRoutine = nullptr; MLAS_POOL_FLOAT_KERNEL* PoolFloatKernel[MlasPoolingKindCount]; MLAS_QLINEAR_BINARY_OP_S8_KERNEL* QLinearAddS8Kernel; MLAS_QLINEAR_BINARY_OP_U8_KERNEL* QLinearAddU8Kernel; - MLAS_REDUCE_MINIMUM_MAXIMUM_FLOAT_KERNEL* ReduceMinimumMaximumF32Kernel; MLAS_QUANTIZE_LINEAR_S8_KERNEL* QuantizeLinearS8Kernel; MLAS_QUANTIZE_LINEAR_U8_KERNEL* QuantizeLinearU8Kernel; MLAS_QUANTIZE_LINEAR_S16_KERNEL* QuantizeLinearS16Kernel; diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 54d47449a23a6..ebeeaa21a45ee 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -729,7 +729,10 @@ Return Value: if (MLAS_CPUIDINFO::GetCPUIDInfo().HasArmSve()) { this->ErfKernelRoutine = MlasSveErfKernel; this->LogisticKernelRoutine = MlasSveLogisticKernel; + this->ComputeExpF32Kernel = MlasSveComputeExpF32Kernel; + this->TanhKernelRoutine = MlasSveTanhKernel; this->ReduceMaximumF32Kernel = MlasSveReduceMaximumF32Kernel; + this->ReduceMinimumMaximumF32Kernel = MlasSveReduceMinimumMaximumF32Kernel; this->ComputeSumExpF32Kernel = MlasSveComputeSumExpF32Kernel; this->ComputeLogSoftmaxOutputF32Kernel = MlasSveComputeLogSoftmaxOutputF32Kernel; this->ComputeSoftmaxOutputF32Kernel = MlasSveComputeSoftmaxOutputF32Kernel; @@ -737,14 +740,23 @@ Return Value: else{ this->ErfKernelRoutine = MlasErfKernel; this->LogisticKernelRoutine = MlasLogisticKernel; + this->ComputeExpF32Kernel = MlasComputeExpF32Kernel; + this->TanhKernelRoutine = MlasTanhKernel; this->ReduceMaximumF32Kernel = MlasReduceMaximumF32Kernel; + this->ReduceMinimumMaximumF32Kernel = MlasReduceMinimumMaximumF32Kernel; this->ComputeSumExpF32Kernel = MlasComputeSumExpF32Kernel; this->ComputeLogSoftmaxOutputF32Kernel = MlasComputeLogSoftmaxOutputF32Kernel; this->ComputeSoftmaxOutputF32Kernel = MlasComputeSoftmaxOutputF32Kernel; } #endif -#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) && !defined(_WIN32) +#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) + // + // The SVE fp16 kernels are portable machine code and available on every + // platform; the NEON fp16 fallbacks are only built on non-Windows + // targets. On Windows without SVE hardware the routines stay null and + // the callers use their scalar fallbacks. + // #if defined(MLAS_USE_SVE) if (MLAS_CPUIDINFO::GetCPUIDInfo().HasArmSve()) { this->ErfFP16KernelRoutine = MlasSveErfFP16Kernel; @@ -752,10 +764,12 @@ Return Value: this->TanhFP16KernelRoutine = MlasSveTanhFP16Kernel; } else{ + #if !defined(_WIN32) this->ErfFP16KernelRoutine = MlasNeonErfFP16Kernel; this->GeluFP16KernelRoutine = MlasNeonGeluFP16Kernel; + #endif } - #else + #elif !defined(_WIN32) this->ErfFP16KernelRoutine = MlasNeonErfFP16Kernel; this->GeluFP16KernelRoutine = MlasNeonGeluFP16Kernel; #endif diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index c5bf2b42bdf49..493437a9efc30 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -2311,7 +2311,7 @@ Return Value: --*/ { -#if defined(MLAS_TARGET_AMD64) +#if defined(MLAS_TARGET_AMD64) || defined(MLAS_USE_SVE) GetMlasPlatform().ReduceMinimumMaximumF32Kernel(Input, Min, Max, N); #else MlasReduceMinimumMaximumF32Kernel(Input, Min, Max, N); diff --git a/onnxruntime/core/mlas/lib/sve/elementwise_sve.cpp b/onnxruntime/core/mlas/lib/sve/elementwise_sve.cpp index fb2972ab7d202..fecefffbf070e 100644 --- a/onnxruntime/core/mlas/lib/sve/elementwise_sve.cpp +++ b/onnxruntime/core/mlas/lib/sve/elementwise_sve.cpp @@ -1,6 +1,9 @@ /*++ Copyright 2025 FUJITSU LIMITED +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. Module Name: @@ -8,169 +11,51 @@ Module Name: Abstract: - This module contains the implementation of SVE-based elementwise operations + This module implements the float32 elementwise kernel bodies using SVE + intrinsics: Erf, Logistic, Exp, SumExp, ReduceMaximum, + ReduceMinimumMaximum and the Softmax/LogSoftmax output steps. + + Only the vector loops live here, as extern "C" *Impl functions that + receive their constant tables by pointer and therefore contain no data + relocations: this lets the machine code be captured verbatim into the + portable assembly variant (elementwise_sve_asm.S), which exports the + same symbols. The public MlasSve* entry points live in + elementwise_sve_dispatch.cpp, which links against exactly one of the two + implementations. + + The kernels share one loop idiom: a whilelt-governed predicate covers the + vector body and the tail in the same loop, and every constant is + broadcast once before the loop. The polynomial constants are the shared + tables from erf.cpp / logistic.cpp / compute.cpp — the values and the + operation order match the scalar/NEON kernels. + + This file is compiled with SVE enabled via its per-file -march compile + flag (see onnxruntime_mlas.cmake), so no pragma/attribute targeting is + needed. + --*/ #include "mlasi_sve.h" -#include - -// -// Bundles the constants for use by kernels written in assembly. -// - -MLAS_INTERNAL_DATA const struct { - float ErfUpperAbsRange; - float ErfSplitBoundary; - float ErfSMALL_P0; - float ErfSMALL_P1; - float ErfSMALL_P2; - float ErfSMALL_P3; - float ErfSMALL_P4; - float ErfSMALL_P5_Minus_One; - float ErfReserved0; - float ErfBIG_P0; - float ErfBIG_P1; - float ErfBIG_P2; - float ErfBIG_P3; - float ErfBIG_P4; - float ErfBIG_P5; - float ErfBIG_P6_Minus_One; - float ErfNegZero; - float ErfOne; - - float Exp_UpperRange; - float Exp_LowerRange; - float Exp_Log2Reciprocal; - float Exp_log2_hi; - float Exp_log2_lo; - float Exp_P0; - float Exp_P1; - float Exp_P2; - float Exp_P3; - float Exp_P4; - float Exp_P5; - float Exp_P6; - float Exp_C; - int32_t Exp_X7F; -} MlasSveErfConstants = { - 3.925f, - 0.921875f, - -5.99104969e-4f, - 4.99339588e-3f, - -2.67667342e-2f, - 1.12818025e-1f, - -3.76124859e-1f, - 1.28379151e-1f, - 0.0f, - 1.72948930e-5f, - -3.83208680e-4f, - 3.88393435e-3f, - -2.42545605e-2f, - 1.06777847e-1f, - 6.34846687e-1f, - 1.28717512e-1f, - -0.0f, - 1.0f, - - // Independent parameters to calculate Exp for Erff() - 88.3762626647950f, - -88.3762626647949f, - 1.44269504088896341f, - -6.93145752e-1f, - -1.42860677e-6f, - 1.38319808e-3f, - 8.37550033e-3f, - 4.16689515e-2f, - 1.66664466e-1f, - 4.99999851e-1f, - 1.00000000e+0f, - 1.00000000e+0f, - 1.25829120e+7f, - 127, -}; - -MLAS_INTERNAL_DATA const struct { - float LowerRange; - float UpperRange; - float alpha_9; - float alpha_7; - float alpha_5; - float alpha_3; - float alpha_1; - float beta_10; - float beta_8; - float beta_6; - float beta_4; - float beta_2; - float beta_0; - float one_half; -} MlasSveLogisticConstants = { - -18.0f, - 18.0f, - 4.37031012579801e-11f, - 1.15627324459942e-07f, - 6.08574864600143e-05f, - 8.51377133304701e-03f, - 2.48287947061529e-01f, - 6.10247389755681e-13f, - 5.76102136993427e-09f, - 6.29106785017040e-06f, - 1.70198817374094e-03f, - 1.16817656904453e-01f, - 9.93151921023180e-01f, - 0.5f, -}; - -MLAS_INTERNAL_DATA const struct { - float LowerRange; - float UpperRange; - float LowerRangeSumExp; - float UpperRangeSumExp; - float RoundingBias; - float Log2Reciprocal; - float Log2High; - float Log2Low; - float poly_0; - float poly_1; - float poly_2; - float poly_3; - float poly_4; - float poly_56; - int32_t MinimumExponent; - int32_t MaximumExponent; -} MlasSveExpConstants = { - -103.9720840454f, - 88.7762626647950f, - -88.3762626647949f, - 88.3762626647949f, - MLAS_ROUNDING_BIAS_MAGIC, - 1.44269504088896341f, - -6.93145752e-1f, - -1.42860677e-6f, - 0x1.694000p-10, - 0x1.125edcp-7, - 0x1.555b5ap-5, - 0x1.555450p-3, - 0x1.fffff6p-2, - 0x1.000000p+0, - int32_t(0xC1000000), - int32_t(0x3F800000), -}; - -MLAS_INTERNAL_DATA const float MlasSveMinimumF32Value = std::numeric_limits::lowest(); - -void + +#include + +extern "C" void MLASCALL -MlasSveErfKernel( +MlasSveErfKernelImpl( const float* Input, float* Output, - size_t N + size_t N, + const MLAS_ERF_CONSTANTS* Constants ) /*++ Routine Description: - This routine implements the generic kernel for the error function. + This routine implements the error function using SVE. + + erf(x) = sign(x) * P_small(|x|) for |x| <= 0.921875, else + sign(x) * (1 - exp(P_big(min(|x|, 3.925)))), with the same split + polynomials as the scalar/NEON kernels. Arguments: @@ -186,85 +71,123 @@ Return Value: --*/ { - MLAS_SVBOOL Pred = svptrue_b32(); - size_t sve_veclen = svcntw(); - size_t stride = sve_veclen; - - while (N > 0) { - // If fewer that SVE vector length elements are remaining, adjust the predicate - if (N < sve_veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - MLAS_SVFLOAT32 Value = MlasSveLoadFloat32(Pred, Input); - MLAS_SVFLOAT32 NegZero = MlasSveBroadcastFloat32(MlasSveErfConstants.ErfNegZero); - MLAS_SVFLOAT32 SignMask = MlasSveAndFloat32(Pred, Value, NegZero); - MLAS_SVFLOAT32 AbsValue = MlasSveAndNotFloat32(Pred, NegZero, Value); - AbsValue = MlasSveMinimumFloat32(Pred, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfUpperAbsRange), AbsValue); - MLAS_SVFLOAT32 SquareValue = MlasSveMultiplyFloat32(Pred, AbsValue, AbsValue); - - MLAS_SVFLOAT32 r_small = MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P0); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, SquareValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P1)); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, SquareValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P2)); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, SquareValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P3)); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, SquareValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P4)); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, SquareValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSMALL_P5_Minus_One)); - r_small = MlasSveMultiplyAddFloat32(Pred, r_small, AbsValue, AbsValue); - MLAS_SVFLOAT32 split_mask = MlasSveGreaterThanFloat32(Pred, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfSplitBoundary)); - r_small = MlasSveAndNotFloat32(Pred, split_mask, r_small); - - AbsValue = MlasSveAndFloat32(Pred, split_mask, AbsValue); - MLAS_SVFLOAT32 r_big = MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P0); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P1)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P2)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P3)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P4)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P5)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfBIG_P6_Minus_One)); - r_big = MlasSveMultiplyAddFloat32(Pred, r_big, AbsValue, AbsValue); - - r_big = MlasSveXorFloat32(Pred, r_big, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfNegZero)); - r_big = MlasSveMaximumFloat32(Pred, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_LowerRange), r_big); - MLAS_SVFLOAT32 exp_c = MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_C); - MLAS_SVFLOAT32 r = MlasSveMultiplyAddFloat32(Pred, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_Log2Reciprocal), r_big, exp_c); - r = MlasSveSubtractFloat32(Pred, r, exp_c); - - MLAS_SVFLOAT32 fx = MlasSveMultiplyAddFloat32(Pred, r, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_log2_hi), r_big); - fx = MlasSveMultiplyAddFloat32(Pred, r, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_log2_lo), fx); - - MLAS_SVFLOAT32 y = MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P0); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P1)); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P2)); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P3)); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P4)); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P5)); - y = MlasSveMultiplyAddFloat32(Pred, y, fx, MlasSveBroadcastFloat32(MlasSveErfConstants.Exp_P6)); - - y = MlasSveMultiplyFloat32(Pred, y, MlasSvePowerOf2Float32(Pred, r)); - y = MlasSveSubtractFloat32(Pred, MlasSveBroadcastFloat32(MlasSveErfConstants.ErfOne), y); - - y = MlasSveOrFloat32(Pred, r_small, y); - y = MlasSveOrFloat32(Pred, y, SignMask); - MlasSveStoreFloat32(Pred, Output, y); - - Input += stride; - Output += stride; - N -= stride; + const svfloat32_t NegZero = svdup_n_f32(Constants->ErfNegZero); + const svfloat32_t UpperAbsRange = svdup_n_f32(Constants->ErfUpperAbsRange); + const svfloat32_t SplitBoundary = svdup_n_f32(Constants->ErfSplitBoundary); + const svfloat32_t One = svdup_n_f32(Constants->ErfOne); + + const svfloat32_t SmallP0 = svdup_n_f32(Constants->ErfSMALL_P0); + const svfloat32_t SmallP1 = svdup_n_f32(Constants->ErfSMALL_P1); + const svfloat32_t SmallP2 = svdup_n_f32(Constants->ErfSMALL_P2); + const svfloat32_t SmallP3 = svdup_n_f32(Constants->ErfSMALL_P3); + const svfloat32_t SmallP4 = svdup_n_f32(Constants->ErfSMALL_P4); + const svfloat32_t SmallP5 = svdup_n_f32(Constants->ErfSMALL_P5_Minus_One); + + const svfloat32_t BigP0 = svdup_n_f32(Constants->ErfBIG_P0); + const svfloat32_t BigP1 = svdup_n_f32(Constants->ErfBIG_P1); + const svfloat32_t BigP2 = svdup_n_f32(Constants->ErfBIG_P2); + const svfloat32_t BigP3 = svdup_n_f32(Constants->ErfBIG_P3); + const svfloat32_t BigP4 = svdup_n_f32(Constants->ErfBIG_P4); + const svfloat32_t BigP5 = svdup_n_f32(Constants->ErfBIG_P5); + const svfloat32_t BigP6 = svdup_n_f32(Constants->ErfBIG_P6_Minus_One); + + const svfloat32_t ExpLowerRange = svdup_n_f32(Constants->Exp_LowerRange); + const svfloat32_t ExpLog2Reciprocal = svdup_n_f32(Constants->Exp_Log2Reciprocal); + const svfloat32_t ExpLog2Hi = svdup_n_f32(Constants->Exp_log2_hi); + const svfloat32_t ExpLog2Lo = svdup_n_f32(Constants->Exp_log2_lo); + const svfloat32_t ExpC = svdup_n_f32(Constants->Exp_C); + const svfloat32_t ExpP0 = svdup_n_f32(Constants->Exp_P0); + const svfloat32_t ExpP1 = svdup_n_f32(Constants->Exp_P1); + const svfloat32_t ExpP2 = svdup_n_f32(Constants->Exp_P2); + const svfloat32_t ExpP3 = svdup_n_f32(Constants->Exp_P3); + const svfloat32_t ExpP4 = svdup_n_f32(Constants->Exp_P4); + const svfloat32_t ExpP5 = svdup_n_f32(Constants->Exp_P5); + const svfloat32_t ExpP6 = svdup_n_f32(Constants->Exp_P6); + const svint32_t ExpX7F = svdup_n_s32(Constants->Exp_X7F); + + for (size_t i = 0; i < N; i += svcntw()) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + + const svfloat32_t Value = svld1_f32(pg, Input + i); + + // Split off the sign so the polynomials operate on |x|. The sign is + // reapplied at the end with a bitwise OR, which also preserves the + // sign of zero. + const svuint32_t SignMask = + svand_u32_x(pg, svreinterpret_u32_f32(Value), svreinterpret_u32_f32(NegZero)); + svfloat32_t AbsValue = + svreinterpret_f32_u32(svbic_u32_x(pg, svreinterpret_u32_f32(Value), svreinterpret_u32_f32(NegZero))); + AbsValue = svmin_f32_x(pg, UpperAbsRange, AbsValue); + + // Small-input polynomial in x^2. + const svfloat32_t SquareValue = svmul_f32_x(pg, AbsValue, AbsValue); + svfloat32_t r_small = svmla_f32_x(pg, SmallP1, SmallP0, SquareValue); + r_small = svmla_f32_x(pg, SmallP2, r_small, SquareValue); + r_small = svmla_f32_x(pg, SmallP3, r_small, SquareValue); + r_small = svmla_f32_x(pg, SmallP4, r_small, SquareValue); + r_small = svmla_f32_x(pg, SmallP5, r_small, SquareValue); + r_small = svmla_f32_x(pg, AbsValue, r_small, AbsValue); + + // Large-input polynomial, then 1 - exp(-poly). + svfloat32_t r_big = svmla_f32_x(pg, BigP1, BigP0, AbsValue); + r_big = svmla_f32_x(pg, BigP2, r_big, AbsValue); + r_big = svmla_f32_x(pg, BigP3, r_big, AbsValue); + r_big = svmla_f32_x(pg, BigP4, r_big, AbsValue); + r_big = svmla_f32_x(pg, BigP5, r_big, AbsValue); + r_big = svmla_f32_x(pg, BigP6, r_big, AbsValue); + r_big = svmla_f32_x(pg, AbsValue, r_big, AbsValue); + r_big = svreinterpret_f32_u32( + sveor_u32_x(pg, svreinterpret_u32_f32(r_big), svreinterpret_u32_f32(NegZero))); + + // exp(-poly) via log2 range reduction, matching the scalar kernel's + // Erf exp step. + r_big = svmax_f32_x(pg, ExpLowerRange, r_big); + + svfloat32_t r_exp = svmla_f32_x(pg, ExpC, r_big, ExpLog2Reciprocal); + r_exp = svsub_f32_x(pg, r_exp, ExpC); + + svfloat32_t fx = svmla_f32_x(pg, r_big, r_exp, ExpLog2Hi); + fx = svmla_f32_x(pg, fx, r_exp, ExpLog2Lo); + + svfloat32_t y_big = svmla_f32_x(pg, ExpP1, ExpP0, fx); + y_big = svmla_f32_x(pg, ExpP2, y_big, fx); + y_big = svmla_f32_x(pg, ExpP3, y_big, fx); + y_big = svmla_f32_x(pg, ExpP4, y_big, fx); + y_big = svmla_f32_x(pg, ExpP5, y_big, fx); + y_big = svmla_f32_x(pg, ExpP6, y_big, fx); + + // Scale by 2^r_exp: convert to integer, bias by 127 and shift into + // the float exponent field. + const svint32_t emm0 = svadd_s32_x(pg, svcvt_s32_f32_x(pg, r_exp), ExpX7F); + const svfloat32_t scale = svreinterpret_f32_s32(svlsl_n_s32_x(pg, emm0, 23)); + + y_big = svmul_f32_x(pg, y_big, scale); + y_big = svsub_f32_x(pg, One, y_big); + + // Select the branch per lane, then reapply the sign. + const svbool_t BigLanes = svcmpgt_f32(pg, AbsValue, SplitBoundary); + svfloat32_t y = svsel_f32(BigLanes, y_big, r_small); + y = svreinterpret_f32_u32(svorr_u32_x(pg, svreinterpret_u32_f32(y), SignMask)); + + svst1_f32(pg, Output + i, y); } } -void +extern "C" void MLASCALL -MlasSveLogisticKernel( +MlasSveLogisticKernelImpl( const float* Input, float* Output, - size_t N + size_t N, + const MLAS_LOGISTIC_CONSTANTS* Constants ) /*++ Routine Description: - This routine implements the generic kernel for the logistic function. + This routine implements the logistic function using SVE, with the same + odd/even rational polynomial as the scalar/NEON kernels. Arguments: @@ -280,404 +203,625 @@ Return Value: --*/ { - MLAS_SVBOOL Pred = svptrue_b32(); - size_t sve_veclen = svcntw(); - size_t stride = sve_veclen; - - while (N > 0) { - // If fewer that SVE vector length elements are remaining, adjust the predicate - if (N < sve_veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - MLAS_SVFLOAT32 Value = MlasSveLoadFloat32(Pred, Input); - - Value = MlasSveMaximumFloat32(Pred, MlasSveBroadcastFloat32(MlasSveLogisticConstants.LowerRange), Value); - Value = MlasSveMinimumFloat32(Pred, MlasSveBroadcastFloat32(MlasSveLogisticConstants.UpperRange), Value); - - MLAS_SVFLOAT32 ValueSquared = MlasSveMultiplyFloat32(Pred, Value, Value); - - MLAS_SVFLOAT32 p; - p = MlasSveMultiplyAddFloat32( - Pred, - ValueSquared, - MlasSveBroadcastFloat32(MlasSveLogisticConstants.alpha_9), - MlasSveBroadcastFloat32(MlasSveLogisticConstants.alpha_7) - ); - p = MlasSveMultiplyAddFloat32(Pred, p, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.alpha_5)); - p = MlasSveMultiplyAddFloat32(Pred, p, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.alpha_3)); - p = MlasSveMultiplyAddFloat32(Pred, p, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.alpha_1)); - p = MlasSveMultiplyFloat32(Pred, p, Value); - - MLAS_SVFLOAT32 q; - q = MlasSveMultiplyAddFloat32( - Pred, - ValueSquared, - MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_10), - MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_8) - ); - q = MlasSveMultiplyAddFloat32(Pred, q, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_6)); - q = MlasSveMultiplyAddFloat32(Pred, q, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_4)); - q = MlasSveMultiplyAddFloat32(Pred, q, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_2)); - q = MlasSveMultiplyAddFloat32(Pred, q, ValueSquared, MlasSveBroadcastFloat32(MlasSveLogisticConstants.beta_0)); - - MlasSveStoreFloat32( - Pred, - Output, - MlasSveClampFloat32(Pred,MlasSveAddFloat32( - Pred, - MlasSveDivideFloat32(Pred, p, q), - MlasSveBroadcastFloat32(0.5f) - ), 0.0f, 1.0f) - ); - - Input += stride; - Output += stride; - N -= stride; + const svfloat32_t LowerRange = svdup_n_f32(Constants->LowerRange); + const svfloat32_t UpperRange = svdup_n_f32(Constants->UpperRange); + + const svfloat32_t Alpha9 = svdup_n_f32(Constants->alpha_9); + const svfloat32_t Alpha7 = svdup_n_f32(Constants->alpha_7); + const svfloat32_t Alpha5 = svdup_n_f32(Constants->alpha_5); + const svfloat32_t Alpha3 = svdup_n_f32(Constants->alpha_3); + const svfloat32_t Alpha1 = svdup_n_f32(Constants->alpha_1); + + const svfloat32_t Beta10 = svdup_n_f32(Constants->beta_10); + const svfloat32_t Beta8 = svdup_n_f32(Constants->beta_8); + const svfloat32_t Beta6 = svdup_n_f32(Constants->beta_6); + const svfloat32_t Beta4 = svdup_n_f32(Constants->beta_4); + const svfloat32_t Beta2 = svdup_n_f32(Constants->beta_2); + const svfloat32_t Beta0 = svdup_n_f32(Constants->beta_0); + + const svfloat32_t OneHalf = svdup_n_f32(Constants->one_half); + const svfloat32_t Zero = svdup_n_f32(0.0f); + const svfloat32_t One = svdup_n_f32(1.0f); + + for (size_t i = 0; i < N; i += svcntw()) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + + svfloat32_t Value = svld1_f32(pg, Input + i); + Value = svmax_f32_x(pg, LowerRange, Value); + Value = svmin_f32_x(pg, UpperRange, Value); + + const svfloat32_t ValueSquared = svmul_f32_x(pg, Value, Value); + + svfloat32_t p = svmla_f32_x(pg, Alpha7, ValueSquared, Alpha9); + p = svmla_f32_x(pg, Alpha5, p, ValueSquared); + p = svmla_f32_x(pg, Alpha3, p, ValueSquared); + p = svmla_f32_x(pg, Alpha1, p, ValueSquared); + p = svmul_f32_x(pg, p, Value); + + svfloat32_t q = svmla_f32_x(pg, Beta8, ValueSquared, Beta10); + q = svmla_f32_x(pg, Beta6, q, ValueSquared); + q = svmla_f32_x(pg, Beta4, q, ValueSquared); + q = svmla_f32_x(pg, Beta2, q, ValueSquared); + q = svmla_f32_x(pg, Beta0, q, ValueSquared); + + svfloat32_t y = svadd_f32_x(pg, svdiv_f32_x(pg, p, q), OneHalf); + y = svmax_f32_x(pg, Zero, y); + y = svmin_f32_x(pg, One, y); + + svst1_f32(pg, Output + i, y); } } -/* -SVE implementation of expf() using a polynomial approximation is taken from ARM Compute Library Repository. -https://github.com/ARM-software/ComputeLibrary/blob/9f7a1fb06bc0435d989a9a6a3c0fd2cebfedbf5f/src/core/NEON/SVEMath.inl#L105 -*/ -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveComputeExpVector( - MLAS_SVBOOL Pred, - MLAS_SVFLOAT32 Vector -) +extern "C" void +MLASCALL +MlasSveComputeExpF32KernelImpl( + const float* Input, + float* Output, + size_t N, + const MLAS_SVE_EXP_CONSTANTS* Constants + ) +/*++ + +Routine Description: + + This routine implements exp(x) using SVE. The polynomial approximation is + taken from the ARM Compute Library: + https://github.com/ARM-software/ComputeLibrary/blob/9f7a1fb06bc0435d989a9a6a3c0fd2cebfedbf5f/src/core/NEON/SVEMath.inl#L105 + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + N - Supplies the number of elements to process. + +Return Value: + + None. + +--*/ { - const uint32_t svexp_f32_coeff[] = { - 0x3f7ffff6, // x^1: 0x1.ffffecp-1f - 0x3efffedb, // x^2: 0x1.fffdb6p-2f - 0x3e2aaf33, // x^3: 0x1.555e66p-3f - 0x3d2b9f17, // x^4: 0x1.573e2ep-5f - 0x3c072010, // x^5: 0x1.0e4020p-7f - }; - - const auto c1 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(svexp_f32_coeff[0])); - const auto c2 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(svexp_f32_coeff[1])); - const auto c3 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(svexp_f32_coeff[2])); - const auto c4 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(svexp_f32_coeff[3])); - const auto c5 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(svexp_f32_coeff[4])); - - const auto shift = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(0x4b00007f)); // 2^23 + 127 = 0x1.0000fep23f - const auto inv_ln2 = MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(0x3fb8aa3b)); // 1 / ln(2) = 0x1.715476p+0f - const auto neg_ln2_hi = - MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(0xbf317200)); // -ln(2) from bits -1 to -19: -0x1.62e400p-1f - const auto neg_ln2_lo = - MlasSveReinterpretAsFLOAT32(MlasSveBroadcastUINT32(0xb5bfbe8e)); // -ln(2) from bits -20 to -42: -0x1.7f7d1cp-20f - - const auto inf = MlasSveBroadcastFloat32(std::numeric_limits::infinity()); - const auto max_input = MlasSveBroadcastFloat32(88.37f); // Approximately ln(2^127.5) - const auto zero = MlasSveZeroFloat32(); - const auto min_input = MlasSveBroadcastFloat32(-86.64f); // Approximately ln(2^-125) - - // Range reduction: - // e^x = 2^n * e^r - // where: - // n = floor(x / ln(2)) - // r = x - n * ln(2) - // - // By adding x / ln(2) with 2^23 + 127 (shift): - // * As FP32 fraction part only has 23-bits, the addition of 2^23 + 127 forces decimal part - // of x / ln(2) out of the result. The integer part of x / ln(2) (i.e. n) + 127 will occupy - // the whole fraction part of z in FP32 format. - // Subtracting 2^23 + 127 (shift) from z will result in the integer part of x / ln(2) - // (i.e. n) because the decimal part has been pushed out and lost. - // * The addition of 127 makes the FP32 fraction part of z ready to be used as the exponent - // in FP32 format. Left shifting z by 23 bits will result in 2^n. - const auto z = MlasSveMultiplyAddFloat32(Pred, Vector, inv_ln2, shift); - const auto n = MlasSveSubtractFloat32(Pred, z, shift); - const auto scale = MlasSveReinterpretAsFLOAT32(MlasSveShiftLeftUInt32<23>(Pred, MlasSveReinterpretAsUInt32(z))); // 2^n - - // The calculation of n * ln(2) is done using 2 steps to achieve accuracy beyond FP32. - // This outperforms longer Taylor series (3-4 tabs) both in term of accuracy and performance. - const auto r_hi = MlasSveMultiplyAddFloat32(Pred, n, neg_ln2_hi, Vector); - const auto r = MlasSveMultiplyAddFloat32(Pred, n, neg_ln2_lo, r_hi); - - // Compute the truncated Taylor series of e^r. - // poly = scale * (1 + c1 * r + c2 * r^2 + c3 * r^3 + c4 * r^4 + c5 * r^5) - const auto r2 = MlasSveMultiplyFloat32(Pred, r, r); - - const auto p1 = MlasSveMultiplyFloat32(Pred, c1, r); - const auto p23 = MlasSveMultiplyAddFloat32(Pred, c3, r, c2); - const auto p45 = MlasSveMultiplyAddFloat32(Pred, c5, r, c4); - const auto p2345 = MlasSveMultiplyAddFloat32(Pred, p45, r2, p23); - const auto p12345 = MlasSveMultiplyAddFloat32(Pred, p2345, r2, p1); - - auto poly = MlasSveMultiplyAddFloat32(Pred, p12345, scale, scale); - - // Handle underflow and overflow. - poly = MlasSveSelect(MlasSveCompareLessThan(Pred, Vector, min_input), zero, poly); - poly = MlasSveSelect(MlasSveCompareGreaterThan(Pred, Vector, max_input), inf, poly); - - return poly; + const svfloat32_t C1 = svreinterpret_f32_u32(svdup_n_u32(Constants->C1)); + const svfloat32_t C2 = svreinterpret_f32_u32(svdup_n_u32(Constants->C2)); + const svfloat32_t C3 = svreinterpret_f32_u32(svdup_n_u32(Constants->C3)); + const svfloat32_t C4 = svreinterpret_f32_u32(svdup_n_u32(Constants->C4)); + const svfloat32_t C5 = svreinterpret_f32_u32(svdup_n_u32(Constants->C5)); + + const svfloat32_t Shift = svreinterpret_f32_u32(svdup_n_u32(Constants->Shift)); + const svfloat32_t InvLn2 = svreinterpret_f32_u32(svdup_n_u32(Constants->InvLn2)); + const svfloat32_t NegLn2Hi = svreinterpret_f32_u32(svdup_n_u32(Constants->NegLn2Hi)); + const svfloat32_t NegLn2Lo = svreinterpret_f32_u32(svdup_n_u32(Constants->NegLn2Lo)); + + const svfloat32_t Inf = svreinterpret_f32_u32(svdup_n_u32(Constants->Inf)); + const svfloat32_t MaxInput = svreinterpret_f32_u32(svdup_n_u32(Constants->MaxInput)); + const svfloat32_t MinInput = svreinterpret_f32_u32(svdup_n_u32(Constants->MinInput)); + const svfloat32_t Zero = svdup_n_f32(0.0f); + + for (size_t i = 0; i < N; i += svcntw()) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + + const svfloat32_t Vector = svld1_f32(pg, Input + i); + + // Range reduction: e^x = 2^n * e^r with n = floor(x / ln(2)). Adding + // the magic shift constant pushes the fractional part of x / ln(2) + // out of the fp32 mantissa, leaving n + 127 in the low mantissa bits + // ready to become the exponent of 2^n. + const svfloat32_t z = svmla_f32_x(pg, Shift, Vector, InvLn2); + const svfloat32_t n = svsub_f32_x(pg, z, Shift); + const svfloat32_t Scale = + svreinterpret_f32_u32(svlsl_n_u32_x(pg, svreinterpret_u32_f32(z), 23)); // 2^n + + // r = x - n * ln(2), computed in two steps for extra accuracy. + const svfloat32_t r_hi = svmla_f32_x(pg, Vector, n, NegLn2Hi); + const svfloat32_t r = svmla_f32_x(pg, r_hi, n, NegLn2Lo); + + // Truncated Taylor series of e^r. + const svfloat32_t r2 = svmul_f32_x(pg, r, r); + + const svfloat32_t p1 = svmul_f32_x(pg, C1, r); + const svfloat32_t p23 = svmla_f32_x(pg, C2, C3, r); + const svfloat32_t p45 = svmla_f32_x(pg, C4, C5, r); + const svfloat32_t p2345 = svmla_f32_x(pg, p23, p45, r2); + const svfloat32_t p12345 = svmla_f32_x(pg, p1, p2345, r2); + + svfloat32_t poly = svmla_f32_x(pg, Scale, p12345, Scale); + + // Handle underflow and overflow. + poly = svsel_f32(svcmplt_f32(pg, Vector, MinInput), Zero, poly); + poly = svsel_f32(svcmpgt_f32(pg, Vector, MaxInput), Inf, poly); + + svst1_f32(pg, Output + i, poly); + } } -void +extern "C" float MLASCALL -MlasSveComputeExpF32Kernel( +MlasSveComputeSumExpF32KernelImpl( const float* Input, float* Output, - size_t N -) + size_t N, + const float* NegativeMaximum, + const MLAS_EXP_CONSTANTS* Constants, + const MLAS_SVE_EXP_CONSTANTS* AclConstants + ) +/*++ + +Routine Description: + + This routine computes the sum of exp(Input[i] + NegativeMaximum), and + optionally stores the individual exponentials. + +Arguments: + + Input - Supplies the input buffer. + + Output - Optionally supplies the output buffer for the exponentials. + + N - Supplies the number of elements to process. + + NegativeMaximum - Supplies the negated maximum of the input values. + +Return Value: + + Returns the sum of the exponentials. + +--*/ { - const size_t veclen = svcntw(); +#if defined(MLAS_SVE_SUMEXP_FEXPA) + // + // EXPERIMENT: exp() via the FEXPA hardware accelerator (as in glibc's + // SVE expf): FEXPA looks up 2^(n/64) from the low bits of the rounded + // scaled input, leaving only a 2-term correction polynomial for the + // residual |r| <= ln2/128. Measured max error 1.16 ulp over + // [-88.5, 88.5] (280M-point sweep) - well inside the 1e-6 test + // tolerance. Inputs below the FEXPA subnormal threshold (-87.34) yield + // ~0 with ~6e-39 absolute error, which is inert in a softmax sum. + // + // FEXPA-safe lower clamp: below -127*ln2 (-88.0297) the FEXPA index + // underflows and wraps into the exponent field, producing NaN (glibc + // special-cases this zone). Clamping at -88.0 instead of the shared + // -88.376 changes the true exp() by at most 2e-39 - inert in any sum. + const svfloat32_t LowerRangeSumExp = svreinterpret_f32_u32(svdup_n_u32(AclConstants->FexpaLowerRange)); + const svfloat32_t InvLn2 = svreinterpret_f32_u32(svdup_n_u32(AclConstants->InvLn2)); + const svfloat32_t NegLn2Hi = svreinterpret_f32_u32(svdup_n_u32(AclConstants->NegLn2Hi)); + const svfloat32_t NegLn2Lo = svreinterpret_f32_u32(svdup_n_u32(AclConstants->NegLn2Lo)); + const svfloat32_t Shift = svreinterpret_f32_u32(svdup_n_u32(AclConstants->FexpaShift)); + const svfloat32_t Half = svreinterpret_f32_u32(svdup_n_u32(AclConstants->OneHalf)); +#else + const svfloat32_t LowerRangeSumExp = svdup_n_f32(Constants->LowerRangeSumExp); + const svfloat32_t RoundingBias = svdup_n_f32(Constants->RoundingBias); + const svfloat32_t Log2Reciprocal = svdup_n_f32(Constants->Log2Reciprocal); + const svfloat32_t Log2High = svdup_n_f32(Constants->Log2High); + const svfloat32_t Log2Low = svdup_n_f32(Constants->Log2Low); + const svfloat32_t Poly0 = svdup_n_f32(Constants->poly_0); + const svfloat32_t Poly1 = svdup_n_f32(Constants->poly_1); + const svfloat32_t Poly2 = svdup_n_f32(Constants->poly_2); + const svfloat32_t Poly3 = svdup_n_f32(Constants->poly_3); + const svfloat32_t Poly4 = svdup_n_f32(Constants->poly_4); + const svfloat32_t Poly56 = svdup_n_f32(Constants->poly_56); + const svint32_t MaximumExponent = svdup_n_s32(Constants->MaximumExponent); +#endif + + const svfloat32_t NegativeMaximumVector = svdup_n_f32(*NegativeMaximum); - // Fast path: Use scalar loop when N is 1 - if (N == 1) { - Output[0] = expf(Input[0]); - return; - } - - // Vectorized path - MLAS_SVBOOL Pred = svptrue_b32(); - size_t stride = veclen; - - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } + // + // Two vectors per iteration, and per-vector accumulators instead of a + // horizontal reduction per iteration (a single reduction happens at the + // end). The second predicate is all-false when only one vector remains; + // the merge-predicated accumulate keeps its lanes unchanged. + // - MLAS_SVFLOAT32 Vector = MlasSveLoadFloat32(Pred, Input); - Vector = MlasSveComputeExpVector(Pred, Vector); - MlasSveStoreFloat32(Pred, Output, Vector); + svfloat32_t Accumulator0 = svdup_n_f32(0.0f); + svfloat32_t Accumulator1 = svdup_n_f32(0.0f); + + const size_t VL = svcntw(); + + for (size_t i = 0; i < N; i += 2 * VL) { + + const svbool_t pg0 = svwhilelt_b32_u64(i, N); + const svbool_t pg1 = svwhilelt_b32_u64(i + VL, N); + + svfloat32_t Vector0 = svld1_f32(pg0, Input + i); + svfloat32_t Vector1 = svld1_f32(pg1, Input + i + VL); + + Vector0 = svadd_f32_x(pg0, Vector0, NegativeMaximumVector); + Vector1 = svadd_f32_x(pg1, Vector1, NegativeMaximumVector); + Vector0 = svmax_f32_x(pg0, LowerRangeSumExp, Vector0); + Vector1 = svmax_f32_x(pg1, LowerRangeSumExp, Vector1); + +#if defined(MLAS_SVE_SUMEXP_FEXPA) + // exp(x) = FEXPA(z) * (1 + r + r^2/2): the shift constant rounds + // x/ln2 to multiples of 1/64 with the FEXPA exponent bias baked into + // its low mantissa bits. + const svfloat32_t z0 = svmla_f32_x(pg0, Shift, Vector0, InvLn2); + const svfloat32_t z1 = svmla_f32_x(pg1, Shift, Vector1, InvLn2); + const svfloat32_t n0 = svsub_f32_x(pg0, z0, Shift); + const svfloat32_t n1 = svsub_f32_x(pg1, z1, Shift); + + svfloat32_t r0 = svmla_f32_x(pg0, Vector0, n0, NegLn2Hi); + svfloat32_t r1 = svmla_f32_x(pg1, Vector1, n1, NegLn2Hi); + r0 = svmla_f32_x(pg0, r0, n0, NegLn2Lo); + r1 = svmla_f32_x(pg1, r1, n1, NegLn2Lo); + + const svfloat32_t scale0 = svexpa_f32(svreinterpret_u32_f32(z0)); + const svfloat32_t scale1 = svexpa_f32(svreinterpret_u32_f32(z1)); + + const svfloat32_t r2_0 = svmul_f32_x(pg0, r0, r0); + const svfloat32_t r2_1 = svmul_f32_x(pg1, r1, r1); + const svfloat32_t poly0 = svmla_f32_x(pg0, r0, r2_0, Half); + const svfloat32_t poly1 = svmla_f32_x(pg1, r1, r2_1, Half); + + svfloat32_t p0 = svmla_f32_x(pg0, scale0, scale0, poly0); + svfloat32_t p1 = svmla_f32_x(pg1, scale1, scale1, poly1); +#else + // exp(x + NegativeMaximum), matching the scalar kernel's operation + // order (including the historical double application of poly_56). + const svfloat32_t biased0 = svmla_f32_x(pg0, RoundingBias, Vector0, Log2Reciprocal); + const svfloat32_t biased1 = svmla_f32_x(pg1, RoundingBias, Vector1, Log2Reciprocal); + const svfloat32_t m0 = svsub_f32_x(pg0, biased0, RoundingBias); + const svfloat32_t m1 = svsub_f32_x(pg1, biased1, RoundingBias); + + Vector0 = svmla_f32_x(pg0, Vector0, m0, Log2High); + Vector1 = svmla_f32_x(pg1, Vector1, m1, Log2High); + Vector0 = svmla_f32_x(pg0, Vector0, m0, Log2Low); + Vector1 = svmla_f32_x(pg1, Vector1, m1, Log2Low); + + svint32_t normal0 = svlsl_n_s32_x(pg0, svreinterpret_s32_f32(biased0), 23); + svint32_t normal1 = svlsl_n_s32_x(pg1, svreinterpret_s32_f32(biased1), 23); + normal0 = svadd_s32_x(pg0, normal0, MaximumExponent); + normal1 = svadd_s32_x(pg1, normal1, MaximumExponent); + + svfloat32_t p0 = svmla_f32_x(pg0, Poly1, Poly0, Vector0); + svfloat32_t p1 = svmla_f32_x(pg1, Poly1, Poly0, Vector1); + p0 = svmla_f32_x(pg0, Poly2, p0, Vector0); + p1 = svmla_f32_x(pg1, Poly2, p1, Vector1); + p0 = svmla_f32_x(pg0, Poly3, p0, Vector0); + p1 = svmla_f32_x(pg1, Poly3, p1, Vector1); + p0 = svmla_f32_x(pg0, Poly4, p0, Vector0); + p1 = svmla_f32_x(pg1, Poly4, p1, Vector1); + p0 = svmla_f32_x(pg0, Poly56, p0, Vector0); + p1 = svmla_f32_x(pg1, Poly56, p1, Vector1); + p0 = svmla_f32_x(pg0, Poly56, p0, Vector0); + p1 = svmla_f32_x(pg1, Poly56, p1, Vector1); + + p0 = svmul_f32_x(pg0, p0, svreinterpret_f32_s32(normal0)); + p1 = svmul_f32_x(pg1, p1, svreinterpret_f32_s32(normal1)); +#endif - Input += stride; - Output += stride; - N -= stride; + if (Output != nullptr) { + svst1_f32(pg0, Output + i, p0); + svst1_f32(pg1, Output + i + VL, p1); + } + + Accumulator0 = svadd_f32_m(pg0, Accumulator0, p0); + Accumulator1 = svadd_f32_m(pg1, Accumulator1, p1); } -} -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveComputeSumExpVector( - MLAS_SVBOOL Pred, - MLAS_SVFLOAT32 Vector, - MLAS_SVFLOAT32 NegativeMaximumVector -) -{ - Vector = MlasSveAddFloat32(Pred, Vector, NegativeMaximumVector); - Vector = MlasSveMaximumFloat32(Pred, MlasSveBroadcastFloat32(MlasSveExpConstants.LowerRangeSumExp), Vector); - - const auto RoundingBias = MlasSveBroadcastFloat32(MlasSveExpConstants.RoundingBias); - auto biased = MlasSveMultiplyAddFloat32(Pred, Vector, MlasSveExpConstants.Log2Reciprocal, RoundingBias); - auto m = MlasSveSubtractFloat32(Pred, biased, RoundingBias); - - Vector = MlasSveMultiplyAddFloat32(Pred, m, MlasSveExpConstants.Log2High, Vector); - Vector = MlasSveMultiplyAddFloat32(Pred, m, MlasSveExpConstants.Log2Low, Vector); - - auto normal = MlasSveShiftLeftInt32<23>(Pred, MlasSveReinterpretAsInt32(biased)); - normal = MlasSveAddInt32(Pred, normal, MlasSveBroadcastInt32(MlasSveExpConstants.MaximumExponent)); - - auto p = MlasSveBroadcastFloat32(MlasSveExpConstants.poly_0); - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_1); - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_2); - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_3); - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_4); - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_56); // <--| - p = MlasSveMultiplyAddFloat32(Pred, p, Vector, MlasSveExpConstants.poly_56); // Twice? - - p = MlasSveMultiplyFloat32(Pred, p, MlasSveReinterpretAsFloat32(normal)); - return p; + const svbool_t ptrue = svptrue_b32(); + return svaddv_f32(ptrue, svadd_f32_x(ptrue, Accumulator0, Accumulator1)); } -float +extern "C" void MLASCALL -MlasSveComputeSumExpF32Kernel( +MlasSveTanhKernelImpl( const float* Input, float* Output, size_t N, - const float* NegativeMaximum -) -/** - * Potential optimization: Consider applying loop unrolling to improve instruction-level - * parallelism (ILP) in this kernel. Evaluate the performance impact using benchmarks - * before and after implementing the optimization. - */ + const MLAS_TANH_CONSTANTS* Constants + ) +/*++ + +Routine Description: + + This routine implements the SVE kernel for the hyperbolic tangent + function: the same clamped P13/Q6 rational polynomial as the generic + kernel, evaluated per whilelt-predicated vector. + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + N - Supplies the number of elements to process. + + Constants - Supplies the coefficient table (passed by pointer so the + frozen machine-code variant needs no data relocations). + +Return Value: + + None. + +--*/ { - if (N == 1) { - float result = expf(Input[0] + *NegativeMaximum); - if (Output != nullptr) { - Output[0] = result; - } - return result; - } - - MLAS_SVBOOL Pred = svptrue_b32(); - size_t veclen = svcntw(); - size_t stride = veclen; - float sum = 0.0f; - - MLAS_SVFLOAT32 NegativeMaximumVector = MlasSveBroadcastFloat32(*NegativeMaximum); - - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - - MLAS_SVFLOAT32 Vector = MlasSveLoadFloat32(Pred, Input); - Vector = MlasSveComputeSumExpVector(Pred, Vector, NegativeMaximumVector); + const svfloat32_t LowerRange = svdup_n_f32(Constants->LowerRange); + const svfloat32_t UpperRange = svdup_n_f32(Constants->UpperRange); - if (Output != nullptr) { - MlasSveStoreFloat32(Pred, Output, Vector); - Output += stride; - } + const svfloat32_t Alpha13 = svdup_n_f32(Constants->alpha_13); + const svfloat32_t Alpha11 = svdup_n_f32(Constants->alpha_11); + const svfloat32_t Alpha9 = svdup_n_f32(Constants->alpha_9); + const svfloat32_t Alpha7 = svdup_n_f32(Constants->alpha_7); + const svfloat32_t Alpha5 = svdup_n_f32(Constants->alpha_5); + const svfloat32_t Alpha3 = svdup_n_f32(Constants->alpha_3); + const svfloat32_t Alpha1 = svdup_n_f32(Constants->alpha_1); + + const svfloat32_t Beta6 = svdup_n_f32(Constants->beta_6); + const svfloat32_t Beta4 = svdup_n_f32(Constants->beta_4); + const svfloat32_t Beta2 = svdup_n_f32(Constants->beta_2); + const svfloat32_t Beta0 = svdup_n_f32(Constants->beta_0); + + for (size_t i = 0; i < N; i += svcntw()) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); - sum += MlasSveReduceAddFloat32(Pred, Vector); + svfloat32_t Value = svld1_f32(pg, Input + i); + Value = svmax_f32_x(pg, LowerRange, Value); + Value = svmin_f32_x(pg, UpperRange, Value); - Input += stride; - N -= stride; + const svfloat32_t ValueSquared = svmul_f32_x(pg, Value, Value); + + svfloat32_t p = svmla_f32_x(pg, Alpha11, ValueSquared, Alpha13); + p = svmla_f32_x(pg, Alpha9, p, ValueSquared); + p = svmla_f32_x(pg, Alpha7, p, ValueSquared); + p = svmla_f32_x(pg, Alpha5, p, ValueSquared); + p = svmla_f32_x(pg, Alpha3, p, ValueSquared); + p = svmla_f32_x(pg, Alpha1, p, ValueSquared); + p = svmul_f32_x(pg, p, Value); + + svfloat32_t q = svmla_f32_x(pg, Beta4, ValueSquared, Beta6); + q = svmla_f32_x(pg, Beta2, q, ValueSquared); + q = svmla_f32_x(pg, Beta0, q, ValueSquared); + + svst1_f32(pg, Output + i, svdiv_f32_x(pg, p, q)); } - return sum; } -float MLASCALL -MlasSveReduceMaximumF32Kernel( +extern "C" float +MLASCALL +MlasSveReduceMaximumF32KernelImpl( const float* Input, - size_t N -) + size_t N, + float MinimumValue + ) +/*++ + +Routine Description: + + This routine computes the maximum value of the supplied buffer. + +Arguments: + + Input - Supplies the input buffer. + + N - Supplies the number of elements to process. + +Return Value: + + Returns the maximum value of the supplied buffer. + +--*/ { - size_t veclen = svcntw(); - MLAS_SVBOOL Pred = svptrue_b32(); + const svbool_t ptrue = svptrue_b32(); + const size_t veclen = svcntw(); + + svfloat32_t MaximumVector0 = svdup_n_f32(MinimumValue); + + // + // Unrolled main loop: four independent maximum accumulators. + // - float Maximum; - MLAS_SVFLOAT32 MaximumVector0 = MlasSveBroadcastFloat32(MlasSveMinimumF32Value); - if (N >= veclen * 4) { - MLAS_SVFLOAT32 MaximumVector1 = MaximumVector0; - MLAS_SVFLOAT32 MaximumVector2 = MaximumVector0; - MLAS_SVFLOAT32 MaximumVector3 = MaximumVector0; - + + svfloat32_t MaximumVector1 = MaximumVector0; + svfloat32_t MaximumVector2 = MaximumVector0; + svfloat32_t MaximumVector3 = MaximumVector0; + while (N >= veclen * 4) { - MaximumVector0 = MlasSveMaximumFloat32(Pred, MaximumVector0, MlasSveLoadFloat32(Pred, Input)); - MaximumVector1 = MlasSveMaximumFloat32(Pred, MaximumVector1, MlasSveLoadFloat32(Pred, Input + veclen)); - MaximumVector2 = MlasSveMaximumFloat32(Pred, MaximumVector2, MlasSveLoadFloat32(Pred, Input + 2 * veclen)); - MaximumVector3 = MlasSveMaximumFloat32(Pred, MaximumVector3, MlasSveLoadFloat32(Pred, Input + 3 * veclen)); + + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, svld1_f32(ptrue, Input)); + MaximumVector1 = svmax_f32_x(ptrue, MaximumVector1, svld1_f32(ptrue, Input + veclen)); + MaximumVector2 = svmax_f32_x(ptrue, MaximumVector2, svld1_f32(ptrue, Input + 2 * veclen)); + MaximumVector3 = svmax_f32_x(ptrue, MaximumVector3, svld1_f32(ptrue, Input + 3 * veclen)); Input += veclen * 4; N -= veclen * 4; } - MaximumVector0 = MlasSveMaximumFloat32(Pred, MaximumVector0, MaximumVector1); - MaximumVector2 = MlasSveMaximumFloat32(Pred, MaximumVector2, MaximumVector3); - MaximumVector0 = MlasSveMaximumFloat32(Pred, MaximumVector0, MaximumVector2); + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, MaximumVector1); + MaximumVector2 = svmax_f32_x(ptrue, MaximumVector2, MaximumVector3); + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, MaximumVector2); } - size_t stride = veclen; - - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - MLAS_SVFLOAT32 Vector = MlasSveLoadFloat32(Pred, Input); - MaximumVector0 = MlasSveMaximumFloat32(Pred, MaximumVector0, Vector); - Input += stride; - N -= stride; + for (size_t i = 0; i < N; i += veclen) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + MaximumVector0 = svmax_f32_m(pg, MaximumVector0, svld1_f32(pg, Input + i)); } - Maximum = MlasSveReduceMaximumFloat32(svptrue_b32(), MaximumVector0); - return Maximum; + return svmaxv_f32(ptrue, MaximumVector0); } -void +extern "C" void MLASCALL -MlasSveReduceMinimumMaximumF32Kernel( +MlasSveReduceMinimumMaximumF32KernelImpl( const float* Input, float* Min, float* Max, size_t N -) + ) +/*++ + +Routine Description: + + This routine computes the minimum and maximum values of the supplied + buffer. + +Arguments: + + Input - Supplies the input buffer. + + Min - Receives the minimum value of the supplied buffer. + + Max - Receives the maximum value of the supplied buffer. + + N - Supplies the number of elements to process. + +Return Value: + + None. + +--*/ { - MLAS_SVBOOL Pred = svptrue_b32(); - size_t veclen = svcntw(); - size_t stride = veclen; + const svbool_t ptrue = svptrue_b32(); + const size_t veclen = svcntw(); - float tmp_min = std::numeric_limits::max(); - float tmp_max = std::numeric_limits::lowest(); + svfloat32_t MinimumVector0 = svdup_n_f32(std::numeric_limits::max()); + svfloat32_t MaximumVector0 = svdup_n_f32(std::numeric_limits::lowest()); - MLAS_SVFLOAT32 MaximumVector = MlasSveBroadcastFloat32(tmp_max); - MLAS_SVFLOAT32 MinimumVector = MlasSveBroadcastFloat32(tmp_min); + // + // Unrolled main loop: four independent accumulator pairs with + // unpredicated operations, matching the generic kernel's structure (a + // single predicated accumulator loop measured 2-3.9x slower than the + // generic NEON kernel on Cortex-A725/X925). + // + + if (N >= veclen * 4) { - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; + svfloat32_t MinimumVector1 = MinimumVector0; + svfloat32_t MinimumVector2 = MinimumVector0; + svfloat32_t MinimumVector3 = MinimumVector0; + + svfloat32_t MaximumVector1 = MaximumVector0; + svfloat32_t MaximumVector2 = MaximumVector0; + svfloat32_t MaximumVector3 = MaximumVector0; + + while (N >= veclen * 4) { + + const svfloat32_t InputVector0 = svld1_f32(ptrue, Input); + const svfloat32_t InputVector1 = svld1_f32(ptrue, Input + veclen); + const svfloat32_t InputVector2 = svld1_f32(ptrue, Input + 2 * veclen); + const svfloat32_t InputVector3 = svld1_f32(ptrue, Input + 3 * veclen); + + MinimumVector0 = svmin_f32_x(ptrue, MinimumVector0, InputVector0); + MinimumVector1 = svmin_f32_x(ptrue, MinimumVector1, InputVector1); + MinimumVector2 = svmin_f32_x(ptrue, MinimumVector2, InputVector2); + MinimumVector3 = svmin_f32_x(ptrue, MinimumVector3, InputVector3); + + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, InputVector0); + MaximumVector1 = svmax_f32_x(ptrue, MaximumVector1, InputVector1); + MaximumVector2 = svmax_f32_x(ptrue, MaximumVector2, InputVector2); + MaximumVector3 = svmax_f32_x(ptrue, MaximumVector3, InputVector3); + + Input += veclen * 4; + N -= veclen * 4; } - MLAS_SVFLOAT32 Vector = MlasSveLoadFloat32(Pred, Input); - MaximumVector = MlasSveMaximumFloat32(Pred, MaximumVector, Vector); - MinimumVector = MlasSveMinimumFloat32(Pred, MinimumVector, Vector); - Input += stride; - N -= stride; + MinimumVector0 = svmin_f32_x(ptrue, MinimumVector0, MinimumVector1); + MinimumVector2 = svmin_f32_x(ptrue, MinimumVector2, MinimumVector3); + MinimumVector0 = svmin_f32_x(ptrue, MinimumVector0, MinimumVector2); + + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, MaximumVector1); + MaximumVector2 = svmax_f32_x(ptrue, MaximumVector2, MaximumVector3); + MaximumVector0 = svmax_f32_x(ptrue, MaximumVector0, MaximumVector2); } - *Min = MlasSveReduceMinimumFloat32(svptrue_b32(), MinimumVector); - *Max = MlasSveReduceMaximumFloat32(svptrue_b32(), MaximumVector); + + for (size_t i = 0; i < N; i += veclen) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + const svfloat32_t Vector = svld1_f32(pg, Input + i); + + MinimumVector0 = svmin_f32_m(pg, MinimumVector0, Vector); + MaximumVector0 = svmax_f32_m(pg, MaximumVector0, Vector); + } + + *Min = svminv_f32(ptrue, MinimumVector0); + *Max = svmaxv_f32(ptrue, MaximumVector0); } -void +extern "C" void MLASCALL -MlasSveComputeSoftmaxOutputF32Kernel( +MlasSveComputeSoftmaxOutputF32KernelImpl( float* Output, size_t N, const float* Parameters -) + ) +/*++ + +Routine Description: + + This routine scales the softmax exponentials in place by the reciprocal + of their sum. + +Arguments: + + Output - Supplies the exponentials, and receives the scaled values. + + N - Supplies the number of elements to process. + + Parameters - Supplies the scale in element 0. + +Return Value: + + None. + +--*/ { - MLAS_SVBOOL Pred = svptrue_b32(); - size_t veclen = svcntw(); - size_t stride = veclen; - - const float Scale = Parameters[0]; - const MLAS_SVFLOAT32 ScaleVector = MlasSveBroadcastFloat32(Scale); - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - MLAS_SVFLOAT32 Vector = MlasSveMultiplyFloat32(Pred, ScaleVector, MlasSveLoadFloat32(Pred, Output)); - MlasSveStoreFloat32(Pred, Output, Vector); + const svfloat32_t ScaleVector = svdup_n_f32(Parameters[0]); + + for (size_t i = 0; i < N; i += svcntw()) { - Output += stride; - N -= stride; + const svbool_t pg = svwhilelt_b32_u64(i, N); + const svfloat32_t Vector = svmul_f32_x(pg, ScaleVector, svld1_f32(pg, Output + i)); + svst1_f32(pg, Output + i, Vector); } } -void +extern "C" void MLASCALL -MlasSveComputeLogSoftmaxOutputF32Kernel( +MlasSveComputeLogSoftmaxOutputF32KernelImpl( const float* Input, float* Output, size_t N, const float* Parameters -) + ) +/*++ + +Routine Description: + + This routine computes Input + NegativeMaximum - log(SumExp) as the final + log-softmax output step. + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + N - Supplies the number of elements to process. + + Parameters - Supplies NegativeMaximum in element 0 and log(SumExp) in + element 1. + +Return Value: + + None. + +--*/ { - MLAS_SVBOOL Pred = svptrue_b32(); - size_t veclen = svcntw(); - size_t stride = veclen; - - const float NegativeMaximum = Parameters[0]; - const float Logarithm = Parameters[1]; - MLAS_SVFLOAT32 NegativeMaximumVector = MlasSveBroadcastFloat32(NegativeMaximum); - MLAS_SVFLOAT32 LogarithmVector = MlasSveBroadcastFloat32(Logarithm); - - while (N > 0) { - if (N < veclen) { - Pred = svwhilelt_b32(0, (int32_t)N); - stride = N; - } - MLAS_SVFLOAT32 Vector = MlasSveLoadFloat32(Pred, Input); - Vector = MlasSveAddFloat32(Pred, Vector, NegativeMaximumVector); - Vector = MlasSveSubtractFloat32(Pred, Vector, LogarithmVector); - MlasSveStoreFloat32(Pred, Output, Vector); - - Input += stride; - Output += stride; - N -= stride; + const svfloat32_t NegativeMaximumVector = svdup_n_f32(Parameters[0]); + const svfloat32_t LogarithmVector = svdup_n_f32(Parameters[1]); + + for (size_t i = 0; i < N; i += svcntw()) { + + const svbool_t pg = svwhilelt_b32_u64(i, N); + + svfloat32_t Vector = svld1_f32(pg, Input + i); + Vector = svadd_f32_x(pg, Vector, NegativeMaximumVector); + Vector = svsub_f32_x(pg, Vector, LogarithmVector); + + svst1_f32(pg, Output + i, Vector); } - } diff --git a/onnxruntime/core/mlas/lib/sve/elementwise_sve_dispatch.cpp b/onnxruntime/core/mlas/lib/sve/elementwise_sve_dispatch.cpp new file mode 100644 index 0000000000000..2acdd2925170e --- /dev/null +++ b/onnxruntime/core/mlas/lib/sve/elementwise_sve_dispatch.cpp @@ -0,0 +1,276 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + elementwise_sve_dispatch.cpp + +Abstract: + + This module contains the public entry points of the SVE elementwise + kernels. It is plain C++ with no SVE intrinsics, so it compiles with any + toolchain (including MSVC, which has no SVE support); the vector loops + are provided by extern "C" *Impl functions that come either from the SVE + intrinsics translation units (GCC/Clang builds) or from the portable + machine-code assembly variant (elementwise_sve_asm.S) — both export the + same symbols, and the build links exactly one of them. + + This layer owns everything that does not belong in assembly: scalar + fast paths, constant-table plumbing, and the multi-pass orchestration of + Gelu. + +--*/ + +#include "mlasi_sve.h" + +#include + +// +// Vector-loop implementations (intrinsics TUs or elementwise_sve_asm.S). +// + +extern "C" { + +void MLASCALL +MlasSveErfKernelImpl(const float* Input, float* Output, size_t N, const MLAS_ERF_CONSTANTS* Constants); + +void MLASCALL +MlasSveLogisticKernelImpl(const float* Input, float* Output, size_t N, const MLAS_LOGISTIC_CONSTANTS* Constants); + +void +MLASCALL +MlasSveTanhKernelImpl(const float* Input, float* Output, size_t N, const MLAS_TANH_CONSTANTS* Constants); + +void MLASCALL +MlasSveComputeExpF32KernelImpl(const float* Input, float* Output, size_t N, const MLAS_SVE_EXP_CONSTANTS* Constants); + +float MLASCALL +MlasSveComputeSumExpF32KernelImpl(const float* Input, float* Output, size_t N, const float* NegativeMaximum, const MLAS_EXP_CONSTANTS* Constants, const MLAS_SVE_EXP_CONSTANTS* AclConstants); + +float MLASCALL +MlasSveReduceMaximumF32KernelImpl(const float* Input, size_t N, float MinimumValue); + +void MLASCALL +MlasSveReduceMinimumMaximumF32KernelImpl(const float* Input, float* Min, float* Max, size_t N); + +void MLASCALL +MlasSveComputeSoftmaxOutputF32KernelImpl(float* Output, size_t N, const float* Parameters); + +void MLASCALL +MlasSveComputeLogSoftmaxOutputF32KernelImpl(const float* Input, float* Output, size_t N, const float* Parameters); + +#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) + +void MLASCALL +MlasSveTanhFP16KernelImpl(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N); + +void MLASCALL +MlasSveErfFP16KernelImpl(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N); + +void MLASCALL +MlasSveGeluTanhArgFP16KernelImpl(const MLAS_FP16* Input, MLAS_FP16* Temp, size_t N); + +void MLASCALL +MlasSveGeluScaleFP16KernelImpl(const MLAS_FP16* Input, MLAS_FP16* Temp, size_t N); + +void MLASCALL +MlasSveGeluCombineFP16KernelImpl(const MLAS_FP16* Input, const MLAS_FP16* Inner, MLAS_FP16* Output, size_t N); + +#endif + +} // extern "C" + +// +// Constants of the ARM Compute Library exp() approximation (see +// elementwise_sve.cpp for the algorithm), as float bit patterns. +// + +extern "C" const MLAS_SVE_EXP_CONSTANTS MlasSveExpConstants = { + 0x3f7ffff6, // x^1: 0x1.ffffecp-1f + 0x3efffedb, // x^2: 0x1.fffdb6p-2f + 0x3e2aaf33, // x^3: 0x1.555e66p-3f + 0x3d2b9f17, // x^4: 0x1.573e2ep-5f + 0x3c072010, // x^5: 0x1.0e4020p-7f + 0x4b00007f, // 2^23 + 127 + 0x3fb8aa3b, // 1 / ln(2) + 0xbf317200, // -ln(2), bits -1..-19 + 0xb5bfbe8e, // -ln(2), bits -20..-42 + 0x7f800000, // +infinity + 0x42b0bd71, // 88.37f, approximately ln(2^127.5) + 0xc2ad47ae, // -86.64f, approximately ln(2^-125) + 0x48481fc0, // 0x1.903f8p17, FEXPA rounding shift with embedded exponent bias + 0x3f000000, // 0.5 + 0xc2b00000, // -88.0, FEXPA-safe SumExp lower clamp (> -127*ln2) +}; + +// +// Float32 entry points. +// + +void +MLASCALL +MlasSveErfKernel( + const float* Input, + float* Output, + size_t N + ) +{ + MlasSveErfKernelImpl(Input, Output, N, &MlasErfConstants); +} + +void +MLASCALL +MlasSveLogisticKernel( + const float* Input, + float* Output, + size_t N + ) +{ + MlasSveLogisticKernelImpl(Input, Output, N, &MlasLogisticConstants); +} + +void +MLASCALL +MlasSveTanhKernel( + const float* Input, + float* Output, + size_t N + ) +{ + MlasSveTanhKernelImpl(Input, Output, N, &MlasTanhConstants); +} + +void +MLASCALL +MlasSveComputeExpF32Kernel( + const float* Input, + float* Output, + size_t N + ) +{ + if (N == 1) { + Output[0] = expf(Input[0]); + return; + } + + MlasSveComputeExpF32KernelImpl(Input, Output, N, &MlasSveExpConstants); +} + +float +MLASCALL +MlasSveComputeSumExpF32Kernel( + const float* Input, + float* Output, + size_t N, + const float* NegativeMaximum + ) +{ + if (N == 1) { + float result = expf(Input[0] + *NegativeMaximum); + if (Output != nullptr) { + Output[0] = result; + } + return result; + } + + return MlasSveComputeSumExpF32KernelImpl(Input, Output, N, NegativeMaximum, &MlasExpConstants, &MlasSveExpConstants); +} + +float +MLASCALL +MlasSveReduceMaximumF32Kernel( + const float* Input, + size_t N + ) +{ + return MlasSveReduceMaximumF32KernelImpl(Input, N, MlasMinimumF32Value); +} + +void +MLASCALL +MlasSveReduceMinimumMaximumF32Kernel( + const float* Input, + float* Min, + float* Max, + size_t N + ) +{ + MlasSveReduceMinimumMaximumF32KernelImpl(Input, Min, Max, N); +} + +void +MLASCALL +MlasSveComputeSoftmaxOutputF32Kernel( + float* Output, + size_t N, + const float* Parameters + ) +{ + MlasSveComputeSoftmaxOutputF32KernelImpl(Output, N, Parameters); +} + +void +MLASCALL +MlasSveComputeLogSoftmaxOutputF32Kernel( + const float* Input, + float* Output, + size_t N, + const float* Parameters + ) +{ + MlasSveComputeLogSoftmaxOutputF32KernelImpl(Input, Output, N, Parameters); +} + +// +// Float16 entry points. +// + +#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) + +void +MLASCALL +MlasSveTanhFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, + size_t N + ) +{ + MlasSveTanhFP16KernelImpl(Input, Output, N); +} + +void +MLASCALL +MlasSveErfFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, + size_t N + ) +{ + MlasSveErfFP16KernelImpl(Input, Output, N); +} + +void +MLASCALL +MlasSveGeluFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, + MLAS_FP16* Temp, + size_t N, + MLAS_GELU_ALGORITHM Algo + ) +{ + if (Algo == MlasGeluTanh) { + MlasSveGeluTanhArgFP16KernelImpl(Input, Temp, N); + MlasSveTanhFP16KernelImpl(Temp, Temp, N); + } else { + MlasSveGeluScaleFP16KernelImpl(Input, Temp, N); + MlasSveErfFP16KernelImpl(Temp, Temp, N); + } + + MlasSveGeluCombineFP16KernelImpl(Input, Temp, Output, N); +} + +#endif // MLAS_F16VEC_INTRINSICS_SUPPORTED diff --git a/onnxruntime/core/mlas/lib/sve/elementwise_sve_fp16.cpp b/onnxruntime/core/mlas/lib/sve/elementwise_sve_fp16.cpp index 26f2a09439910..c40954138a1f8 100644 --- a/onnxruntime/core/mlas/lib/sve/elementwise_sve_fp16.cpp +++ b/onnxruntime/core/mlas/lib/sve/elementwise_sve_fp16.cpp @@ -7,17 +7,42 @@ Licensed under the MIT License. Module Name: - Elementwise_sve_fp16.cpp + elementwise_sve_fp16.cpp Abstract: - This module contains the SVE Elementwise functions . + This module implements the float16 elementwise kernel bodies using SVE + intrinsics: Tanh, Erf and the Gelu passes. + + Only the vector loops live here, as extern "C" *Impl functions with no + data relocations (constants are materialized from immediates), so the + machine code can be captured verbatim into the portable assembly variant + (elementwise_sve_asm.S), which exports the same symbols. The public + MlasSve* entry points live in elementwise_sve_dispatch.cpp. + + The kernels share one loop idiom: a whilelt-governed predicate covers the + vector body and the tail in the same loop, and every constant is + broadcast once before the loop. Divisions are computed with the + reciprocal-estimate + two Newton-Raphson steps idiom. + + This file is compiled with SVE and fp16 enabled via its per-file -march + compile flag (see onnxruntime_mlas.cmake). --*/ -#include "mlas_sve_fp16.h" -using _mlas_fp16_ = uint16_t; -struct MlasTanhConstants_fp16_scalar { +#include "mlasi_sve.h" + +#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) + +#include + +namespace { + +// +// Coefficients of the rational tanh approximation. +// + +constexpr struct { __fp16 LowerRange; __fp16 UpperRange; __fp16 alpha_7; @@ -28,9 +53,7 @@ struct MlasTanhConstants_fp16_scalar { __fp16 beta_4; __fp16 beta_2; __fp16 beta_0; -}; - -constexpr MlasTanhConstants_fp16_scalar TanhConstantsFp16 = { +} MlasSveTanhConstantsFp16 = { -3.515625f, 3.515625f, 5.960464477539063e-08f, @@ -40,217 +63,377 @@ constexpr MlasTanhConstants_fp16_scalar TanhConstantsFp16 = { 1.1920928955078125e-06f, 0.00011855363845825195f, 0.0022678375244140625f, - 0.004894256591796875f + 0.004894256591796875f, }; -static inline MLAS_SVFLOAT16 -Tanh_Vector_SVE_fp16(MLAS_SVFLOAT16 x, MLAS_SVBOOL pg) +// +// Coefficients of the Abramowitz-Stegun erf approximation: +// erf(x) ~= 1 - (a1*t + a2*t^2 + ... + a5*t^5) * exp(-x^2), t = 1/(1 + p*x), +// with inputs saturated to +-1 beyond |x| = 4. +// + +constexpr struct { + __fp16 p; + __fp16 a1; + __fp16 a2; + __fp16 a3; + __fp16 a4; + __fp16 a5; + __fp16 SaturationThreshold; +} MlasSveErfConstantsFp16 = { + 0.328f, + 0.2505f, + -0.2881f, + 1.4102f, + -1.423f, + 1.0547f, + 4.0f, +}; + +// +// Coefficients of the rational approximation of exp(-x) over [0, 6], used by +// the erf kernel for the exp(-x^2) factor. +// + +constexpr struct { + __fp16 UpperRange; + __fp16 c0; + __fp16 c1; + __fp16 c2; + __fp16 d0; + __fp16 d1; + __fp16 d2; +} MlasSveExpNegConstantsFp16 = { + 6.0f, + 1.330f, + -0.390f, + 0.0288f, + 1.338f, + 0.848f, + 0.467f, +}; + +// +// Gelu constants: sqrt(1/2) for the erf form, and the tanh-form inner +// polynomial coefficients sqrt(2/pi) and 0.044715 * sqrt(2/pi), with the +// tanh argument clamped to +-5. +// + +constexpr struct { + __fp16 OneHalf; + __fp16 One; + __fp16 Sqrt1_2; + __fp16 TanhB; + __fp16 TanhC; + __fp16 TanhArgLowerRange; + __fp16 TanhArgUpperRange; +} MlasSveGeluConstantsFp16 = { + 0.5f, + 1.0f, + static_cast(M_SQRT1_2), + 0.7978845608028654f, + 0.035677408136300125f, + -5.0f, + 5.0f, +}; + +// +// Computes 1/x with reciprocal estimate plus two Newton-Raphson steps. +// + +MLAS_FORCEINLINE +svfloat16_t +MlasSveReciprocalFp16( + svbool_t pg, + svfloat16_t x + ) { - MLAS_SVFLOAT16 g_LowerRange_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.LowerRange); - MLAS_SVFLOAT16 g_UpperRange_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.UpperRange); - MLAS_SVFLOAT16 g_alpha_7_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.alpha_7); - MLAS_SVFLOAT16 g_alpha_5_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.alpha_5); - MLAS_SVFLOAT16 g_alpha_3_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.alpha_3); - MLAS_SVFLOAT16 g_alpha_1_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.alpha_1); - MLAS_SVFLOAT16 g_beta_6_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.beta_6); - MLAS_SVFLOAT16 g_beta_4_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.beta_4); - MLAS_SVFLOAT16 g_beta_2_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.beta_2); - MLAS_SVFLOAT16 g_beta_0_vec = MlasSveBroadcastfloat16(TanhConstantsFp16.beta_0); - - x = MlasSveMinfloat16(pg, x, g_UpperRange_vec); - x = MlasSveMaxfloat16(pg, x, g_LowerRange_vec); - - MLAS_SVFLOAT16 x2 = MlasSveMulfloat16(pg, x, x); - MLAS_SVFLOAT16 p = MlasSveMLAfloat16(pg, g_alpha_5_vec, g_alpha_7_vec, x2); - p = MlasSveMLAfloat16(pg, g_alpha_3_vec, p, x2); - p = MlasSveMLAfloat16(pg, g_alpha_1_vec, p, x2); - p = MlasSveMulfloat16(pg, p, x); - - svfloat16_t q = MlasSveMLAfloat16(pg, g_beta_4_vec, g_beta_6_vec, x2); - q = MlasSveMLAfloat16(pg, g_beta_2_vec, q, x2); - q = MlasSveMLAfloat16(pg, g_beta_0_vec, q, x2); - - MLAS_SVFLOAT16 res = MlasSveDivfloat16(pg, p, q); - - return res; + svfloat16_t recip = svrecpe_f16(x); + recip = svmul_f16_x(pg, recip, svrecps_f16(x, recip)); + recip = svmul_f16_x(pg, recip, svrecps_f16(x, recip)); + return recip; } -void -MlasSveTanhFP16Kernel(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) +} // namespace + +extern "C" void +MLASCALL +MlasSveTanhFP16KernelImpl( + const MLAS_FP16* Input, + MLAS_FP16* Output, + size_t N + ) +/*++ + +Routine Description: + + This routine implements tanh(x) for fp16 using the rational polynomial + approximation p(x^2)*x / q(x^2) on the clamped input. + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + N - Supplies the number of elements to process. + +Return Value: + + None. + +--*/ +{ + const __fp16* input = reinterpret_cast(Input); + __fp16* output = reinterpret_cast<__fp16*>(Output); + + const svfloat16_t LowerRange = svdup_n_f16(MlasSveTanhConstantsFp16.LowerRange); + const svfloat16_t UpperRange = svdup_n_f16(MlasSveTanhConstantsFp16.UpperRange); + const svfloat16_t Alpha7 = svdup_n_f16(MlasSveTanhConstantsFp16.alpha_7); + const svfloat16_t Alpha5 = svdup_n_f16(MlasSveTanhConstantsFp16.alpha_5); + const svfloat16_t Alpha3 = svdup_n_f16(MlasSveTanhConstantsFp16.alpha_3); + const svfloat16_t Alpha1 = svdup_n_f16(MlasSveTanhConstantsFp16.alpha_1); + const svfloat16_t Beta6 = svdup_n_f16(MlasSveTanhConstantsFp16.beta_6); + const svfloat16_t Beta4 = svdup_n_f16(MlasSveTanhConstantsFp16.beta_4); + const svfloat16_t Beta2 = svdup_n_f16(MlasSveTanhConstantsFp16.beta_2); + const svfloat16_t Beta0 = svdup_n_f16(MlasSveTanhConstantsFp16.beta_0); + + for (size_t i = 0; i < N; i += svcnth()) { + + const svbool_t pg = svwhilelt_b16_u64(i, N); + + svfloat16_t x = svld1_f16(pg, input + i); + x = svmin_f16_x(pg, x, UpperRange); + x = svmax_f16_x(pg, x, LowerRange); + + const svfloat16_t x2 = svmul_f16_x(pg, x, x); + + svfloat16_t p = svmla_f16_x(pg, Alpha5, Alpha7, x2); + p = svmla_f16_x(pg, Alpha3, p, x2); + p = svmla_f16_x(pg, Alpha1, p, x2); + p = svmul_f16_x(pg, p, x); + + svfloat16_t q = svmla_f16_x(pg, Beta4, Beta6, x2); + q = svmla_f16_x(pg, Beta2, q, x2); + q = svmla_f16_x(pg, Beta0, q, x2); + + svst1_f16(pg, output + i, svdiv_f16_x(pg, p, q)); + } +} + +extern "C" void +MLASCALL +MlasSveErfFP16KernelImpl( + const MLAS_FP16* Input, + MLAS_FP16* Output, + size_t N + ) +/*++ + +Routine Description: + + This routine implements erf(x) for fp16 using the Abramowitz-Stegun + approximation, saturating to sign(x) beyond |x| = 4. + +Arguments: + + Input - Supplies the input buffer. + + Output - Supplies the output buffer. + + N - Supplies the number of elements to process. + +Return Value: + + None. + +--*/ { - size_t offset = 0; - const auto* input = reinterpret_cast(Input); - auto* output = reinterpret_cast<_mlas_fp16_*>(Output); - while (offset < N) { - MLAS_SVBOOL pg = MlasSveSelPredicatefloat16(offset, N); - MLAS_SVFLOAT16 x = MlasSvereinterpretf16_u16(MlasSveLoadUint16(pg, &input[offset])); - MLAS_SVFLOAT16 y = Tanh_Vector_SVE_fp16(x, pg); - MlasSveStoreUint16(pg, &output[offset], MlasSvereinterpretu16_f16(y)); - offset += svcnth(); + const __fp16* input = reinterpret_cast(Input); + __fp16* output = reinterpret_cast<__fp16*>(Output); + + const svfloat16_t P = svdup_n_f16(MlasSveErfConstantsFp16.p); + const svfloat16_t A1 = svdup_n_f16(MlasSveErfConstantsFp16.a1); + const svfloat16_t A2 = svdup_n_f16(MlasSveErfConstantsFp16.a2); + const svfloat16_t A3 = svdup_n_f16(MlasSveErfConstantsFp16.a3); + const svfloat16_t A4 = svdup_n_f16(MlasSveErfConstantsFp16.a4); + const svfloat16_t A5 = svdup_n_f16(MlasSveErfConstantsFp16.a5); + const svfloat16_t Threshold = svdup_n_f16(MlasSveErfConstantsFp16.SaturationThreshold); + + const svfloat16_t ExpUpperRange = svdup_n_f16(MlasSveExpNegConstantsFp16.UpperRange); + const svfloat16_t ExpC0 = svdup_n_f16(MlasSveExpNegConstantsFp16.c0); + const svfloat16_t ExpC1 = svdup_n_f16(MlasSveExpNegConstantsFp16.c1); + const svfloat16_t ExpC2 = svdup_n_f16(MlasSveExpNegConstantsFp16.c2); + const svfloat16_t ExpD0 = svdup_n_f16(MlasSveExpNegConstantsFp16.d0); + const svfloat16_t ExpD1 = svdup_n_f16(MlasSveExpNegConstantsFp16.d1); + const svfloat16_t ExpD2 = svdup_n_f16(MlasSveExpNegConstantsFp16.d2); + + const svfloat16_t One = svdup_n_f16(__fp16(1.0f)); + const svfloat16_t NegOne = svdup_n_f16(__fp16(-1.0f)); + const svfloat16_t Zero = svdup_n_f16(__fp16(0.0f)); + + for (size_t i = 0; i < N; i += svcnth()) { + + const svbool_t pg = svwhilelt_b16_u64(i, N); + + const svfloat16_t x = svld1_f16(pg, input + i); + + const svbool_t NegativeLanes = svcmplt_f16(pg, x, Zero); + const svfloat16_t Sign = svsel_f16(NegativeLanes, NegOne, One); + + const svfloat16_t AbsX = svabs_f16_x(pg, x); + const svbool_t ApproxLanes = svcmplt_f16(pg, AbsX, Threshold); + const svfloat16_t AbsXClamped = svmin_f16_x(pg, AbsX, Threshold); + + // t = 1 / (1 + p * |x|) + const svfloat16_t Denominator = svmla_f16_x(pg, One, P, AbsXClamped); + const svfloat16_t t = MlasSveReciprocalFp16(pg, Denominator); + + const svfloat16_t t2 = svmul_f16_x(pg, t, t); + const svfloat16_t t3 = svmul_f16_x(pg, t2, t); + const svfloat16_t t4 = svmul_f16_x(pg, t3, t); + const svfloat16_t t5 = svmul_f16_x(pg, t4, t); + + svfloat16_t Poly = svmul_f16_x(pg, A1, t); + Poly = svmla_f16_x(pg, Poly, A2, t2); + Poly = svmla_f16_x(pg, Poly, A3, t3); + Poly = svmla_f16_x(pg, Poly, A4, t4); + Poly = svmla_f16_x(pg, Poly, A5, t5); + + // exp(-x^2) via the rational approximation over [0, 6]. + svfloat16_t x2 = svmul_f16_x(pg, AbsXClamped, AbsXClamped); + x2 = svmin_f16_x(pg, x2, ExpUpperRange); + + const svfloat16_t x4 = svmul_f16_x(pg, x2, x2); + + svfloat16_t Numerator = svmla_f16_x(pg, ExpC0, ExpC1, x2); + Numerator = svmla_f16_x(pg, Numerator, ExpC2, x4); + + svfloat16_t Denominator2 = svmla_f16_x(pg, ExpD0, ExpD1, x2); + Denominator2 = svmla_f16_x(pg, Denominator2, ExpD2, x4); + + const svfloat16_t ExpNegX2 = + svmul_f16_x(pg, Numerator, MlasSveReciprocalFp16(pg, Denominator2)); + + // erf ~= sign * (1 - poly * exp(-x^2)), clamped to [-1, 1]; saturated + // lanes produce sign(x) directly. + svfloat16_t Erf = svmul_f16_x(pg, Sign, svsub_f16_x(pg, One, svmul_f16_x(pg, Poly, ExpNegX2))); + Erf = svmin_f16_x(pg, Erf, One); + Erf = svmax_f16_x(pg, Erf, NegOne); + + const svfloat16_t Result = svsel_f16(ApproxLanes, Erf, Sign); + + svst1_f16(pg, output + i, Result); } } -static inline MLAS_SVFLOAT16 -exp_neg_rational_approx_f16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x) +// +// Gelu is orchestrated from elementwise_sve_dispatch.cpp as three vector +// passes (argument preparation, tanh/erf, combine) so that each pass is a +// single self-contained loop. +// + +extern "C" void +MLASCALL +MlasSveGeluTanhArgFP16KernelImpl( + const MLAS_FP16* Input, + MLAS_FP16* Temp, + size_t N + ) +/*++ + +Routine Description: + + This routine computes the clamped tanh-form Gelu inner argument + sqrt(2/pi) * (x + 0.044715 * x^3). + +--*/ { - const __fp16 a0 = 6.0f; - MLAS_SVFLOAT16 max_x = MlasSveBroadcastfloat16(a0); - x = MlasSveMinfloat16(pg, x, max_x); - - const __fp16 c0 = 1.330f; - const __fp16 c1 = -0.390f; - const __fp16 c2 = 0.0288f; - - const __fp16 d0 = 1.338f; - const __fp16 d1 = 0.848f; - const __fp16 d2 = 0.467f; - - MLAS_SVFLOAT16 c0v = MlasSveBroadcastfloat16(c0); - MLAS_SVFLOAT16 c1v = MlasSveBroadcastfloat16(c1); - MLAS_SVFLOAT16 c2v = MlasSveBroadcastfloat16(c2); - MLAS_SVFLOAT16 d0v = MlasSveBroadcastfloat16(d0); - MLAS_SVFLOAT16 d1v = MlasSveBroadcastfloat16(d1); - MLAS_SVFLOAT16 d2v = MlasSveBroadcastfloat16(d2); - MLAS_SVFLOAT16 x2 = MlasSveMulfloat16(pg, x, x); - - MLAS_SVFLOAT16 num = MlasSveMLAfloat16(pg, c0v, c1v, x); - num = MlasSveMLAfloat16(pg, num, c2v, x2); - - MLAS_SVFLOAT16 den = MlasSveMLAfloat16(pg, d0v, d1v, x); - den = MlasSveMLAfloat16(pg, den, d2v, x2); - - MLAS_SVFLOAT16 recip = MlasSveReciprocalfloat16(den); - recip = MlasSveMulfloat16(pg, recip, MlasSveReciprocalStepfloat16(den, recip)); - recip = MlasSveMulfloat16(pg, recip, MlasSveReciprocalStepfloat16(den, recip)); - - MLAS_SVFLOAT16 result = MlasSveMulfloat16(pg, num, recip); - return result; + const __fp16* input = reinterpret_cast(Input); + __fp16* temp = reinterpret_cast<__fp16*>(Temp); + + const svfloat16_t TanhB = svdup_n_f16(MlasSveGeluConstantsFp16.TanhB); + const svfloat16_t TanhC = svdup_n_f16(MlasSveGeluConstantsFp16.TanhC); + const svfloat16_t ArgLower = svdup_n_f16(MlasSveGeluConstantsFp16.TanhArgLowerRange); + const svfloat16_t ArgUpper = svdup_n_f16(MlasSveGeluConstantsFp16.TanhArgUpperRange); + + for (size_t i = 0; i < N; i += svcnth()) { + + const svbool_t pg = svwhilelt_b16_u64(i, N); + + const svfloat16_t x = svld1_f16(pg, input + i); + const svfloat16_t x2 = svmul_f16_x(pg, x, x); + + svfloat16_t TanhArg = svmul_f16_x(pg, x, svmla_f16_x(pg, TanhB, TanhC, x2)); + TanhArg = svmax_f16_x(pg, ArgLower, svmin_f16_x(pg, TanhArg, ArgUpper)); + + svst1_f16(pg, temp + i, TanhArg); + } } -void MLASCALL -MlasSveErfFP16Kernel(const MLAS_FP16* Input, MLAS_FP16* Output, size_t N) +extern "C" void +MLASCALL +MlasSveGeluScaleFP16KernelImpl( + const MLAS_FP16* Input, + MLAS_FP16* Temp, + size_t N + ) +/*++ + +Routine Description: + + This routine scales the input by sqrt(1/2) for the exact (erf) Gelu form. + +--*/ { - const auto* input = reinterpret_cast(Input); - auto* output = reinterpret_cast<_mlas_fp16_*>(Output); - const __fp16 p = 0.328f; - const __fp16 a1 = 0.2505f; - const __fp16 a2 = -0.2881f; - const __fp16 a3 = 1.4102f; - const __fp16 a4 = -1.423f; - const __fp16 a5 = 1.0547f; - - MLAS_SVFLOAT16 vp = MlasSveBroadcastfloat16(p); - MLAS_SVFLOAT16 va1 = MlasSveBroadcastfloat16(a1); - MLAS_SVFLOAT16 va2 = MlasSveBroadcastfloat16(a2); - MLAS_SVFLOAT16 va3 = MlasSveBroadcastfloat16(a3); - MLAS_SVFLOAT16 va4 = MlasSveBroadcastfloat16(a4); - MLAS_SVFLOAT16 va5 = MlasSveBroadcastfloat16(a5); - - const __fp16 v1 = 1.0f; - const __fp16 v2 = -1.0f; - const __fp16 v3 = 0.0f; - const __fp16 v4 = 4.0f; - MLAS_SVFLOAT16 vone = MlasSveBroadcastfloat16(v1); - MLAS_SVFLOAT16 vneg_one = MlasSveBroadcastfloat16(v2); - MLAS_SVFLOAT16 vzero = MlasSveBroadcastfloat16(v3); - MLAS_SVFLOAT16 vth = MlasSveBroadcastfloat16(v4); - - size_t i = 0; - while (i < N) { - MLAS_SVBOOL pg = MlasSveSelPredicatefloat16(i, N); - MLAS_SVFLOAT16 x = MlasSvereinterpretf16_u16(MlasSveLoadUint16(pg, &input[i])); - MLAS_SVBOOL neg_mask = MlasSveComparelessthanfloat16(pg, x, vzero); - MLAS_SVFLOAT16 sign = MlasSveSelectfloat16(neg_mask, vneg_one, vone); - MLAS_SVFLOAT16 absx = MlasSveAbsolutefloat16(MlasSveBroadcastfloat16(v3), pg, x); - svbool_t use_mask = MlasSveComparelessthanfloat16(pg, absx, vth); - MLAS_SVFLOAT16 absx_clamped = MlasSveMinfloat16(pg, absx, vth); - MLAS_SVFLOAT16 denom = MlasSveMLAfloat16(pg, vone, vp, absx_clamped); - MLAS_SVFLOAT16 t = MlasSveReciprocalfloat16(denom); - t = MlasSveMulfloat16(pg, t, MlasSveReciprocalStepfloat16(denom, t)); - t = MlasSveMulfloat16(pg, t, MlasSveReciprocalStepfloat16(denom, t)); - MLAS_SVFLOAT16 t2 = MlasSveMulfloat16(pg, t, t); - MLAS_SVFLOAT16 t3 = MlasSveMulfloat16(pg, t2, t); - MLAS_SVFLOAT16 t4 = MlasSveMulfloat16(pg, t3, t); - MLAS_SVFLOAT16 t5 = MlasSveMulfloat16(pg, t4, t); - svfloat16_t poly = MlasSveMulfloat16(pg, va1, t); - poly = MlasSveMLAfloat16(pg, poly, va2, t2); - poly = MlasSveMLAfloat16(pg, poly, va3, t3); - poly = MlasSveMLAfloat16(pg, poly, va4, t4); - poly = MlasSveMLAfloat16(pg, poly, va5, t5); - MLAS_SVFLOAT16 x2 = MlasSveMulfloat16(pg, absx_clamped, absx_clamped); - MLAS_SVFLOAT16 exp_neg_x2 = exp_neg_rational_approx_f16(pg, x2); - MLAS_SVFLOAT16 poly_mul_exp = MlasSveMulfloat16(pg, poly, exp_neg_x2); - MLAS_SVFLOAT16 one_minus_term = MlasSveSubtractfloat16(pg, vone, poly_mul_exp); - MLAS_SVFLOAT16 erf_approx = MlasSveMulfloat16(pg, sign, one_minus_term); - erf_approx = MlasSveMinfloat16(pg, erf_approx, vone); - erf_approx = MlasSveMaxfloat16(pg, erf_approx, vneg_one); - MLAS_SVFLOAT16 result = MlasSveSelectfloat16(use_mask, erf_approx, sign); - MlasSveStoreUint16(pg, &output[i], MlasSvereinterpretu16_f16(result)); - i += svcntp_b16(svptrue_b16(), pg); + const __fp16* input = reinterpret_cast(Input); + __fp16* temp = reinterpret_cast<__fp16*>(Temp); + + const svfloat16_t Sqrt1_2 = svdup_n_f16(MlasSveGeluConstantsFp16.Sqrt1_2); + + for (size_t i = 0; i < N; i += svcnth()) { + + const svbool_t pg = svwhilelt_b16_u64(i, N); + + const svfloat16_t x = svld1_f16(pg, input + i); + svst1_f16(pg, temp + i, svmul_f16_x(pg, x, Sqrt1_2)); } } -void MLASCALL -MlasSveGeluFP16Kernel(const MLAS_FP16* input, MLAS_FP16* output, MLAS_FP16* temp, size_t count, MLAS_GELU_ALGORITHM algo) +extern "C" void +MLASCALL +MlasSveGeluCombineFP16KernelImpl( + const MLAS_FP16* Input, + const MLAS_FP16* Inner, + MLAS_FP16* Output, + size_t N + ) +/*++ + +Routine Description: + + This routine combines the Gelu factors: 0.5 * x * (1 + tanh/erf value). + +--*/ { - const __fp16 r1 = 0.5f; - const __fp16 r2 = 1.0f; - const __fp16 r3 = static_cast(M_SQRT1_2); - const __fp16 r4 = 0.7978845608028654f; - const __fp16 r5 = 0.035677408136300125f; - - const MLAS_SVFLOAT16 v_half = MlasSveBroadcastfloat16(r1); - const MLAS_SVFLOAT16 v_one = MlasSveBroadcastfloat16(r2); - const MLAS_SVFLOAT16 v_sqrt1_2 = MlasSveBroadcastfloat16(r3); - const MLAS_SVFLOAT16 v_B = MlasSveBroadcastfloat16(r4); - const MLAS_SVFLOAT16 v_C = MlasSveBroadcastfloat16(r5); - - const __fp16 c1 = -5.0f; - const __fp16 c2 = 5.0f; - if (algo == MlasGeluTanh) { - size_t i = 0; - while (i < count) { - svbool_t pg = MlasSveSelPredicatefloat16(i, count); - MLAS_SVFLOAT16 v_x = MlasSveLoadFloat16(pg, &input[i]); - MLAS_SVFLOAT16 v_x2 = MlasSveMulfloat16(pg, v_x, v_x); - MLAS_SVFLOAT16 v_inner = MlasSveMLAfloat16(pg, v_B, v_C, v_x2); - MLAS_SVFLOAT16 v_tanh_arg = MlasSveMulfloat16(pg, v_x, v_inner); - v_tanh_arg = MlasSveMaxfloat16(pg, MlasSveBroadcastfloat16(c1), MlasSveMinfloat16(pg, v_tanh_arg, MlasSveBroadcastfloat16(c2))); - MlasSveStoreF16(pg, &temp[i], v_tanh_arg); - i += svcnth(); - } - - MlasSveTanhFP16Kernel(reinterpret_cast(temp), reinterpret_cast(temp), count); - - size_t j = 0; - while (j < (count)) { - svbool_t pg = MlasSveSelPredicatefloat16(j, count); - MLAS_SVFLOAT16 v_x = MlasSveLoadFloat16(pg, &input[j]); - MLAS_SVFLOAT16 v_tanh = MlasSveLoadFloat16(pg, &temp[j]); - MLAS_SVFLOAT16 v_result = MlasSveMulfloat16(pg, v_half, MlasSveMulfloat16(pg, v_x, svadd_f16_m(pg, v_one, v_tanh))); - MlasSveStoreF16(pg, &output[j], v_result); - j += svcnth(); - } - } else { - size_t i = 0; - while (i < (count)) { - svbool_t pg = MlasSveSelPredicatefloat16(i, count); - MLAS_SVFLOAT16 v_x = MlasSveLoadFloat16(pg, &input[i]); - MLAS_SVFLOAT16 v_scaled = MlasSveMulfloat16(pg, v_x, v_sqrt1_2); - MlasSveStoreF16(pg, &temp[i], v_scaled); - i += svcnth(); - } - - MlasSveErfFP16Kernel(temp, temp, count); - - size_t j = 0; - while (j < (count)) { - svbool_t pg = MlasSveSelPredicatefloat16(j, count); - MLAS_SVFLOAT16 v_x = MlasSveLoadFloat16(pg, &input[j]); - MLAS_SVFLOAT16 v_erf = MlasSveLoadFloat16(pg, &temp[j]); - MLAS_SVFLOAT16 v_result = MlasSveMulfloat16(pg, v_half, MlasSveMulfloat16(pg, v_x, MlasSveAddfloat16(pg, v_one, v_erf))); - MlasSveStoreF16(pg, &output[j], v_result); - j += svcnth(); - } + const __fp16* input = reinterpret_cast(Input); + const __fp16* inner = reinterpret_cast(Inner); + __fp16* output = reinterpret_cast<__fp16*>(Output); + + const svfloat16_t OneHalf = svdup_n_f16(MlasSveGeluConstantsFp16.OneHalf); + const svfloat16_t One = svdup_n_f16(MlasSveGeluConstantsFp16.One); + + for (size_t i = 0; i < N; i += svcnth()) { + + const svbool_t pg = svwhilelt_b16_u64(i, N); + + const svfloat16_t x = svld1_f16(pg, input + i); + const svfloat16_t InnerValue = svld1_f16(pg, inner + i); + const svfloat16_t Result = + svmul_f16_x(pg, OneHalf, svmul_f16_x(pg, x, svadd_f16_x(pg, One, InnerValue))); + + svst1_f16(pg, output + i, Result); } } + +#endif // fp16 vector intrinsics supported diff --git a/onnxruntime/core/mlas/lib/sve/gen_sve_asm.py b/onnxruntime/core/mlas/lib/sve/gen_sve_asm.py new file mode 100644 index 0000000000000..87942a6f769f1 --- /dev/null +++ b/onnxruntime/core/mlas/lib/sve/gen_sve_asm.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +"""Freeze SVE intrinsics translation units into KleidiAI-style machine code. + +Compiles an SVE intrinsics reference TU with gcc, extracts the requested +extern "C" functions from the object file, and emits a portable .S file in +which every instruction is a raw word (KAI_ASM_INST -> GAS ".inst" / +armasm64 "DCD" via aarch64/kai_asm_macros.h). The output assembles with +toolchains that have no SVE support (e.g. MSVC armasm64) and the bytes are +identical on every platform. + +Safety checks (the generator aborts if any fails): + * no relocations against any frozen function's .text.* section + (compile with -fno-stack-protector; .eh_frame relocs are ignored -- + unwind info is not carried into the frozen file); + * no adrp/adr (page-relative addressing of data), no bl/blr (calls), + and no PC-relative literal-pool loads -- every input must arrive via + the argument registers/stack so the code is position-independent; + * the section contains only 4-byte instruction words (no embedded data). + +Example (QGEMM svmmla kernels): + python3 gen_sve_asm.py \ + --src sve/qgemm_mmla_sve_impl.cpp \ + --out aarch64/qgemm_mmla_sve_asm.S \ + --march armv8.2-a+sve+i8mm \ + --define MLAS_SVE_QGEMM_TILE_12X8=1 --define MLAS_SVE_QGEMM_TILE_8X12=1 \ + --symbols MlasGemmS8S8KernelSmmlaSveImpl,MlasGemmU8X8KernelUmmlaSveImpl \ + --module qgemm_mmla_sve +""" + +import argparse +import os +import re +import subprocess +import sys +import tempfile + +INSN_RE = re.compile(r"^\s*([0-9a-f]+):\s+([0-9a-f]{8})\s+(.*)$") +FORBIDDEN_MNEMONIC_RE = re.compile(r"^(adrp|adr|bl|blr)\s") +LITERAL_LOAD_RE = re.compile(r"^ldr\s+[^,]+,\s+0x[0-9a-f]+\s*$") + + +def run(cmd): + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + sys.exit(f"FAILED: {' '.join(cmd)}\n{result.stderr}") + return result.stdout + + +def check_relocations(obj, symbols): + out = run(["readelf", "-r", obj]) + section = None + for line in out.splitlines(): + m = re.match(r"Relocation section '(\S+)'", line) + if m: + section = m.group(1) + continue + if section and any(f".text.{s}" in section for s in symbols): + if re.match(r"^[0-9a-f]{6,}", line.strip()): + sys.exit(f"reloc in frozen section {section}: {line.strip()}") + + +def extract_functions(obj, symbols, allow_missing=False): + # Parsing is driven by sections (-ffunction-sections puts each frozen + # function in its own .text.), NOT by symbol header lines: gcc + # emits local labels inside a function (e.g. the .SVLPSPL/.SVLPEND SVE + # stack-probe loop), which objdump renders as symbol headers that must + # not terminate extraction. + out = run(["objdump", "-d", obj]) + functions = {} + current = None + for line in out.splitlines(): + m = re.match(r"^Disassembly of section \.text\.(\S+):$", line) + if m: + current = m.group(1) if m.group(1) in symbols else None + if current: + functions[current] = [] + continue + if current is None: + continue + if not line.strip(): + continue + if re.match(r"^[0-9a-f]+ <[^>]+>:$", line): + continue + m = INSN_RE.match(line) + if not m: + sys.exit(f"non-instruction line in {current}: {line!r}") + _, word, disasm = m.groups() + disasm = disasm.split("//")[0].rstrip() + if FORBIDDEN_MNEMONIC_RE.match(disasm): + sys.exit(f"forbidden instruction in {current}: {disasm}") + if LITERAL_LOAD_RE.match(disasm): + sys.exit(f"literal-pool load in {current}: {disasm}") + functions[current].append((word, disasm)) + missing = [s for s in symbols if s not in functions] + if missing and not allow_missing: + sys.exit(f"symbols not found in object: {missing}") + return functions + + +HEADER = """\ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + {module}_asm.S + +Abstract: + + Portable machine-code variant of the SVE {module} kernels, in the style + of Arm's KleidiAI library: every instruction is emitted as a raw + instruction word (GAS ".inst" / armasm64 "DCD" via KAI_ASM_INST), so the + file assembles with toolchains that have no SVE support, and the bytes + are identical on every platform. + + GENERATED FILE - DO NOT EDIT BY HAND. + + Generated from the SVE intrinsics translation unit(s) ({src}), which + remain the reference implementation and regeneration source (script: + sve/gen_sve_asm.py). The functions here export the same extern "C" + symbols; the build links exactly one of the two implementations. The + code is fully position-independent (the generator verifies there are no + relocations, calls, or literal pools). + +--*/ + +#include "kai_asm_macros.h" + + KAI_ASM_CODE({module}) +""" + + +def emit(functions, symbols, module, src, out_path): + lines = [HEADER.format(module=module, src=src)] + for sym in symbols: + lines.append("") + lines.append(" KAI_ASM_ALIGN") + lines.append(f" KAI_ASM_GLOBAL({sym})") + lines.append(f" KAI_ASM_FUNCTION_TYPE({sym})") + lines.append(f"KAI_ASM_FUNCTION_LABEL({sym})") + for word, disasm in functions[sym]: + lines.append(f" KAI_ASM_INST(0x{word}) // {disasm}") + lines.append(f" KAI_ASM_FUNCTION_END({sym})") + lines.append("") + lines.append(" KAI_ASM_END") + lines.append("") + with open(out_path, "w") as f: + f.write("\n".join(lines)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--src", required=True, action="append", + help="intrinsics TU; repeatable, objects are scanned in order") + ap.add_argument("--out", required=True) + ap.add_argument("--march", required=True) + ap.add_argument("--opt", default="O3", help="optimization level (O2/O3)") + ap.add_argument("--include", action="append", default=[], + help="extra -I directory; repeatable") + ap.add_argument("--define", action="append", default=[]) + ap.add_argument("--symbols", required=True) + ap.add_argument("--module", required=True) + ap.add_argument("--cxx", default="g++") + args = ap.parse_args() + + symbols = args.symbols.split(",") + src_dir = os.path.dirname(os.path.abspath(args.src[0])) + + functions = {} + with tempfile.TemporaryDirectory() as tmp: + for i, src in enumerate(args.src): + obj = os.path.join(tmp, f"frozen{i}.o") + cmd = [args.cxx, "-std=c++17", f"-{args.opt}", f"-march={args.march}", + "-fno-stack-protector", "-ffunction-sections", + f"-I{os.path.dirname(os.path.abspath(src))}"] + cmd += [f"-I{d}" for d in args.include] + cmd += [f"-D{d}" for d in args.define] + cmd += ["-c", "-o", obj, src] + run(cmd) + wanted = [s for s in symbols if s not in functions] + check_relocations(obj, wanted) + for name, body in extract_functions(obj, wanted, allow_missing=True).items(): + functions[name] = body + missing = [s for s in symbols if s not in functions] + if missing: + sys.exit(f"symbols not found in any object: {missing}") + emit(functions, symbols, args.module, ", ".join(args.src), args.out) + + total = sum(len(v) for v in functions.values()) + print(f"wrote {args.out}: {len(symbols)} functions, {total} instructions") + + +if __name__ == "__main__": + main() diff --git a/onnxruntime/core/mlas/lib/sve/mlas_sve_fp16.h b/onnxruntime/core/mlas/lib/sve/mlas_sve_fp16.h deleted file mode 100644 index cde88c5517873..0000000000000 --- a/onnxruntime/core/mlas/lib/sve/mlas_sve_fp16.h +++ /dev/null @@ -1,181 +0,0 @@ -/*++ - -Copyright 2025 FUJITSU LIMITED -Copyright (c) Microsoft Corporation. All rights reserved. - -Licensed under the MIT License. - -Module Name: - - mlas_sve_fp16.h - -Abstract: - - This module contains the procedure prototypes for the SVE FP16 intrinsics. - ---*/ - -#pragma once -#include "mlasi_sve.h" - -#if defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC) && defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveBroadcastfloat16(__fp16 Value) -{ - return svdup_f16(Value); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveMinfloat16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 range) -{ - return svmin_f16_m(pg, x, range); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveMaxfloat16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 range) -{ - return svmax_f16_m(pg, x, range); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveMulfloat16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svmul_f16_m(pg, x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveMLAfloat16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y, MLAS_SVFLOAT16 z) -{ - return svmla_f16_m(pg, x, y, z); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveDivfloat16(MLAS_SVBOOL pg, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svdiv_f16_m(pg, x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVBOOL -MlasSveSelPredicatefloat16(size_t x, size_t y) -{ - return svwhilelt_b16(x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSvereinterpretf16_u16(MLAS_SVUINT16 x) -{ - return svreinterpret_f16_u16(x); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT16 -MlasSveLoadUint16(MLAS_SVBOOL pg, const uint16_t* x) -{ - return svld1_u16(pg, x); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveLoadFloat16(MLAS_SVBOOL pg, const MLAS_FP16* x) -{ - return svld1_f16(pg, reinterpret_cast(x)); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreUint16(MLAS_SVBOOL pg, uint16_t* Buffer, MLAS_SVUINT16 Vector) -{ - return svst1_u16(pg, Buffer, Vector); -} -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreF16(MLAS_SVBOOL pg, MLAS_FP16* Buffer, MLAS_SVFLOAT16 Vector) -{ - return svst1_f16(pg, reinterpret_cast<__fp16*>(Buffer), Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT16 -MlasSvereinterpretu16_f16(MLAS_SVFLOAT16 x) -{ - return svreinterpret_u16_f16(x); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveReciprocalfloat16(MLAS_SVFLOAT16 x) -{ - return svrecpe_f16(x); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveReciprocalStepfloat16(MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svrecps_f16(x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveSelectfloat16(MLAS_SVBOOL Pred, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svsel_f16(Pred, x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveSubtractfloat16(MLAS_SVBOOL Pred, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svsub_f16_m(Pred, x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVBOOL -MlasSveComparelessthanfloat16(MLAS_SVBOOL Pred, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svcmplt_f16(Pred, x, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveAbsolutefloat16(MLAS_SVFLOAT16 inactive, MLAS_SVBOOL Pred, MLAS_SVFLOAT16 y) -{ - return svabs_f16_m(inactive, Pred, y); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT16 -MlasSveAddfloat16(MLAS_SVBOOL Pred, MLAS_SVFLOAT16 x, MLAS_SVFLOAT16 y) -{ - return svadd_f16_m(Pred, x, y); -} -#endif diff --git a/onnxruntime/core/mlas/lib/sve/mlasi_sve.h b/onnxruntime/core/mlas/lib/sve/mlasi_sve.h index 3660bc108165c..22336928b311d 100644 --- a/onnxruntime/core/mlas/lib/sve/mlasi_sve.h +++ b/onnxruntime/core/mlas/lib/sve/mlasi_sve.h @@ -1,6 +1,9 @@ /*++ Copyright 2025 FUJITSU LIMITED +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. Module Name: @@ -8,66 +11,177 @@ Module Name: Abstract: - This module contains the procedure prototypes for the SVE intrinsics. + This module contains the procedure prototypes for the SVE elementwise + kernels. + + This header is deliberately free of SVE types and intrinsics so that it + can be included from translation units that are not compiled with SVE + support (platform.cpp today, and eventually MSVC, which has no SVE + intrinsics at all). The kernel implementations get their SVE code + generation from their per-file -march compile flags. --*/ #pragma once #include "../mlasi.h" -#include // SVE intrinsic header - -#ifndef __clang__ -#pragma GCC push_options -#pragma GCC target("arch=armv8.2-a+sve") -#endif - -// Use Clang-specific per-function attribute -#ifdef __clang__ -#define MLAS_SVE_TARGET __attribute__((target("arch=armv8.2-a+sve"))) -#else -#define MLAS_SVE_TARGET -#endif - -typedef svfloat32_t MLAS_SVFLOAT32; -typedef svint32_t MLAS_SVINT32; -typedef svuint32_t MLAS_SVUINT32; -typedef svbool_t MLAS_SVBOOL; -typedef svfloat16_t MLAS_SVFLOAT16; -typedef svuint16_t MLAS_SVUINT16; + +// +// Layouts of the shared constant tables defined in erf.cpp, logistic.cpp and +// compute.cpp ("bundles the constants for use by kernels written in +// assembly"). The tables have C linkage precisely so other implementations of +// the same kernels can reference them; the SVE kernels bind to the same +// symbols instead of maintaining duplicate copies. +// + +struct MLAS_ERF_CONSTANTS { + float ErfUpperAbsRange; + float ErfSplitBoundary; + float ErfSMALL_P0; + float ErfSMALL_P1; + float ErfSMALL_P2; + float ErfSMALL_P3; + float ErfSMALL_P4; + float ErfSMALL_P5_Minus_One; + float ErfReserved0; + float ErfBIG_P0; + float ErfBIG_P1; + float ErfBIG_P2; + float ErfBIG_P3; + float ErfBIG_P4; + float ErfBIG_P5; + float ErfBIG_P6_Minus_One; + float ErfNegZero; + float ErfOne; + + float Exp_UpperRange; + float Exp_LowerRange; + float Exp_Log2Reciprocal; + float Exp_log2_hi; + float Exp_log2_lo; + float Exp_P0; + float Exp_P1; + float Exp_P2; + float Exp_P3; + float Exp_P4; + float Exp_P5; + float Exp_P6; + float Exp_C; + int32_t Exp_X7F; +}; + +struct MLAS_LOGISTIC_CONSTANTS { + float LowerRange; + float UpperRange; + float alpha_9; + float alpha_7; + float alpha_5; + float alpha_3; + float alpha_1; + float beta_10; + float beta_8; + float beta_6; + float beta_4; + float beta_2; + float beta_0; + float one_half; +}; + +struct MLAS_EXP_CONSTANTS { + float LowerRange; + float UpperRange; + float LowerRangeSumExp; + float UpperRangeSumExp; + float RoundingBias; + float Log2Reciprocal; + float Log2High; + float Log2Low; + float poly_0; + float poly_1; + float poly_2; + float poly_3; + float poly_4; + float poly_56; + int32_t MinimumExponent; + int32_t MaximumExponent; +}; + +struct MLAS_TANH_CONSTANTS { + float LowerRange; + float UpperRange; + float alpha_13; + float alpha_11; + float alpha_9; + float alpha_7; + float alpha_5; + float alpha_3; + float alpha_1; + float beta_6; + float beta_4; + float beta_2; + float beta_0; +}; + +extern "C" const MLAS_ERF_CONSTANTS MlasErfConstants; +extern "C" const MLAS_LOGISTIC_CONSTANTS MlasLogisticConstants; +extern "C" const MLAS_TANH_CONSTANTS MlasTanhConstants; +extern "C" const MLAS_EXP_CONSTANTS MlasExpConstants; +extern "C" const float MlasMinimumF32Value; + +// +// Constants of the ARM Compute Library exp() approximation used by the SVE +// Exp kernel, as float bit patterns. Defined in elementwise_sve_dispatch.cpp +// and passed to the kernel by pointer so its code stays free of data +// relocations. +// + +struct MLAS_SVE_EXP_CONSTANTS { + uint32_t C1; + uint32_t C2; + uint32_t C3; + uint32_t C4; + uint32_t C5; + uint32_t Shift; + uint32_t InvLn2; + uint32_t NegLn2Hi; + uint32_t NegLn2Lo; + uint32_t Inf; + uint32_t MaxInput; + uint32_t MinInput; + uint32_t FexpaShift; + uint32_t OneHalf; + uint32_t FexpaLowerRange; +}; + +extern "C" const MLAS_SVE_EXP_CONSTANTS MlasSveExpConstants; + +// +// Float32 elementwise kernels (elementwise_sve.cpp). +// void MLASCALL -MlasSveErfFP16Kernel( - const MLAS_FP16* Input, - MLAS_FP16* Output, +MlasSveErfKernel( + const float* Input, + float* Output, size_t N -); + ); -void -MLASCALL -MlasSveTanhFP16Kernel( - const MLAS_FP16* Input, - MLAS_FP16* Output, +void +MLASCALL +MlasSveLogisticKernel( + const float* Input, + float* Output, size_t N -); + ); -void -MLASCALL -MlasSveGeluFP16Kernel( - const MLAS_FP16* Input, - MLAS_FP16* Output, - MLAS_FP16* Temp, - size_t N, - MLAS_GELU_ALGORITHM Algo -); -// function declarations -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveComputeExpVector( - MLAS_SVBOOL Pred, - MLAS_SVFLOAT32 Vector -); +void +MLASCALL +MlasSveTanhKernel( + const float* Input, + float* Output, + size_t N + ); void MLASCALL @@ -75,15 +189,7 @@ MlasSveComputeExpF32Kernel( const float* Input, float* Output, size_t N -); - -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveComputeSumExpVector( - MLAS_SVBOOL Pred, - MLAS_SVFLOAT32 Vector, - MLAS_SVFLOAT32 NegativeMaximumVector -); + ); float MLASCALL @@ -92,13 +198,14 @@ MlasSveComputeSumExpF32Kernel( float* Output, size_t N, const float* NegativeMaximum -); + ); -float MLASCALL +float +MLASCALL MlasSveReduceMaximumF32Kernel( const float* Input, size_t N -); + ); void MLASCALL @@ -107,7 +214,7 @@ MlasSveReduceMinimumMaximumF32Kernel( float* Min, float* Max, size_t N -); + ); void MLASCALL @@ -115,7 +222,7 @@ MlasSveComputeSoftmaxOutputF32Kernel( float* Output, size_t N, const float* Parameters -); + ); void MLASCALL @@ -124,557 +231,34 @@ MlasSveComputeLogSoftmaxOutputF32Kernel( float* Output, size_t N, const float* Parameters -); + ); + +// +// Float16 elementwise kernels (elementwise_sve_fp16.cpp). +// void MLASCALL -MlasSveErfKernel( - const float* Input, - float* Output, +MlasSveErfFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, size_t N -); + ); -void +void MLASCALL -MlasSveLogisticKernel( - const float* Input, - float* Output, +MlasSveTanhFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, size_t N -); - -//MLAS API for SVE intrinsics - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveReinterpretAsInt32(MLAS_SVFLOAT32 Vector) -{ - return svreinterpret_s32_f32(Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT32 -MlasSveReinterpretAsUInt32(MLAS_SVFLOAT32 Vector) -{ - return svreinterpret_u32_f32(Vector); -} - -// Reinterprets an unsigned 32-bit vector as a 32-bit floating-point vector. -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveReinterpretAsFLOAT32(MLAS_SVUINT32 Vector) -{ - return svreinterpret_f32_u32(Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveCastToInt32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - return svcvt_s32_f32_z(Pred, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveCastToFloat32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector) -{ - return svcvt_f32_s32_z(Pred, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveBroadcastInt32(int32_t Value) -{ - return svdup_n_s32(Value); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveLoadInt32(MLAS_SVBOOL Pred, const int32_t* Buffer) -{ - return svld1_s32(Pred, Buffer); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreInt32(MLAS_SVBOOL Pred, int32_t* Buffer, MLAS_SVINT32 Vector) -{ - svst1_s32(Pred, Buffer, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveAddInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svadd_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveSubtractInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svsub_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveAndInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svand_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT32 -MlasSveAndUInt32(MLAS_SVBOOL Pred, MLAS_SVUINT32 Vector1, MLAS_SVUINT32 Vector2) -{ - return svand_u32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveOrInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svorr_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveAndNotInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 VectorNot, MLAS_SVINT32 Vector) -{ - return svand_s32_m(Pred, svnot_s32_z(Pred, VectorNot), Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveXorInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return sveor_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveBlendInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2, MLAS_SVINT32 Selection) -{ - return MlasSveOrInt32( - Pred, - MlasSveAndInt32(Pred, Vector2, Selection), - MlasSveAndNotInt32(Pred, Selection, Vector1) ); -} - -template -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT32 -MlasSveShiftLeftUInt32(MLAS_SVBOOL Pred, MLAS_SVUINT32 Vector) -{ - return svlsl_n_u32_z(Pred, Vector, ShiftCount); -} - -template -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveShiftLeftInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector) -{ - return svlsl_n_s32_z(Pred, Vector, ShiftCount); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT32 -MlasSveShiftRightInt32(MLAS_SVBOOL Pred, MLAS_SVUINT32 Vector, uint ShiftCount) -{ - return svlsr_n_u32_m(Pred, Vector, ShiftCount); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveMaximumInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svmax_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVINT32 -MlasSveMinimumInt32(MLAS_SVBOOL Pred, MLAS_SVINT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svmin_s32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveReinterpretAsFloat32(MLAS_SVINT32 Vector) -{ - return svreinterpret_f32_s32(Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveBroadcastFloat32(float Value) -{ - return svdup_n_f32(Value); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVUINT32 -MlasSveBroadcastUINT32(uint Value) -{ - return svdup_n_u32(Value); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveBroadcastFloat32(const float* Value) -{ - return svld1_f32(svptrue_b32(), Value); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveZeroFloat32(void) -{ - return svdup_n_f32(0.0f); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveLoadFloat32(MLAS_SVBOOL Pred, const float* Buffer) -{ - return svld1_f32(Pred, Buffer); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreFloat32(MLAS_SVBOOL Pred, float* Buffer, MLAS_SVFLOAT32 Vector) -{ - svst1_f32(Pred, Buffer, Vector); -} - -template -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreLaneFloat32(float* Buffer, MLAS_SVFLOAT32 Vector) -{ - svbool_t Pred = svwhilelt_b32(Lane, Lane + 1); - svst1_f32(Pred, Buffer, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -void -MlasSveStoreLowHalfFloat32(float* Buffer, MLAS_SVFLOAT32 Vector) -{ - svbool_t Pred = svwhilelt_b32(0, (int32_t)svcntw() / 2); - svst1_f32(Pred, Buffer, Vector); -} - -template -MLAS_SVE_TARGET -MLAS_FORCEINLINE -float -MlasSveExtractLaneFloat32(MLAS_SVFLOAT32 Vector) -{ - float TmpBuffer[1]; - svbool_t Pred = svwhilelt_b32(Lane, Lane + 1); - svst1_f32(Pred, TmpBuffer, Vector); - return TmpBuffer[0]; -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveInterleaveLowFloat32(MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svzip1_f32(Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveInterleaveHighFloat32(MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svzip2_f32(Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveAddFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svadd_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveSubtractFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svsub_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMultiplyFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svmul_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveExpFloat32(MLAS_SVUINT32 Vector) -{ - return svexpa_f32(Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveScaleFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVINT32 Vector2) -{ - return svscale_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveRoundINTFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - return svrintm_f32_z(Pred, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMultiplyAddFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2, MLAS_SVFLOAT32 Vector3) -{ - return svmla_f32_m(Pred, Vector3, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMultiplyAddFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, float Scalar2, MLAS_SVFLOAT32 Vector3) -{ - return MlasSveMultiplyAddFloat32(Pred, Vector1, MlasSveBroadcastFloat32(Scalar2), Vector3); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMultiplyAddFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2, float Scalar3) -{ - return MlasSveMultiplyAddFloat32(Pred, Vector1, Vector2, MlasSveBroadcastFloat32(Scalar3)); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveDivideFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svdiv_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveGreaterThanFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - // Compare Vector1 and Vector2, return a predicate vector - svbool_t cmp_mask = svcmpgt_f32(Pred, Vector1, Vector2); - - //Convert predicate to uint32_t mask - svuint32_t mask_bits = svdup_u32_z(cmp_mask, 0xFFFFFFFF); - - //Reinterpret to float32 - return svreinterpret_f32_u32(mask_bits); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveAndFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return MlasSveReinterpretAsFloat32( - MlasSveAndInt32( - Pred, - MlasSveReinterpretAsInt32(Vector1), - MlasSveReinterpretAsInt32(Vector2) - ) - ); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveOrFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return MlasSveReinterpretAsFloat32( - MlasSveOrInt32( - Pred, - MlasSveReinterpretAsInt32(Vector1), - MlasSveReinterpretAsInt32(Vector2) - ) - ); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveAndNotFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return MlasSveReinterpretAsFloat32( - MlasSveAndNotInt32( - Pred, - MlasSveReinterpretAsInt32(Vector1), - MlasSveReinterpretAsInt32(Vector2) - ) - ); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveXorFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return MlasSveReinterpretAsFloat32( - MlasSveXorInt32( - Pred, - MlasSveReinterpretAsInt32(Vector1), - MlasSveReinterpretAsInt32(Vector2) - ) - ); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveBlendFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2, MLAS_SVFLOAT32 Selection) -{ - return MlasSveOrFloat32( - Pred, - MlasSveAndFloat32(Pred, Vector2, Selection), - MlasSveAndFloat32(Pred, Vector1, Selection) - ); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMaximumFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svmax_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveMinimumFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector1, MLAS_SVFLOAT32 Vector2) -{ - return svmin_f32_m(Pred, Vector1, Vector2); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveClampFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Value, float LowerRange, float UpperRange) -{ - Value = MlasSveMaximumFloat32(Pred, MlasSveBroadcastFloat32(LowerRange), Value); - Value = MlasSveMinimumFloat32(Pred, MlasSveBroadcastFloat32(UpperRange), Value); - return Value; -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -float -MlasSveReduceAddFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - return svaddv_f32(Pred, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -float -MlasSveReduceMaximumFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - return svmaxv_f32(Pred, Vector); -} -MLAS_SVE_TARGET -MLAS_FORCEINLINE -float -MlasSveReduceMinimumFloat32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - return svminv_f32(Pred, Vector); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSvePowerOf2Float32(MLAS_SVBOOL Pred, MLAS_SVFLOAT32 Vector) -{ - MLAS_SVINT32 emm0 = MlasSveAddInt32( - Pred, - MlasSveCastToInt32(Pred, Vector), - MlasSveBroadcastInt32(127) +void +MLASCALL +MlasSveGeluFP16Kernel( + const MLAS_FP16* Input, + MLAS_FP16* Output, + MLAS_FP16* Temp, + size_t N, + MLAS_GELU_ALGORITHM Algo ); - return MlasSveReinterpretAsFloat32(MlasSveShiftLeftInt32<23>(Pred, emm0)); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVFLOAT32 -MlasSveSelect(svbool_t Pred, MLAS_SVFLOAT32 TrueValue, MLAS_SVFLOAT32 FalseValue) -{ - return svsel_f32(Pred, TrueValue, FalseValue); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVBOOL -MlasSveCompareLessThan(svbool_t Pred, MLAS_SVFLOAT32 A, MLAS_SVFLOAT32 B) -{ - return svcmplt_f32(Pred, A, B); -} - -MLAS_SVE_TARGET -MLAS_FORCEINLINE -MLAS_SVBOOL -MlasSveCompareGreaterThan(svbool_t Pred, MLAS_SVFLOAT32 A, MLAS_SVFLOAT32 B) -{ - return svcmpgt_f32(Pred, A, B); -} - -// GCC: Pop options after SVE-specific functions -#ifndef __clang__ -#pragma GCC pop_options -#endif - - diff --git a/onnxruntime/core/mlas/lib/tanh.cpp b/onnxruntime/core/mlas/lib/tanh.cpp index 2b99b416deb63..691e54653ab03 100644 --- a/onnxruntime/core/mlas/lib/tanh.cpp +++ b/onnxruntime/core/mlas/lib/tanh.cpp @@ -178,7 +178,7 @@ Return Value: --*/ { -#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64) +#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_RISCV64) || defined(MLAS_USE_SVE) GetMlasPlatform().TanhKernelRoutine(Input, Output, N); #else MlasTanhKernel(Input, Output, N);