diff --git a/.github/workflows/_runner-gap9-w-ne16-tiled.yml b/.github/workflows/_runner-gap9-w-ne16-tiled.yml new file mode 100644 index 0000000000..fdffe33618 --- /dev/null +++ b/.github/workflows/_runner-gap9-w-ne16-tiled.yml @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: _runner-gap9-w-ne16-tiled + +"on": + workflow_call: + inputs: + runner: + required: true + type: string + docker-image: + required: true + type: string + pytest-markers: + required: true + type: string + +jobs: + test-runner-gap9-w-ne16-tiled: + runs-on: ${{ inputs.runner }} + container: + image: ${{ inputs.docker-image }} + steps: + - name: Mark workspace as safe + run: git config --global --add safe.directory '*' + - name: Checkout Repo + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Build Deeploy + shell: bash + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + pip install -e . || true + deactivate + - name: Cache ccache + uses: actions/cache/restore@v4 + with: + path: /app/.ccache + key: ccache-gap9 + - name: Run Test + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + export GVSOC_INSTALL_DIR=/app/install/gap9-sdk/install/workstation + export GAP_RISCV_GCC_TOOLCHAIN=/app/install/gcc/gap9 + cd DeeployTest + mkdir -p /app/.ccache + export CCACHE_DIR=/app/.ccache + pytest test_platforms.py -v -m "${{ inputs.pytest-markers }}" + deactivate + shell: bash + - name: NE16 Profiling (cycle counts) + if: always() + run: | + source /app/install/gap9-sdk/.gap9-venv/bin/activate + source /app/install/gap9-sdk/configs/gap9_evk_audio.sh || true + export GVSOC_INSTALL_DIR=/app/install/gap9-sdk/install/workstation + export GAP_RISCV_GCC_TOOLCHAIN=/app/install/gcc/gap9 + mkdir -p /app/.ccache + export CCACHE_DIR=/app/.ccache + cd DeeployTest + rm -rf TEST_GAP9_W_NE16/build_master + for test in \ + "Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench --l1 32000" \ + "Tests/Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ --l1 32000" \ + "Tests/Kernels/Integer/Conv/Dense_2D_RQ --l1 32000"; do + dir=$(echo $test | awk '{print $1}') + l1=$(echo $test | awk '{print $3}') + echo "========================================" + echo "PROFILING: $dir (L1=$l1)" + echo "========================================" + python3 deeployRunner_tiled_gap9_w_ne16.py \ + -t "$dir" --l1 "$l1" \ + --toolchain GCC --toolchain-install-dir /app/install/gcc/gap9 \ + --cores 8 --enable-3x3 --profileTiling -v \ + -D CMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF 2>&1 || true + done + deactivate + shell: bash diff --git a/.github/workflows/_runner-snitch-tiled-sequential.yml b/.github/workflows/_runner-snitch-tiled-sequential.yml index bcdd58a166..2fdd0ec839 100644 --- a/.github/workflows/_runner-snitch-tiled-sequential.yml +++ b/.github/workflows/_runner-snitch-tiled-sequential.yml @@ -33,10 +33,10 @@ jobs: - name: Build Deeploy shell: bash run: pip install -e . - - name: Run Test # VJUNG: Run tests with 4 parallel threads as GitHub action VM has 4 cores. + - name: Run Test # 2-way parallel: 4-way OOMs the GitHub runner on the FP32 GEMM/TransB build. run: | cd DeeployTest mkdir -p /app/.ccache export CCACHE_DIR=/app/.ccache - pytest test_platforms.py -v -n 4 -m "snitch_tiled and ${{ inputs.pytest-marker }}" + pytest test_platforms.py -v -n 2 -m "snitch_tiled and ${{ inputs.pytest-marker }}" shell: bash diff --git a/.github/workflows/ci-platform-gap9-tiled.yml b/.github/workflows/ci-platform-gap9-tiled.yml index 61cab4ea70..721cd5a365 100644 --- a/.github/workflows/ci-platform-gap9-tiled.yml +++ b/.github/workflows/ci-platform-gap9-tiled.yml @@ -25,6 +25,9 @@ concurrency: jobs: select-env: + # ghcr.io/pulp-platform/deeploy-gap9 is private; only upstream's + # self-hosted runners have credentials. Skip cleanly on forks. + if: github.repository == 'pulp-platform/Deeploy' uses: ./.github/workflows/_select-env.yml with: docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} diff --git a/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml b/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml new file mode 100644 index 0000000000..5411e4b930 --- /dev/null +++ b/.github/workflows/ci-platform-gap9-w-ne16-tiled.yml @@ -0,0 +1,68 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +--- +name: CI • GAP9 + NE16 (Tiled) + +"on": + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + workflow_dispatch: + inputs: + docker_image_deeploy: + description: "Deeploy Image to use" + required: false + default: "ghcr.io/pulp-platform/deeploy-gap9:devel" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + select-env: + # The GAP9 + NE16 image is hosted in pulp-platform's private ghcr.io + # registry; only upstream's self-hosted runners have credentials to + # pull it. On forks the docker pull always returns "denied", so skip + # the whole pipeline cleanly there. (Same constraint as the existing + # ci-platform-gap9{,-tiled}.yml jobs.) + if: github.repository == 'pulp-platform/Deeploy' + uses: ./.github/workflows/_select-env.yml + with: + docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} + + gap9-w-ne16-kernels-tiled-singlebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and kernels and singlebuffer and l2" + + gap9-w-ne16-kernels-tiled-doublebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and kernels and doublebuffer and l2" + + gap9-w-ne16-models-tiled-singlebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and models and singlebuffer and l2" + + gap9-w-ne16-models-tiled-doublebuffer-L2: + needs: select-env + uses: ./.github/workflows/_runner-gap9-w-ne16-tiled.yml + with: + runner: ${{ needs.select-env.outputs.runner }} + docker-image: ${{ needs.select-env.outputs.image }} + pytest-markers: "gap9_w_ne16_tiled and models and doublebuffer and l2" diff --git a/.github/workflows/ci-platform-gap9.yml b/.github/workflows/ci-platform-gap9.yml index 014828d6ce..597c0f40ef 100644 --- a/.github/workflows/ci-platform-gap9.yml +++ b/.github/workflows/ci-platform-gap9.yml @@ -26,6 +26,9 @@ concurrency: jobs: select-env: + # ghcr.io/pulp-platform/deeploy-gap9 is private; only upstream's + # self-hosted runners have credentials. Skip cleanly on forks. + if: github.repository == 'pulp-platform/Deeploy' uses: ./.github/workflows/_select-env.yml with: docker_image_deeploy: ${{ github.event.inputs.docker_image_deeploy || 'ghcr.io/pulp-platform/deeploy-gap9:devel' }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 20a85bccf8..1699025e27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,8 +20,8 @@ if(TOOLCHAIN STREQUAL GCC) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) endif() -set(platform MemPool CACHE STRING "Platform (MemPool, SoftHier, QEMU, Siracusa, Siracusa_w_neureka, PULP-Open, GAP9, Generic, Snitch)") -set_property(CACHE platform PROPERTY STRINGS MemPool SoftHier QEMU Siracusa Siracusa_w_neureka PULP-Open GAP9 Generic Snitch) +set(platform MemPool CACHE STRING "Platform (MemPool, SoftHier, QEMU, Siracusa, Siracusa_w_neureka, PULP-Open, GAP9, GAP9_w_NE16, Generic, Snitch)") +set_property(CACHE platform PROPERTY STRINGS MemPool SoftHier QEMU Siracusa Siracusa_w_neureka PULP-Open GAP9 GAP9_w_NE16 Generic Snitch) if(platform STREQUAL MemPool) message(STATUS "Building for platform 'MemPool'") @@ -33,8 +33,8 @@ elseif(platform STREQUAL Siracusa_w_neureka) message(STATUS "Building for platform 'Siracusa_w_neureka'") elseif(platform STREQUAL PULPOpen) message(STATUS "Building for platform 'PULP-Open'") -elseif(platform STREQUAL GAP9) - message(STATUS "Building for platform 'GAP9'") +elseif(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) + message(STATUS "Building for platform '${platform}'") set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) # Select SDK config based on simulator type @@ -64,7 +64,7 @@ endif() # Import useful functions / macros include(${CMAKE_CURRENT_LIST_DIR}/cmake/Util.cmake) # Only if not GAP9 -if(NOT platform STREQUAL GAP9) +if(NOT platform STREQUAL GAP9 AND NOT platform STREQUAL GAP9_w_NE16) include(${CMAKE_CURRENT_LIST_DIR}/cmake/common.cmake) endif() include(${CMAKE_CURRENT_LIST_DIR}/cmake/simulation.cmake) @@ -233,7 +233,7 @@ if(platform STREQUAL Siracusa OR platform STREQUAL Siracusa_w_neureka OR platfor endif() -if(platform STREQUAL GAP9) +if(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) project(${TESTNAME} LANGUAGES C ASM) include(${CMAKE_CURRENT_LIST_DIR}/cmake/gap9/gap9_gvsoc.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cmake/gap9/gap9_board.cmake) diff --git a/Deeploy/Targets/GAP9/Bindings.py b/Deeploy/Targets/GAP9/Bindings.py index 2bda98af8f..ad215b9193 100644 --- a/Deeploy/Targets/GAP9/Bindings.py +++ b/Deeploy/Targets/GAP9/Bindings.py @@ -18,11 +18,12 @@ from Deeploy.DeeployTypes import CodeTransformation, NodeBinding from Deeploy.FutureExtension.Bindings.AutoFutureBinding import AutoFutureBinding from Deeploy.FutureExtension.CodeTransformationPasses.FutureCodeTransformation import FutureGeneration -from Deeploy.Targets.GAP9.DMA.L3Dma import gap9L3DmaHack +from Deeploy.Targets.GAP9.DMA.L3Dma import GAP9L3Dma from Deeploy.Targets.GAP9.DMA.MchanDma import GAP9MchanDma +from Deeploy.Targets.GAP9.Templates import GAP9SDKDequantQuantTemplate, NE16GEMMTemplate # Import templates from PULPOpen and Generic from Deeploy.Targets.Generic.Templates import AddTemplate, ConcatTemplate, DequantTemplate, FloatReduceMeanTemplate, \ - FloatReduceSumTemplate, GatherTemplate, QuantTemplate, RQSiGELUTemplate, SliceTemplate, iHardswishTemplate + FloatReduceSumTemplate, GatherTemplate, RQSiGELUTemplate, SliceTemplate, iHardswishTemplate from Deeploy.Targets.Generic.TypeCheckers import AddChecker, ConcatChecker, ConvChecker, DequantChecker, \ GatherChecker, GELUChecker, GEMMChecker, HardswishChecker, LayerNormChecker, MatMulChecker, MulChecker, \ QuantChecker, ReduceMeanChecker, ReluChecker, ReshapeChecker, RQAddChecker, RQHardswishChecker, SGDChecker, \ @@ -57,7 +58,7 @@ MemoryManagementGeneration("L1"), TilingVariableReplacement("L2"), MemoryAwareFunctionCallClosure(writeback = False, generateStruct = True), - PULPL3Tiling("L3", "L2", gap9L3DmaHack), # Use GAP9-specific L3 DMA + PULPL3Tiling("L3", "L2", GAP9L3Dma()), # Use GAP9-specific L3 DMA PULPProfileUntiled(), ArgumentStructGeneration(), L3MemoryAwareFunctionCallClosure(writeback = False), @@ -76,7 +77,7 @@ MemoryManagementGeneration("L1"), TilingVariableReplacement("L2"), MemoryAwareFunctionCallClosure(writeback = False, generateStruct = True), - PULPL3Tiling("L3", "L2", gap9L3DmaHack), # Use GAP9-specific L3 DMA + PULPL3Tiling("L3", "L2", GAP9L3Dma()), # Use GAP9-specific L3 DMA PULPProfileUntiled(), ArgumentStructGeneration(), L3MemoryAwareFunctionCallClosure(writeback = False), @@ -183,6 +184,26 @@ GAP9Transformer) for type1, type2 in zip([int8_t, uint8_t, int8_t, uint8_t], [int8_t, uint8_t, uint8_t, int8_t]) ] +GAP9NE16RQSGEMMBindings = [ + NodeBinding( + PULPLinearChecker([ + PointerClass(type1), + PointerClass(int8_t), + PointerClass(int32_t), + PointerClass(uint8_t), + PointerClass(uint8_t) + ], [PointerClass(type2)]), NE16GEMMTemplate.referenceTemplate, GAP9ClusterTransformer) + for type1 in [int8_t, uint8_t] + for type2 in [int8_t, uint8_t] +] + +GAP9NE16GEMMInt32Bindings = [ + NodeBinding( + GEMMChecker([PointerClass(type1), PointerClass(int8_t), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16GEMMTemplate.int32OutputTemplate, + GAP9ClusterTransformer) for type1 in [int8_t, uint8_t] +] + GAP9FloatGEMMBindings = [ NodeBinding( GEMMChecker([PointerClass(float32_t), PointerClass(float32_t), @@ -386,14 +407,17 @@ ] GAP9QuantBindings = [ - NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, - GAP9Transformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), + GAP9SDKDequantQuantTemplate.fp32QuantI8Template, GAP9Transformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), + GAP9SDKDequantQuantTemplate.fp32QuantU8Template, GAP9Transformer), ] GAP9DequantBindings = [ - NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, - GAP9Transformer), -] + [ + NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), + GAP9SDKDequantQuantTemplate.fp32DequantI8Template, GAP9Transformer), + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), + GAP9SDKDequantQuantTemplate.fp32DequantU8Template, GAP9Transformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, GAP9Transformer), ] diff --git a/Deeploy/Targets/GAP9/DMA/L3Dma.py b/Deeploy/Targets/GAP9/DMA/L3Dma.py index adbf161328..aadc5974b9 100644 --- a/Deeploy/Targets/GAP9/DMA/L3Dma.py +++ b/Deeploy/Targets/GAP9/DMA/L3Dma.py @@ -6,8 +6,7 @@ from typing import Dict, Tuple from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation, VariableBuffer -from Deeploy.TilingExtension.AsyncDma import AsyncDma, BlockingDmaFromAsyncDmaAdapter, DmaDirection, Future, \ - PerTensorWaitingStrategy +from Deeploy.TilingExtension.AsyncDma import AsyncDma, DmaDirection, Future, PerTensorWaitingStrategy class GAP9L3DmaFuture(Future): @@ -29,7 +28,7 @@ class GAP9L3Dma(AsyncDma): _transferTemplates = { 2: NodeTemplate( - "pi_cl_ram_copy_2d(get_ram_ptr(), ${ext}, ${loc}, ${transfer_size}, ${stride}, ${length}, ${ext2loc}, &${future});" + "pi_cl_ram_copy_2d(get_ram_ptr(), (uint32_t)${ext}, (void *)${loc}, (uint32_t)${transfer_size}, (uint32_t)${stride}, (uint32_t)${length}, ${ext2loc}, &${future});" ) } _waitingStrategy = PerTensorWaitingStrategy(GAP9L3DmaFuture) @@ -58,7 +57,3 @@ def transferOpRepr(self, externalBuffer: VariableBuffer, localBuffer: VariableBu "stride": strideExt[0], }) return operatorRepresentation - - -# Blocking adapter for L3 DMA (used in GAP9 L3 tiling) -gap9L3DmaHack = BlockingDmaFromAsyncDmaAdapter(GAP9L3Dma()) diff --git a/Deeploy/Targets/GAP9/Parsers.py b/Deeploy/Targets/GAP9/Parsers.py new file mode 100644 index 0000000000..4d730b7cae --- /dev/null +++ b/Deeploy/Targets/GAP9/Parsers.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.Parsers import GEMMParser, RQSParserInterface + + +class NE16GEMMParser(GEMMParser, RQSParserInterface): + """Parser for NE16 RequantizedGemm nodes with 5 inputs [A, B, C, mul, scale_n].""" + + def __init__(self): + super().__init__(noBiasHoisting = True) + + def parseNode(self, node: gs.Node) -> bool: + ret_rqs = RQSParserInterface.parseNode(self, node) + ret_matmul = GEMMParser.parseNode(self, node) + ret = all([ret_rqs, ret_matmul, 'shift' in node.attrs, len(node.inputs) == 5]) + if ret: + self.operatorRepresentation['shift'] = int(node.attrs['shift'].values) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = GEMMParser.parseNodeCtxt(self, ctxt, node, channels_first) + if ret: + inputs = ['A', 'B', 'C', 'mul', 'scale_n'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = newCtxt.lookup(inputNode.name).name + return newCtxt, True + return ctxt, False diff --git a/Deeploy/Targets/GAP9/Platform.py b/Deeploy/Targets/GAP9/Platform.py index bad6f8d859..8234d23a16 100644 --- a/Deeploy/Targets/GAP9/Platform.py +++ b/Deeploy/Targets/GAP9/Platform.py @@ -5,24 +5,29 @@ import numpy as np import onnx_graphsurgeon as gs +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RemoveEmptyConvBiasPass, RemoveOnlySingletonReduceMeanPass from Deeploy.DeeployTypes import ConstantBuffer, DeploymentEngine, DeploymentPlatform, NetworkContext, NodeMapper, \ - NodeTemplate, StructBuffer, TransientBuffer, VariableBuffer + NodeTemplate, StructBuffer, TopologyOptimizer, TransientBuffer, VariableBuffer from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel from Deeploy.MemoryLevelExtension.NetworkDeployers.MemoryLevelDeployer import MemoryPlatform, MemoryPlatformWrapper +from Deeploy.Targets.GAP9.Parsers import NE16GEMMParser from Deeploy.Targets.GAP9.Templates import AllocateTemplate, FreeTemplate # Import GAP9-specific tiler bindings -from Deeploy.Targets.GAP9.Tiler import GAP9AddTilingReadyBindings, GAP9ConcatTilingReadyBindings, \ - GAP9Conv2DTilingReadyBindings, GAP9DWConv2DTilingReadyBindings, GAP9FlattenTilingReadyBindings, \ - GAP9FPGELUTilingReadyBindings, GAP9FPGEMMTilingReadyBindings, GAP9GatherTilingReadyBindings, \ - GAP9iHardswishTilingReadyBindings, GAP9iRMSNormTilingReadyBindings, GAP9iRQSGELUTilingReadyBindings, \ - GAP9LayernormTilingReadyBindings, GAP9MatMulTilingReadyBindings, GAP9MaxPool2DTilingReadyBindings, \ - GAP9MulTilingReadyBindings, GAP9ReduceSumTilingReadyBindings, GAP9ReluTilingReadyBindings, \ +from Deeploy.Targets.GAP9.Tiler import DeQuantTilingReadyBindings, GAP9AddTilingReadyBindings, \ + GAP9ConcatTilingReadyBindings, GAP9Conv2DTilingReadyBindings, GAP9DWConv2DTilingReadyBindings, \ + GAP9FlattenTilingReadyBindings, GAP9FPGELUTilingReadyBindings, GAP9FPGEMMTilingReadyBindings, \ + GAP9GatherTilingReadyBindings, GAP9iHardswishTilingReadyBindings, GAP9iRMSNormTilingReadyBindings, \ + GAP9iRQSGELUTilingReadyBindings, GAP9LayernormTilingReadyBindings, GAP9MatMulTilingReadyBindings, \ + GAP9MaxPool2DTilingReadyBindings, GAP9MulTilingReadyBindings, GAP9NE16GEMMInt32TilingReadyBindings, \ + GAP9NE16RQSGEMMTilingReadyBindings, GAP9ReduceSumTilingReadyBindings, GAP9ReluTilingReadyBindings, \ GAP9RQAddTilingReadyBindings, GAP9RQSConv2DTilingReadyBindings, GAP9RQSDWConv2DTilingReadyBindings, \ GAP9RQSGEMMTilingReadyBindings, GAP9RQSiHardswishTilingReadyBindings, GAP9RQSMatrixVecTilingReadyBindings, \ GAP9RQSTallGEMMTilingReadyBindings, GAP9RQSTilingReadyBindings, GAP9SGDTilingReadyBindings, \ GAP9SoftmaxCrossEntropyGradTilingReadyBindings, GAP9SoftmaxCrossEntropyTilingReadyBindings, \ GAP9SoftmaxGradTilingReadyBindings, GAP9SoftmaxTilingReadyBindings, GAP9TransposeTilingReadyBindings, \ - GAP9UniformRQSTilingReadyBindings + GAP9UniformRQSTilingReadyBindings, QuantTilingReadyBindings +from Deeploy.Targets.GAP9.TopologyOptimizationPasses.Passes import NE16AdjustGEMMWeightLayoutPass from Deeploy.Targets.Generic.Bindings import BasicGEMMBindings, BasicPad1DBindings, BasicPad2DBindings, \ BasicRQIntegerDivBinding from Deeploy.Targets.Generic.Layers import AddLayer, ConcatLayer, ConvLayer, GatherLayer, GELULayer, GEMMLayer, \ @@ -37,12 +42,17 @@ SoftmaxCrossEntropyLossGradParser, SoftmaxCrossEntropyLossParser, SoftmaxGradParser, SoftmaxParser, \ TransposeParser, UniformRequantShiftParser, UnsqueezeParser, iHardswishParser, iRMSNormParser, iSoftmaxParser from Deeploy.Targets.Generic.Templates import AllocateTemplate as BasicAllocateTemplate -from Deeploy.Targets.PULPOpen.Bindings import BasicDequantBindings, BasicQuantBindings, PULPDMASliceBindings, \ - PULPDWConv1DBinding, PULPReduceMeanBindings, PULPRQSConv1DBindings, PULPSliceBindings +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import DequantPatternPass, DequantQuantMergePass, \ + IntegerDivRequantMergePass, MergeConstAddAndRequantPass, MergeTrueIntegerDivRequantShiftPass, QuantPatternPass, \ + RQSSplitPass, SkipEmptyConcatPass, SkipUnityRequantPass, iGELURequantMergePass, iHardswishRequantMergePass +from Deeploy.Targets.PULPOpen.Bindings import PULPDMASliceBindings, PULPDWConv1DBinding, PULPReduceMeanBindings, \ + PULPRQSConv1DBindings, PULPSliceBindings from Deeploy.Targets.PULPOpen.Layers import PULPRQSConvLayer, PULPRQSGEMMLayer from Deeploy.Targets.PULPOpen.Parsers import PULPConv1DParser, PULPConv2DParser, PULPDWConv1DParser, \ PULPDWConv2DParser, PULPFPConv2DParser, PULPFPDWConv2DParser, PULPGEMMParser, PULPMatrixVecParser, \ PULPTallGEMMParser +from Deeploy.Targets.PULPOpen.TopologyOptimizationPasses.Passes import PULPAddRequantMergePass, \ + PULPConvRequantMergePass, PULPGEMMRequantMergePass, PULPMatMulRequantMergePass # Create GAP9-specific NodeMappers GAP9_RQAddMapper = NodeMapper(RQAddParser(), GAP9RQAddTilingReadyBindings) @@ -90,9 +100,40 @@ GAP9_SoftmaxCrossEntropyLossGradMapper = NodeMapper(SoftmaxCrossEntropyLossGradParser(), GAP9SoftmaxCrossEntropyGradTilingReadyBindings) GAP9_SGDMapper = NodeMapper(SGDParser(), GAP9SGDTilingReadyBindings) -GAP9_QuantMapper = NodeMapper(QuantParser(), BasicQuantBindings) -GAP9_DequantMapper = NodeMapper(DequantParser(), BasicDequantBindings) +GAP9_QuantMapper = NodeMapper(QuantParser(), QuantTilingReadyBindings) +GAP9_DequantMapper = NodeMapper(DequantParser(), DeQuantTilingReadyBindings) GAP9_GEMMDequantMapper = NodeMapper(PULPGEMMParser(), BasicGEMMBindings) +GAP9_NE16GEMMMapper = NodeMapper(NE16GEMMParser(), GAP9NE16RQSGEMMTilingReadyBindings) +GAP9_NE16GEMMInt32Mapper = NodeMapper(GEMMParser(), GAP9NE16GEMMInt32TilingReadyBindings) + +GAP9Optimizer = TopologyOptimizer( + [ + QuantPatternPass(), + DequantPatternPass(), + DequantQuantMergePass(), + # MatMulAddMergePass(), # fuses to Gemm with transA=transB=0 — wrong layout + # for MatMul inputs that don't share Gemm semantics; FP32 + # SkipConnection regressed from 0/16 to 16/16 errors under it. + # Leave MatMul and Add separate (matches devel base behavior). + SkipEmptyConcatPass(), + SkipUnityRequantPass(previous_op_regex = "Concat", num_inputs = 2), + SkipUnityRequantPass(previous_op_regex = "Reshape|Transpose", num_inputs = 1), + SkipUnityRequantPass(previous_op_regex = "Reshape|Transpose", num_inputs = 1), + RQSSplitPass(), + MergeTrueIntegerDivRequantShiftPass(), + IntegerDivRequantMergePass(), + iGELURequantMergePass(), + iHardswishRequantMergePass(), + PULPConvRequantMergePass(), + MergeConstAddAndRequantPass(), + PULPGEMMRequantMergePass(), + PULPMatMulRequantMergePass(), + PULPAddRequantMergePass(), + RemoveEmptyConvBiasPass(), + RemoveOnlySingletonReduceMeanPass(), + NE16AdjustGEMMWeightLayoutPass(), + ], + name = "GAP9Optimizer") # GAP9-specific mapping using ClDma GAP9Mapping = { @@ -101,8 +142,14 @@ 'RequantizedConv': PULPRQSConvLayer([GAP9_Conv2DMapper, GAP9_DWConv2DMapper, GAP9_Conv1DMapper, GAP9_DWConv1DMapper]), 'RequantizedGemm': - PULPRQSGEMMLayer([GAP9_MatrixVecMapper, GAP9_TallGEMMMapper, GAP9_GEMMMapper]), - 'Gemm': + PULPRQSGEMMLayer([GAP9_NE16GEMMMapper, GAP9_MatrixVecMapper, GAP9_TallGEMMMapper, GAP9_GEMMMapper]), + 'Gemm': # GAP9_NE16GEMMInt32Mapper would also belong here for int8/uint8 Gemm, + # but it shares the same GEMMParser class as the other mappers; the + # deployer keys candidate-bindings by parser class, so listing it + # alongside FloatGEMM / GEMMDequant masks them for FP32 / dequant + # paths and the whole graph fails to map. The int8/uint8 path is + # already covered by RequantizedGemm above; keep plain Gemm for FP + # and dequant flavours only. GEMMLayer([GAP9_FloatGEMMMapper, GAP9_GEMMDequantMapper]), 'Gelu': GELULayer([GAP9_GELUMapper]), @@ -244,7 +291,10 @@ class GAP9StructBuffer(StructBuffer): deallocTemplate = NodeTemplate("") -_includeList = ["pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h"] +_includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CNN_BasicKernels_NE16.h", "CNN_Copy.h", "ne16_utils.h", "CycleCounter.h" +] class GAP9ClusterEngine(DeploymentEngine): diff --git a/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py b/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py new file mode 100644 index 0000000000..cd4374466e --- /dev/null +++ b/Deeploy/Targets/GAP9/Templates/GAP9SDKDequantQuantTemplate.py @@ -0,0 +1,168 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +# +# Quant/Dequant templates using GAP9 SDK kernels (CNN_Copy.c). +# All called via GAP9Transformer which handles pi_cl_team_fork. + +from Deeploy.DeeployTypes import NodeTemplate + +# ============================================================ +# Dequant templates: int → fp16 (SDK kernels from CNN_Copy.c) +# ============================================================ + +# int8 → fp16: SDK kernel CNN_FpsIEEE16 +fp16DequantI8Template = NodeTemplate(""" +// FP16 Dequant int8→fp16 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _dq_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_FpsIEEE16(&_dq_arg); +} +""") + +# uint8 → fp16: SDK kernel CNN_UFpsIEEE16 +fp16DequantU8Template = NodeTemplate(""" +// FP16 Dequant uint8→fp16 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _dq_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_UFpsIEEE16(&_dq_arg); +} +""") + +# ============================================================ +# Dequant templates: int → fp32 (SDK kernels from CNN_Copy.c) +# ============================================================ + +# int8 → fp32: SDK kernel CNN_FpsFloat32 +fp32DequantI8Template = NodeTemplate(""" +// FP32 Dequant int8→fp32 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_FpsFloat32_T _dq_arg = { + .In = (signed char *)${data_in}, + .Out = (float *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_FpsFloat32(&_dq_arg); +} +""") + +# uint8 → fp32: SDK kernel CNN_UFpsFloat32 +fp32DequantU8Template = NodeTemplate(""" +// FP32 Dequant uint8→fp32 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _dq_infos[8]; + *((float *)(_dq_infos + 0)) = (float)(-(${zero_point})); + *((float *)(_dq_infos + 4)) = (float)(${scale}); + CNN_UFpsFloat32_T _dq_arg = { + .In = (unsigned char *)${data_in}, + .Out = (float *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _dq_infos, + }; + CNN_UFpsFloat32(&_dq_arg); +} +""") + +# ============================================================ +# Quant templates: fp16 → int (SDK kernels from CNN_Copy.c) +# ============================================================ + +# fp16 → int8: SDK kernel CNN_IEEE16Fps +fp16QuantI8Template = NodeTemplate(""" +// FP16 Quant fp16→int8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _q_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_IEEE16Fps(&_q_arg); +} +""") + +# fp16 → uint8: SDK kernel CNN_IEEE16UFps +fp16QuantU8Template = NodeTemplate(""" +// FP16 Quant fp16→uint8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Quantize_T _q_arg = { + .In = (void *)${data_in}, + .Out = (void *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_IEEE16UFps(&_q_arg); +} +""") + +# ============================================================ +# Quant templates: fp32 → int (SDK kernels from CNN_Copy.c) +# ============================================================ + +# fp32 → int8: SDK kernel CNN_Float32Fps +fp32QuantI8Template = NodeTemplate(""" +// FP32 Quant fp32→int8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Float32Fps_T _q_arg = { + .In = (float *)${data_in}, + .Out = (signed char *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_Float32Fps(&_q_arg); +} +""") + +# fp32 → uint8: SDK kernel CNN_Float32UFps +fp32QuantU8Template = NodeTemplate(""" +// FP32 Quant fp32→uint8 (Name: ${nodeName}, Op: ${nodeOp}) +{ + signed char _q_infos[8]; + *((float *)(_q_infos + 0)) = (float)(${zero_point}); + *((float *)(_q_infos + 4)) = (float)(${scale}); + CNN_Float32UFps_T _q_arg = { + .In = (float *)${data_in}, + .Out = (unsigned char *)${data_out}, + .W = ${size}, + .H = 1, + .Infos = _q_infos, + }; + CNN_Float32UFps(&_q_arg); +} +""") diff --git a/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py b/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py new file mode 100644 index 0000000000..8acb27ce25 --- /dev/null +++ b/Deeploy/Targets/GAP9/Templates/NE16GEMMTemplate.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation + + +def _ne16_conv_1x1_weight_layout(W, w_bits = 8): + """Pack int8 weights for NE16 1x1 conv mode. + W: int8 [Ko, Ki] -> uint8 [Ko, Nb_KI, Qw, 2] (bitplane packed) + Weights stored as uint8 = int8 + 128. + """ + tp_in = 16 + Ko_, Ki_ = W.shape + W_uint8 = (W.astype(np.int32) + 128).astype(np.uint8) + nb_ki = (Ki_ + tp_in - 1) // tp_in + w_binary = np.zeros((Ko_ * nb_ki, w_bits, 8, tp_in // 8), dtype = np.uint8) + for ko in range(Ko_): + for ki_maj in range(nb_ki): + for ki_min in range(tp_in): + idx = ko * nb_ki + ki_maj + ki = ki_maj * tp_in + ki_min + val = int(W_uint8[ko, ki]) if ki < Ki_ else 0 + for q in range(w_bits): + w_binary[idx, q, ki_min % 8, ki_min // 8] = (val >> q) & 1 + space = np.logspace(0, 7, num = 8, base = 2, dtype = np.int32).reshape((8, 1)) + w_layout = np.sum(w_binary * space, axis = 2, dtype = np.uint8) + return w_layout.reshape((Ko_, nb_ki, w_bits, tp_in // 8)) + + +class NE16GEMMTemplate(NodeTemplate): + + def __init__(self, templateStr): + super().__init__(templateStr) + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + + A = ctxt.lookup(operatorRepresentation['A']) + B = ctxt.lookup(operatorRepresentation['B']) + C = ctxt.lookup(operatorRepresentation['C']) + data_out = ctxt.lookup(operatorRepresentation['data_out']) + + # Determine signedness from type system (reliable, post-type-inference) + input_signed = A._type.referencedType.typeMin < 0 + output_bits = data_out._type.referencedType.typeWidth + output_signed = data_out._type.referencedType.typeMin < 0 + + operatorRepresentation['input_signed'] = input_signed + operatorRepresentation['output_bits'] = output_bits + operatorRepresentation['quant_bits'] = 2 if output_bits == 32 else 0 + operatorRepresentation['quant_norect'] = 1 if (output_bits == 32 or output_signed) else 0 + + # Weight packing and signed bias compensation + w_int8 = B.values.astype(np.int8) # [Ko, Ki], still int8 at this point + Ko, Ki = w_int8.shape + + # Truncate broadcast bias [M, O] → per-channel [Ko] for NE16 + bias_flat = C.values.flatten() + if bias_flat.size > Ko: + C.values = bias_flat[:Ko].copy() + + # Signed input bias compensation + if input_signed: + # Compute w_sum BEFORE packing (needed for signed bias compensation) + w_sum = w_int8.astype(np.int64).sum(axis = 1) # [Ko] + bias_values = C.values.flatten().astype(np.int64) + if 'mul' in operatorRepresentation: + # RequantizedGemm: bias -= 128 * w_sum * scale + scale_buf = ctxt.lookup(operatorRepresentation['mul']) + scale_values = scale_buf.values.flatten().astype(np.int64) + bias_values -= 128 * w_sum * scale_values + else: + # Gemm int32: bias -= 128 * w_sum (no scale) + bias_values -= 128 * w_sum + C.values = bias_values.astype(np.int32) + + # Pack weights to NE16 bitplane format + ne16_weights = _ne16_conv_1x1_weight_layout(w_int8) + B.values = ne16_weights.reshape(Ko, -1) + + return ctxt, operatorRepresentation, [] + + +# 8-bit output template (RequantizedGemm) — uses tiled ${mul} and ${scale_n} +referenceTemplate = NE16GEMMTemplate(""" +// NE16 Linear 8-bit (Name: ${nodeName}, Op: ${nodeOp}) + +% if input_signed: +// Signed input: add 128 offset to convert int8 -> uint8 (multi-core SIMD) +{ + ne16_int8_to_uint8_T _offset_arg = { + .In = (int8_t *)${A}, + .Out = (uint8_t *)${A}, + .size = ${batch} * ${M} * ${N} + }; + pi_cl_team_fork(NUM_CORES, (void *)ne16_int8_to_uint8, &_offset_arg); +} +% endif + +{ + unsigned int _ne16_cfg = 0; + _ne16_cfg |= ((8 - 1) & NE16_MASK_WBITS_M1) << NE16_SHIFT_WBITS_M1; + _ne16_cfg |= (0 & NE16_MASK_MODE16) << NE16_SHIFT_MODE16; + _ne16_cfg |= (1 & NE16_MASK_OUTQUANT) << NE16_SHIFT_OUTQUANT; + _ne16_cfg |= (NE16_FILTER_MODE_1x1 & NE16_MASK_FILTER_MODE) << NE16_SHIFT_FILTER_MODE; + _ne16_cfg |= (0 & NE16_MASK_LINEAR_MODE) << NE16_SHIFT_LINEAR_MODE; + _ne16_cfg |= (0 & NE16_MASK_STRIDED_MODE) << NE16_SHIFT_STRIDED_MODE; + _ne16_cfg |= (NE16_BITS_8BIT & NE16_MASK_NORM_BITS) << NE16_SHIFT_NORM_BITS; + _ne16_cfg |= (0 & NE16_MASK_STREAMIN) << NE16_SHIFT_STREAMIN; + _ne16_cfg |= (1 & NE16_MASK_WEIGHT_OFFSET_CFG) << NE16_SHIFT_WEIGHT_OFFSET_CFG; + _ne16_cfg |= (0 & NE16_MASK_QUANT_RIGHT_SHIFT) << NE16_SHIFT_QUANT_RIGHT_SHIFT; + _ne16_cfg |= (${quant_bits} & NE16_MASK_QUANT_BITS) << NE16_SHIFT_QUANT_BITS; + _ne16_cfg |= (${quant_norect} & NE16_MASK_QUANT_NORECT) << NE16_SHIFT_QUANT_NORECT; + _ne16_cfg |= (1 & NE16_MASK_NORM_SHIFT) << NE16_SHIFT_NORM_SHIFT; + _ne16_cfg |= (1 & NE16_MASK_NORM_BIAS) << NE16_SHIFT_NORM_BIAS; + + NE16_Enable(); + NE16_SoftReset(); + + KerConv_NE16_T _ne16_arg = { + .In = (void *)${A}, + .Filter = (unsigned short *)${B}, + .Bias = (int *)${C}, + .Out = (void *)${data_out}, + .Scale = (unsigned char *)${mul}, + .ScaleN = (unsigned char *)${scale_n}, + .Tile_InFeat = ${N}, + .TotalInFeatures = ${N}, + .Tile_InH = 1, + .Tile_InW = ${batch} * ${M}, + .Tile_OutFeat = ${O}, + .Tile_OutH = 1, + .Tile_OutW = ${batch} * ${M}, + .FilterSize = 1, + .Pad_Val = 0, + .Pad = (v4u){0, 0, 0, 0}, + .W_Offset = -128, + .Qw = 8, + .Mode16 = 0, + .FirstD0 = 1, + .LastD0 = 1, + .Default_NE16_Job_Cfg = _ne16_cfg, + .Fx = 1, + .Fy = 1, + .Sx = 1, + .Sy = 1, + .Dx = 1, + .Dy = 1, + .BuffOut = NULL, + .Infos = NULL, + .Extra = NULL, + }; + KerConv1x1_SmallHW_Stride1_NE16(&_ne16_arg); + + NE16_Disable(); +} +""") + +# Int32 output template (plain Gemm) — hardcoded scale=1, scale_n=0 +int32OutputTemplate = NE16GEMMTemplate(""" +// NE16 Linear Int32 (Name: ${nodeName}, Op: ${nodeOp}) + +% if input_signed: +// Signed input: add 128 offset to convert int8 -> uint8 (multi-core SIMD) +{ + ne16_int8_to_uint8_T _offset_arg = { + .In = (int8_t *)${A}, + .Out = (uint8_t *)${A}, + .size = ${batch} * ${M} * ${N} + }; + pi_cl_team_fork(NUM_CORES, (void *)ne16_int8_to_uint8, &_offset_arg); +} +% endif + +{ + unsigned char _ne16_ones[${O}]; + unsigned char _ne16_zeros[${O}]; + memset(_ne16_ones, 1, ${O}); + memset(_ne16_zeros, 0, ${O}); + + unsigned int _ne16_cfg = 0; + _ne16_cfg |= ((8 - 1) & NE16_MASK_WBITS_M1) << NE16_SHIFT_WBITS_M1; + _ne16_cfg |= (0 & NE16_MASK_MODE16) << NE16_SHIFT_MODE16; + _ne16_cfg |= (1 & NE16_MASK_OUTQUANT) << NE16_SHIFT_OUTQUANT; + _ne16_cfg |= (NE16_FILTER_MODE_1x1 & NE16_MASK_FILTER_MODE) << NE16_SHIFT_FILTER_MODE; + _ne16_cfg |= (0 & NE16_MASK_LINEAR_MODE) << NE16_SHIFT_LINEAR_MODE; + _ne16_cfg |= (0 & NE16_MASK_STRIDED_MODE) << NE16_SHIFT_STRIDED_MODE; + _ne16_cfg |= (NE16_BITS_8BIT & NE16_MASK_NORM_BITS) << NE16_SHIFT_NORM_BITS; + _ne16_cfg |= (0 & NE16_MASK_STREAMIN) << NE16_SHIFT_STREAMIN; + _ne16_cfg |= (1 & NE16_MASK_WEIGHT_OFFSET_CFG) << NE16_SHIFT_WEIGHT_OFFSET_CFG; + _ne16_cfg |= (0 & NE16_MASK_QUANT_RIGHT_SHIFT) << NE16_SHIFT_QUANT_RIGHT_SHIFT; + _ne16_cfg |= (${quant_bits} & NE16_MASK_QUANT_BITS) << NE16_SHIFT_QUANT_BITS; + _ne16_cfg |= (${quant_norect} & NE16_MASK_QUANT_NORECT) << NE16_SHIFT_QUANT_NORECT; + _ne16_cfg |= (1 & NE16_MASK_NORM_SHIFT) << NE16_SHIFT_NORM_SHIFT; + _ne16_cfg |= (1 & NE16_MASK_NORM_BIAS) << NE16_SHIFT_NORM_BIAS; + + NE16_Enable(); + NE16_SoftReset(); + + KerConv_NE16_T _ne16_arg = { + .In = (void *)${A}, + .Filter = (unsigned short *)${B}, + .Bias = (int *)${C}, + .Out = (void *)${data_out}, + .Scale = _ne16_ones, + .ScaleN = _ne16_zeros, + .Tile_InFeat = ${N}, + .TotalInFeatures = ${N}, + .Tile_InH = 1, + .Tile_InW = ${batch} * ${M}, + .Tile_OutFeat = ${O}, + .Tile_OutH = 1, + .Tile_OutW = ${batch} * ${M}, + .FilterSize = 1, + .Pad_Val = 0, + .Pad = (v4u){0, 0, 0, 0}, + .W_Offset = -128, + .Qw = 8, + .Mode16 = 0, + .FirstD0 = 1, + .LastD0 = 1, + .Default_NE16_Job_Cfg = _ne16_cfg, + .Fx = 1, + .Fy = 1, + .Sx = 1, + .Sy = 1, + .Dx = 1, + .Dy = 1, + .BuffOut = NULL, + .Infos = NULL, + .Extra = NULL, + }; + KerConv1x1_SmallHW_Stride1_NE16(&_ne16_arg); + + NE16_Disable(); +} +""") diff --git a/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py b/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py new file mode 100644 index 0000000000..6f490b9803 --- /dev/null +++ b/Deeploy/Targets/GAP9/TileConstraints/NE16GEMMTileConstraint.py @@ -0,0 +1,196 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import math +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + + +class NE16GEMMTileConstraint(TileConstraint): + """Tile constraint for NE16 GEMM with bitplane-packed weights stored as 2D [Ko, Ki].""" + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + bufferC = ctxt.lookup(name = parseDict['C']) + outputBuffer = ctxt.lookup(name = parseDict['data_out']) + + bufferNames = [bufferA.name, bufferB.name, bufferC.name, outputBuffer.name] + hasMul = 'mul' in parseDict and isinstance(parseDict['mul'], str) + if hasMul: + mulBuffer = ctxt.lookup(name = parseDict['mul']) + bufferNames.append(mulBuffer.name) + hasScaleN = 'scale_n' in parseDict and isinstance(parseDict['scale_n'], str) + if hasScaleN: + scaleNBuffer = ctxt.lookup(name = parseDict['scale_n']) + bufferNames.append(scaleNBuffer.name) + + for bufferName in bufferNames: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + dimOffsetOut = len(outputBuffer.shape) - 2 + + AFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, dimIdx = dimOffsetA + parseDict['transA']) + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + outputFirstDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut) + outputSecondDimVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = dimOffsetOut + 1) + + tilerModel.addConstraint(outputFirstDimVar == AFirstDimVar) + tilerModel.addConstraint(outputSecondDimVar == BSecondDimVar) + tilerModel.addConstraint(ASecondDimVar == BFirstDimVar) + + addDimVar = tilerModel.getTensorDimVar(tensorName = bufferC.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == addDimVar) + + if hasMul: + mulDimVar = tilerModel.getTensorDimVar(tensorName = mulBuffer.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == mulDimVar) + + if hasScaleN: + scaleNDimVar = tilerModel.getTensorDimVar(tensorName = scaleNBuffer.name, dimIdx = 0) + tilerModel.addConstraint(outputSecondDimVar == scaleNDimVar) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + bufferA = ctxt.lookup(name = parseDict['A']) + bufferB = ctxt.lookup(name = parseDict['B']) + + dimOffsetA = len(bufferA.shape) - 2 + dimOffsetB = len(bufferB.shape) - 2 + + # Don't tile N (reduction dimension) — NE16 needs full input channels + ASecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferA.name, + dimIdx = dimOffsetA + 1 - parseDict['transA']) + BFirstDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, dimIdx = dimOffsetB + parseDict['transB']) + tilerModel.addConstraint(ASecondDimVar == parseDict['N']) + tilerModel.addConstraint(BFirstDimVar == parseDict['N']) + + # O (output channels) should be divisible by 32 (NE16 TP_OUT) + BSecondDimVar = tilerModel.getTensorDimVar(tensorName = bufferB.name, + dimIdx = dimOffsetB + 1 - parseDict['transB']) + if parseDict["O"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + 'O', + BSecondDimVar, + 32, + strategy = PerformanceHint(priority = 1)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + hasMul = 'mul' in operatorRepresentation and isinstance(operatorRepresentation['mul'], str) + hasScaleN = 'scale_n' in operatorRepresentation and isinstance(operatorRepresentation['scale_n'], str) + addrNames = ['A', 'B', 'C', 'data_out'] + if hasMul: + addrNames.insert(2, 'mul') + if hasScaleN: + addrNames.insert(-1, 'scale_n') + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + transA = operatorRepresentation['transA'] + transB = operatorRepresentation['transB'] + + buffA = ctxt.lookup(operatorRepresentation['A']) + buffB = ctxt.lookup(operatorRepresentation['B']) + + NSize = buffA.shape[-1] + + inputACubes = [] + inputBCubes = [] + inputMulCubes = [] + inputAddCubes = [] + + replacements = {"M": [], "O": [], "batch": []} + + for cube in outputCubes: + MOffset, OOffset = cube.offset[-2:] + MSize, OSize = cube.dims[-2:] + + if len(cube.offset) > 2: + BatchSize = math.prod(cube.dims[:-2]) + else: + BatchSize = 1 + + replacements["M"].append(MSize) + replacements["O"].append(OSize) + replacements["batch"].append(BatchSize) + + if transA == 0: + AMatrixOffsets = (MOffset, 0) + AMatrixShape = (MSize, NSize) + else: + AMatrixOffsets = (0, MOffset) + AMatrixShape = (NSize, MSize) + + if len(buffA.shape) > 2: + batchDimCount = len(buffA.shape) - 2 + AMatrixOffsets = tuple(cube.offset[:-2][-batchDimCount:]) + AMatrixOffsets + AMatrixShape = tuple(cube.dims[:-2][-batchDimCount:]) + AMatrixShape + + inputACubes.append(HyperRectangle(AMatrixOffsets, AMatrixShape)) + + if transB == 0: + BMatrixOffsets = (0, OOffset) + BMatrixShape = (NSize, OSize) + else: + BMatrixOffsets = (OOffset, 0) + BMatrixShape = (OSize, NSize) + + inputBCubes.append(HyperRectangle(BMatrixOffsets, BMatrixShape)) + + RequantCube = HyperRectangle((OOffset,), (OSize,)) + inputMulCubes.append(RequantCube) + inputAddCubes.append(RequantCube) + + replacements["N"] = [NSize] * len(outputCubes) + + replacementTypes = { + "M": PointerClass(uint16_t), + "N": PointerClass(uint16_t), + "O": PointerClass(uint16_t), + "batch": PointerClass(uint8_t) + } + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for idx, (a, b, c) in enumerate(zip(inputACubes, inputBCubes, inputAddCubes)): + load = {"A": a, "B": b, "C": c} + if hasMul: + load["mul"] = inputMulCubes[idx] + if hasScaleN: + load["scale_n"] = inputMulCubes[idx] # same per-channel slice as mul/C + inputLoadSchedule.append(load) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + schedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + + return VariableReplacementScheme(replacements, replacementTypes), schedule diff --git a/Deeploy/Targets/GAP9/Tiler.py b/Deeploy/Targets/GAP9/Tiler.py index fefe12b6d7..b93aacb9db 100644 --- a/Deeploy/Targets/GAP9/Tiler.py +++ b/Deeploy/Targets/GAP9/Tiler.py @@ -10,14 +10,16 @@ import copy -from Deeploy.Targets.GAP9.Bindings import GAP9AddBindings, GAP9ConcatBindings, GAP9FloatConv2DBindings, \ - GAP9FloatDWConv2DBindings, GAP9FloatGELUBinding, GAP9FloatGEMMBindings, GAP9GatherBindings, \ - GAP9iHardswishBindings, GAP9iRMSNormBindings, GAP9iRQSGELUBindings, GAP9LayernormBinding, GAP9MatMulBindings, \ - GAP9MaxPool2DBindings, GAP9MulBindings, GAP9ReduceSumBindings, GAP9ReluBinding, GAP9ReshapeBindings, \ - GAP9RQAddBindings, GAP9RQSBindings, GAP9RQSConv2DBindings, GAP9RQSDWConv2DBindings, GAP9RQSGEMMBindings, \ - GAP9RQSiHardswishBindings, GAP9RQSMatrixVecBindings, GAP9RQSTallGEMMBindings, GAP9SGDBindings, \ - GAP9SoftmaxBindings, GAP9SoftmaxCrossEntropyLossBindings, GAP9SoftmaxCrossEntropyLossGradBindings, \ - GAP9SoftmaxGradBindings, GAP9TransposeBindings, GAP9UniformRQSBindings +from Deeploy.Targets.GAP9.Bindings import GAP9AddBindings, GAP9ConcatBindings, GAP9DequantBindings, \ + GAP9FloatConv2DBindings, GAP9FloatDWConv2DBindings, GAP9FloatGELUBinding, GAP9FloatGEMMBindings, \ + GAP9GatherBindings, GAP9iHardswishBindings, GAP9iRMSNormBindings, GAP9iRQSGELUBindings, GAP9LayernormBinding, \ + GAP9MatMulBindings, GAP9MaxPool2DBindings, GAP9MulBindings, GAP9NE16GEMMInt32Bindings, GAP9NE16RQSGEMMBindings, \ + GAP9QuantBindings, GAP9ReduceSumBindings, GAP9ReluBinding, GAP9ReshapeBindings, GAP9RQAddBindings, \ + GAP9RQSBindings, GAP9RQSConv2DBindings, GAP9RQSDWConv2DBindings, GAP9RQSGEMMBindings, GAP9RQSiHardswishBindings, \ + GAP9RQSMatrixVecBindings, GAP9RQSTallGEMMBindings, GAP9SGDBindings, GAP9SoftmaxBindings, \ + GAP9SoftmaxCrossEntropyLossBindings, GAP9SoftmaxCrossEntropyLossGradBindings, GAP9SoftmaxGradBindings, \ + GAP9TransposeBindings, GAP9UniformRQSBindings +from Deeploy.Targets.GAP9.TileConstraints.NE16GEMMTileConstraint import NE16GEMMTileConstraint from Deeploy.Targets.Generic.TileConstraints.AddTileConstraint import AddTileConstraint from Deeploy.Targets.Generic.TileConstraints.ConcatTileConstraint import ConcatTileConstraint from Deeploy.Targets.Generic.TileConstraints.iHardswishTileConstraint import iHardswishTileConstraint @@ -60,6 +62,12 @@ GAP9RQSGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9RQSGEMMBindings, tileConstraint = GEMMTileConstraint()) +GAP9NE16RQSGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9NE16RQSGEMMBindings, + tileConstraint = NE16GEMMTileConstraint()) + +GAP9NE16GEMMInt32TilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9NE16GEMMInt32Bindings, + tileConstraint = NE16GEMMTileConstraint()) + GAP9FPGEMMTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9FloatGEMMBindings, tileConstraint = FloatGEMMTileConstraint()) @@ -142,3 +150,9 @@ GAP9SGDTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9SGDBindings, tileConstraint = SGDTileConstraint()) + +QuantTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9QuantBindings, + tileConstraint = UnaryTileConstraint()) + +DeQuantTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = GAP9DequantBindings, + tileConstraint = UnaryTileConstraint()) diff --git a/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py new file mode 100644 index 0000000000..3043c700d1 --- /dev/null +++ b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/Passes.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import math + +import numpy as np +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.Matchers import Match, NonBranchingMatcher +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, contextagnostic + + +def _compute_ne16_scale_shift(mul_values, log2D): + """Convert Deeploy's mul/log2D to NE16's per-channel scale/scale_n.""" + Ko = len(mul_values) + ne16_scale = np.zeros(Ko, dtype = np.uint8) + ne16_scale_n = np.zeros(Ko, dtype = np.uint8) + for ko in range(Ko): + sf = float(mul_values[ko]) / float(2**log2D) + if sf >= 1.0: + sn = 0 + sc = min(255, max(1, int(round(sf)))) + elif sf > 0: + sn = min(31, max(0, int(math.floor(math.log2(127.0 / sf))))) + sc = min(255, max(1, int(round(sf * (1 << sn))))) + else: + sn = 0 + sc = 0 + ne16_scale[ko] = sc + ne16_scale_n[ko] = sn + return ne16_scale, ne16_scale_n + + +def _ne16_adjust_gemm_weight_layout_fun(graph: gs.Graph, match: Match, name: str): + """Prepare GEMM node for NE16 execution. + + Handles transB normalization, scale/scale_n computation, and bias rescaling. + Weight bitplane packing and signed bias compensation are deferred to alignToContext + where input signedness is known from the type system. + """ + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + # Only act on NE16-colored nodes. Cluster-bound Gemm/RequantizedGemm (e.g. + # AnomalyDetection's 10 Gemm+RQ layers in the MLPerf gap9-tiled model + # tests) must keep Deeploy's original mul / bias / no-scale_n layout so + # pulp_nn_linear stays bit-exact with the int8 reference outputs. + if node.attrs.get("engine") != "NE16": + return graph + + # Weight is input[1] for both Gemm and RequantizedGemm + weightTensor = node.inputs[1] + + if not isinstance(weightTensor, gs.Constant): + return graph + + values = weightTensor.values + + # Skip true float weights (Deeploy stores int8 weights as float32) + if not np.array_equal(values, np.round(values)): + return graph + + # Check shape is 2D + if len(values.shape) != 2: + return graph + + # Determine actual Ko, Ki based on transB + transB = node.attrs.get('transB', 0) + if transB: + Ko, Ki = values.shape + else: + Ki, Ko = values.shape + + # Check NE16 compatibility BEFORE modifying the node + if Ki % 16 != 0: + return graph + + # Transpose weight to [Ko, Ki] if needed — keep as int8 + if not transB: + transposed = values.T.astype(np.int8) + newWeightTensor = gs.Constant(f"{name}_{weightTensor.name}", transposed) + node.inputs[1] = newWeightTensor + node.attrs['transB'] = 1 + + # For RequantizedGemm: transform mul → ne16_scale, create scale_n, rescale bias + if node.op == 'RequantizedGemm' and len(node.inputs) >= 4: + mulTensor = node.inputs[3] + biasTensor = node.inputs[2] + + if isinstance(mulTensor, gs.Constant) and isinstance(biasTensor, gs.Constant): + mul_values = mulTensor.values.flatten().astype(np.int32) + log2D = int(np.log2(node.attrs['div'].values)) + + # Broadcast scalar mul to per-channel if needed + if len(mul_values) == 1: + mul_values = np.full(Ko, mul_values[0], dtype = np.int32) + + ne16_scale, ne16_scale_n = _compute_ne16_scale_shift(mul_values, log2D) + + # Rescale bias from mul/log2D domain to scale/scale_n domain + # bias_merged is already *= mul from PULPGEMMRequantMergePass + # NE16 needs: bias_ne16 = bias_merged * 2^(scale_n - log2D) + bias_values = biasTensor.values.flatten().astype(np.int64) + ne16_bias = np.zeros(Ko, dtype = np.int64) + for ko in range(Ko): + shift_diff = int(ne16_scale_n[ko]) - log2D + if shift_diff >= 0: + ne16_bias[ko] = bias_values[ko] << shift_diff + else: + ne16_bias[ko] = bias_values[ko] >> (-shift_diff) + + ne16_bias = ne16_bias.astype(np.int32) + + # Overwrite mul tensor with ne16_scale + mulTensor.values = ne16_scale + + # Overwrite bias tensor + biasTensor.values = ne16_bias + + # Append scale_n as new input[4] + scale_n_tensor = gs.Constant(f"{name}_scale_n", ne16_scale_n) + node.inputs.append(scale_n_tensor) + + return graph + + +@contextagnostic +class NE16AdjustGEMMWeightLayoutPass(ReplaceSequentialPatternPass): + + def __init__(self): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedGemm|Gemm', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__(graph, _ne16_adjust_gemm_weight_layout_fun, "_NE16_ADJUST_GEMM_WEIGHT_LAYOUT_PASS", + NonBranchingMatcher(regex_op = True)) diff --git a/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py new file mode 100644 index 0000000000..4694b67df5 --- /dev/null +++ b/Deeploy/Targets/GAP9/TopologyOptimizationPasses/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/Deeploy/Targets/Generic/Bindings.py b/Deeploy/Targets/Generic/Bindings.py index 1ed08bda52..b82eb6b3a1 100644 --- a/Deeploy/Targets/Generic/Bindings.py +++ b/Deeploy/Targets/Generic/Bindings.py @@ -308,12 +308,15 @@ BasicQuantBindings = [ NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, BasicTransformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), QuantTemplate.referenceTemplate, + BasicTransformer), ] BasicDequantBindings = [ NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, BasicTransformer), -] + [ + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, + BasicTransformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, BasicTransformer), ] diff --git a/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py index 146bcf699e..ff1b539aad 100644 --- a/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/Generic/TopologyOptimizationPasses/Passes.py @@ -844,7 +844,18 @@ def _split_rqs_fun(graph: gs.Graph, match: Match, name: str, splitSet: List[str] if isinstance(var, gs.Variable): postSplitInput = var else: - postSplitInput = gs.Constant(name = f"{t1.name}_split_{idx}", values = var.values.copy().reshape(-1,)) + # The standalone RequantShift kernel binding expects integer mul/add + # (int32). Source ONNX often stores them as float32 even when the + # values are integer-valued, because they get folded into the + # preceding Conv/Gemm's bias path which handles the cast inline. + # After splitting, these constants survive standalone, so cast the + # values to int32 when they are integer-valued. Otherwise the + # parser-side type checker rejects every integer binding and + # parsing backtracks out of the whole graph. + values = var.values.copy().reshape(-1,) + if values.dtype != np.int32 and np.array_equal(values, np.round(values)): + values = values.astype(np.int32) + postSplitInput = gs.Constant(name = f"{t1.name}_split_{idx}", values = values) postSplitInputs.append(postSplitInput) for idx, node in enumerate(originalNode.outputs.copy()): @@ -1177,3 +1188,85 @@ def __init__(self): name = "_RECOGNIZE_DEQUANT_PASS" super().__init__(graph, _recognize_dequant_fun, name) + + +def _merge_dequant_quant_fun(graph: gs.Graph, match: Match, name: str): + matched_nodes = [m for k, m in match.nodes_map.items()] + dequant_node = matched_nodes[0] + quant_node = matched_nodes[1] + + # Skip if dequant output has multiple consumers or is a graph output + dequant_out = dequant_node.outputs[0] + if len(dequant_out.outputs) > 1 or dequant_out in graph.outputs: + return graph + + # Extract Dequant parameters (stored as Python floats) + s_d = float(dequant_node.attrs['scale']) + zp_d = float(dequant_node.attrs['zero_point']) + + # Extract Quant parameters (stored as numpy arrays) + s_q = float(np.array(quant_node.attrs['scale']).item()) + zp_q = float(np.array(quant_node.attrs['zero_point']).item()) + + signed_val = int(np.array(quant_node.attrs['signed']).item()) if 'signed' in quant_node.attrs else 1 + signed = bool(signed_val) + bit_width = int(np.array(quant_node.attrs['bit_width']).item()) if 'bit_width' in quant_node.attrs else 8 + n_levels = 2**bit_width + + # Compute effective ratio: y_float = (x_int - zp_d) * s_d * s_q + zp_q + ratio = s_d * s_q + + # Identity case: ratio ~= 1.0, both zero points == 0 + EPSILON = 1e-6 + if abs(ratio - 1.0) < EPSILON and abs(zp_d) < EPSILON and abs(zp_q) < EPSILON and signed: + input_tensor = dequant_node.inputs[0] + output_tensor = quant_node.outputs[0] + for downstream_node in list(output_tensor.outputs): + for i, inp in enumerate(downstream_node.inputs): + if inp == output_tensor: + downstream_node.inputs[i] = input_tensor + dequant_node.inputs.clear() + dequant_node.outputs.clear() + quant_node.inputs.clear() + quant_node.outputs.clear() + graph.cleanup().toposort() + return graph + + # Requantization case: convert to RequantShift + shift = 16 + div_val = 2**shift + + mul_val = int(np.round(ratio * div_val)) + add_val = int(np.round((-zp_d * ratio + zp_q) * div_val)) + + mul_const = gs.Constant(name = f'{name}_mul', values = np.array([mul_val], dtype = np.int32)) + add_const = gs.Constant(name = f'{name}_add', values = np.array([add_val], dtype = np.int32)) + + rqs_attrs = { + 'div': gs.Constant(f'{name}_div', np.array(div_val)), + 'n_levels_out': gs.Constant(f'{name}_n_levels', np.array(n_levels)), + 'signed': gs.Constant(f'{name}_signed', np.array([signed_val])), + } + + _inputs = [dequant_node.inputs[0], mul_const, add_const] + _outputs = quant_node.outputs + + rqs_node = gs.Node(op = 'RequantShift', name = name, attrs = rqs_attrs) + graph.replaceInsertNode(_inputs, _outputs, rqs_node) + + return graph + + +@contextagnostic +class DequantQuantMergePass(ReplaceSequentialPatternPass): + + def __init__(self): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['dequant_out'], op = 'Dequant', name = 'dequant') + output = graph.layer(inputs = output, outputs = ['quant_out'], op = 'Quant', name = 'quant') + graph.outputs.append(output) + graph.inputs.append(_input) + + name = "_MERGE_DEQUANT_QUANT_PASS" + super().__init__(graph, _merge_dequant_quant_fun, name) diff --git a/Deeploy/Targets/Generic/TypeCheckers.py b/Deeploy/Targets/Generic/TypeCheckers.py index 48fda7a00c..dd843199ce 100644 --- a/Deeploy/Targets/Generic/TypeCheckers.py +++ b/Deeploy/Targets/Generic/TypeCheckers.py @@ -542,6 +542,15 @@ class QuantChecker(SignPropTypeChecker): def __init__(self, input_types: Sequence[Type[Pointer]], output_types: Sequence[Type[Pointer]]): super().__init__(input_types, output_types) + def checkOutputType(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> bool: + outputTypeSigned = self.output_types[0].referencedType.typeMin < 0 + opSigned = bool(operatorRepresentation['signed']) + if opSigned and outputTypeSigned: + return True + if (not opSigned) and (not outputTypeSigned): + return True + return False + def _inferNumLevels(self, inputs: List[VariableBuffer], operatorRepresentation: OperatorRepresentation) -> List[int]: # Calculate number of levels based on bit_width diff --git a/Deeploy/Targets/NE16/Bindings.py b/Deeploy/Targets/NE16/Bindings.py new file mode 100644 index 0000000000..58db14aee3 --- /dev/null +++ b/Deeploy/Targets/NE16/Bindings.py @@ -0,0 +1,72 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import int8_t, int32_t, uint8_t +from Deeploy.DeeployTypes import NodeBinding +from Deeploy.Targets.GAP9.Bindings import GAP9ClusterTransformer as ClusterTransformer +from Deeploy.Targets.Generic.TypeCheckers import ConvChecker +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE16DenseConv2D_Template, NE16DWConv2D_Template, \ + NE16PWConv2D_Template, NE16RqntDenseConv2D_Template, NE16RqntDWConv2D_Template, NE16RqntPWConv2D_Template +from Deeploy.Targets.PULPOpen.TypeCheckers import PULPConvChecker + +NE16RQSPWConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntPWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16PWConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16PWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] + +NE16RQSDWConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntDWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16DWConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16DWConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] + +NE16RQSDenseConv2DBindings = [ + NodeBinding( + PULPConvChecker( + [PointerClass(data_in_type), + PointerClass(weight_type), + PointerClass(int32_t), + PointerClass(int32_t)], [PointerClass(data_out_type)]), NE16RqntDenseConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for data_out_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] +NE16DenseConv2DBindings = [ + NodeBinding( + ConvChecker( + [PointerClass(data_in_type), PointerClass(weight_type), + PointerClass(int32_t)], [PointerClass(int32_t)]), NE16DenseConv2D_Template, ClusterTransformer) + for data_in_type in [uint8_t, int8_t] + for weight_type in [uint8_t, int8_t] +] diff --git a/Deeploy/Targets/NE16/Deployer.py b/Deeploy/Targets/NE16/Deployer.py new file mode 100644 index 0000000000..368222f9e4 --- /dev/null +++ b/Deeploy/Targets/NE16/Deployer.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Callable, Dict, Type + +import onnx_graphsurgeon as gs + +from Deeploy.AbstractDataTypes import Pointer +from Deeploy.DeeployTypes import DeploymentPlatform, TopologyOptimizer +from Deeploy.Targets.GAP9.Deployer import GAP9Deployer +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import TransposeConstOptPass, TransposeMergePass, \ + TransposeNoPermOptPass, TransposeSplitPass +from Deeploy.Targets.NE16.TopologyOptimizationPasses.Passes import ConvEngineDiscolorationPass, NE16OptimizationPass + + +class NE16Deployer(GAP9Deployer): + + def __init__(self, + graph: gs.Graph, + deploymentPlatform: DeploymentPlatform, + inputTypes: Dict[str, Type[Pointer]], + loweringOptimizer: TopologyOptimizer, + scheduler: Callable = lambda graph: list(graph.nodes), + name: str = 'DeeployNetwork', + default_channels_first = False, + deeployStateDir: str = "DeeployStateDir", + inputOffsets = {}): + super().__init__(graph, deploymentPlatform, inputTypes, loweringOptimizer, scheduler, name, + default_channels_first, deeployStateDir, inputOffsets) + + # Keep the global PULPNCHWtoNHWCPass for DW convs (cluster-compatible NHWC). + # NE16-colored DW convs are fixed up to NE16 NHWC layout inside + # NE16OptimizationPass below. This avoids breaking cluster-fallback DW convs + # (stride-2 layers) when --enable-3x3 is on for mixed-engine graphs. + + self.loweringOptimizer.passes += [ + ConvEngineDiscolorationPass(), + NE16OptimizationPass(self.default_channels_first, "NE16"), + # NE16OptimizationPass appends its own layout transposes (see + # _appendTranspose in the NE16 passes). It runs *after* the + # PULPOpen deployer's transpose clean-up chain, so without + # re-running that chain here those transposes survive to codegen: + # consecutive NE16 convs end up separated by a HWC->CHW followed by + # a CHW->HWC pair that is an identity and should cancel. + TransposeSplitPass(), + TransposeMergePass(), + TransposeConstOptPass(), + TransposeNoPermOptPass(), + ] diff --git a/Deeploy/Targets/NE16/Engine.py b/Deeploy/Targets/NE16/Engine.py new file mode 100644 index 0000000000..48f5bca284 --- /dev/null +++ b/Deeploy/Targets/NE16/Engine.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import List + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import DeploymentEngine, NodeMapper +from Deeploy.Targets.Generic.Layers import ConvLayer +from Deeploy.Targets.NE16.Parsers import NE16DenseConv2DParser, NE16DWConv2DParser, NE16PWConv2DParser, \ + NE16RQSDenseConv2DParser, NE16RQSDWConv2DParser, NE16RQSPWConv2DParser +from Deeploy.Targets.NE16.Tiler import NE16DenseConv2DTilingReadyBindings, NE16DWConv2DTilingReadyBindings, \ + NE16PWConv2DTilingReadyBindings, NE16RQSDenseConv2DTilingReadyBindings, NE16RQSDWConv2DTilingReadyBindings, \ + NE16RQSPWConv2DTilingReadyBindings +from Deeploy.Targets.PULPOpen.Layers import PULPRQSConvLayer + +NE16RqntPWConv2DMapper = NodeMapper(NE16RQSPWConv2DParser(), NE16RQSPWConv2DTilingReadyBindings) +NE16PWConv2DMapper = NodeMapper(NE16PWConv2DParser(), NE16PWConv2DTilingReadyBindings) + +NE16RqntDWConv2DMapper = NodeMapper(NE16RQSDWConv2DParser(), NE16RQSDWConv2DTilingReadyBindings) +NE16DWConv2DMapper = NodeMapper(NE16DWConv2DParser(), NE16DWConv2DTilingReadyBindings) + +NE16RqntDenseConv2DMapper = NodeMapper(NE16RQSDenseConv2DParser(), NE16RQSDenseConv2DTilingReadyBindings) +NE16DenseConv2DMapper = NodeMapper(NE16DenseConv2DParser(), NE16DenseConv2DTilingReadyBindings) + +NE16Mapping = { + 'RequantizedConv': PULPRQSConvLayer([NE16RqntPWConv2DMapper, NE16RqntDWConv2DMapper, NE16RqntDenseConv2DMapper]), + 'Conv': ConvLayer([NE16PWConv2DMapper, NE16DWConv2DMapper, NE16DenseConv2DMapper]), +} + +_includeList = ["pulp_nnx_ne16.h", "pulp_nnx_util.h", "ne16_pulp_bsp.h", "ne16.h", "ne16_task.h"] + +_ne16InitCode = r""" +ne16_pulp_conf_t conf = {.max_stall = 8}; +ne16_nnx_init(ne16_pulp_get_dev(), &conf); +""" + + +class NE16Engine(DeploymentEngine): + + def __init__(self, + name: str, + Mapping = NE16Mapping, + initCode: str = _ne16InitCode, + includeList: List[str] = _includeList, + enable3x3: bool = False, + enableStrides: bool = False) -> None: + super().__init__(name, Mapping, initCode, includeList) + + self.enable3x3 = enable3x3 + self.enableStrides = enableStrides + + def isDenseConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [3, 3] and \ + node.attrs['dilations'] == [1, 1] and \ + node.attrs['group'] == 1 and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def isPWConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [1, 1] and \ + node.attrs['dilations'] == [1, 1] and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def isDWConv(self, node) -> bool: + return node.op in ["Conv", "RequantizedConv"] and \ + isinstance(node.inputs[1], gs.Constant) and \ + node.attrs['kernel_shape'] == [3, 3] and \ + node.attrs['dilations'] == [1, 1] and \ + node.attrs['group'] != 1 and \ + (node.attrs['strides'] == [1, 1] or self.enableStrides) + + def canExecute(self, node: gs.Node) -> bool: + if self.enable3x3: + return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) + else: + return self.isPWConv(node) diff --git a/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py b/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py new file mode 100644 index 0000000000..b6a530a319 --- /dev/null +++ b/Deeploy/Targets/NE16/OptimizationPasses/MemoryLevelAnnotationPasses.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import numpy as np +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import SequentialPass +from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryLevel + + +class AnnotateNE16WeightMemoryLevel(SequentialPass): + + def __init__(self, ne16EngineName: str, weightMemoryLevel: MemoryLevel): + self._weightMemoryLevel = weightMemoryLevel + self.ne16EngineName = ne16EngineName + super().__init__() + + def apply(self, ctxt: NetworkContext, graph: gs.Graph) -> Tuple[NetworkContext, gs.Graph]: + + def _ne16WeightBufferSize(buffer: ConstantBuffer) -> int: + return int(np.prod(buffer.shape)) # Weights are encoded as bytes so no need to check for typeWidth + + weightMemoryOccupation = 0 + + # Current weight memory occupation + for buffer in {**ctxt.globalObjects, **ctxt.localObjects}.values(): + if hasattr(buffer, "_memoryLevel") and buffer._memoryLevel == self._weightMemoryLevel.name: + weightMemoryOccupation += _ne16WeightBufferSize(buffer) + + ne16Nodes = [node for node in graph.nodes if node.attrs["engine"] == self.ne16EngineName] + for node in ne16Nodes: + if node.op in ["Conv", "RequantizedConv"]: + + if not (ctxt.is_local(node.inputs[1].name) or ctxt.is_global(node.inputs[1].name)): + continue + + buffer = ctxt.lookup(node.inputs[1].name) + if weightMemoryOccupation + _ne16WeightBufferSize(buffer) < self._weightMemoryLevel.size: + buffer._memoryLevel = self._weightMemoryLevel.name + weightMemoryOccupation += _ne16WeightBufferSize(buffer) + return ctxt, graph diff --git a/Deeploy/Targets/NE16/OptimizationPasses/__init__.py b/Deeploy/Targets/NE16/OptimizationPasses/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/OptimizationPasses/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/Parsers.py b/Deeploy/Targets/NE16/Parsers.py new file mode 100644 index 0000000000..3d157114fc --- /dev/null +++ b/Deeploy/Targets/NE16/Parsers.py @@ -0,0 +1,203 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Tuple + +import onnx_graphsurgeon as gs + +from Deeploy.DeeployTypes import NetworkContext +from Deeploy.Targets.Generic.Parsers import Conv2DParser, ConvParser, RQSParserInterface + + +class NE16Conv2DBaseParser(Conv2DParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + # No dilation support + self.operatorRepresentation['dilations'] == [1, 1], + # Channels have to be last + 'channels_first' in self.operatorRepresentation and not self.operatorRepresentation['channels_first'], + # Expect "weight_offset" attribute in the node + "weight_offset" in node.attrs, + ]): + return False + + self.operatorRepresentation['padding_y_top'] = int(self.operatorRepresentation['pads'][0]) + self.operatorRepresentation['padding_x_left'] = int(self.operatorRepresentation['pads'][1]) + self.operatorRepresentation['padding_y_bottom'] = int(self.operatorRepresentation['pads'][2]) + self.operatorRepresentation['padding_x_right'] = int(self.operatorRepresentation['pads'][3]) + self.operatorRepresentation['weight_offset'] = int(node.attrs["weight_offset"]) + + return True + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + # LMACAN: Cannot reuse the Conv2DParser's parserNodeCtxt because it requires the weight shape + # to be of length 4 whereas ne16 does a specific weight encoding so the shape + # ends up being equal to 3. + newCtxt, ret = ConvParser.parseNodeCtxt(self, ctxt, node, channels_first) + + if not ret: + return ctxt, False + + # LMACAN: c/p of Conv2DParser's parserNodeCtxt but with a different weight shape check + # and enforcing that the channels_first is false + data_in = newCtxt.lookup(self.operatorRepresentation['data_in']) + data_out = newCtxt.lookup(self.operatorRepresentation['data_out']) + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + + if not all([ + channels_first == False, + len(data_in.shape) == 4, + # LMACAN: weight shape should be equal to 3 because we have to do the ne16's + # special weight encoding. Dense 3x3 uses rank 4, + # PW/DW use rank 3. + len(weight.shape) in (3, 4), + ]): + return newCtxt, False + + self.operatorRepresentation['batch'] = data_in.shape[0] + self.operatorRepresentation['dim_im_in_x'] = data_in.shape[1] + self.operatorRepresentation['dim_im_in_y'] = data_in.shape[2] + self.operatorRepresentation['ch_im_in'] = data_in.shape[3] + self.operatorRepresentation['dim_im_out_x'] = data_out.shape[1] + self.operatorRepresentation['dim_im_out_y'] = data_out.shape[2] + self.operatorRepresentation['ch_im_out'] = data_out.shape[3] + + # No requantization + self.operatorRepresentation['mul'] = 'NULL' + self.operatorRepresentation['add'] = 'NULL' + self.operatorRepresentation['shift'] = 'NULL' + + return newCtxt, True + + +class NE16DWConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + # After NE16 weight encoding for DW, the encoded weight shape no longer + # carries cout==group (all channels are packed into the cinMinor + # dimension). Trust the ONNX `group` attribute alone: for DW, + # group > 1 AND group == channel_out AND kernel_shape == [3,3]. + if not all([ + self.operatorRepresentation['kernel_shape'] == [3, 3], + self.operatorRepresentation['group'] > 1, + ]): + return False + + return True + + +class NE16RQSDWConv2DParser(NE16DWConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16DWConv2DParser.parseNode(self, node), + ]) + + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True + + +class NE16PWConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + self.operatorRepresentation['kernel_shape'] == [1, 1], + self.operatorRepresentation['group'] == 1, + ]): + return False + + return True + + +class NE16RQSPWConv2DParser(NE16PWConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16PWConv2DParser.parseNode(self, node), + ]) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True + + +class NE16DenseConv2DParser(NE16Conv2DBaseParser): + + def parseNode(self, node: gs.Node) -> bool: + if not super().parseNode(node): + return False + + if not all([ + self.operatorRepresentation['kernel_shape'] == [3, 3], + self.operatorRepresentation['group'] == 1, + ]): + return False + + return True + + +class NE16RQSDenseConv2DParser(NE16DenseConv2DParser, RQSParserInterface): + + def parseNode(self, node: gs.Node) -> bool: + ret = all([ + RQSParserInterface.parseNode(self, node), + NE16DenseConv2DParser.parseNode(self, node), + ]) + return ret + + def parseNodeCtxt(self, + ctxt: NetworkContext, + node: gs.Node, + channels_first: bool = True) -> Tuple[NetworkContext, bool]: + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + + if not ret: + return ctxt, False + + inputs = ['data_in', 'weight', 'mul', 'add'] + for idx, inputNode in enumerate(node.inputs): + self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + + return newCtxt, True diff --git a/Deeploy/Targets/NE16/Platform.py b/Deeploy/Targets/NE16/Platform.py new file mode 100644 index 0000000000..665146030a --- /dev/null +++ b/Deeploy/Targets/NE16/Platform.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Optional + +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RequantizedGemmToPwPass +from Deeploy.DeeployTypes import TopologyOptimizer +from Deeploy.MemoryLevelExtension.MemoryLevels import MemoryHierarchy, MemoryLevel +from Deeploy.Targets.GAP9.Platform import GAP9ClusterEngine, GAP9ConstantBuffer, GAP9Platform, GAP9StructBuffer, \ + GAP9TransientBuffer, GAP9VariableBuffer, MemoryGAP9Platform, MemoryGAP9PlatformWrapper +from Deeploy.Targets.NE16.Engine import NE16Engine +from Deeploy.Targets.PULPOpen.Platform import PULPOptimizer + +NE16Optimizer = TopologyOptimizer([ + *PULPOptimizer.passes, + RequantizedGemmToPwPass(), +], name = "NE16Optimizer") + + +class NE16Platform(GAP9Platform): + + def __init__(self, + engines = None, + variableBuffer = GAP9VariableBuffer, + constantBuffer = GAP9ConstantBuffer, + structBuffer = GAP9StructBuffer, + transientBuffer = GAP9TransientBuffer) -> None: + if engines is None: + # Drop SDK NE16 headers from the cluster engine include list so the + # generated Network.c does not pull in CNN_BasicKernels_NE16.h / + # ne16_utils.h alongside pulp-nnx's ne16_task_defs.h + # (NE16_REG_* macros are defined in both, causing -Werror redefs). + cluster = GAP9ClusterEngine( + "GAP9Cluster", + includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CycleCounter.h" + ], + ) + engines = [NE16Engine("NE16"), cluster] + super().__init__(engines, variableBuffer, constantBuffer, structBuffer, transientBuffer) + + +class MemoryNE16Platform(MemoryGAP9Platform): + + def __init__(self, + memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel, + weightMemoryLevel: Optional[MemoryLevel] = None, + engines = None, + variableBuffer = GAP9VariableBuffer, + constantBuffer = GAP9ConstantBuffer, + structBuffer = GAP9StructBuffer, + transientBuffer = GAP9TransientBuffer) -> None: + if engines is None: + # Drop SDK NE16 headers from the cluster engine include list so the + # generated Network.c does not pull in CNN_BasicKernels_NE16.h / + # ne16_utils.h alongside pulp-nnx's ne16_task_defs.h + # (NE16_REG_* macros are defined in both, causing -Werror redefs). + cluster = GAP9ClusterEngine( + "GAP9Cluster", + includeList = [ + "pmsis.h", "DeeployGAP9Math.h", "pulp_nn_kernels.h", "DeeployMchan.h", "CNN_BasicKernels_fp32.h", + "CycleCounter.h" + ], + ) + engines = [NE16Engine("NE16"), cluster] + super().__init__(memoryHierarchy, defaultTargetMemoryLevel, engines, variableBuffer, constantBuffer, + structBuffer, transientBuffer) + self.weightMemoryLevel = weightMemoryLevel + + +class MemoryNE16PlatformWrapper(MemoryGAP9PlatformWrapper): + + def __init__(self, + platform: NE16Platform, + memoryHierarchy: MemoryHierarchy, + defaultTargetMemoryLevel: MemoryLevel, + weightMemoryLevel: Optional[MemoryLevel] = None): + assert isinstance(platform, NE16Platform), \ + f"Given platform is not an instance of NE16Platform. Platform type: {type(platform).__name__}" + super().__init__(platform, memoryHierarchy, defaultTargetMemoryLevel) + self.weightMemoryLevel = weightMemoryLevel diff --git a/Deeploy/Targets/NE16/Templates/AllocateTemplate.py b/Deeploy/Targets/NE16/Templates/AllocateTemplate.py new file mode 100644 index 0000000000..502b5af578 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/AllocateTemplate.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2023 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from Deeploy.DeeployTypes import NodeTemplate + +ne16GenericGlobalInitTemplate = NodeTemplate(""" +% if _memoryLevel == "L1": +static PI_L1 ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% elif _memoryLevel == "L2" or _memoryLevel is None: +static PI_L2 ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% elif _memoryLevel == "L3": +// ${name} is allocated in L3 \n +static PI_L2 ${type.referencedType.typeName}* ${name}; +% elif _memoryLevel == "WeightMemory_SRAM": +static __attribute__((section(".weightmem_sram"))) ${type.referencedType.typeName} ${name}[${size}] = {${values}};\n +% endif +""") diff --git a/Deeploy/Targets/NE16/Templates/ConvTemplate.py b/Deeploy/Targets/NE16/Templates/ConvTemplate.py new file mode 100644 index 0000000000..337f5e10c4 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/ConvTemplate.py @@ -0,0 +1,398 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from abc import abstractmethod +from typing import Dict, List, Tuple + +import numpy as np + +from Deeploy.DeeployTypes import ConstantBuffer, NetworkContext, NodeTemplate, OperatorRepresentation + + +def _getNumTiles(fullDim: int, tileDim: int) -> int: + return int(np.ceil(fullDim / tileDim)) + + +def _getBorderTileSize(fullDim: int, tileDim: int) -> int: + return fullDim % tileDim if fullDim % tileDim > 0 else tileDim + + +def ioStridesFromDimensions(width: int, channel: int, bits: int) -> Tuple[int, int]: + """stridesFromDimensions + Returns strides in bytes. + """ + width_stride = channel * bits // 8 + height_stride = width * width_stride + return height_stride, width_stride + + +def getNormQuantConf0(use_relu: bool, layerwise_output_shift: int, scale_bits: int, use_bias: bool, + use_shift: bool) -> int: + conf0 = 0 + conf0 |= 1 << 4 # Use Normalization and quantization + if scale_bits == 32: + conf0 |= 2 << 12 + conf0 |= layerwise_output_shift << 16 + if not use_relu: + conf0 |= 1 << 23 + if use_shift: + conf0 |= 1 << 24 + if use_bias: + conf0 |= 1 << 25 + return conf0 + + +def getInputAddrOffset(width_in: int, width_in_stride: int, padding_top: int, padding_left: int) -> int: + return (padding_top * width_in + padding_left) * width_in_stride + + +class NE16ConvTemplate(NodeTemplate): + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + @abstractmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + pass + + @classmethod + @abstractmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + pass + + @classmethod + @abstractmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + pass + + def alignToContext(self, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[NetworkContext, Dict, List[str]]: + data_in: ConstantBuffer = ctxt.lookup(operatorRepresentation['data_in']) + data_out: ConstantBuffer = ctxt.lookup(operatorRepresentation['data_out']) + weight: ConstantBuffer = ctxt.lookup(operatorRepresentation['weight']) + + operatorRepresentation['input_signed'] = data_in._type.referencedType.typeMin < 0 + operatorRepresentation['use_relu'] = data_out._type.referencedType.typeMin >= 0 + + operatorRepresentation['input_bits'] = data_in._type.referencedType.typeWidth + operatorRepresentation['output_bits'] = data_out._type.referencedType.typeWidth + operatorRepresentation['weight_bits'] = weight._type.referencedType.typeWidth + + operatorRepresentation["input_typeWidth_bytes"] = int(np.ceil(data_in._type.referencedType.typeWidth / 8)) + operatorRepresentation["output_typeWidth_bytes"] = int(np.ceil(data_out._type.referencedType.typeWidth / 8)) + + operatorRepresentation["weight_addr_offset"] = 0 + + operatorRepresentation["use_wmem"] = hasattr(weight, + "_memoryLevel") and weight._memoryLevel == "WeightMemory_SRAM" + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(operatorRepresentation["dim_im_in_y"], + operatorRepresentation["ch_im_in"], + operatorRepresentation["input_bits"]) + operatorRepresentation["dim_im_in_y_stride"] = dim_im_in_y_stride + operatorRepresentation["dim_im_in_x_stride"] = dim_im_in_x_stride + + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(operatorRepresentation["dim_im_out_y"], + operatorRepresentation["ch_im_out"], + operatorRepresentation["output_bits"]) + operatorRepresentation["dim_im_out_y_stride"] = dim_im_out_y_stride + operatorRepresentation["dim_im_out_x_stride"] = dim_im_out_x_stride + + operatorRepresentation["input_addr_offset"] = getInputAddrOffset(operatorRepresentation["dim_im_in_y"], + operatorRepresentation["dim_im_in_y_stride"], + operatorRepresentation["padding_y_top"], + operatorRepresentation["padding_x_left"]) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = self.getCounters( + operatorRepresentation["ch_im_in"], operatorRepresentation["dim_im_out_x"], + operatorRepresentation["dim_im_out_y"], operatorRepresentation["ch_im_out"], + operatorRepresentation["padding_y_bottom"], operatorRepresentation["padding_x_right"], + operatorRepresentation) + + operatorRepresentation["nKo"] = nKo + operatorRepresentation["nKi"] = nKi + operatorRepresentation["nHo"] = nHo + operatorRepresentation["nWo"] = nWo + operatorRepresentation["bKo"] = bKo + operatorRepresentation["bKi"] = bKi + operatorRepresentation["bHo"] = bHo + operatorRepresentation["bWo"] = bWo + operatorRepresentation["bHi"] = bHi + operatorRepresentation["bWi"] = bWi + + weightStrideD0, weightStrideD1, weightStrideD2 = self.getWeightStrides(operatorRepresentation["ch_im_in"]) + + operatorRepresentation["weightStrideD0"] = weightStrideD0 + operatorRepresentation["weightStrideD1"] = weightStrideD1 + operatorRepresentation["weightStrideD2"] = weightStrideD2 + + operatorRepresentation["conf0"] = self.getConf0(operatorRepresentation["output_bits"], + operatorRepresentation["weight_bits"], + operatorRepresentation["input_signed"], + operatorRepresentation["use_wmem"]) + + operatorRepresentation["wmem_addr_offset"] = 0x10400000 if operatorRepresentation["use_wmem"] else 0 + + operatorRepresentation["ne16_kernel_shape"] = self.NE16_KERNEL_SHAPE + operatorRepresentation["ne16_depthwise"] = self.NE16_IS_DEPTHWISE + operatorRepresentation["ne16_subtile_output_channel"] = self.NE16_SUBTILE_OUTPUT_CHANNEL + + # If requantized + if operatorRepresentation["mul"] != "NULL": + mulBuff = ctxt.lookup(operatorRepresentation["mul"]) + mulBits = mulBuff._type.referencedType.typeWidth + operatorRepresentation["conf0"] |= getNormQuantConf0(operatorRepresentation["use_relu"], + operatorRepresentation["log2D"], mulBits, "add" + in operatorRepresentation, False) + return ctxt, operatorRepresentation, [] + + +class NE162DPWConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 1 + NE16_IS_DEPTHWISE = 0 + NE16_SUBTILE_OUTPUT_CHANNEL = 32 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + # NE16 subtiles: INPUT_CHANNEL=16, OUTPUT_HxW=3x3, OUTPUT_CHANNEL=32 + n_channel_out_subtiles = _getNumTiles(channel_out, 32) + n_channel_in_subtiles = _getNumTiles(channel_in, 16) + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 32) + channel_in_border = _getBorderTileSize(channel_in, 16) + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border - padding_bottom + width_in_border = width_out_border - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # NE16 PW 1x1: per (cout, cinMajor) block = bits * H*W * cinMinorBytes + # = 8 * 1 * 2 = 16 bytes for 8-bit weights with CIN_SUBTILE=16 + n_channel_in = _getNumTiles(channel_in, 16) + _NE16_PW_WEIGHT_BYTES = 16 # bits * HW * cinMinorBytes = 8*1*2 + return _NE16_PW_WEIGHT_BYTES, _NE16_PW_WEIGHT_BYTES * n_channel_in, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + conf0 |= 2 << 5 # PW MODE + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +class NE162DDWConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 3 + NE16_IS_DEPTHWISE = 1 + # For DW, hardware replicates input channels as output channels, so the + # output-channel subtile size equals the input-channel subtile (16). + NE16_SUBTILE_OUTPUT_CHANNEL = 16 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels + + # NE16 DW 3x3: CIN_SUBTILE=16 single mode, output 3x3 + n_channel_out_subtiles = _getNumTiles(channel_out, 16) + n_channel_in_subtiles = n_channel_out_subtiles + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 16) + channel_in_border = channel_out_border + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border + 2 - padding_bottom + width_in_border = width_out_border + 2 - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # Match ne16_task_set_strides for depthwise 3x3: + # d0 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride + # = 3 * 3 * 2 = 18 + # d1 = 0 (DW has no cin-major striding from the HW's perspective). + _NE16_FILTER_SIZE = 3 + _NE16_WEIGHT_D0_STRIDE_MODE8 = 2 + d0 = _NE16_FILTER_SIZE * _NE16_FILTER_SIZE * _NE16_WEIGHT_D0_STRIDE_MODE8 + return d0, 0, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + conf0 |= 1 << 5 # DW MODE + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +class NE162DDenseConvTemplate(NE16ConvTemplate): + + NE16_KERNEL_SHAPE = 3 + NE16_IS_DEPTHWISE = 0 + NE16_SUBTILE_OUTPUT_CHANNEL = 32 + + def __init__(self, templateStr: str): + super().__init__(templateStr) + + @classmethod + def getCounters( + cls, channel_in: int, height_out: int, width_out: int, channel_out: int, padding_bottom: int, + padding_right: int, + operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: + _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels + + # NE16 Dense 3x3: CIN_SUBTILE=16, OUTPUT 3x3x32 + n_channel_out_subtiles = _getNumTiles(channel_out, 32) + n_channel_in_subtiles = _getNumTiles(channel_in, 16) + n_height_out_subtiles = _getNumTiles(height_out, 3) + n_width_out_subtiles = _getNumTiles(width_out, 3) + + channel_out_border = _getBorderTileSize(channel_out, 32) + channel_in_border = _getBorderTileSize(channel_in, 16) + height_out_border = _getBorderTileSize(height_out, 3) + width_out_border = _getBorderTileSize(width_out, 3) + height_in_border = height_out_border + 2 - padding_bottom + width_in_border = width_out_border + 2 - padding_right + + return (n_channel_out_subtiles, n_channel_in_subtiles, n_height_out_subtiles, n_width_out_subtiles, + channel_out_border, channel_in_border, height_out_border, width_out_border, height_in_border, + width_in_border) + + @classmethod + def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: + # Match ne16_task_set_strides for dense 3x3 (non-DW): + # d0 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride = 18 + # d1 = NE16_FILTER_SIZE * NE16_FILTER_SIZE * weight_d0_stride * qw * num_k_in + # = 18 * 8 * num_k_in + _NE16_FILTER_SIZE = 3 + _NE16_WEIGHT_D0_STRIDE_MODE8 = 2 + _QW = 8 + n_channel_in = _getNumTiles(channel_in, 16) + d0 = _NE16_FILTER_SIZE * _NE16_FILTER_SIZE * _NE16_WEIGHT_D0_STRIDE_MODE8 + d1 = d0 * _QW * n_channel_in + return d0, d1, 0 + + @classmethod + def getConf0(cls, output_bits: int, weight_bits: int, input_signed: bool, use_wmem: bool) -> int: + conf0 = 0 + conf0 |= weight_bits - 1 + if use_wmem: + conf0 |= 1 << 9 + conf0 |= 1 << 15 # Layerwise weight offset mode + if output_bits == 32: + conf0 |= 2 << 21 + if input_signed: + conf0 |= 1 << 26 + return conf0 + + +NE16TaskInitTemplateStr = """ +// N-EUREKA Task Init +ne16_task_t task = { + .data = (ne16_task_data_t) { + .weights_addr = (uint32_t)${weight} - ${wmem_addr_offset} + ${weight_addr_offset}, + .infeat_addr = (uint32_t)${data_in} - ${input_addr_offset}, + .outfeat_addr = (uint32_t)${data_out}, + .scale_addr = (uint32_t)${mul}, + .scale_shift_addr = (uint32_t)${shift}, + .scale_bias_addr = (uint32_t)${add}, + .cfg = (ne16_cfg_t) { + .input_stride = (ne16_stride_t) { + .d0 = ${dim_im_in_y_stride}, + .d1 = ${dim_im_in_x_stride}, + .d2 = 0 + }, + .output_stride = (ne16_stride_t) { + .d0 = NE16_OUTPUT_BANDWIDTH_BYTES, + .d1 = ${dim_im_out_y_stride}, + .d2 = ${dim_im_out_x_stride} + }, + task.data.cfg.weights_stride = (ne16_stride_t) { + .d0 = ${weightStrideD0}, + .d1 = ${weightStrideD1}, + .d2 = ${weightStrideD2} + }, + .subtile = (ne16_subtile_t) { + .number = { + .KoKi = nnx_concat_half(${nKo}, ${nKi}), + .HoWo = nnx_concat_half(${nHo}, ${nWo}) + }, + .remainder = { + .KoKi = nnx_concat_half(${bKo}, ${bKi}), + .HoWo = nnx_concat_half(${bHo}, ${bWo}), + .HiWi = nnx_concat_half(${bHi}, ${bWi}) + } + }, + .padding = (${padding_y_top} << 28) + (${padding_x_right} << 24) + (${padding_y_bottom} << 20) + (${padding_x_left} << 16), + .weight_offset_factor = ${weight_offset}, + .filter_mask = 0, + .conf0 = ${conf0}, + } + } +}; +// NE16 top-level task struct fields (required by HAL helpers and NE16 HW for +// non-1x1 paths). Kept consistent with ne16_task_set_op_to_conv/_set_bits. +task.weight_d0_stride = NE16_WEIGHT_D0_STRIDE_MODE8; +task.qw = ${weight_bits}; +task.subtile_output_channel = ${ne16_subtile_output_channel}; +task.kernel_shape = ${ne16_kernel_shape}; +task.depthwise = ${ne16_depthwise}; +""" + +NE16TaskExecutionTemplateStr = """ +// N-EUREKA Task Execution +ne16_nnx_dispatch_wait(ne16_pulp_get_dev()); +ne16_nnx_dispatch(ne16_pulp_get_dev(), &task); +ne16_nnx_resolve_wait(ne16_pulp_get_dev(), &task); +""" + +NE16RqntPWConv2D_Template = NE162DPWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16PWConv2D_Template = NE162DPWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) + +NE16RqntDWConv2D_Template = NE162DDWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16DWConv2D_Template = NE162DDWConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) + +NE16RqntDenseConv2D_Template = NE162DDenseConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) +NE16DenseConv2D_Template = NE162DDenseConvTemplate(NE16TaskInitTemplateStr + NE16TaskExecutionTemplateStr) diff --git a/Deeploy/Targets/NE16/Templates/__init__.py b/Deeploy/Targets/NE16/Templates/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/Templates/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py new file mode 100644 index 0000000000..f6ce899ff5 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16DenseConstraint.py @@ -0,0 +1,316 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DDenseConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme, calculateFlatOffsetInBytes + + +class NE16DenseConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + strides = parseDict["strides"] + padding = parseDict["pads"] + dilation = parseDict["dilations"] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 3) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + + weightBuffer = ctxt.lookup(weightBufferName) + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + else: + tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + + # serializeTilingSolution always emits the weight tile as + # HyperRectangle((COffset, 0, ...), (CSize,) + weightShape[1:]) -- only the + # output-channel dimension is tiled, the NE16-encoded tail is always moved + # whole. Leaving that tail unconstrained lets the solver reserve less L1 + # than the DMA actually writes: for 64x64 dense the tail is + # (cinMajor=4, bits=8, H*W*cinMinorBytes=18) and the solver picked 16 for + # the last dim, reserving 32*4*8*16 = 16384 B while the transfer is + # 32*4*8*18 = 18432 B. The extra 2048 B ran straight over the mul/add + # requant parameters that follow the weight buffer in the arena, so a + # matching 2048 B of the output came out wrong. + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + + inputBuffer = ctxt.lookup(inputBufferName) + + effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) + effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) + + tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) + tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 3) + + strides = parseDict["strides"] + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) + tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) + + tilerModel.addConstraint(inputHeightVar >= parseDict['dim_kernel_x']) + tilerModel.addConstraint(inputWidthVar >= parseDict['dim_kernel_y']) + + # NE16 computes TP_OUT=32 output channels per pass, so an output-channel + # tile that is not a multiple of 32 leaves the remaining lanes idle for + # the whole tile. Without a hint the solver is free to pick any Ko that + # fits (Ko=3 and Ko=56 have both been observed), which costs far more + # than the L1 it saves. GAP9's own AutoTiler feeds its solver the same + # preference via PreferedTileSize (Ki=16, Ko=32, spatial=3). This is a + # hint, not a hard constraint: shapes with Co < 32, or too tight an L1 + # budget, must still be tileable. + outputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 3) + tilerModel.addConstraint(outputChannelVar % 32 == 0, strategy = PerformanceHint(2)) + + # NE16's 9 columns retire one 3x3 output patch per pass, so an output + # tile whose H or W is not a multiple of 3 wastes part of the patch on + # its border pass -- GAP9's AutoTiler passes the same spatial + # PreferedTileSize of 3. Deliberately a *weaker* hint than the channel + # alignment above: splitting the spatial dimensions costs a 2-row/2-col + # halo re-fetch per extra tile, while splitting output channels costs + # nothing, so when L1 is tight this must give way. Measured: forcing it + # at the same priority as the channel hint made the double-buffered + # DW_2D_RQ kernel go 11,926 -> 19,391 cycles. + outHVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 1) + outWVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 2) + tilerModel.addConstraint(outHVar % 3 == 0, strategy = PerformanceHint(0)) + tilerModel.addConstraint(outWVar % 3 == 0, strategy = PerformanceHint(0)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, outputBuffer.shape) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DDenseConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + # NE16-encoded weight rank depends on conv kind: PW/DW are rank 3 + # (cout, cinMajor, bits * H*W * cinMinorBytes), Dense 3x3 is rank 4 + # (cout, cinMajor, bits, H*W * cinMinorBytes). The rectangle must match + # the buffer rank — otherwise _legalizeTransfers left-pads with size-1 + # dims and the cout offset shifts onto cinMajor, breaking double-buffer + # tiling on stride-2 Dense (cout > tile size). + weightRank = len(weightShape) + weightFullTail = tuple(weightShape[1:]) + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for absoluteCube in absoluteOutputCubes: + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + WeightCube = HyperRectangle((COffset,) + (0,) * (weightRank - 1), (CSize,) + weightFullTail) + replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + for cube, load in zip(outputCubes, inputLoadSchedule): + COffset, CSize = cube.offset[-1], cube.dims[-1] + load['weight'] = HyperRectangle((COffset,) + (0,) * (weightRank - 1), (CSize,) + weightFullTail) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSDenseConv2DTileConstraint(NE16DenseConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16DenseConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py new file mode 100644 index 0000000000..085bab8a00 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16DepthwiseConstraint.py @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DDWConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + + +class NE16DWConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + strides = parseDict["strides"] + padding = parseDict["pads"] + dilation = parseDict["dilations"] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + tilerModel.addConstraint(outputChannelVar == inputChannelVar) + + weightBuffer = ctxt.lookup(weightBufferName) + # NE16 DW weight is packed as a single (1, 1, packed_bytes) block + # containing all output channels (up to NE16_SUBTILE_INPUT_CHANNEL=16). + # Keep the outermost dim fixed at its full (=1) value regardless of + # the output channel tiling. + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + + tilerModel.addConstraint(inputHeightVar >= 3) + tilerModel.addConstraint(inputWidthVar >= 3) + + inputBuffer = ctxt.lookup(inputBufferName) + + effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) + effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) + + tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) + tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + + strides = parseDict["strides"] + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) + tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) + + # NE16 retires TP_OUT=32 output channels per pass; a channel tile that is + # not a multiple of 32 leaves the remaining lanes idle for the whole tile. + # GAP9's AutoTiler feeds its solver the same preference -- see + # CNN_Generators_NE16.c: `OutTileCons = CannotTileChannels ? OutFeat : 32`, + # which applies to depthwise and pointwise alike. A hint, not a hard + # constraint: shapes with C < 32, or a tight L1 budget, must stay tileable. + outputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 3) + tilerModel.addConstraint(outputChannelVar % 32 == 0, strategy = PerformanceHint(2)) + + # NE16's 9 columns retire one 3x3 output patch per pass, so an output + # tile whose H or W is not a multiple of 3 wastes part of the patch on + # its border pass -- GAP9's AutoTiler passes the same spatial + # PreferedTileSize of 3. Deliberately a *weaker* hint than the channel + # alignment above: splitting the spatial dimensions costs a 2-row/2-col + # halo re-fetch per extra tile, while splitting output channels costs + # nothing, so when L1 is tight this must give way. Measured: forcing it + # at the same priority as the channel hint made the double-buffered + # DW_2D_RQ kernel go 11,926 -> 19,391 cycles. + outHVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 1) + outWVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_out'], dimIdx = 2) + tilerModel.addConstraint(outHVar % 3 == 0, strategy = PerformanceHint(0)) + tilerModel.addConstraint(outWVar % 3 == 0, strategy = PerformanceHint(0)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, + ctxt.lookup(varOut).shape) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DDWConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for _ in absoluteOutputCubes: + # DW weight is a single packed block — no per-cout offset. + replacements['weight_addr_offset'].append(0) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + # DW weight is a single packed (1, 1, packed_bytes) block used + # across all output-channel tiles — same cube every iteration. + for _cube, load in zip(outputCubes, inputLoadSchedule): + load['weight'] = HyperRectangle((0, 0, 0), (weightShape[0], weightShape[1], weightShape[2])) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSDWConv2DTileConstraint(NE16DWConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16DWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py b/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py new file mode 100644 index 0000000000..60e4cb5327 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/NE16PointwiseConstraint.py @@ -0,0 +1,318 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple + +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.NE16.Templates.ConvTemplate import NE162DPWConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.NE16.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme, calculateFlatOffsetInBytes + + +class NE16PWConv2DTileConstraint(TileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + inputBufferName = parseDict['data_in'] + weightBufferName = parseDict['weight'] + outputBufferName = parseDict['data_out'] + + for bufferName in [inputBufferName, weightBufferName, outputBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + inputBatchVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 0) + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) + + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + # Map output dims to inputs dims + tilerModel.addConstraint(outputBatchVar == inputBatchVar) + tilerModel.addConstraint(outputHeightVar == inputHeightVar) + tilerModel.addConstraint(outputWidthVar == inputWidthVar) + + weightBuffer = ctxt.lookup(weightBufferName) + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) + else: + tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + + tilerModel.addConstraint(inputHeightVar >= 1) + tilerModel.addConstraint(inputWidthVar >= 1) + + return tilerModel + + @staticmethod + def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + + # Get to-be-tiled tensor's buffers + inputBuffer = ctxt.lookup(name = parseDict['data_in']) + weightBuffer = ctxt.lookup(name = parseDict['weight']) + outputBuffer = ctxt.lookup(name = parseDict['data_out']) + + inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 1) + inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 2) + inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 3) + + weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 0) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 1) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = weightBuffer.name, dimIdx = 2) + + outputHeightVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 1) + outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 2) + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 3) + + strides = parseDict["strides"] + padding = parseDict["pads"] + + # LMACAN: Force full input channel to avoid partial results + tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + + tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) + tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + + # N-EUREKA tile constraints to align with N-EUREKA's hardware subtiling + if parseDict["dim_im_out_x"] > 6: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_x", + outputHeightVar, + 6, + strategy = PerformanceHint(priority = 3)) + else: + tilerModel.addConstraint(outputHeightVar == outputHeightVar.Max(), strategy = PerformanceHint(priority = 3)) + + if parseDict["dim_im_out_y"] > 6: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "dim_im_out_y", + outputWidthVar, + 6, + strategy = PerformanceHint(priority = 2)) + else: + tilerModel.addConstraint(outputWidthVar == outputWidthVar.Max(), strategy = PerformanceHint(priority = 2)) + + if parseDict["ch_im_out"] > 32: + tilerModel.addTileSizeDivisibleConstraint(parseDict, + "ch_im_out", + outputChannelVar, + 32, + strategy = PerformanceHint(priority = 1)) + else: + tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), + strategy = PerformanceHint(priority = 1)) + + # NE16 retires TP_OUT=32 output channels per pass; a channel tile that is + # not a multiple of 32 leaves the remaining lanes idle for the whole tile. + # GAP9's AutoTiler feeds its solver the same preference -- see + # CNN_Generators_NE16.c: `OutTileCons = CannotTileChannels ? OutFeat : 32`, + # which applies to depthwise and pointwise alike. A hint, not a hard + # constraint: shapes with C < 32, or a tight L1 budget, must stay tileable. + tilerModel.addConstraint(outputChannelVar % 32 == 0, strategy = PerformanceHint(2)) + + # NE16's 9 columns retire one 3x3 output patch per pass, so an output + # tile whose H or W is not a multiple of 3 wastes part of the patch on + # its border pass -- GAP9's AutoTiler passes the same spatial + # PreferedTileSize of 3. Deliberately a *weaker* hint than the channel + # alignment above: splitting the spatial dimensions costs a 2-row/2-col + # halo re-fetch per extra tile, while splitting output channels costs + # nothing, so when L1 is tight this must give way. Measured: forcing it + # at the same priority as the channel hint made the double-buffered + # DW_2D_RQ kernel go 11,926 -> 19,391 cycles. + tilerModel.addConstraint(outputHeightVar % 3 == 0, strategy = PerformanceHint(0)) + tilerModel.addConstraint(outputWidthVar % 3 == 0, strategy = PerformanceHint(0)) + + return tilerModel + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + addrNames = ['data_in', 'data_out'] + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, addrNames) + + varWeight = operatorRepresentation['weight'] + varOut = operatorRepresentation['data_out'] + + inputInCubes = [] + replacements: Dict[str, List[int]] = { + "padding_y_top": [], + "padding_y_bottom": [], + "padding_x_left": [], + "padding_x_right": [], + "dim_im_in_x_stride": [], + "dim_im_in_y_stride": [], + "dim_im_out_x_stride": [], + "dim_im_out_y_stride": [], + "input_addr_offset": [], + "nKo": [], + "nKi": [], + "nHo": [], + "nWo": [], + "bKo": [], + "bKi": [], + "bHo": [], + "bWo": [], + "bHi": [], + "bWi": [], + } + + replacementTypes = { + "padding_y_top": PointerClass(uint8_t), + "padding_y_bottom": PointerClass(uint8_t), + "padding_x_left": PointerClass(uint8_t), + "padding_x_right": PointerClass(uint8_t), + "dim_im_in_x_stride": PointerClass(uint32_t), + "dim_im_in_y_stride": PointerClass(uint32_t), + "dim_im_out_x_stride": PointerClass(uint32_t), + "dim_im_out_y_stride": PointerClass(uint32_t), + "input_addr_offset": PointerClass(uint32_t), + "nKo": PointerClass(uint16_t), + "nKi": PointerClass(uint16_t), + "nHo": PointerClass(uint16_t), + "nWo": PointerClass(uint16_t), + "bKo": PointerClass(uint16_t), + "bKi": PointerClass(uint16_t), + "bHo": PointerClass(uint16_t), + "bWo": PointerClass(uint16_t), + "bHi": PointerClass(uint16_t), + "bWi": PointerClass(uint16_t), + } + + weightH = operatorRepresentation['dim_kernel_y'] + weightW = operatorRepresentation['dim_kernel_x'] + weightC = operatorRepresentation['ch_im_in'] + + pads = operatorRepresentation['pads'] + strides = operatorRepresentation['strides'] + + outputBuffer = ctxt.lookup(varOut) + assert isinstance(outputBuffer, VariableBuffer) + + for cube in outputCubes: + (BatchOffset, HOffset, WOffset, COffset) = cube.offset + (BatchSize, HSize, WSize, CSize) = cube.dims + + InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, + cube, outputBuffer.shape) + padding_left, padding_right, padding_top, padding_bottom = padding_tuple + + replacements['padding_y_top'].append(padding_top) + replacements['padding_y_bottom'].append(padding_bottom) + replacements['padding_x_left'].append(padding_left) + replacements['padding_x_right'].append(padding_right) + + inBSize, inHSize, inWSize, inCSize = InCube.dims + + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, + operatorRepresentation["input_bits"]) + replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) + replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, + operatorRepresentation["output_bits"]) + replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) + replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) + + replacements['input_addr_offset'].append( + getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) + + nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = NE162DPWConvTemplate.getCounters( + inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) + + replacements["nKo"].append(nKo) + replacements["nKi"].append(nKi) + replacements["nHo"].append(nHo) + replacements["nWo"].append(nWo) + replacements["bKo"].append(bKo) + replacements["bKi"].append(bKi) + replacements["bHo"].append(bHo) + replacements["bWo"].append(bWo) + replacements["bHi"].append(bHi) + replacements["bWi"].append(bWi) + + inputInCubes.append(InCube) + + inputLoadSchedule = [] + outputLoadSchedule = [] + + for a in inputInCubes: + inputLoadSchedule.append({"data_in": a}) + + for out in outputCubes: + outputLoadSchedule.append({"data_out": out}) + + weightBuffer = ctxt.lookup(varWeight) + assert isinstance(weightBuffer, VariableBuffer) + weightShape = weightBuffer.shape + + if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + replacements['weight_addr_offset'] = [] + replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) + for absoluteCube in absoluteOutputCubes: + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + else: + inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['weight']) + inputBaseOffsets.update(inputWeightBaseOffsets) + outputBaseOffsets.update(outputWeightBaseOffsets) + + for cube, load in zip(outputCubes, inputLoadSchedule): + COffset, CSize = cube.offset[-1], cube.dims[-1] + load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) + + return variableReplacementSchedule, tilingSchedule + + +class NE16RQSPWConv2DTileConstraint(NE16PWConv2DTileConstraint): + + @staticmethod + def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: + tilerModel = NE16PWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + addrNames = ['mul', 'add'] + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + addrNames) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py b/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py new file mode 100644 index 0000000000..e1e4b16aea --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/RequantHelpers.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List + +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation +from Deeploy.TilingExtension.TilerModel import TilerModel +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle + + +def requantAddGeometricalConstraint(tilerModel: TilerModel, operatorRepresentation: OperatorRepresentation, + ctxt: NetworkContext) -> TilerModel: + outputBufferName = operatorRepresentation['data_out'] + mulBufferName = operatorRepresentation['mul'] + addBufferName = operatorRepresentation['add'] + + # Add I/O dimensions to the model as variables + for bufferName in [mulBufferName, addBufferName]: + tilerModel.addTensorDimToModel(ctxt, bufferName) + + outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 3) + + addBuffer = ctxt.lookup(addBufferName) + addChannelVar = tilerModel.getTensorDimVar(tensorName = addBufferName, dimIdx = len(addBuffer.shape) - 1) + mulBuffer = ctxt.lookup(mulBufferName) + mulChannelVar = tilerModel.getTensorDimVar(tensorName = mulBufferName, dimIdx = len(mulBuffer.shape) - 1) + + tilerModel.addConstraint(outputChannelVar == addChannelVar) + tilerModel.addConstraint(outputChannelVar == mulChannelVar) + + return tilerModel + + +def requantLoadSchedule( + absoluteOutputCubes: List[AbsoluteHyperRectangle], + ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation, +) -> List[Dict[str, HyperRectangle]]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + shapeMul = ctxt.lookup(operatorRepresentation["mul"]).shape + shapeAdd = ctxt.lookup(operatorRepresentation["add"]).shape + + schedule = [] + for cube in outputCubes: + (_, _, _, COffset) = cube.offset + (_, _, _, CSize) = cube.dims + MulCube = HyperRectangle((0,) * (len(shapeMul) - 1) + (COffset,), (1,) * (len(shapeMul) - 1) + (CSize,)) + AddCube = HyperRectangle((0,) * (len(shapeAdd) - 1) + (COffset,), (1,) * (len(shapeAdd) - 1) + (CSize,)) + schedule.append({"mul": MulCube, "add": AddCube}) + + return schedule diff --git a/Deeploy/Targets/NE16/TileConstraints/__init__.py b/Deeploy/Targets/NE16/TileConstraints/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/TileConstraints/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/Tiler.py b/Deeploy/Targets/NE16/Tiler.py new file mode 100644 index 0000000000..2bc53a441a --- /dev/null +++ b/Deeploy/Targets/NE16/Tiler.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + + +from Deeploy.Targets.NE16.Bindings import NE16DenseConv2DBindings, NE16DWConv2DBindings, NE16PWConv2DBindings, \ + NE16RQSDenseConv2DBindings, NE16RQSDWConv2DBindings, NE16RQSPWConv2DBindings +from Deeploy.Targets.NE16.TileConstraints.NE16DenseConstraint import NE16DenseConv2DTileConstraint, \ + NE16RQSDenseConv2DTileConstraint +from Deeploy.Targets.NE16.TileConstraints.NE16DepthwiseConstraint import NE16DWConv2DTileConstraint, \ + NE16RQSDWConv2DTileConstraint +from Deeploy.Targets.NE16.TileConstraints.NE16PointwiseConstraint import NE16PWConv2DTileConstraint, \ + NE16RQSPWConv2DTileConstraint +from Deeploy.TilingExtension.TilerExtension import TilingReadyNodeBindings + +NE16RQSPWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSPWConv2DBindings, + tileConstraint = NE16RQSPWConv2DTileConstraint()) +NE16PWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16PWConv2DBindings, + tileConstraint = NE16PWConv2DTileConstraint()) + +NE16RQSDWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSDWConv2DBindings, + tileConstraint = NE16RQSDWConv2DTileConstraint()) +NE16DWConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16DWConv2DBindings, + tileConstraint = NE16DWConv2DTileConstraint()) + +NE16RQSDenseConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16RQSDenseConv2DBindings, + tileConstraint = NE16RQSDenseConv2DTileConstraint()) +NE16DenseConv2DTilingReadyBindings = TilingReadyNodeBindings(nodeBindings = NE16DenseConv2DBindings, + tileConstraint = NE16DenseConv2DTileConstraint()) diff --git a/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py new file mode 100644 index 0000000000..5b83fa7f8f --- /dev/null +++ b/Deeploy/Targets/NE16/TopologyOptimizationPasses/Passes.py @@ -0,0 +1,334 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import itertools +import math +from functools import partial +from typing import Generator, List, Tuple + +import numpy as np +import numpy.typing as npt +import onnx_graphsurgeon as gs + +from Deeploy.CommonExtensions.OptimizationPasses.Matchers import Match, NonBranchingMatcher +from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, SequentialPass, \ + contextagnostic +from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ + RemoveGlobalOutputReshapePass, _appendTranspose, _createReshape, _transformLayoutPermutation +from Deeploy.EngineExtension.OptimizationPasses.TopologyOptimizationPasses.EngineColoringPasses import \ + EngineDiscolorationPass +from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import ReshapeConstOptPass, ReshapeMergePass + + +def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = False) -> npt.NDArray[np.uint8]: + """NE16 weight encoder, ported from pulp-nnx/test/Ne16Weight.py. + + Expected weight shape: (cout, cin, H, W). + Output layout: (cout, cinMajor, Bits, H*W, cinMinorBytes) where + CIN_SUBTILE = 16 (single mode, no 1x1 vs 3x3 split like Neureka). + """ + _NE16_CIN_SUBTILE = 16 + + if depthwise: + weight = weight.transpose(1, 0, 2, 3) # Swap cout and cin + + cout, cin, height, width = weight.shape + + # Pad cin to be divisible with CIN_SUBTILE + if cin % _NE16_CIN_SUBTILE != 0: + cinPad = _NE16_CIN_SUBTILE - cin % _NE16_CIN_SUBTILE + weight = np.pad( + weight, + ((0, 0), (0, cinPad), (0, 0), (0, 0)), + "constant", + constant_values = 0, + ) + cin = cin + cinPad + + cinMajor = cin // _NE16_CIN_SUBTILE + cinMinor = _NE16_CIN_SUBTILE + + # (cout, cinMajor, cinMinor, H*W, 1) + weight = weight.reshape(cout, cinMajor, cinMinor, height * width, 1) + # (cout, cinMajor, cinMinor, H*W, Bits) + weight = np.unpackbits(weight, axis = -1, count = bits, bitorder = "little") + # (cout, cinMajor, Bits, H*W, cinMinor) + weight = weight.transpose(0, 1, 4, 3, 2) + # Pack cinMinor bits into bytes — 16 bits = 2 bytes + weight = weight.reshape(-1, 8) + weight = np.packbits(weight, axis = -1, bitorder = "little") + cinMinorBytes = cinMinor // 8 + # Layout rank varies by conv kind: + # - Dense 3x3 (!depthwise, kernel 3x3): rank 4 + # (cout, cinMajor, Bits, H*W*cinMinorBytes) + # — NE16DenseConstraint tiles over weight.shape[3]. + # - PW 1x1 and DW 3x3: rank 3 + # (cout, cinMajor, Bits*H*W*cinMinorBytes) + # — NE16{Pointwise,Depthwise}Constraint don't need a bits dim. + if not depthwise and height == 3 and width == 3: + return weight.reshape(cout, cinMajor, bits, height * width * cinMinorBytes) + return weight.reshape(cout, cinMajor, bits * height * width * cinMinorBytes) + + +def _ne16_adjust_weight_memory_layout_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + ne16EngineName: str): + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + if not ("engine" in node.attrs and node.attrs["engine"] == ne16EngineName): + return graph + + weightTensor = node.inputs[1] + + if not isinstance(weightTensor, gs.Constant): + return graph + + # Adjust N-EUREKA's weights + values = weightTensor.values + + # Extract weight offset and translate weights by the offset + weight_offset = values.min() + values = values - weight_offset + node.attrs["weight_offset"] = weight_offset + + if "channels_first" in node.attrs: + channels_first = node.attrs["channels_first"] + else: + channels_first = default_channels_first + + # Weight encode expects channels-first (cout, cin_per_group, H, W) + if not channels_first: + values = values.transpose(0, 3, 1, 2) + + bits = 8 # Support only 8 bit weights for now + if node.attrs['group'] == 1: + weightTensor.values = _weightEncode(values.astype(np.uint8), bits, depthwise = False) + else: + # Depthwise: Deeploy's NHWC pass leaves weight as + # (cin_per_group=1, cout=group, H, W) after the transpose above; + # Ne16Weight.py's encode expects standard (cout, cin_per_group, H, W) + # — swap axes 0/1 before encoding so the result is a single packed + # (1, 1, packed_bytes) block across up to NE16_SUBTILE_INPUT_CHANNEL=16 + # parallel output channels. + values = values.transpose(1, 0, 2, 3) + weightTensor.values = _weightEncode(values.astype(np.uint8), bits, depthwise = True) + weightTensor.name = f"{name}_{weightTensor.name}" + + return graph + + +@contextagnostic +class NE16AdjustWeightMemoryLayoutPass(ReplaceSequentialPatternPass): + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__( + graph, + partial(_ne16_adjust_weight_memory_layout_fun, + default_channels_first = default_channels_first, + ne16EngineName = ne16EngineName), "_NE16_ADJUST_WEIGHT_MEMORY_LAYOUT_PASS", + NonBranchingMatcher(regex_op = True)) + + +def _findAllMultiplicands(x: int) -> List[int]: + multiplicands = [] + tmpX = x + for i in range(2, math.ceil(math.sqrt(x))): # Ceil cause range doesn't include the last number + while tmpX % i == 0: + multiplicands.append(i) + tmpX = tmpX / i + + if x // math.prod(multiplicands) > 1: + multiplicands.append(x // math.prod(multiplicands)) + + return multiplicands + + +def _findAllReshapeOptions(dim: int) -> Generator[Tuple[int, int], None, None]: + multiplicands = _findAllMultiplicands(dim) + for combLen in range(1, 1 + (len(multiplicands) // 2)): + for comb in itertools.combinations(multiplicands, combLen): + a = math.prod(comb) + b = dim // a + yield a, b + + +def _nSubtiles(dims: Tuple[int, int]): + return math.ceil(dims[0] / 6) * math.ceil(dims[1] / 6) + + +def _findLowestNumberOfSubtilesReshapeOptions(dim: int) -> List[Tuple[int, int]]: + lowestNumberOfSubtiles = dim + bestOptions: List[Tuple[int, int]] = [(dim, 1)] + for option in _findAllReshapeOptions(dim): + nSubtiles = _nSubtiles(option) + if nSubtiles < lowestNumberOfSubtiles: + lowestNumberOfSubtiles = nSubtiles + bestOptions = [option] + elif nSubtiles == lowestNumberOfSubtiles: + bestOptions.append(option) + return bestOptions + + +def _bestReshapeOption(dim: int) -> Tuple[int, int]: + smallestDim = dim + biggestDim = 1 + for option in _findLowestNumberOfSubtilesReshapeOptions(dim): + if option[0] < smallestDim: + smallestDim = option[0] + biggestDim = option[1] + elif option[1] < smallestDim: + smallestDim = option[1] + biggestDim = option[0] + return biggestDim, smallestDim + + +def _ne16_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + ne16EngineName: str): + matched_nodes = list(match.nodes_map.values()) + node = matched_nodes[0] + + if not ("engine" in node.attrs and node.attrs["engine"] == ne16EngineName): + return graph + + if not (node.attrs["kernel_shape"] == [1, 1]): + return graph + + if "channels_first" in node.attrs: + channels_first = node.attrs["channels_first"] + else: + channels_first = default_channels_first + + def extractSpatialDims(shape: List[int]) -> List[int]: + if channels_first: + return shape[-2:] + else: + return shape[-3:-1] + + def replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int]) -> List[int]: + if channels_first: + return shape[:-2] + list(newSpatialDims) + else: + return shape[:-3] + list(newSpatialDims) + shape[-1:] + + _input = node.inputs[0] + spatialDims = extractSpatialDims(_input.shape) + newSpatialDims = _bestReshapeOption(math.prod(spatialDims)) + newInputShape = replaceSpatialDims(_input.shape, newSpatialDims) + + inputReshapeNode, reshapedInput = _createReshape(_input, name, newInputShape) + graph.nodes.append(inputReshapeNode) + node.inputs[0] = reshapedInput + + output = node.outputs[0] + newOutputShape = replaceSpatialDims(output.shape, newSpatialDims) + reshapedOutput = gs.Variable(output.name + "_Reshaped", dtype = output.dtype, shape = newOutputShape) + outputReshapeNode, _ = _createReshape(reshapedOutput, name, output.shape, output) + graph.nodes.append(outputReshapeNode) + node.outputs[0] = reshapedOutput + + return graph + + +@contextagnostic +class NE16ReshapePointwiseConvolutionPass(ReplaceSequentialPatternPass): + """Reshape pointwise convolution's spatial dimensions so that they work better for N-EUREKA's hardware tiling""" + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__( + graph, + partial(_ne16_reshape_pointwise_convolution_fun, + default_channels_first = default_channels_first, + ne16EngineName = ne16EngineName), "_NE16_RESHAPE_POINTWISE_CONVOLUTION_PASS", + NonBranchingMatcher(regex_op = True)) + + +class ConvEngineDiscolorationPass(EngineDiscolorationPass): + + def __init__(self): + pattern = gs.Graph() + _input = gs.Variable(name = 'input') + output = pattern.layer(inputs = [_input], outputs = ['output'], op = 'RequantizedConv|Conv', name = 'conv') + pattern.outputs.append(output) + pattern.inputs.append(_input) + super().__init__(pattern, "_CONV_ENGINE_DISCOLORATION_PASS", matcher = NonBranchingMatcher(regex_op = True)) + + +def _ne16_dw_layout_fixup_fun(graph: gs.Graph, match: Match, name: str, ne16EngineName: str): + """Convert NE16-colored DW conv from PULP NHWC layout to NE16 NHWC layout. + + After PULPNCHWtoNHWCPass runs, every DW conv has: + - weight in PULP NHWC layout (cout, H, W, cin/g) + - input NOT transposed (PULP DW kernel convention) + + NE16 DW expects: + - weight in NE16 NHWC layout (cin/g=1, H, W, cout) + - input in NHWC layout + + For NE16-colored DW convs we do both adjustments here. Cluster-colored + DW convs (e.g. stride-2 fallbacks when --enable-3x3 is on) are left + untouched, so the PULP cluster path still works. + """ + node = list(match.nodes_map.values())[0] + if node.op not in ("Conv", "RequantizedConv"): + return graph + if node.attrs.get("group", 1) == 1: + return graph + if node.attrs.get("engine") != ne16EngineName: + return graph + if len(node.inputs) < 2 or not isinstance(node.inputs[1], gs.Constant): + return graph + + weightTensor = node.inputs[1] + if weightTensor.values.ndim != 4: + return graph + + # Weight: (cout, H, W, cin/g=1) -> (cin/g=1, H, W, cout) + weightTensor.values = weightTensor.values.transpose(3, 1, 2, 0) + + # PULP DW NHWC doesn't insert an input transpose; NE16 DW needs NHWC input. + tensorIn = node.inputs[0] + spatialDims = 2 + permuteIn = _transformLayoutPermutation(len(tensorIn.shape), spatialDims, False) + graph.nodes.append(_appendTranspose(tensorIn, node, permuteIn)) + + return graph + + +@contextagnostic +class NE16DwLayoutFixupPass(ReplaceSequentialPatternPass): + + def __init__(self, ne16EngineName: str): + graph = gs.Graph() + _input = gs.Variable(name = 'input_1') + output = graph.layer(inputs = [_input], outputs = ['out'], op = 'RequantizedConv|Conv', name = 'node') + graph.outputs.append(output) + graph.inputs.append(_input) + + super().__init__(graph, partial(_ne16_dw_layout_fixup_fun, ne16EngineName = ne16EngineName), + "_NE16_DW_LAYOUT_FIXUP_PASS", NonBranchingMatcher(regex_op = True)) + + +@contextagnostic +class NE16OptimizationPass(SequentialPass): + + def __init__(self, default_channels_first: bool, ne16EngineName: str): + super().__init__(NE16DwLayoutFixupPass(ne16EngineName), + NE16AdjustWeightMemoryLayoutPass(default_channels_first, ne16EngineName), + NE16ReshapePointwiseConvolutionPass(default_channels_first, ne16EngineName), + ReshapeMergePass(), + ReshapeConstOptPass(), + RemoveGlobalOutputReshapePass(), + name_prefix = '') diff --git a/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py b/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/TopologyOptimizationPasses/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/NE16/__init__.py b/Deeploy/Targets/NE16/__init__.py new file mode 100644 index 0000000000..be436b64a3 --- /dev/null +++ b/Deeploy/Targets/NE16/__init__.py @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from . import * diff --git a/Deeploy/Targets/PULPOpen/Bindings.py b/Deeploy/Targets/PULPOpen/Bindings.py index 2c78978e23..2a68c3333c 100644 --- a/Deeploy/Targets/PULPOpen/Bindings.py +++ b/Deeploy/Targets/PULPOpen/Bindings.py @@ -453,12 +453,15 @@ BasicQuantBindings = [ NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(int8_t)]), QuantTemplate.referenceTemplate, ForkTransformer), + NodeBinding(QuantChecker([PointerClass(float32_t)], [PointerClass(uint8_t)]), QuantTemplate.referenceTemplate, + ForkTransformer), ] BasicDequantBindings = [ NodeBinding(DequantChecker([PointerClass(int8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, ForkTransformer), -] + [ + NodeBinding(DequantChecker([PointerClass(uint8_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, + ForkTransformer), NodeBinding(DequantChecker([PointerClass(int32_t)], [PointerClass(float32_t)]), DequantTemplate.referenceTemplate, ForkTransformer), ] diff --git a/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py b/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py index 59499706e5..280cb4ff6e 100644 --- a/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py +++ b/Deeploy/Targets/PULPOpen/Templates/FloatGemmTemplate.py @@ -4,7 +4,10 @@ from typing import Dict, List, Tuple -from Deeploy.AbstractDataTypes import float32_tPtr +from Deeploy.AbstractDataTypes import PointerClass +from Deeploy.CommonExtensions.DataTypes import float32_t + +float32_tPtr = PointerClass(float32_t) from Deeploy.DeeployTypes import NetworkContext, NodeTemplate, OperatorRepresentation diff --git a/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py b/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py index 8a0c1b9b54..5ec56e375f 100644 --- a/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py +++ b/Deeploy/TilingExtension/CodeTransformationPasses/TilingHoistingMixIn.py @@ -60,7 +60,18 @@ def _hoistValues(self, else: cb._type = PointerClass(BasicDataTypes.minimalIntegerType(values)) cb._instance = cb._type(cb.name, ctxt) - cb._memoryLevel = self.memory + # These are constant tile *control* tables (numTiles / DMA cmd / size / + # dims / offsets) read by the (cluster) controller to drive the tiling + # loop and program DMAs -- not bulk tile data. Putting them in the + # innermost tile memory (L1/TCDM) wastes scarce L1 and, on GAP9, places + # them in the contended L1 region next to the cluster master stack: a + # deep stack write can clobber a single table entry, turning a DMA `cmd` + # into a garbage code pointer so mchan_transfer_wait() hangs forever + # (observed on MobileNetV1 training). Keep them in the controller- + # addressable outer memory (L2) instead. Only redirect the L2->L1 pass; + # the L3->L2 pass keeps its tables in L2 (== self.memory), never L3. + # Platforms that don't tile into a level named "L1" are unaffected. + cb._memoryLevel = "L2" if self.memory == "L1" else self.memory return cb def _hoistReference(self, diff --git a/DeeployTest/CMakeLists.txt b/DeeployTest/CMakeLists.txt index b7f3535790..4e45904541 100644 --- a/DeeployTest/CMakeLists.txt +++ b/DeeployTest/CMakeLists.txt @@ -50,7 +50,7 @@ elseif(DEEPLOY_ARCH STREQUAL SNITCH) add_subdirectory(Platforms/Snitch) elseif(DEEPLOY_ARCH STREQUAL CHIMERA) add_subdirectory(Platforms/Chimera) -elseif(platform STREQUAL GAP9) +elseif(platform STREQUAL GAP9 OR platform STREQUAL GAP9_w_NE16) # Search for hex files generated by Python code generator # These files indicate L3 mode (external memory with readfs) diff --git a/DeeployTest/Platforms/GAP9/CMakeLists.txt b/DeeployTest/Platforms/GAP9/CMakeLists.txt index cbb6382329..19308b2082 100644 --- a/DeeployTest/Platforms/GAP9/CMakeLists.txt +++ b/DeeployTest/Platforms/GAP9/CMakeLists.txt @@ -17,6 +17,9 @@ add_deeploy_executable(${ProjectId} EXCLUDE_FROM_ALL ${SOURCES}) # add_executable(${ProjectId} ${SOURCES}) target_include_directories(${ProjectId} PRIVATE ${CMAKE_CURRENT_LIST_DIR}/inc) +# Network.c needs CycleCounter.h when --profileTiling is enabled +target_include_directories(network PRIVATE ${CMAKE_CURRENT_LIST_DIR}/inc) + target_link_libraries(${ProjectId} PRIVATE network deeploylib) target_compile_options(${ProjectId} INTERFACE network) add_gvsoc_emulation(${ProjectId} "gap9.evk") @@ -26,12 +29,17 @@ if(POWER_MEASUREMENT) target_compile_definitions(${ProjectId} PRIVATE POWER_MEASUREMENT) endif() +if(SLAVESTACKSIZE) + target_compile_definitions(${ProjectId} PRIVATE SLAVESTACKSIZE=${SLAVESTACKSIZE}) +endif() + # RW: Waive sign comparison warnings from pulp_nn_utils.h target_compile_options(network PRIVATE -Wno-sign-compare -Wno-pointer-sign -Wno-unknown-pragmas -Wno-error + -O3 ) target_link_options(${ProjectId} PRIVATE diff --git a/DeeployTest/Platforms/GAP9/src/deeploytest.c b/DeeployTest/Platforms/GAP9/src/deeploytest.c index 77fe46a4e9..511ff9e333 100644 --- a/DeeployTest/Platforms/GAP9/src/deeploytest.c +++ b/DeeployTest/Platforms/GAP9/src/deeploytest.c @@ -14,7 +14,12 @@ #include "testoutputs.h" // RW: Remove MAINSTACKSIZE because gap9-sdk does not use it +// Allow -DSLAVESTACKSIZE= from CMake to override this; an unconditional +// #define here would shadow the command-line one and trip "redefined" under +// -Werror. +#ifndef SLAVESTACKSIZE #define SLAVESTACKSIZE 3800 +#endif #ifdef POWER_MEASUREMENT unsigned int GPIOs = 89; diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz new file mode 100644 index 0000000000..cfd8568bf1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx new file mode 100644 index 0000000000..c357ce307e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz new file mode 100644 index 0000000000..76f10c49d9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz new file mode 100644 index 0000000000..ffde610d3a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx new file mode 100644 index 0000000000..8ef6c9a35f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz new file mode 100644 index 0000000000..adada12411 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Dense_2D_RQ_NE16Bench/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx new file mode 100644 index 0000000000..05a2c65c6b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz new file mode 100644 index 0000000000..e70621d012 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_DW_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx new file mode 100644 index 0000000000..489a855d1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz new file mode 100644 index 0000000000..d30262cefc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_Dense_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx new file mode 100644 index 0000000000..8b3e63eb1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz new file mode 100644 index 0000000000..d43a29edfc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/NE16Bench_PW_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..ebdff4c497 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx new file mode 100644 index 0000000000..cc2034f982 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..834ebeebf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_128x128_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..17ad32958c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx new file mode 100644 index 0000000000..8d66631f17 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..dae1e7f7c6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..4689dac206 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx new file mode 100644 index 0000000000..939d536f4c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..afbcae147d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l1116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz new file mode 100644 index 0000000000..73c859a6b7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx new file mode 100644 index 0000000000..6a3b7e6c12 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz new file mode 100644 index 0000000000..04aa02ef34 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_64x64_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..210c550f10 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx new file mode 100644 index 0000000000..3fa75b2ee4 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..1165e217dd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_16x16_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz new file mode 100644 index 0000000000..e67bd709cb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx new file mode 100644 index 0000000000..b103e9ebe2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz new file mode 100644 index 0000000000..671f169239 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz new file mode 100644 index 0000000000..e67bd709cb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx new file mode 100644 index 0000000000..b103e9ebe2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz new file mode 100644 index 0000000000..671f169239 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_128x128_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..bf0e6ad49f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..ed513954f6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..06ca0a0378 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..e4890b3de9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..a12370a7f3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..f1cea1b787 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..962c196cf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..5cb3651be7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..dd3adf1f2b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..962c196cf3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx new file mode 100644 index 0000000000..5cb3651be7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..dd3adf1f2b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_2x2_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1120000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..a1c43ee634 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..4570dd25b2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx new file mode 100644 index 0000000000..b3a32d8643 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..2d9dbb8d1e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..7bf6342548 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx new file mode 100644 index 0000000000..673e60171f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..f0c45e85bc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..332fac9c5a Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx new file mode 100644 index 0000000000..d6f4675e81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..ea229ed0a2 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_32x32_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz new file mode 100644 index 0000000000..1b9569d852 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx new file mode 100644 index 0000000000..a3a4cb44be Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz new file mode 100644 index 0000000000..23c670a428 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_112x112_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz new file mode 100644 index 0000000000..ab2bb9f524 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx new file mode 100644 index 0000000000..9b0088c7ca Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz new file mode 100644 index 0000000000..7325c82ef5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz new file mode 100644 index 0000000000..ab2bb9f524 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx new file mode 100644 index 0000000000..9b0088c7ca Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz new file mode 100644 index 0000000000..7325c82ef5 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_128x128_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..e09452502f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx new file mode 100644 index 0000000000..c6df448027 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..2bfb1830bb Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..92bebd6e72 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx new file mode 100644 index 0000000000..d12015ff31 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..b470b747ac Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..bf2c667ca1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..eaf6df52f1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..9f85bc0c1c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz new file mode 100644 index 0000000000..80390b2417 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx new file mode 100644 index 0000000000..111288276c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz new file mode 100644 index 0000000000..122dfec516 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_80x80_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..783b7d2858 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..c07e5887aa Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx new file mode 100644 index 0000000000..bbbefa7ca0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..7ce97bdee0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_4x4_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz new file mode 100644 index 0000000000..1698249444 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx new file mode 100644 index 0000000000..c9c3a30a81 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz new file mode 100644 index 0000000000..3daa36a4f3 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_24x24_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1100000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l1116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx new file mode 100644 index 0000000000..79793f965f Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz new file mode 100644 index 0000000000..8d37dca455 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_32x32_l180000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1100000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..98abda321c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx new file mode 100644 index 0000000000..888ddf2040 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..ebfa462b70 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..890c46f1a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx new file mode 100644 index 0000000000..b1b6960c73 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..4552f19b8e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_64x64_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..924cf47e2c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx new file mode 100644 index 0000000000..81e3d462fe Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..033ff1a224 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_6x6_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz new file mode 100644 index 0000000000..4c8df3f3d6 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx new file mode 100644 index 0000000000..3f0f1b0cec Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz new file mode 100644 index 0000000000..0179ec58dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_32x32_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz new file mode 100644 index 0000000000..b0cc432f09 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx new file mode 100644 index 0000000000..73d37d12a7 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz new file mode 100644 index 0000000000..5fc8e6e2c0 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_48x48_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz new file mode 100644 index 0000000000..d4ba139b99 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx new file mode 100644 index 0000000000..efb385c5c9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz new file mode 100644 index 0000000000..1c2e61aafd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz new file mode 100644 index 0000000000..d4ba139b99 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx new file mode 100644 index 0000000000..efb385c5c9 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz new file mode 100644 index 0000000000..1c2e61aafd Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_64x64_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l1115000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l116000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz new file mode 100644 index 0000000000..4f78d15de8 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx new file mode 100644 index 0000000000..e870c7083d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz new file mode 100644 index 0000000000..f7197789df Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dense_8x8_96x96_l164000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1112000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l1128000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..a3aeaeba02 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx new file mode 100644 index 0000000000..b1520cfe66 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..164283f70b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_128x128_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx new file mode 100644 index 0000000000..05a2c65c6b Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz new file mode 100644 index 0000000000..e70621d012 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_32x32_8x8_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz new file mode 100644 index 0000000000..f5a162ca11 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx new file mode 100644 index 0000000000..2a034837ee Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz new file mode 100644 index 0000000000..8208f0e71e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_dw_64x64_32x32_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz new file mode 100644 index 0000000000..281aba2cbe Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx new file mode 100644 index 0000000000..d4a1806bc1 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz new file mode 100644 index 0000000000..055d919693 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_256x128_16x16_l132000/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz new file mode 100644 index 0000000000..a5fd02380e Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx new file mode 100644 index 0000000000..8b3e63eb1d Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz new file mode 100644 index 0000000000..d43a29edfc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Sw_pw_32x32_8x8_l132000/outputs.npz differ diff --git a/DeeployTest/conftest.py b/DeeployTest/conftest.py index e37ddcf99b..e9ebb90553 100644 --- a/DeeployTest/conftest.py +++ b/DeeployTest/conftest.py @@ -68,6 +68,7 @@ def pytest_configure(config: pytest.Config) -> None: "siracusa_neureka_tiled: mark test as a Siracusa + Neureka platform test (tiled)") config.addinivalue_line("markers", "gap9: mark test as a GAP9 platform test") config.addinivalue_line("markers", "gap9_tiled: mark test as a GAP9 platform test (tiled)") + config.addinivalue_line("markers", "gap9_w_ne16_tiled: mark test as a GAP9 + NE16 platform test (tiled)") config.addinivalue_line("markers", "xdna2: mark test as an XDNA2 (AIE2p) platform test") config.addinivalue_line("markers", "kernels: mark test as a kernel test (individual operators)") config.addinivalue_line("markers", "models: mark test as a model test (full networks)") diff --git a/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py b/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py new file mode 100644 index 0000000000..63c2277789 --- /dev/null +++ b/DeeployTest/deeployRunner_tiled_gap9_w_ne16.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +import sys + +from testUtils.deeployRunner import main + +if __name__ == "__main__": + + # Define parser setup callback to add GAP9+NE16-specific arguments + def setup_parser(parser): + parser.add_argument('--cores', type = int, default = 8, help = 'Number of cores (default: 8)\n') + parser.add_argument('--ne16-wmem', action = 'store_true', help = 'Enable NE16 weight memory\n') + parser.add_argument('--enable-3x3', action = 'store_true', help = 'Enable 3x3 convolutions\n') + + sys.exit( + main(default_platform = "GAP9_w_NE16", + default_simulator = "gvsoc", + tiling_enabled = True, + parser_setup_callback = setup_parser)) diff --git a/DeeployTest/generateNetwork.py b/DeeployTest/generateNetwork.py index 159cfd19e9..c9ae63c0c2 100644 --- a/DeeployTest/generateNetwork.py +++ b/DeeployTest/generateNetwork.py @@ -85,6 +85,13 @@ def generateNetwork(args): platform, signProp = mapPlatform(args.platform) + # Enable NE16 3x3 convolutions (DW and Dense) if requested + if hasattr(args, 'enable_3x3') and args.enable_3x3: + from Deeploy.Targets.NE16.Engine import NE16Engine + for engine in platform.engines: + if isinstance(engine, NE16Engine): + engine.enable3x3 = True + clusters = [engine for engine in platform.engines if isinstance(engine, PULPClusterEngine)] for cluster in clusters: cluster.n_cores = args.cores @@ -194,6 +201,11 @@ def generateNetwork(args): help = '(Optional) mapping of input names to offsets. ' 'If not specified, offsets are set to 0. ' 'Example: --input-offset-map input_0=0 input_1=128 ...') + parser.add_argument('--enable-3x3', + action = 'store_true', + dest = 'enable_3x3', + default = False, + help = 'Enable NE16 3x3 convolutions (DW and Dense)\n') parser.add_argument('--shouldFail', action = 'store_true') parser.add_argument( "--cores", diff --git a/DeeployTest/testMVP.py b/DeeployTest/testMVP.py index 998412772b..3871392f6e 100644 --- a/DeeployTest/testMVP.py +++ b/DeeployTest/testMVP.py @@ -75,6 +75,8 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg platform, signProp = mapPlatform(args.platform) + if args.enable_3x3: + platform.engines[0].enable3x3 = True if args.enableStrides: platform.engines[0].enableStrides = True @@ -96,7 +98,7 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg scheduler = _mockScheduler) # Make the deployer engine-color-aware - if args.platform == "Siracusa_w_neureka": + if args.platform in ("Siracusa_w_neureka", "GAP9_w_NE16"): deployer = EngineColoringDeployerWrapper(deployer) # Make platform memory-aware after mapDeployer because it requires the platform to be an instance of an unwrapped platform @@ -158,6 +160,11 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg action = "store_true", default = False, help = 'Adds weight memory and neureka engine color\n') + parser.add_argument('--enable-3x3', + dest = "enable_3x3", + action = "store_true", + default = False, + help = 'Adds EXPERIMENTAL support for 3x3 convolutions on N-EUREKA\n') parser.add_argument('--enableStrides', dest = "enableStrides", action = "store_true", diff --git a/DeeployTest/testUtils/core/execution.py b/DeeployTest/testUtils/core/execution.py index 5b9cacd409..cfb0938660 100644 --- a/DeeployTest/testUtils/core/execution.py +++ b/DeeployTest/testUtils/core/execution.py @@ -134,7 +134,7 @@ def build_binary(config: DeeployTestConfig) -> None: ] # GAP9 requires the 'image' target to generate MRAM .bin files for GVSOC - if config.platform == 'GAP9': + if config.platform in ('GAP9', 'GAP9_w_NE16'): cmd.append("image") env = os.environ.copy() diff --git a/DeeployTest/testUtils/deeployRunner.py b/DeeployTest/testUtils/deeployRunner.py index 00ec496ed9..8d6b6c4231 100644 --- a/DeeployTest/testUtils/deeployRunner.py +++ b/DeeployTest/testUtils/deeployRunner.py @@ -249,6 +249,9 @@ def create_config_from_args(args: argparse.Namespace, if hasattr(args, 'neureka_wmem') and args.neureka_wmem: gen_args_list.append("--neureka-wmem") + if getattr(args, 'enable_3x3', False): + gen_args_list.append("--enable-3x3") + if not tiling and getattr(args, 'profileUntiled', False): gen_args_list.append("--profileUntiled") diff --git a/DeeployTest/testUtils/platformMapping.py b/DeeployTest/testUtils/platformMapping.py index 9155ed77ae..7ce6467859 100644 --- a/DeeployTest/testUtils/platformMapping.py +++ b/DeeployTest/testUtils/platformMapping.py @@ -15,11 +15,13 @@ from Deeploy.Targets.CortexM.Deployer import CMSISDeployer from Deeploy.Targets.CortexM.Platform import CMSISOptimizer, CMSISPlatform from Deeploy.Targets.GAP9.Deployer import GAP9Deployer -from Deeploy.Targets.GAP9.Platform import GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper +from Deeploy.Targets.GAP9.Platform import GAP9Optimizer, GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper from Deeploy.Targets.Generic.Deployer import GenericDeployer from Deeploy.Targets.Generic.Platform import GenericOptimizer, GenericPlatform from Deeploy.Targets.MemPool.Deployer import MemPoolDeployer from Deeploy.Targets.MemPool.Platform import MemPoolOptimizer, MemPoolPlatform +from Deeploy.Targets.NE16.Deployer import NE16Deployer +from Deeploy.Targets.NE16.Platform import MemoryNE16Platform, MemoryNE16PlatformWrapper, NE16Optimizer, NE16Platform from Deeploy.Targets.Neureka.Deployer import NeurekaDeployer from Deeploy.Targets.Neureka.Platform import MemoryNeurekaPlatform, MemoryNeurekaPlatformWrapper, NeurekaOptimizer, \ NeurekaPlatform @@ -31,7 +33,9 @@ from Deeploy.Targets.SoftHier.Platform import SoftHierOptimizer, SoftHierPlatform _SIGNPROP_PLATFORMS = ["Apollo3", "Apollo4", "QEMU-ARM", "Generic", "MemPool", "SoftHier"] -_NONSIGNPROP_PLATFORMS = ["Siracusa", "Siracusa_w_neureka", "PULPOpen", "Snitch", "Chimera", "GAP9", "XDNA2"] +_NONSIGNPROP_PLATFORMS = [ + "Siracusa", "Siracusa_w_neureka", "PULPOpen", "Snitch", "Chimera", "GAP9", "GAP9_w_NE16", "XDNA2" +] _PLATFORMS = _SIGNPROP_PLATFORMS + _NONSIGNPROP_PLATFORMS @@ -67,6 +71,9 @@ def mapPlatform(platformName: str) -> Tuple[DeploymentPlatform, bool]: elif platformName == "Siracusa_w_neureka": Platform = NeurekaPlatform() + elif platformName == "GAP9_w_NE16": + Platform = NE16Platform() + elif platformName == "Snitch": Platform = SnitchPlatform() @@ -94,6 +101,8 @@ def setupMemoryPlatform(platform: DeploymentPlatform, memoryHierarchy: MemoryHie weightMemoryLevel = memoryHierarchy.memoryLevels["WeightMemory_SRAM"] \ if "WeightMemory_SRAM" in memoryHierarchy.memoryLevels else None return MemoryNeurekaPlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel, weightMemoryLevel) + elif isinstance(platform, NE16Platform): + return MemoryNE16PlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel) if isinstance(platform, GAP9Platform): return MemoryGAP9PlatformWrapper(platform, memoryHierarchy, defaultTargetMemoryLevel) else: @@ -211,10 +220,27 @@ def mapDeployer(platform: DeploymentPlatform, default_channels_first = default_channels_first, deeployStateDir = deeployStateDir) + elif isinstance(platform, (NE16Platform, MemoryNE16Platform, MemoryNE16PlatformWrapper)): + + if loweringOptimizer is None: + loweringOptimizer = NE16Optimizer + + if default_channels_first is None: + default_channels_first = False + + deployer = NE16Deployer(graph, + platform, + inputTypes, + loweringOptimizer, + scheduler, + name = name, + default_channels_first = default_channels_first, + deeployStateDir = deeployStateDir) + elif isinstance(platform, (GAP9Platform, MemoryGAP9Platform, MemoryGAP9PlatformWrapper)): if loweringOptimizer is None: - loweringOptimizer = PULPOptimizer + loweringOptimizer = GAP9Optimizer if default_channels_first is None: default_channels_first = False diff --git a/DeeployTest/test_gap9_ne16_tiled_config.py b/DeeployTest/test_gap9_ne16_tiled_config.py new file mode 100644 index 0000000000..7d4c4e819c --- /dev/null +++ b/DeeployTest/test_gap9_ne16_tiled_config.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: 2026 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 +"""Test configuration for GAP9 platform with NE16 accelerator (tiled). + +NE16-supported convolution kernels verified to dispatch to NE16 +(`ne16_nnx_dispatch` appears in generated Network.c) and PASS on +gvsoc gap9.evk: + +- PW 1x1 RQ Conv (PW_2D_RQ/Regular_RQ) +- PW 1x1 Conv (PW_2D) +- DW 3x3 RQ Conv (DW_2D_RQ, with --enable-3x3) +- 3x3 strided RQ (StriddedPadded_2D_RQ) — falls back to cluster + because stride 2x2 requires --enableStrides which isn't wired into + the tiled runner today; still PASS via the cluster kernel. +""" + +DEFAULT_CORES = 8 + +L2_SINGLEBUFFER_KERNELS = { + "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000, 16000], + "Kernels/Integer/Conv/PW_2D": [32000], + "Kernels/Integer/Conv/DW_2D_RQ": [32000, 16000], + "Kernels/Integer/Conv/Dense_2D_RQ": [32000], + "Kernels/Integer/Conv/StriddedPadded_2D_RQ": [32000], +} + +L2_DOUBLEBUFFER_KERNELS = { + "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], + "Kernels/Integer/Conv/DW_2D_RQ": [32000], +} + +L2_SINGLEBUFFER_MODELS = { + "Models/MLPerf/VisualWakeWords": [128000], +} + +L2_DOUBLEBUFFER_MODELS = { + # L1 budget is below the full 128KB because the double-buffered tiles plus + # the runtime allocator overhead do not fit. Two things moved this ceiling: + # SLAVESTACKSIZE became overridable (freeing ~20KB of L1), and the weight + # tile's NE16-encoded tail is now pinned, so the solver reserves the full + # 18432 B the DMA actually writes instead of 16384 B. The latter removes + # slack that was only ever available by overrunning the requant parameters, + # so the previous 110000 no longer allocates. 100000 is measured to run. + # 100000 -> 567,140 cycles, bit-exact + # 110000 -> "Allocation failed for allocator 2" + "Models/MLPerf/VisualWakeWords": [100000], +} + +L3_SINGLEBUFFER_MODELS = {} +L3_DOUBLEBUFFER_MODELS = {} +L2_SINGLEBUFFER_KERNELS_WMEM = {} +L3_DOUBLEBUFFER_MODELS_WMEM = {} diff --git a/DeeployTest/test_platforms.py b/DeeployTest/test_platforms.py index e473cdc704..d9a6c4c44f 100644 --- a/DeeployTest/test_platforms.py +++ b/DeeployTest/test_platforms.py @@ -11,6 +11,11 @@ from test_gap9_config import DEFAULT_NUM_CORES as GAP9_DEFAULT_NUM_CORES from test_gap9_config import KERNEL_TESTS as GAP9_KERNEL_TESTS from test_gap9_config import MODEL_TESTS as GAP9_MODEL_TESTS +from test_gap9_ne16_tiled_config import DEFAULT_CORES as GAP9_NE16_TILED_DEFAULT_CORES +from test_gap9_ne16_tiled_config import L2_DOUBLEBUFFER_KERNELS as GAP9_NE16_L2_DOUBLEBUFFER_KERNELS +from test_gap9_ne16_tiled_config import L2_DOUBLEBUFFER_MODELS as GAP9_NE16_L2_DOUBLEBUFFER_MODELS +from test_gap9_ne16_tiled_config import L2_SINGLEBUFFER_KERNELS as GAP9_NE16_L2_SINGLEBUFFER_KERNELS +from test_gap9_ne16_tiled_config import L2_SINGLEBUFFER_MODELS as GAP9_NE16_L2_SINGLEBUFFER_MODELS from test_gap9_tiled_config import DEFAULT_CORES as GAP9_TILED_DEFAULT_CORES from test_gap9_tiled_config import L2_DOUBLEBUFFER_KERNELS as GAP9_L2_DOUBLEBUFFER_KERNELS from test_gap9_tiled_config import L2_DOUBLEBUFFER_MODELS as GAP9_L2_DOUBLEBUFFER_MODELS @@ -140,6 +145,7 @@ def param_id(param): # siracusa_neureka_tiled: tests from the Siracusa + Neureka platform (tiled) # gap9: tests from the GAP9 platform (untiled) # gap9_tiled: tests from the GAP9 platform (tiled) +# gap9_w_ne16_tiled: tests from the GAP9 + NE16 platform (tiled) # Test type markers: # kernels: single kernel (or single layer) tests # models: full model (multiple layer) tests @@ -1046,6 +1052,140 @@ def test_gap9_tiled_models_l3_doublebuffer(test_params, deeploy_test_dir, toolch run_and_assert_test(test_name, config, skipgen, skipsim) +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.kernels +@pytest.mark.singlebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_SINGLEBUFFER_KERNELS, "L2-singlebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_kernels_l2_singlebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}"] + + # --enable-3x3 is additive (extends NE16Engine.canExecute to DW/Dense 3x3); + # safe to enable for all three kernel cases (PW 1x1 + DW 3x3 + Dense 3x3). + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = False, + gen_args = ["--enable-3x3"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.models +@pytest.mark.singlebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_SINGLEBUFFER_MODELS, "L2-singlebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_models_l2_singlebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}"] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = False, + gen_args = ["--enable-3x3", "--enableStrides"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.kernels +@pytest.mark.doublebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_DOUBLEBUFFER_KERNELS, "L2-doublebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_kernels_l2_doublebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}"] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = True, + gen_args = ["--enable-3x3"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + +@pytest.mark.gap9_w_ne16_tiled +@pytest.mark.models +@pytest.mark.doublebuffer +@pytest.mark.l2 +@pytest.mark.parametrize( + "test_params", + generate_test_params(GAP9_NE16_L2_DOUBLEBUFFER_MODELS, "L2-doublebuffer"), + ids = param_id, +) +def test_gap9_w_ne16_tiled_models_l2_doublebuffer(test_params, deeploy_test_dir, toolchain, toolchain_dir, cmake_args, + skipgen, skipsim) -> None: + test_name, l1, config_name = test_params + + ne16_cmake_args = cmake_args + [f"NUM_CORES={GAP9_NE16_TILED_DEFAULT_CORES}"] + + config = create_test_config( + test_name = test_name, + platform = "GAP9_w_NE16", + simulator = "gvsoc", + deeploy_test_dir = deeploy_test_dir, + toolchain = toolchain, + toolchain_dir = toolchain_dir, + cmake_args = ne16_cmake_args, + tiling = True, + cores = GAP9_NE16_TILED_DEFAULT_CORES, + l1 = l1, + default_mem_level = "L2", + double_buffer = True, + gen_args = ["--enable-3x3", "--enableStrides"], + ) + run_and_assert_test(test_name, config, skipgen, skipsim) + + @pytest.mark.xdna2 @pytest.mark.kernels @pytest.mark.parametrize("test_name", XDNA2_KERNEL_TESTS, ids = XDNA2_KERNEL_TESTS) diff --git a/README_NE16_PERFORMANCE.md b/README_NE16_PERFORMANCE.md new file mode 100644 index 0000000000..b4a830edfc --- /dev/null +++ b/README_NE16_PERFORMANCE.md @@ -0,0 +1,316 @@ + + +# NE16 on GAP9: Where the Peak Is, and How We Got Close To It + +This note collects (1) the primary sources for the NE16 accelerator, (2) what its peak +throughput actually is and what you must do to approach it, and (3) the changes we made in +Deeploy to take MobileNetV1 from **9.43 to 13.96 MAC/cycle**, past the **10.3 MAC/cycle** the +GAP9 SDK reaches on a comparable model. + +Every number below was measured on GVSoC (`gap9.evk`) through the Deeploy CI configuration. +Where a claim comes from reading source rather than from a measurement, it says so. + +--- + +## 1. Sources + +### 1.1 The accelerator itself + +| What | Where | Notes | +|---|---|---| +| **NE16 RTL + docs** | [`pulp-platform/ne16`](https://github.com/pulp-platform/ne16) | The official repository. Maintained by Francesco Conti (University of Bologna / GreenWaves Technologies). | +| **RBE** (predecessor) | [`pulp-platform/rbe`](https://github.com/pulp-platform/rbe) | Reconfigurable Binary Engine, by Gianna Paulin and Francesco Conti. NE16 derives from it. | +| **pulp-nnx** (driver / HAL) | [`pulp-platform/pulp-nnx`](https://github.com/pulp-platform/pulp-nnx) | The task-descriptor layer Deeploy generates calls into (`ne16_task.c`, subtile counters, strides). | + +There is **no paper describing the NE16 microarchitecture itself**. It is a productised IP in +GAP9, and its details live in the repository and in the SDK. The publication the NE16 README +cites is the ancestor design: + +> F. Conti, P. D. Schiavone, L. Benini, *"XNOR Neural Engine: A Hardware Accelerator IP for +> 21.6-fJ/op Binary Neural Network Inference"*, IEEE Transactions on Computer-Aided Design of +> Integrated Circuits and Systems, vol. 37, no. 11, 2018, pp. 2940–2951. + +Systems papers that use NE16 and report end-to-end numbers (useful for calibration, not for +microarchitecture): *Flexible and Fully Quantized Ultra-Lightweight TinyissimoYOLO* +([arXiv:2307.05999](https://arxiv.org/pdf/2307.05999)), *GAP9Shield* +([arXiv:2407.13706](https://arxiv.org/html/2407.13706v1)). + +### 1.2 The behavioural model — the most useful source in practice + +GVSoC ships a cycle-level C++ model of NE16. When the documentation is ambiguous, **this is the +ground truth**, and it is readable: + +``` +$GAP_SDK/gvsoc/gvsoc_gap/gap/ne16/src/ +├── ne16_regfile.cpp ← CONFIG0 bit decode. Authoritative bit map. +├── ne16_matrixvec.cpp ← the MAC array itself +├── ne16_normquant.cpp ← scale/bias/shift. Note: plain `>> shift`, no rounding term. +├── ne16_streamout.cpp ← output saturation (signed [-128,127] / unsigned [0,255]) +└── ne16_load.cpp ← input fetch +``` + +Two examples of questions we answered by reading it rather than guessing: + +* **Does NE16 round or truncate in requantisation?** `ne16_normquant.cpp` does + `accum32[i] >> shift` with no rounding term added first. A golden model copied from the + *software* `RequantShift_s8.c` kernel (which rounds half-up) will disagree with hardware by + up to 1 LSB. +* **Which CONFIG0 bits exist?** `ne16_regfile.cpp` decodes `[4]` outquant, `[6:5]` filter mode, + `[7]` linear, `[8]` strided-2x2, `[11:9]` **reserved**, `[13:12]` norm bits, `[14]` streamin, + `[15]` weight-offset (marked *"FIXME not implemented"*), `[20:16]` quant shift, `[22:21]` + quant bits, `[23]` quant-norect, `[24]` norm shift, `[25]` norm bias. Anything you set + outside that map is a no-op. + +Register bit names are in `$GAP_SDK/tools/autotiler_v3/CNN_Libraries_HWPE/hal_ne16.h` +(`NE16_REG_CONFIG 0x5c`, `NE16_SHIFT_*`). + +### 1.3 The analytical performance model — read this before optimising + +[`dory/Hardware_targets/PULP/GAP9_NE16/Tiler/Ne16PerfModel.py`](https://github.com/pulp-platform/dory/blob/master/dory/Hardware_targets/PULP/GAP9_NE16/Tiler/Ne16PerfModel.py) + +DORY's model decomposes one NE16 job into pipeline stages and gives a closed-form cycle count: + +``` +total = n_spatial × [ n_out_body × iteration_latency(k_out_body) + iteration_latency(k_out_rem) ] + +FIFO_LATENCY = 6 SHIFTER_COUNT = 4 ADDER_COUNT = 8 +MULTIPLIER_COUNT = 4 MEMORY_THROUGHPUT = 256 bit/cycle +``` + +The term that matters is **`k_out_rem`**: every job pays a fixed setup cost, and a tile that +does not fill the array pays it for partial work. Utilisation is the ratio of real MACs to +`max_ops`, which the model derives from the *padded-up* tile counts. + +### 1.4 What the SDK's own tiler does + +`$GAP_SDK/tools/autotiler_v3/CNN_Generators_NE16/CNN_Generators_NE16.c` — worth reading because +it encodes GreenWaves' own answer to "how do I keep NE16 busy": + +| Line | Code | Meaning | +|---|---|---| +| `669`, `1115` | `OutTileCons = CannotTileChannels ? OutFeat : 32` | prefer output-channel tiles that are a multiple of **32** | +| `687`, `1185` | `InTileCons = Mode16 ? 8 : 16` | input-channel tiles multiple of **16** for 8-bit | +| `1132` | `Fcx==3 && Fcy==3 && (s==1 \|\| s==2)` → `O_NE16_3X3` | 3x3 **stride 2 is native**, not a fallback | +| `1186` | `AllowActFusion && ActOper != KOP_NONE` | conv + activation fused into one kernel | + +Note `CannotTileChannels ? OutFeat : 32` — the alignment is a *preference that degrades*, never +a hard constraint. Layers with fewer than 32 output channels must still be tileable. + +--- + +## 2. Peak performance, and why you will not reach it + +### 2.1 The number + +The array is **9 × 9 engines × 16 input channels**, each engine performing one binary +multiply-accumulate per cycle: + +``` +9 × 9 × 16 = 1296 binary MAC/cycle +``` + +NE16 is bit-serial in the weights: an 8-bit weight takes 8 passes. So for the usual +8-bit × 8-bit case: + +``` +1296 / 8 = 162 MAC/cycle ← theoretical peak, 8-bit weights +``` + +At GAP9's 370 MHz that is ~60 GMAC/s; the commonly quoted **32.2 GMAC/s** figure corresponds to +sustained real-workload throughput, not the array bound. + +### 2.2 The three alignment rules + +Derived from the array geometry, and independently confirmed by what the SDK's tiler enforces: + +| Rule | Why | +|---|---| +| `Ci % 16 == 0` | `TP_IN = 16`: each engine consumes 16 **contiguous** input channels per cycle. This also forces a **channels-last (HWC)** layout — it is not a preference, it is how the datapath is fed. | +| `Co % 32 == 0` | `TP_OUT = 32` output channels retire per pass. A tile of 3 or 56 output channels wastes most of the output lanes for the *whole* tile. | +| `Ho % 3 == 0`, `Wo % 3 == 0` | the 9 columns produce a 3×3 output patch per pass. | + +Plus: stride 1 or 2 only, `qw = 8` for the 162 figure (lower weight precision scales linearly — +4-bit weights double it). + +### 2.3 What actually happens on a real network + +Measured, MLPerf Tiny VisualWakeWords (MobileNetV1 0.25×, 96×96, 7,489,664 MAC): + +| Configuration | MAC/cycle | % of 162 | +|---|---|---| +| Single-layer dense conv, 64→64 ch, 32×32 | **74.33** | 45.9 % | +| Full network, this work | **13.96** | 8.6 % | +| Full network, GAP9 SDK (comparable size) | 10.3 | 6.4 % | + +**A whole network runs at roughly a fifth of what a single well-shaped layer achieves, and both +are far from 162.** The reasons are structural, not fixable by tuning: + +* MobileNetV1 0.25× has layers with **8, 16, 32** channels against `TP_IN=16` / `TP_OUT=32`. + The first layer uses 8 of 32 output lanes. No tiler can fix a model that is narrower than the + datapath. +* Depthwise layers have one input channel per output channel by construction, so the 16-wide + input dimension is inherently underfilled. +* Everything that is not a MAC — layout conversion, tile DMA, job setup — is pure overhead. + +The practical consequence, and the point worth making to anyone tuning this: **beyond the +alignment rules, the remaining wins are in removing non-compute work, not in feeding the array +better.** Our 1.48× came entirely from the former — the NE16 dispatch count did not change at +all. + +--- + +## 3. What we changed, and why + +Four changes, in descending order of impact. All were verified bit-exact against the +pre-existing golden outputs; the NE16 CI jobs (`kernels`/`models` × `singlebuffer`/`doublebuffer`, +L2) pass. + +### 3.1 Fold the redundant layout transposes — 1.48× on the full network + +**Symptom.** The generated `Network.c` for VisualWakeWords contained **26 transpose passes for +27 convolutions**: a `_pre_transpose` / `_transpose` pair wrapped around essentially every +convolution, 232 cluster forks, each half carrying its own tiling loop and L2↔L1 DMA round trip. + +**Cause.** ONNX is NCHW; NE16 requires HWC (§2.2). `PULPNCHWtoNHWCPass` inserts the conversions, +and `PULPOpenDeployer` already ends its lowering chain with the clean-up that folds them: + +```python +PULPNCHWtoNHWCPass(...) TransposeSplitPass() RQAddTransposeSquashPass() +TransposeSplitPass() TransposeMergePass() TransposeConstOptPass() +ReshapeConstOptPass() TransposeNoPermOptPass() +``` + +But `NE16Deployer` then does `self.loweringOptimizer.passes += [...]`, so +`NE16OptimizationPass` — which inserts layout transposes of its own via `_appendTranspose` — +runs **after** that clean-up. Its transposes were never folded. Two consecutive NE16 convs were +therefore separated by a `HWC→CHW` followed by a `CHW→HWC`: an identity pair that survived all +the way into generated code. + +**Fix.** Re-run the same clean-up chain after `NE16OptimizationPass` (4 lines, +`Targets/NE16/Deployer.py`). + +| | before | after | +|---|---|---| +| transpose cluster forks | 232 | **16** | +| tiling loops | 114 | **62** | +| NE16 dispatches | 56 | **56** (compute untouched) | +| cycles | 794,080 | **536,521** | +| MAC/cycle | 9.43 | **13.96** | + +The general lesson: when a subclass appends passes with `+=`, whatever the base class ran as a +*final* clean-up is no longer final. + +### 3.2 Pin the weight tile's encoded tail — a 2 KB buffer overrun + +**Symptom.** Many tiled dense configurations produced wrong results: 2047 wrong outputs for +`64/64 @32×32`, 8097 for `16/16 @64×64`, 10121 for `4/4 @128×128`. Error counts scaled with the +tile count, were invariant to the `--l1` value and to arena placement, and untiled runs were +always correct. + +**Diagnosis.** The wrong outputs formed **two contiguous 1024-byte runs exactly 32768 bytes +apart** — one output tile's worth, same offset within each tile. That shape says *memory +overwrite*, not arithmetic. Dumping the generated L1 layout: + +``` +data_in @ 0 size 65536 → 0 – 65535 +data_out @ 65536 size 32768 → 65536 – 98303 +weight @ 98304 size 18432 → 98304 – 116735 ← overruns +mul @ 114688 size 128 +add @ 114816 size 128 +``` + +`serializeTilingSolution` always emits the weight tile as `(CSize,) + weightShape[1:]` — only the +output-channel dimension is tiled, the NE16-encoded tail is always moved whole. But +`addGeometricalConstraint` only pinned `weightOutChannelVar`, leaving `(cinMajor, bits, +H*W*cinMinorBytes) = (4, 8, 18)` free. The solver shrank the last dimension to 16 and reserved +`32×4×8×16 = 16384` B while the DMA writes `32×4×8×18 = 18432` B. The extra 2048 B landed on the +requantisation `mul`/`add` parameters — and exactly 2048 B of output came out wrong. + +**Fix.** Constrain the three tail dimensions to their maximum (3 lines, +`NE16DenseConstraint.addGeometricalConstraint`). All three failing shapes go to **0 errors**; the +control shape is cycle-identical. + +### 3.3 Prefer output-channel tiles that are a multiple of 32 + +Nothing expressed §2.2's `Co % 32` rule to the tiler, and the solver was free to pick whatever +fit — `Ko = 3` and `Ko = 56` were both observed in generated code. Added as a +`PerformanceHint` (not a hard constraint, matching the SDK's `CannotTileChannels` degradation) +to the dense, depthwise and pointwise constraints. This also makes shapes tileable that +previously could not be tiled at all — e.g. `8/8 @96×96`, which now runs bit-exact. + +### 3.4 Make `SLAVESTACKSIZE` overridable + +The cluster slave stacks were pinned at 3800 B/core by an unconditional `#define`, so ~32 KB of +the 128 KB L1 was gone before the tiling arena started — while the tiler was still told it had +the full `--l1` budget. Values above ~98000 either failed to allocate or, worse, produced a tile +layout that overran L1 and showed up only as a DMA out-of-bound trace at run time. + +Two edits are needed and **either one alone is a silent no-op**: the GAP9 `CMakeLists.txt` must +turn the `-D` cache variable into a compile definition, *and* the `#define` in `deeploytest.c` +must be wrapped in `#ifndef` or it shadows the command-line one. Arena goes from 98,176 to +~121,000 B. + +Caveat: this is a knob, not a free win. Slave stacks below ~1024 B crash the cluster kernels +that VisualWakeWords still runs (`Invalid fetch request (addr: 0x0)` — a clobbered return +address), and on this model the extra L1 buys nothing, because single-buffer is not L1-bound: +`--l1` 128000→131000 × stack 3800/1280/1024 all give an identical 794,080 cycles. + +--- + +## 4. Result + +| Configuration | cycles | MAC/cycle | vs SDK | +|---|---|---|---| +| Baseline (before this work) | 794,080 | 9.43 | 92 % | +| **Single-buffer, L1 128000** | **536,521** | **13.96** | **136 %** | +| Double-buffer, L1 100000 | 567,140 | 13.21 | 128 % | +| GAP9 SDK, comparable model size | — | 10.3 | 100 % | + +Single-buffer is the better configuration here: double-buffering must fit two of every tile in +L1, so it tiles more finely, and the extra splits cost more than the overlapped DMA saves. + +--- + +## 5. Reproducing + +```bash +source $GAP_SDK/.gap9-venv/bin/activate +source $GAP_SDK/configs/gap9_evk_audio.sh # NOT gap9_v2.sh — the target must + # match --target=gap9.evk or the chip + # never boots and gvsoc hangs silently +export GVSOC_INSTALL_DIR=$GAP_SDK/install/workstation +export GAP_RISCV_GCC_TOOLCHAIN=/path/to/gcc/gap9 +export CCACHE_DIR= + +cd DeeployTest +pytest test_platforms.py -v -s -m "gap9_w_ne16_tiled and models and singlebuffer and l2" +``` + +Use `pytest` with markers, never the single-kernel runner with hand-picked flags: the per-model +overrides (L1 budget, `gen_args`) live in `test_gap9_ne16_tiled_config.py` and +`test_platforms.py`. Bypassing them cost us a 44 % discrepancy on the same nominal `--l1`, +because the direct runner does not accept `--enableStrides` and the stride-2 layers silently +fell back to the cluster. + +Also: **wipe `DeeployTest/TEST_GAP9_W_NE16` between experiments.** CMake uses `file(GLOB)` at +configure time, and a stale `build_master` will happily re-run a previous binary — which +produced four consecutive wrong diagnoses in the course of this work. + +--- + +## 6. Open items + +* **Signed activations are not supported.** `ConvTemplate.getConf0` sets `conf0 |= 1 << 26` for + `input_signed`, but bit 26 is not part of CONFIG0 (§1.2) — it is a no-op, so int8 activations + are consumed as uint8. NE16 has no signed-input mode; the correct approach is an offset + correction (+128 on the input, `-128 × Σweights` folded into the bias). Post-ReLU networks + such as MobileNet are unaffected, which is why this has gone unnoticed. +* **Activation fusion.** The SDK fuses conv + activation into a single kernel + (`CNN_Generators_NE16.c:1186`); Deeploy runs them as separate passes. This is the most likely + source of the remaining gap on networks where the transposes are already folded. +* **Narrow layers.** Nothing in the tiler exploits the fact that a `Ci = 8` layer wastes half + the input datapath. The SDK does not appear to either, but it bounds what either can achieve. diff --git a/STRIDE_HANDOFF.md b/STRIDE_HANDOFF.md new file mode 100644 index 0000000000..3987b758ab --- /dev/null +++ b/STRIDE_HANDOFF.md @@ -0,0 +1,94 @@ +# NE16: what "the stride bug" actually was, and where the real headroom is +(2026-08-04, second pass) + +## 1. Stride is NOT broken, and NOT the bottleneck. Premise disproven. + +VisualWakeWords (MobileNetV1) generated code: + NE16 dispatches (ne16_nnx_dispatch): 56 + cluster conv kernels (pulp_nn_conv*): 0 +All 27 conv passes -- including every stride-2 downsample -- already run on NE16. +There is no speedup available from "implementing stride"; it is already done. +pytest passes `--enableStrides` via gen_args; only the *direct runner* lacked +the CLI flag (now added, see uncommitted changes). + +## 2. What StriddedPadded_2D_RQ actually exposes: signed inputs are unsupported + +Error detail (this is what I should have read first): + Expected: 127 Actual: -128 Diff: -1 x6 of 8 + +Expected is saturated +127, actual is saturated -128 -- the accumulator's SIGN +is wrong, not its addressing. + +Input ranges across the NE16 test suite: +| test | input range | signed | result | +|---|---|---|---| +| StriddedPadded_2D_RQ | -128..124 | YES | 6/8 wrong | +| DW_2D_RQ | 0..255 | no | 0 errors | +| PW_2D_RQ/Regular_RQ | 0..254 | no | 0 errors | +| Dense_2D_RQ | 0..3 | no | 0 errors | + +Every passing NE16 test is unsigned; the only signed one fails. NE16 has **no +signed-input config bit**: gvsoc `ne16_regfile.cpp:200-225` decodes CONFIG0 as +[4] outquant, [6:5] filter mode, [7] linear, [8] strided2x2, [11:9] RESERVED, +[13:12] norm bits, [14] streamin, [15] weight-offset (marked "FIXME not +implemented"), [20:16] quant shift, [22:21] quant bits, [23] quant norect, +[24] norm shift, [25] norm bias. **Bit 26 is not decoded at all** -- and +`ConvTemplate.getConf0` sets `conf0 |= 1 << 26` for `input_signed`. That is a +phantom bit; signedness never reaches the hardware, so int8 is consumed as +uint8. Bit 9 (`use_wmem` in Deeploy) also lands in the reserved [11:9] field +and deserves a separate look. + +Correct approach for signed input on NE16 is an offset correction: shift the +input by +128 (making it unsigned) and subtract 128*sum(weights) through the +bias. Not implemented anywhere in Deeploy today. + +MobileNet/VWW is unaffected: its activations are post-ReLU and unsigned. + +## 3. The real gap vs the GAP9 SDK: per-layer layout transposes + +VisualWakeWords generated code contains **26 transpose passes for 27 conv +passes** -- a `_pre_transpose` / `_transpose` pair wrapped around essentially +every convolution, 232 cluster-fork calls in total, plus their own L2<->L1 DMA +round trips. Pure data movement, zero MACs. + +GAP9's AutoTiler converts the whole network to HWC **once at import**, so there +are no per-layer conversions at all. That is the structural difference behind +our 9.44 MAC/cycle vs the SDK's 10.3 on this model size. + +Puzzle to start from: `Deeploy/Targets/NE16/Deployer.py:24` already sets +`default_channels_first = False`, so the graph should be channels-last globally +and these transposes should not exist. Find what re-introduces them (a topology +pass? a non-conv op that demands NCHW? the network's own ONNX?). + +## Measurements to compare against (all gvsoc, VWW = MobileNetV1, 7.49 MMAC) +| config | cycles | MAC/cycle | +|---|---|---| +| singlebuffer @128000 (CI default, best) | 794,080 | 9.43-9.44 | +| double buffer @110000 + SLAVESTACKSIZE=1280 | 830,146 | 9.02 | +| double buffer @90000 (old CI default) | 860,577 | 8.70 | +| GAP9 SDK, closest model size | -- | 10.3 | + +Single-buffer is NOT L1-bound: l1 128000..131000 x slave stack 3800/1280/1024 +all give an identical 794,080 cycles. Memory tuning is exhausted. + +## Uncommitted working-tree changes (all backed up) +- `Targets/NE16/Templates/ConvTemplate.py` (/tmp/ConvTemplate.bak) + Stride-aware input extent in getCounters (dense + DW): + `(height_out_border-1)*strideH + 3 - padding_bottom`. Collapses to the old + `+2` when S=1, so stride-1 is bit-identical (verified). Correct in principle, + still UNVALIDATED for S=2 -- needs a case whose *border* subtile has Ho>=2. + (The conf0 `1 << 8` strided-mode bit was tried and REVERTED: it changed 10680 + -> 10598 cycles and zero errors, i.e. irrelevant to the failure.) +- `DeeployTest/deeployRunner_tiled_gap9_w_ne16.py` (/tmp/rn.bak) and + `testUtils/deeployRunner.py` (/tmp/dr.bak): expose/forward `--enableStrides`. + +## Already on PR #183 (pushed, CI models test passes 0 errors) + d8bef5e9 test(NE16): raise VWW double-buffer L1 budget to 110KB + 4e88a10a perf(NE16): prefer output-channel tiles multiple of TP_OUT=32 + 1c37e813 fix(GAP9): hoist L2->L1 tile-control tables to L2 + a47e812d fix(GAP9): make SLAVESTACKSIZE overridable from CMake + +## Also open +per-tile boundary bug: error count scales with tile count, invariant to L1 and +arena placement. Localise by reducing failing output indices modulo the tile +output geometry. diff --git a/TargetLibraries/GAP9/CMakeLists.txt b/TargetLibraries/GAP9/CMakeLists.txt index ca4c3ffbeb..26d10d9b0c 100644 --- a/TargetLibraries/GAP9/CMakeLists.txt +++ b/TargetLibraries/GAP9/CMakeLists.txt @@ -4,22 +4,69 @@ file(GLOB_RECURSE SOURCES "src/**" + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_fp32/CNN_Bias_Linear_Activation_fp32.c" + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries/CNN_Copy.c" ) +# CNN_BasicKernels_NE16 from gap9-sdk redefines NE16_REG_* macros that +# pulp-nnx's ne16 hal also defines. For GAP9_w_NE16 we use the pulp-nnx +# NE16 stack; for plain GAP9 (Pu DENG's NE16-Linear path) we use the SDK's. +if(NOT platform STREQUAL "GAP9_w_NE16") + list(APPEND SOURCES + "$ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_HWPE/CNN_BasicKernels_NE16.c" + ) +endif() + + +# Exclude dory_mem and dory_dma from SOURCES (they need different optimization) +list(FILTER SOURCES EXCLUDE REGEX ".*dory_(mem|dma).*") + # RW: Include PULPOpen sources but exclude dory_mem related files file(GLOB_RECURSE PULPOPEN_SOURCES "../PULPOpen/src/**") list(FILTER PULPOPEN_SOURCES EXCLUDE REGEX ".*dory_mem.*") list(APPEND SOURCES ${PULPOPEN_SOURCES}) +# Separate dory library compiled without -O3 +add_library(dory_lib STATIC + ${CMAKE_CURRENT_LIST_DIR}/src/dory_mem.c + ${CMAKE_CURRENT_LIST_DIR}/src/dory_dma.c +) +target_include_directories(dory_lib PUBLIC + ${CMAKE_CURRENT_LIST_DIR}/inc + ${CMAKE_CURRENT_LIST_DIR}/../PULPOpen/inc +) +target_compile_options(dory_lib PRIVATE + -Wno-implicit-function-declaration + -Wno-sign-conversion + -Wno-sign-compare + -Wno-type-limits + -Wno-attributes + -Wno-incompatible-pointer-types + -Og +) +target_compile_definitions(dory_lib PUBLIC NUM_CORES=${NUM_CORES}) +target_link_libraries(dory_lib PUBLIC pmsis) + add_deeploy_library(deeploygap9 STATIC ${SOURCES}) target_include_directories(deeploygap9 PUBLIC ${CMAKE_CURRENT_LIST_DIR}/inc ${CMAKE_CURRENT_LIST_DIR}/../PULPOpen/inc + ${TILER_INC} + ${TILER_EMU_INC} + ${TILER_CNN_KERNEL_PATH_FP32} + ${TILER_CNN_KERNEL_PATH_FP16} + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_SQ8 + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries + $ENV{GAP_SDK_HOME}/tools/autotiler_v3/CNN_Libraries_HWPE + ${TILER_DSP_KERNEL_V2_PATH} + ${TILER_DSP_KERNEL_V2_PATH}/FastMathFunctions ) + target_compile_options(deeploygap9 PUBLIC -DNUM_CORES=${NUM_CORES} + -DSTD_FLOAT ) target_compile_options(deeploygap9 PRIVATE @@ -27,10 +74,10 @@ target_compile_options(deeploygap9 PRIVATE -Wno-sign-compare -Wno-type-limits -Wno-attributes + -Wno-incompatible-pointer-types + -O3 ) -target_link_libraries(deeploygap9 PUBLIC pmsis) - #RW: Link PULP-NN #RW: Set PULP-NN version and bitwidth for pulp-nn-mixed set(PULPNNVERSION XPULPV2) @@ -80,5 +127,22 @@ endif() target_link_libraries(deeploygap9 PUBLIC pulp-nn-mixed) -target_link_libraries(deeploygap9 PUBLIC m) +# NE16 accelerator (via pulp-nnx) for GAP9_w_NE16 platform +if(platform STREQUAL "GAP9_w_NE16") + set(USE_NE16 ON CACHE BOOL "Use the NE16 accelerator." FORCE) + add_subdirectory(../third_party/pulp-nnx ${CMAKE_CURRENT_BINARY_DIR}/pulp-nnx) + target_link_libraries(pulp-nnx PUBLIC pmsis) + target_compile_options(pulp-nnx PRIVATE + -Wno-error + -Wno-implicit-int-conversion + -Wno-sign-conversion + -Wno-typedef-redefinition + -Wno-unused-parameter + -Wno-incompatible-pointer-types-discards-qualifiers + ) + target_link_libraries(deeploygap9 PUBLIC pulp-nnx) +endif() +target_link_libraries(deeploygap9 PUBLIC pmsis) +target_link_libraries(deeploygap9 PUBLIC m) +target_link_libraries(deeploygap9 PUBLIC dory_lib) diff --git a/TargetLibraries/GAP9/inc/ne16_utils.h b/TargetLibraries/GAP9/inc/ne16_utils.h new file mode 100644 index 0000000000..4d041c75dc --- /dev/null +++ b/TargetLibraries/GAP9/inc/ne16_utils.h @@ -0,0 +1,23 @@ +/* + * SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna + * SPDX-License-Identifier: Apache-2.0 + * + * NE16 utility kernels for GAP9 + */ + +#ifndef __NE16_UTILS_GAP9__ +#define __NE16_UTILS_GAP9__ + +#include "CNN_BasicKernels_fp32.h" +#include "pmsis.h" + +typedef struct { + int8_t *In; + uint8_t *Out; + int size; +} ne16_int8_to_uint8_T; + +/* Multi-core SIMD int8 → uint8 conversion (+128 offset) */ +void ne16_int8_to_uint8(ne16_int8_to_uint8_T *Arg); + +#endif diff --git a/TargetLibraries/GAP9/src/ne16_utils.c b/TargetLibraries/GAP9/src/ne16_utils.c new file mode 100644 index 0000000000..c19c119b25 --- /dev/null +++ b/TargetLibraries/GAP9/src/ne16_utils.c @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2025 ETH Zurich and University of Bologna + * SPDX-License-Identifier: Apache-2.0 + * + * NE16 utility kernels for GAP9 + */ + +#include "ne16_utils.h" + +void ne16_int8_to_uint8(ne16_int8_to_uint8_T *Arg) { + int8_t *In = Arg->In; + uint8_t *Out = Arg->Out; + int size = Arg->size; + + unsigned int CoreId = gap_coreid(); + unsigned int NCore = gap_ncore(); + unsigned int total_quads = size / 4; + unsigned int Chunk = (total_quads + NCore - 1) / NCore; + unsigned int First = Chunk * CoreId; + unsigned int Last = First + Chunk; + if (Last > total_quads) + Last = total_quads; + + v4s offset = {-128, -128, -128, -128}; + for (unsigned int q = First; q < Last; q++) { + *((v4s *)&Out[q * 4]) = *((v4s *)&In[q * 4]) + offset; + } + + /* Handle remaining elements (size not multiple of 4) */ + if (CoreId == 0) { + for (int i = total_quads * 4; i < size; i++) { + Out[i] = (uint8_t)((int32_t)In[i] + 128); + } + } +}