diff --git a/CHANGELOG.md b/CHANGELOG.md index bd1b4c8b8e..1da9a5d5a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid ### List of Pull Requests +- Fix Neureka [#188](https://github.com/pulp-platform/Deeploy/pull/188) - XDNA2 Platform Support [#179](https://github.com/pulp-platform/Deeploy/pull/179) - Add Microbenchmarking Infrastructure and CI Using GVSoC CSR [#162](https://github.com/pulp-platform/Deeploy/pull/162) - Fix CI Cache Generation [#176](https://github.com/pulp-platform/Deeploy/pull/176) @@ -21,6 +22,8 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Fix GAP9 L3 Board Tests: readfs Flash Ordering and Duplicate Input Data [#196](https://github.com/pulp-platform/Deeploy/pull/196) ### Added +- tests for Regular and DW Conv2D with 3x3 kernel +- Neureka's engine-aware DW lowering pass `NeurekaNCHWtoNHWCDwConvPass` - XDNA2 (AIE2p) platform beta: first MLIR backend for Deeploy, targeting AMD/Xilinx NPU2 with a single BF16 Add kernel - `MLIRNodeTemplate` and `MLIRCodeTransformation` base classes for MLIR-emitting backends - Auto-tiling with L1 memory constraints for XDNA2 @@ -38,6 +41,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Add support for the Generic target for the following operators [Ceil](https://onnx.ai/onnx/operators/onnx__Ceil.html), [Floor](https://onnx.ai/onnx/operators/onnx__Floor.html), [Clip](https://onnx.ai/onnx/operators/onnx__Clip.html), [Sub](https://onnx.ai/onnx/operators/onnx__Sub.html), [Exp](https://onnx.ai/onnx/operators/onnx__Exp.html), [Sigmoid](https://onnx.ai/onnx/operators/onnx__Sigmoid.html), [Swish](https://onnx.ai/onnx/operators/onnx__Swish.html), [HardSigmoid](https://onnx.ai/onnx/operators/onnx__HardSigmoid.html), [HardSwish](https://onnx.ai/onnx/operators/onnx__HardSwish.html), [InstanceNormalization](https://onnx.ai/onnx/operators/onnx__InstanceNormalization.html), [GroupNormalization](https://onnx.ai/onnx/operators/onnx__GroupNormalization.html), [AveragePool](https://onnx.ai/onnx/operators/onnx__AveragePool.html), [GlobalAveragePool](https://onnx.ai/onnx/operators/onnx__GlobalAveragePool.html), [GlobalMaxPool](https://onnx.ai/onnx/operators/onnx__GlobalMaxPool.html). ### Changed +- Refactor the topology optimization pass `NeurekaReshapePointwiseConvolutionPass` and Neureka's Tile constraints - `aie.dialects` API: move `link_with` from `aie_d.core()` to `aie_d.external_func()` (mlir-aie v1.3.2) - Decouple XDNA requirements (`requirements-xdna.txt`) from base dev requirements - Make `aie` import optional to not enforce mlir-aie package installation for non-XDNA users @@ -56,6 +60,9 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Skip emitting duplicate `testInputVector` data for inputs placed in L3 (loaded at runtime from the readfs hex instead), reducing test binary size ### Fixed +- Fix Neureka's output-channels subtile size (in ConvTemplate) and Dense/DW/PW tile constraints +- in `NetworkContainer._createIOBindings`, set `_live = True` on network input and output buffers so that any buffer aliasing a network I/O tensor is no longer deallocated while the I/O tensor is still in use. +- Fix latent bug in `VariableBuffer.has_live_aliases` where `visited` variable was storing buffer names as a set of characters instead of strings. - Add missing `shell: bash` directive to CI cache generation steps to ensure correct shell execution - Fix wrong test case in GAP9 ccache workflow (`test_gap9_tiled_kernels_l2_singlebuffer` using `MatMul/Regular` instead of `Add/Large`) - Fix Docker flow to fetch `*.so` git lfs files @@ -69,6 +76,7 @@ This file contains the changelog for the Deeploy project. The changelog is divid - Fix GAP9 board tests with `--defaultMemLevel L3` reading garbage inputs: place all gapy `--flash-property` options before the positional subcommand and use `image flash run` so the readfs partition (input hex files) is flashed to the device ### Removed +- removed experimental `enable3x3` flag, from Neureka Engine. Now, 3x3 mode is enabled by default. - `testDMA.py` was an old test; we now have `test_dmas.py` instead. ## Release v0.2.1 (2026-02-05) [#158](https://github.com/pulp-platform/Deeploy/pull/158) diff --git a/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py b/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py index aba6740d49..1380462324 100644 --- a/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py +++ b/Deeploy/CommonExtensions/OptimizationPasses/TopologyOptimizationPasses/LoweringOptimizationPasses.py @@ -106,7 +106,7 @@ def _prependSqueezeDims(tensor: gs.Tensor, name: str, axis: Union[int, Sequence[ # Permute (0,1,2,3,...,N-2,N-1) -> (0,1,2,3,...,N-1,N-2) def _swapLastTwoDimsPermutation(N: int) -> List[int]: - assert N >= 2, "N needs to be larger then 2" + assert N >= 2, "N needs to be larger than 2" return [*range(N - 2), N - 1, N - 2] @@ -393,12 +393,18 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): if not isinstance(matrixB, gs.Constant): return graph - assert len(matrixA.shape) in [ - 2, 3 - ], f"Unsupported number of dimensions for input matrix A of GEMM operation: {len(matrixA.shape)}; shape: {matrixA.shape}" - assert len(matrixY.shape) in [ - 2, 3 - ], f"Unsupported number of dimensions for output matrix of GEMM operation: {len(matrixY.shape)}; shape: {matrixY.shape}" + assert len(matrixA.shape) in (2, 3), f"Unsupported GEMM's input matrix A with shape {matrixA.shape}" + assert len(matrixY.shape) in (2, 3), f"Unsupported GEMM's output matrix with shape {matrixY.shape}" + + # The pointwise conv this node is about to be lowered into only supports + # per-tensor or per-output-channel requantization via its RQS unit. If + # mul/add don't fit that (e.g. per-row requantization, where M becomes a + # spatial dimension of the convolution rather than a channel), bail out + # here and leave the RequantizedGemm for another lowering pass to handle. + add, mul = node.inputs[2], node.inputs[3] + out_channels = matrixY.shape[-1] + if int(np.prod(mul.shape)) not in (1, out_channels) or int(np.prod(add.shape)) not in (1, out_channels): + return graph # Pointwise with HWC layout (channels_first == False) @@ -408,13 +414,15 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): node.attrs['alpha'] = node.attrs.get('alpha', 1.0) node.attrs['beta'] = node.attrs.get('beta', 1.0) - # If transA is set then the matrix is of shape [B x K x M] and it needs to be transposed, otherwise its shape is [B x M x K] + # If transA is set then the matrix is of shape [B x K x M] and it needs to + # be transposed, otherwise its shape is [B x M x K] if node.attrs['transA'] == 1: perm = _swapLastTwoDimsPermutation(len(matrixA.shape)) graph.nodes.append(_appendTranspose(matrixA, node, perm)) matrixA = node.inputs[0] - # If transB is set then the matrix is of shape [N x K] and it doesn't need to be transposed, otherwise its shape is [K x N] and it has to be transposed + # If transB is set then the matrix is of shape [N x K] and it doesn't need + # to be transposed, otherwise its shape is [K x N] and it has to be transposed if node.attrs['transB'] == 0: perm = _swapLastTwoDimsPermutation(len(matrixB.shape)) matrixB.values = matrixB.values.transpose(perm) @@ -444,6 +452,7 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): matrixYSqueezeDimsNode, pwOut = _prependSqueezeDims(matrixY, name, squeezeDims) graph.nodes.append(matrixYSqueezeDimsNode) + n_levels = node.attrs['n_levels_out'] if 'n_levels_out' in node.attrs else node.attrs['n_levels'] pwAttrs = { 'channels_first': False, 'dilations': [1, 1], @@ -452,14 +461,11 @@ def _requantized_gemm_to_pw_fun(graph: gs.Graph, match: Match, name: str): 'pads': [0, 0, 0, 0], 'strides': [1, 1], 'div': node.attrs['div'], - 'n_levels_out': node.attrs['n_levels_out'], + 'n_levels_out': n_levels, 'shift': node.attrs['shift'], 'signed': node.attrs['signed'], } - add = node.inputs[2] - mul = node.inputs[3] - _inputs = [pwIn, pwWeight, mul, add] pw = gs.Node(op = 'RequantizedConv', diff --git a/Deeploy/DeeployTypes.py b/Deeploy/DeeployTypes.py index 44abe85112..d054ffea8c 100644 --- a/Deeploy/DeeployTypes.py +++ b/Deeploy/DeeployTypes.py @@ -339,7 +339,7 @@ def has_live_aliases(self, ctxt: NetworkContext) -> bool: # Do a breadth-first search across the aliasing double-linked list live = self._live queue = set(self.aliases) - visited = set(self.name) + visited = {self.name} while len(queue) > 0: next = queue.pop() buffNext = ctxt.lookup(next) @@ -2499,6 +2499,12 @@ def _createIOBindings(self, ctxt: NetworkContext, graph: gs.Graph): data_type = self.inputTypes[node.name] nb = ctxt.VariableBuffer(data_name, data_size) nb.is_input = True + # Global network I/O buffers are externally allocated and live for + # the entire inference. Marking them live ensures has_live_aliases + # protects them: a buffer that aliases a network input/output (e.g. + # a no-op Reshape view) must never be deallocated, or the free would + # lead to overwrite the still-needed I/O memory. + nb._live = True ctxt.add(nb, 'global') ctxt.annotateType(data_name, data_type) @@ -2509,6 +2515,7 @@ def _createIOBindings(self, ctxt: NetworkContext, graph: gs.Graph): # WIESEP: The shape and type will be parsed from the graph nb = ctxt.VariableBuffer(data_name, data_size) nb.is_output = True + nb._live = True ctxt.add(nb, 'global') return ctxt diff --git a/Deeploy/Targets/Neureka/Deployer.py b/Deeploy/Targets/Neureka/Deployer.py index be34e1f4d3..27b1e8231c 100644 --- a/Deeploy/Targets/Neureka/Deployer.py +++ b/Deeploy/Targets/Neureka/Deployer.py @@ -8,10 +8,10 @@ from Deeploy.AbstractDataTypes import Pointer from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ - NCHWtoNHWCPass, PULPNCHWtoNHWCPass + PULPNCHWtoNHWCPass from Deeploy.DeeployTypes import DeploymentPlatform, TopologyOptimizer from Deeploy.Targets.Neureka.TopologyOptimizationPasses.Passes import ConvEngineDiscolorationPass, \ - NeurekaOptimizationPass + NeurekaNCHWtoNHWCPass, NeurekaOptimizationPass from Deeploy.Targets.PULPOpen.Deployer import PULPDeployer @@ -30,10 +30,10 @@ def __init__(self, super().__init__(graph, deploymentPlatform, inputTypes, loweringOptimizer, scheduler, name, default_channels_first, deeployStateDir, inputOffsets) - if self.Platform.engines[0].enable3x3: - for idx in range(len(self.loweringOptimizer.passes)): - if isinstance(self.loweringOptimizer.passes[idx], PULPNCHWtoNHWCPass): - self.loweringOptimizer.passes[idx] = NCHWtoNHWCPass(self.default_channels_first) + engine_name = self.Platform.engines[0].name + for idx in range(len(self.loweringOptimizer.passes)): + if isinstance(self.loweringOptimizer.passes[idx], PULPNCHWtoNHWCPass): + self.loweringOptimizer.passes[idx] = NeurekaNCHWtoNHWCPass(self.default_channels_first, engine_name) self.loweringOptimizer.passes += [ ConvEngineDiscolorationPass(), diff --git a/Deeploy/Targets/Neureka/Engine.py b/Deeploy/Targets/Neureka/Engine.py index 2585b1a688..67cc2d8c56 100644 --- a/Deeploy/Targets/Neureka/Engine.py +++ b/Deeploy/Targets/Neureka/Engine.py @@ -31,11 +31,14 @@ ConvLayer([NeurekaPWConv2DMapper, NeurekaDWConv2DMapper, NeurekaDenseConv2DMapper]), } -_includeList = ["pulp_nnx_neureka.h", "pulp_nnx_util.h", "neureka_siracusa_bsp.h", "neureka.h", "neureka_task.h"] +_includeList = [ + "pulp_nnx_neureka.h", "pulp_nnx_util.h", "neureka_siracusa_bsp.h", "neureka.h", "neureka_task.h", "neureka_gvsoc.h" +] _neurekaInitCode = r""" neureka_siracusa_conf_t conf = {.max_stall = 8}; neureka_nnx_init(neureka_siracusa_get_dev(), &conf); +// neureka_gvsoc_log_activate(neureka_siracusa_get_dev(), NEUREKA_GVSOC_LOG_LEVEL_ALL, NEUREKA_GVSOC_LOG_FORMAT_HEXADECIMAL); """ @@ -46,11 +49,9 @@ def __init__(self, Mapping = NeurekaMapping, initCode: str = _neurekaInitCode, includeList: List[str] = _includeList, - enable3x3: bool = False, enableStrides: bool = False) -> None: super().__init__(name, Mapping, initCode, includeList) - self.enable3x3 = enable3x3 self.enableStrides = enableStrides def isDenseConv(self, node) -> bool: @@ -77,7 +78,4 @@ def isDWConv(self, node) -> bool: (node.attrs['strides'] == [1, 1] or self.enableStrides) def canExecute(self, node: gs.Node) -> bool: - if self.enable3x3: - return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) - else: - return self.isPWConv(node) + return self.isPWConv(node) or self.isDWConv(node) or self.isDenseConv(node) diff --git a/Deeploy/Targets/Neureka/Parsers.py b/Deeploy/Targets/Neureka/Parsers.py index 3c564c10b2..a587ff8013 100644 --- a/Deeploy/Targets/Neureka/Parsers.py +++ b/Deeploy/Targets/Neureka/Parsers.py @@ -4,6 +4,7 @@ from typing import Tuple +import numpy as np import onnx_graphsurgeon as gs from Deeploy.DeeployTypes import NetworkContext @@ -50,14 +51,18 @@ def parseNodeCtxt(self, # and enforcing that the channels_first is false data_in = newCtxt.lookup(self.operatorRepresentation['data_in']) data_out = newCtxt.lookup(self.operatorRepresentation['data_out']) - weight = newCtxt.lookup(self.operatorRepresentation['weight']) + # MARCHIOA: weight depends on the type of convolution so it requires to be parsed by the child parsers + # - PW -> 3-dim + # - DW -> 4-dim + # - Dense -> 4-dim + # weight = newCtxt.lookup(self.operatorRepresentation['weight']) if not all([ channels_first == False, len(data_in.shape) == 4, - # LMACAN: weight shape should be equal to 3 because we have to do the neureka's - # special weight encoding - len(weight.shape) == 3, + # # LMACAN: weight shape should be equal to 3 because we have to do the neureka's + # # special weight encoding + # len(weight.shape) == 3, ]): return newCtxt, False @@ -83,18 +88,36 @@ def parseNode(self, node: gs.Node) -> bool: if not super().parseNode(node): return False - ch_im_out = node.inputs[1].shape[0] - ch_im_in = node.inputs[1].shape[1] + weights = node.inputs[1] + # weigths reshaped by the weigths encoder into + # (cout, cinMajor, bits, weightBandwidthBytes) + # where: + # - cout: 1 by definition (it is cin from ONNX) + # - cinMajor: number of tiles over the channels + # - bits: weight bit width (only 8 is supported) + # - weightBandwidthBytes: which is 32 in Siracusa if not all([ self.operatorRepresentation['kernel_shape'] == [3, 3], - self.operatorRepresentation['group'] == ch_im_out, - self.operatorRepresentation['group'] == ch_im_in, + len(weights.shape) == 4, + weights.shape[0] == 1, # ch_im_out ]): return False return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 4): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSDWConv2DParser(NeurekaDWConv2DParser, RQSParserInterface): @@ -136,6 +159,18 @@ def parseNode(self, node: gs.Node) -> bool: return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 3): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSPWConv2DParser(NeurekaPWConv2DParser, RQSParserInterface): @@ -155,9 +190,24 @@ def parseNodeCtxt(self, if not ret: return ctxt, False - inputs = ['data_in', 'weight', 'mul', 'add'] - for idx, inputNode in enumerate(node.inputs): - self.operatorRepresentation[inputs[idx]] = ctxt.lookup(inputNode.name).name + data_in = ctxt.lookup(node.inputs[0].name) + weight = ctxt.lookup(node.inputs[1].name) + mul = ctxt.lookup(node.inputs[2].name) + add = ctxt.lookup(node.inputs[3].name) + + # The Neureka PW conv's RQS unit only supports per-tensor or + # per-output-channel requantization: mul/add must have either 1 + # element or one element per output channel (weight's dim 0). + out_channels = weight.shape[0] + for tensor in (mul, add): + size = int(np.prod(tensor.shape)) + if size not in (1, out_channels): + return ctxt, False + + self.operatorRepresentation['data_in'] = data_in.name + self.operatorRepresentation['weight'] = weight.name + self.operatorRepresentation['mul'] = mul.name + self.operatorRepresentation['add'] = add.name return newCtxt, True @@ -176,6 +226,18 @@ def parseNode(self, node: gs.Node) -> bool: return True + def parseNodeCtxt(self, ctxt, node, channels_first = True): + + newCtxt, ret = super().parseNodeCtxt(ctxt, node, channels_first) + if not ret: + return newCtxt, False + + weight = newCtxt.lookup(self.operatorRepresentation['weight']) + if not (len(weight.shape) == 4): + return newCtxt, False + + return newCtxt, True + class NeurekaRQSDenseConv2DParser(NeurekaDenseConv2DParser, RQSParserInterface): diff --git a/Deeploy/Targets/Neureka/Templates/ConvTemplate.py b/Deeploy/Targets/Neureka/Templates/ConvTemplate.py index 97253d6e12..c7d39cf851 100644 --- a/Deeploy/Targets/Neureka/Templates/ConvTemplate.py +++ b/Deeploy/Targets/Neureka/Templates/ConvTemplate.py @@ -225,7 +225,6 @@ def getCounters( @classmethod def getWeightStrides(cls, channel_in: int) -> Tuple[int, int, int]: - n_channel_in = _getNumTiles(channel_in, 28) _NEUREKA_WEIGHT_BANDWIDTH_BYTES = 32 return _NEUREKA_WEIGHT_BANDWIDTH_BYTES, 0, 0 @@ -256,12 +255,12 @@ def getCounters( operatorRepresentation: OperatorRepresentation) -> Tuple[int, int, int, int, int, int, int, int, int, int]: _ = operatorRepresentation # operatorRepresentation not accessed for now because it's just for pointwise kernels - n_channel_out_subtiles = _getNumTiles(channel_out, 28) + n_channel_out_subtiles = _getNumTiles(channel_out, 32) n_channel_in_subtiles = _getNumTiles(channel_in, 28) n_height_out_subtiles = _getNumTiles(height_out, 6) n_width_out_subtiles = _getNumTiles(width_out, 6) - channel_out_border = _getBorderTileSize(channel_out, 28) + channel_out_border = _getBorderTileSize(channel_out, 32) channel_in_border = _getBorderTileSize(channel_in, 28) height_out_border = _getBorderTileSize(height_out, 6) width_out_border = _getBorderTileSize(width_out, 6) diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py new file mode 100644 index 0000000000..840f608290 --- /dev/null +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaConvTileConstraint.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2024 ETH Zurich and University of Bologna +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Dict, List, Tuple, Type + +from Deeploy.AbstractDataTypes import Pointer, PointerClass +from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer +from Deeploy.Targets.Neureka.Templates.ConvTemplate import NeurekaConvTemplate, getInputAddrOffset, \ + ioStridesFromDimensions +from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule +from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint +from Deeploy.TilingExtension.TileConstraint import TileConstraint +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ + VariableReplacementScheme + +# Order in which the N-EUREKA hardware subtile counters are returned by every template's getCounters(). +_COUNTER_NAMES = ("nKo", "nKi", "nHo", "nWo", "bKo", "bKi", "bHo", "bWo", "bHi", "bWi") + + +class PerTileReplacements: + """Accumulate per-tile template-variable replacements together with their C type. + + Each ``append`` records both the value and the variable's pointer type. + Call ``scheme`` once at the end to materialize the :class:`VariableReplacementScheme` the tiler expects. + """ + + def __init__(self) -> None: + self._types: Dict[str, Type] = {} + self._values: Dict[str, List] = {} + + def append(self, name: str, dtype: Type, value) -> None: + if name not in self._types: + self._types[name] = dtype + self._values[name] = [] + self._values[name].append(value) + + def scheme(self) -> VariableReplacementScheme: + replacementTypes: Dict[str, Type[Pointer]] = {name: PointerClass(dtype) for name, dtype in self._types.items()} + return VariableReplacementScheme(self._values, replacementTypes) + + +class NeurekaConvTileConstraint(TileConstraint): + """Shared tiling logic for the N-EUREKA convolution variants (pointwise, depthwise, dense). + + The serialization skeleton (input-cube computation, I/O strides, subtile counters, load + schedules) is identical across the three; the parts that genuinely differ are exposed as hooks: + + - ``_ConvTemplate`` : the template class providing ``getCounters`` for this variant. + - ``_adjustInputCube``: post-process the computed input cube (depthwise slices channels). + - ``_addWeightSchedule``: emit the weight base offset / load schedule (packing differs per variant). + """ + + # Set by each concrete variant to its Neureka2D*ConvTemplate subclass. + _ConvTemplate: Type[NeurekaConvTemplate] + + @classmethod + def _adjustInputCube(cls, inCube: HyperRectangle, outputCube: HyperRectangle) -> HyperRectangle: + """Adjust the input cube derived from an output tile. Identity by default.""" + return inCube + + @classmethod + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + """Emit the per-tile weight addressing (offset and/or load schedule). Variant-specific.""" + raise NotImplementedError(f"{cls.__name__} must implement _addWeightSchedule") + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + outputCubes = [cube.rectangle for cube in absoluteOutputCubes] + + inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, + operatorRepresentation, ['data_in', 'data_out']) + + outputBuffer = ctxt.lookup(operatorRepresentation['data_out']) + assert isinstance(outputBuffer, VariableBuffer) + + weightH: int = operatorRepresentation['dim_kernel_y'] + weightW: int = operatorRepresentation['dim_kernel_x'] + weightC: int = operatorRepresentation['ch_im_in'] + pads: tuple[int, int, int, int] = operatorRepresentation['pads'] + strides: tuple[int, int] = operatorRepresentation['strides'] + + input_bits: int = operatorRepresentation["input_bits"] + output_bits: int = operatorRepresentation["output_bits"] + + rep = PerTileReplacements() + inputCubes = [] + + for cube in outputCubes: + (_, _, _, COffset) = cube.offset + (_, HSize, WSize, CSize) = cube.dims + + inCube, pads_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, cube, + outputBuffer.shape) + inCube = cls._adjustInputCube(inCube, cube) + + pad_left, pad_right, pad_top, pad_bottom = pads_tuple + rep.append('padding_y_top', uint8_t, pad_top) + rep.append('padding_y_bottom', uint8_t, pad_bottom) + rep.append('padding_x_left', uint8_t, pad_left) + rep.append('padding_x_right', uint8_t, pad_right) + + _, _, inWSize, inCSize = inCube.dims + dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, input_bits) + rep.append('dim_im_in_x_stride', uint32_t, dim_im_in_x_stride) + rep.append('dim_im_in_y_stride', uint32_t, dim_im_in_y_stride) + dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, output_bits) + rep.append('dim_im_out_x_stride', uint32_t, dim_im_out_x_stride) + rep.append('dim_im_out_y_stride', uint32_t, dim_im_out_y_stride) + + rep.append('input_addr_offset', uint32_t, getInputAddrOffset(inWSize, dim_im_in_y_stride, pad_top, + pad_left)) + + counters = cls._ConvTemplate.getCounters(inCSize, HSize, WSize, CSize, pad_bottom, pad_right, + operatorRepresentation) + for name, value in zip(_COUNTER_NAMES, counters): + rep.append(name, uint16_t, value) + + inputCubes.append(inCube) + + inputLoadSchedule = [{"data_in": cube} for cube in inputCubes] + outputLoadSchedule = [{"data_out": cube} for cube in outputCubes] + + cls._addWeightSchedule(rep, inputLoadSchedule, inputBaseOffsets, outputBaseOffsets, absoluteOutputCubes, + tilingSolution, targetMemLevel, ctxt, operatorRepresentation) + + tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) + + return rep.scheme(), tilingSchedule + + +class NeurekaRQSConvTileConstraint(NeurekaConvTileConstraint): + """Mixin adding requantization to any :class:`NeurekaConvTileConstraint` variant. + + Combine it (listed first) with a concrete variant, e.g.:: + + class NeurekaRQSDWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDWConv2DTileConstraint): + pass + + Cooperative ``super()`` dispatch then routes to the variant's geometrical constraint and + serialization before layering the requant offsets/loads on top. + """ + + @classmethod + def addGeometricalConstraint(cls, tilerModel, parseDict: Dict, ctxt: NetworkContext): + tilerModel = super().addGeometricalConstraint(tilerModel, parseDict, ctxt) + return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) + + @classmethod + def serializeTilingSolution( + cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: + + variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( + tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) + + inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, + ['mul', 'add']) + newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} + + requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) + newInputLoadSchedule = [{ + **load, + **rqLoad + } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] + + newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, + tilingSchedule.outputLoadSchedule) + + return variableReplacementSchedule, newTilingSchedule diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py index 814024a877..91857b45fd 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDenseConstraint.py @@ -2,23 +2,21 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDenseConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDenseConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes -class NeurekaDenseConv2DTileConstraint(TileConstraint): +class NeurekaDenseConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DDenseConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -53,6 +51,7 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw weightBuffer = ctxt.lookup(weightBufferName) if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": + # No tiling. Weight tensor is a constant statically placed in the weight memory (wmem) tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) else: tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) @@ -73,6 +72,10 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 3) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 3) + strides = parseDict["strides"] tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) @@ -80,6 +83,11 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) + # Force the weight tensor's non-tiled dims to their full size + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) @@ -89,180 +97,33 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, outputBuffer.shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DDenseConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) for absoluteCube in absoluteOutputCubes: COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] - WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + WeightCube = HyperRectangle((COffset, 0, 0, 0), + (CSize, weightShape[-3], weightShape[-2], weightShape[-1])) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) else: inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] - load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - - return variableReplacementSchedule, tilingSchedule - - -class NeurekaRQSDenseConv2DTileConstraint(NeurekaDenseConv2DTileConstraint): + for absoluteCube, load in zip(absoluteOutputCubes, inputLoadSchedule): + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] + load['weight'] = HyperRectangle((COffset, 0, 0, 0), + (CSize, weightShape[-3], weightShape[-2], weightShape[-1])) - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaDenseConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSDenseConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDenseConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py index fd5d791119..6e131e23e8 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaDepthwiseConstraint.py @@ -2,23 +2,28 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDWConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DDWConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes +# Neureka packs depthwise weights into "cinMajor" blocks of this many channels (see +# NeurekaAdjustWeightMemoryLayoutPass). A channel tile can therefore only start on a boundary that +# is a multiple of this value. +_NEUREKA_CIN_SUBTILE_3x3 = 28 +_NEUREKA_KERNEL_HEIGHT_3x3 = 3 +_NEUREKA_KERNEL_WIDTH_3x3 = 3 -class NeurekaDWConv2DTileConstraint(TileConstraint): + +class NeurekaDWConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DDWConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -27,8 +32,7 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw outputBufferName = parseDict['data_out'] strides = parseDict["strides"] - padding = parseDict["pads"] - dilation = parseDict["dilations"] + pads = parseDict["pads"] for bufferName in [inputBufferName, weightBufferName, outputBufferName]: tilerModel.addTensorDimToModel(ctxt, bufferName) @@ -38,6 +42,8 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBufferName, dimIdx = 3) + # In depthwise this axis is degenerate (cout == 1): the actual channels are folded into `cinMajor`, + # not into cout. it is just the size-1 output-channel axis of the weight blob. weightOutChannelVar = tilerModel.getTensorDimVar(tensorName = weightBufferName, dimIdx = 0) outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBufferName, dimIdx = 0) @@ -49,22 +55,24 @@ def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: Netw tilerModel.addConstraint(outputBatchVar == inputBatchVar) tilerModel.addConstraint(outputChannelVar == inputChannelVar) - weightBuffer = ctxt.lookup(weightBufferName) - if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) - else: - tilerModel.addConstraint(weightOutChannelVar == outputChannelVar) + tilerModel.addConstraint(weightOutChannelVar == weightOutChannelVar.Max()) # dummy since cout=1 - tilerModel.addConstraint(inputHeightVar >= 3) - tilerModel.addConstraint(inputWidthVar >= 3) + # Since channels are packed in blocks of _NEUREKA_CIN_SUBTILE_3x3 channels, either + # - channels are not tiled (single tile == full size) or + # - channels are tiles with a tile size multiple of _NEUREKA_CIN_SUBTILE_3x3 + tilerModel.addConstraint((outputChannelVar == outputChannelVar.Max()) + + ((outputChannelVar % _NEUREKA_CIN_SUBTILE_3x3) == 0) >= 1) - inputBuffer = ctxt.lookup(inputBufferName) + tilerModel.addConstraint(inputHeightVar >= _NEUREKA_KERNEL_HEIGHT_3x3) + tilerModel.addConstraint(inputWidthVar >= _NEUREKA_KERNEL_WIDTH_3x3) - effectiveHeight = inputHeightVar + ((padding[0] + padding[2]) * (inputHeightVar == inputBuffer.shape[1])) - effectiveWidth = inputWidthVar + ((padding[1] + padding[3]) * (inputWidthVar == inputBuffer.shape[2])) - - tilerModel.addConstraint((outputHeightVar == (effectiveHeight - (3 - 1) - 1) // strides[0] + 1)) - tilerModel.addConstraint((outputWidthVar == (effectiveWidth - (3 - 1) - 1) // strides[1] + 1)) + _, Hin, Win, _ = ctxt.lookup(inputBufferName).shape + effectiveHeight = inputHeightVar + ((pads[0] + pads[2]) * (inputHeightVar == Hin)) + effectiveWidth = inputWidthVar + ((pads[1] + pads[3]) * (inputWidthVar == Win)) + outputHeight = (effectiveHeight - (_NEUREKA_KERNEL_HEIGHT_3x3 - 1) - 1) // strides[0] + 1 + outputWidth = (effectiveWidth - (_NEUREKA_KERNEL_WIDTH_3x3 - 1) - 1) // strides[1] + 1 + tilerModel.addConstraint(outputHeightVar == outputHeight) + tilerModel.addConstraint(outputWidthVar == outputWidth) return tilerModel @@ -73,192 +81,65 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo inputHeightVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 1) inputWidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['data_in'], dimIdx = 2) + weightInChannelMajorVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 1) + weightBitsVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 2) + weightBandwidthVar = tilerModel.getTensorDimVar(tensorName = parseDict['weight'], dimIdx = 3) + strides = parseDict["strides"] tilerModel.addConstraint((inputHeightVar % strides[0]) == 0) tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) + # Force the weight tensor's non-tiled dims to their full size + tilerModel.addConstraint(weightInChannelMajorVar == weightInChannelMajorVar.Max()) + tilerModel.addConstraint(weightBitsVar == weightBitsVar.Max()) + tilerModel.addConstraint(weightBandwidthVar == weightBandwidthVar.Max()) + tilerModel.addConstraint(inputHeightVar == inputHeightVar.Max(), strategy = PerformanceHint(1)) tilerModel.addConstraint(inputWidthVar == inputWidthVar.Max(), strategy = PerformanceHint(1)) return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, - ctxt.lookup(varOut).shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DDWConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _adjustInputCube(cls, inCube: HyperRectangle, outputCube: HyperRectangle) -> HyperRectangle: + # In DW, each output channel only depends on the corresponding input channel. + # Therefore we can tile the input channels exactly as the output channels. + COffset = outputCube.offset[-1] + CSize = outputCube.dims[-1] + return HyperRectangle(inCube.offset[:-1] + (COffset,), inCube.dims[:-1] + (CSize,)) + + @classmethod + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape - if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) - for absoluteCube in absoluteOutputCubes: - COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] - WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) - else: + # The DW weight is never tiled: it is always resident in full (in SRAM for the wmem case, or DMA'd + # whole into L1 otherwise). It is packed as (cout=1, cinMajor, bits, bandwidthBytes), where the + # channels live in the cinMajor dimension in blocks of _NEUREKA_CIN_SUBTILE_3x3. A channel tile + # starting at COffset (guaranteed to be a multiple of _NEUREKA_CIN_SUBTILE_3x3 by the geometrical + # constraint) therefore starts at cinMajor block COffset // _NEUREKA_CIN_SUBTILE_3x3, so we offset + # the weight base to that block. + for absoluteCube in absoluteOutputCubes: + COffset = absoluteCube.absoluteOffset[-1] + cinMajorOffset = COffset // _NEUREKA_CIN_SUBTILE_3x3 + WeightCube = HyperRectangle((0, cinMajorOffset, 0, 0), + (weightShape[0], 1, weightShape[-2], weightShape[-1])) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + + if not (hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM"): inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] - load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) + for load in inputLoadSchedule: + load['weight'] = HyperRectangle((0,) * len(weightShape), tuple(weightShape)) - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - return variableReplacementSchedule, tilingSchedule - - -class NeurekaRQSDWConv2DTileConstraint(NeurekaDWConv2DTileConstraint): - - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaDWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSDWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaDWConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py b/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py index 61a5b8756a..89da39d0d8 100644 --- a/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py +++ b/Deeploy/Targets/Neureka/TileConstraints/NeurekaPointwiseConstraint.py @@ -2,23 +2,26 @@ # # SPDX-License-Identifier: Apache-2.0 -from typing import Dict, List, Tuple +from typing import Dict, List -from Deeploy.AbstractDataTypes import PointerClass -from Deeploy.CommonExtensions.DataTypes import uint8_t, uint16_t, uint32_t +from Deeploy.CommonExtensions.DataTypes import uint32_t from Deeploy.DeeployTypes import NetworkContext, OperatorRepresentation, VariableBuffer -from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DPWConvTemplate, getInputAddrOffset, \ - ioStridesFromDimensions -from Deeploy.Targets.Neureka.TileConstraints.RequantHelpers import requantAddGeometricalConstraint, requantLoadSchedule -from Deeploy.Targets.PULPOpen.TileConstraints.ConvTileConstraint import Conv2DTileConstraint +from Deeploy.Targets.Neureka.Templates.ConvTemplate import Neureka2DPWConvTemplate +from Deeploy.Targets.Neureka.TileConstraints.NeurekaConvTileConstraint import NeurekaConvTileConstraint, \ + NeurekaRQSConvTileConstraint, PerTileReplacements from Deeploy.TilingExtension.MemoryConstraints import NodeMemoryConstraint -from Deeploy.TilingExtension.TileConstraint import TileConstraint from Deeploy.TilingExtension.TilerModel import PerformanceHint, TilerModel -from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, TilingSchedule, \ - VariableReplacementScheme, calculateFlatOffsetInBytes +from Deeploy.TilingExtension.TilingCodegen import AbsoluteHyperRectangle, HyperRectangle, calculateFlatOffsetInBytes +_NEUREKA_PE_H = 6 +_NEUREKA_PE_W = 6 +_NEUREKA_TP_IN = 32 # input channel parallelism +_NEUREKA_TP_OUT = 32 # output channel parallelism -class NeurekaPWConv2DTileConstraint(TileConstraint): + +class NeurekaPWConv2DTileConstraint(NeurekaConvTileConstraint): + + _ConvTemplate = Neureka2DPWConvTemplate @staticmethod def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: @@ -64,6 +67,7 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo weightBuffer = ctxt.lookup(name = parseDict['weight']) outputBuffer = ctxt.lookup(name = parseDict['data_out']) + outputBatchVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 0) inputHeightVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 1) inputWidthVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 2) inputChannelVar = tilerModel.getTensorDimVar(tensorName = inputBuffer.name, dimIdx = 3) @@ -76,8 +80,10 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo outputWidthVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 2) outputChannelVar = tilerModel.getTensorDimVar(tensorName = outputBuffer.name, dimIdx = 3) + # Neureka has no batch counter: process one batch element per dispatch + tilerModel.addConstraint(outputBatchVar == 1) + strides = parseDict["strides"] - padding = parseDict["pads"] # LMACAN: Force full input channel to avoid partial results tilerModel.addConstraint(inputChannelVar == inputChannelVar.Max()) @@ -88,29 +94,29 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo tilerModel.addConstraint((inputWidthVar % strides[1]) == 0) # N-EUREKA tile constraints to align with N-EUREKA's hardware subtiling - if parseDict["dim_im_out_x"] > 6: + if parseDict["dim_im_out_x"] > _NEUREKA_PE_W: tilerModel.addTileSizeDivisibleConstraint(parseDict, "dim_im_out_x", outputHeightVar, - 6, + _NEUREKA_PE_W, strategy = PerformanceHint(priority = 3)) else: tilerModel.addConstraint(outputHeightVar == outputHeightVar.Max(), strategy = PerformanceHint(priority = 3)) - if parseDict["dim_im_out_y"] > 6: + if parseDict["dim_im_out_y"] > _NEUREKA_PE_H: tilerModel.addTileSizeDivisibleConstraint(parseDict, "dim_im_out_y", outputWidthVar, - 6, + _NEUREKA_PE_H, strategy = PerformanceHint(priority = 2)) else: tilerModel.addConstraint(outputWidthVar == outputWidthVar.Max(), strategy = PerformanceHint(priority = 2)) - if parseDict["ch_im_out"] > 32: + if parseDict["ch_im_out"] > _NEUREKA_TP_OUT: tilerModel.addTileSizeDivisibleConstraint(parseDict, "ch_im_out", outputChannelVar, - 32, + _NEUREKA_TP_OUT, strategy = PerformanceHint(priority = 1)) else: tilerModel.addConstraint(outputChannelVar == outputChannelVar.Max(), @@ -119,180 +125,31 @@ def addPolicyConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkCo return tilerModel @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - outputCubes = [cube.rectangle for cube in absoluteOutputCubes] - - addrNames = ['data_in', 'data_out'] - inputBaseOffsets, outputBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, - operatorRepresentation, addrNames) - - varWeight = operatorRepresentation['weight'] - varOut = operatorRepresentation['data_out'] - - inputInCubes = [] - replacements: Dict[str, List[int]] = { - "padding_y_top": [], - "padding_y_bottom": [], - "padding_x_left": [], - "padding_x_right": [], - "dim_im_in_x_stride": [], - "dim_im_in_y_stride": [], - "dim_im_out_x_stride": [], - "dim_im_out_y_stride": [], - "input_addr_offset": [], - "nKo": [], - "nKi": [], - "nHo": [], - "nWo": [], - "bKo": [], - "bKi": [], - "bHo": [], - "bWo": [], - "bHi": [], - "bWi": [], - } - - replacementTypes = { - "padding_y_top": PointerClass(uint8_t), - "padding_y_bottom": PointerClass(uint8_t), - "padding_x_left": PointerClass(uint8_t), - "padding_x_right": PointerClass(uint8_t), - "dim_im_in_x_stride": PointerClass(uint32_t), - "dim_im_in_y_stride": PointerClass(uint32_t), - "dim_im_out_x_stride": PointerClass(uint32_t), - "dim_im_out_y_stride": PointerClass(uint32_t), - "input_addr_offset": PointerClass(uint32_t), - "nKo": PointerClass(uint16_t), - "nKi": PointerClass(uint16_t), - "nHo": PointerClass(uint16_t), - "nWo": PointerClass(uint16_t), - "bKo": PointerClass(uint16_t), - "bKi": PointerClass(uint16_t), - "bHo": PointerClass(uint16_t), - "bWo": PointerClass(uint16_t), - "bHi": PointerClass(uint16_t), - "bWi": PointerClass(uint16_t), - } - - weightH = operatorRepresentation['dim_kernel_y'] - weightW = operatorRepresentation['dim_kernel_x'] - weightC = operatorRepresentation['ch_im_in'] - - pads = operatorRepresentation['pads'] - strides = operatorRepresentation['strides'] - - outputBuffer = ctxt.lookup(varOut) - assert isinstance(outputBuffer, VariableBuffer) - - for cube in outputCubes: - (BatchOffset, HOffset, WOffset, COffset) = cube.offset - (BatchSize, HSize, WSize, CSize) = cube.dims - - InCube, padding_tuple = Conv2DTileConstraint.computeInputCube((weightH, weightW), pads, strides, weightC, - cube, outputBuffer.shape) - padding_left, padding_right, padding_top, padding_bottom = padding_tuple - - replacements['padding_y_top'].append(padding_top) - replacements['padding_y_bottom'].append(padding_bottom) - replacements['padding_x_left'].append(padding_left) - replacements['padding_x_right'].append(padding_right) - - inBSize, inHSize, inWSize, inCSize = InCube.dims - - dim_im_in_x_stride, dim_im_in_y_stride = ioStridesFromDimensions(inWSize, inCSize, - operatorRepresentation["input_bits"]) - replacements['dim_im_in_x_stride'].append(dim_im_in_x_stride) - replacements['dim_im_in_y_stride'].append(dim_im_in_y_stride) - dim_im_out_x_stride, dim_im_out_y_stride = ioStridesFromDimensions(WSize, CSize, - operatorRepresentation["output_bits"]) - replacements['dim_im_out_x_stride'].append(dim_im_out_x_stride) - replacements['dim_im_out_y_stride'].append(dim_im_out_y_stride) - - replacements['input_addr_offset'].append( - getInputAddrOffset(inWSize, dim_im_in_y_stride, padding_top, padding_left)) - - nKo, nKi, nHo, nWo, bKo, bKi, bHo, bWo, bHi, bWi = Neureka2DPWConvTemplate.getCounters( - inCSize, HSize, WSize, CSize, padding_bottom, padding_right, operatorRepresentation) - - replacements["nKo"].append(nKo) - replacements["nKi"].append(nKi) - replacements["nHo"].append(nHo) - replacements["nWo"].append(nWo) - replacements["bKo"].append(bKo) - replacements["bKi"].append(bKi) - replacements["bHo"].append(bHo) - replacements["bWo"].append(bWo) - replacements["bHi"].append(bHi) - replacements["bWi"].append(bWi) - - inputInCubes.append(InCube) - - inputLoadSchedule = [] - outputLoadSchedule = [] - - for a in inputInCubes: - inputLoadSchedule.append({"data_in": a}) - - for out in outputCubes: - outputLoadSchedule.append({"data_out": out}) - - weightBuffer = ctxt.lookup(varWeight) + def _addWeightSchedule(cls, rep: PerTileReplacements, inputLoadSchedule: List[Dict[str, HyperRectangle]], + inputBaseOffsets: Dict[str, List[int]], outputBaseOffsets: Dict[str, List[int]], + absoluteOutputCubes: List[AbsoluteHyperRectangle], tilingSolution: NodeMemoryConstraint, + targetMemLevel: str, ctxt: NetworkContext, + operatorRepresentation: OperatorRepresentation) -> None: + + weightBuffer = ctxt.lookup(operatorRepresentation['weight']) assert isinstance(weightBuffer, VariableBuffer) weightShape = weightBuffer.shape if hasattr(weightBuffer, "_memoryLevel") and weightBuffer._memoryLevel == "WeightMemory_SRAM": - replacements['weight_addr_offset'] = [] - replacementTypes['weight_addr_offset'] = PointerClass(uint32_t) for absoluteCube in absoluteOutputCubes: COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] WeightCube = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - replacements['weight_addr_offset'].append(calculateFlatOffsetInBytes(WeightCube, weightBuffer)) + rep.append('weight_addr_offset', uint32_t, calculateFlatOffsetInBytes(WeightCube, weightBuffer)) else: inputWeightBaseOffsets, outputWeightBaseOffsets = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, ['weight']) inputBaseOffsets.update(inputWeightBaseOffsets) outputBaseOffsets.update(outputWeightBaseOffsets) - for cube, load in zip(outputCubes, inputLoadSchedule): - COffset, CSize = cube.offset[-1], cube.dims[-1] + for absoluteCube, load in zip(absoluteOutputCubes, inputLoadSchedule): + COffset, CSize = absoluteCube.absoluteOffset[-1], absoluteCube.rectangle.dims[-1] load['weight'] = HyperRectangle((COffset, 0, 0), (CSize, weightShape[-2], weightShape[-1])) - tilingSchedule = TilingSchedule(inputBaseOffsets, outputBaseOffsets, inputLoadSchedule, outputLoadSchedule) - variableReplacementSchedule = VariableReplacementScheme(replacements, replacementTypes) - - return variableReplacementSchedule, tilingSchedule - -class NeurekaRQSPWConv2DTileConstraint(NeurekaPWConv2DTileConstraint): - - @staticmethod - def addGeometricalConstraint(tilerModel: TilerModel, parseDict: Dict, ctxt: NetworkContext) -> TilerModel: - tilerModel = NeurekaPWConv2DTileConstraint.addGeometricalConstraint(tilerModel, parseDict, ctxt) - return requantAddGeometricalConstraint(tilerModel, parseDict, ctxt) - - @classmethod - def serializeTilingSolution( - cls, tilingSolution: NodeMemoryConstraint, absoluteOutputCubes: List[AbsoluteHyperRectangle], - targetMemLevel: str, ctxt: NetworkContext, - operatorRepresentation: OperatorRepresentation) -> Tuple[VariableReplacementScheme, TilingSchedule]: - variableReplacementSchedule, tilingSchedule = super().serializeTilingSolution( - tilingSolution, absoluteOutputCubes, targetMemLevel, ctxt, operatorRepresentation) - - addrNames = ['mul', 'add'] - inputRequantBaseOffsets, _ = cls.extractBaseAddr(tilingSolution, targetMemLevel, operatorRepresentation, - addrNames) - newInputBaseOffsets = {**tilingSchedule.inputBaseOffsets, **inputRequantBaseOffsets} - - requantSchedule = requantLoadSchedule(absoluteOutputCubes, ctxt, operatorRepresentation) - newInputLoadSchedule = [{ - **load, - **rqLoad - } for load, rqLoad in zip(tilingSchedule.inputLoadSchedule, requantSchedule)] - - newTilingSchedule = TilingSchedule(newInputBaseOffsets, tilingSchedule.outputBaseOffsets, newInputLoadSchedule, - tilingSchedule.outputLoadSchedule) - - return variableReplacementSchedule, newTilingSchedule +class NeurekaRQSPWConv2DTileConstraint(NeurekaRQSConvTileConstraint, NeurekaPWConv2DTileConstraint): + pass diff --git a/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py index 84e0565b97..61eccb0a15 100644 --- a/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/Neureka/TopologyOptimizationPasses/Passes.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -import itertools import math from functools import partial from typing import Generator, List, Tuple @@ -15,7 +14,8 @@ from Deeploy.CommonExtensions.OptimizationPasses.PassClasses import ReplaceSequentialPatternPass, SequentialPass, \ contextagnostic from Deeploy.CommonExtensions.OptimizationPasses.TopologyOptimizationPasses.LoweringOptimizationPasses import \ - RemoveGlobalOutputReshapePass, _createReshape + NCHWtoNHWCConvPass, NCHWtoNHWCMaxPoolPass, NCHWtoNHWCPadPass, RemoveGlobalOutputReshapePass, _createReshape, \ + _isDepthwise, _NCWHtoNHWC_dw_fun, _PULP_NCHWtoNHWC_dw_fun, _singleNodePattern from Deeploy.EngineExtension.OptimizationPasses.TopologyOptimizationPasses.EngineColoringPasses import \ EngineDiscolorationPass from Deeploy.Targets.Generic.TopologyOptimizationPasses.Passes import ReshapeConstOptPass, ReshapeMergePass @@ -34,9 +34,6 @@ def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = Fa _NEUREKA_CIN_SUBTILE_1x1 = 32 _NEUREKA_CIN_SUBTILE_3x3 = 28 - if depthwise: - weight = weight.transpose(1, 0, 2, 3) # Swap cout and cin - cout, cin, height, width = weight.shape cinSubtile = (_NEUREKA_CIN_SUBTILE_3x3 if height == 3 else _NEUREKA_CIN_SUBTILE_1x1) @@ -106,10 +103,7 @@ def _weightEncode(weight: npt.NDArray[np.uint8], bits: int, depthwise: bool = Fa if height == 1 and width == 1: # (cout, cinMajor, Weight Bandwidth Bytes) return weight.reshape(cout, cinMajor, weightBandwidthBytes) - elif depthwise: - return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) - else: - return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) + return weight.reshape(cout, cinMajor, bits, weightBandwidthBytes) def _neureka_adjust_weight_memory_layout_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, @@ -170,57 +164,45 @@ def __init__(self, default_channels_first: bool, neurekaEngineName: str): NonBranchingMatcher(regex_op = True)) -def _findAllMultiplicands(x: int) -> List[int]: - multiplicands = [] - tmpX = x - for i in range(2, math.ceil(math.sqrt(x))): # Ceil cause range doesn't include the last number - while tmpX % i == 0: - multiplicands.append(i) - tmpX = tmpX / i +def _spatialFactorPairs(n: int) -> Generator[Tuple[int, int], None, None]: + """Yield every (a, b) with a * b == n and a >= b.""" + for b in range(1, math.isqrt(n) + 1): + if n % b == 0: + yield n // b, b - if x // math.prod(multiplicands) > 1: - multiplicands.append(x // math.prod(multiplicands)) - return multiplicands +def _nSubtiles(height: int, width: int) -> int: + """Number of 6x6 N-EUREKA HW subtiles needed to cover a (height, width) plane.""" + return math.ceil(height / 6) * math.ceil(width / 6) -def _findAllReshapeOptions(dim: int) -> Generator[Tuple[int, int], None, None]: - multiplicands = _findAllMultiplicands(dim) - for combLen in range(1, 1 + (len(multiplicands) // 2)): - for comb in itertools.combinations(multiplicands, combLen): - a = math.prod(comb) - b = dim // a - yield a, b +def _bestSpatialReshape(n: int) -> Tuple[int, int]: + """Find the (height, width) factorization of n needing the fewest 6x6 HW subtiles. + Ties are broken toward the more square-like factorization, since that also + tends to reduce padding waste in the border subtiles. + """ + best = (n, 1) + bestCost = _nSubtiles(*best) + bestBalance = abs(best[0] - best[1]) -def _nSubtiles(dims: Tuple[int, int]): - return math.ceil(dims[0] / 6) * math.ceil(dims[1] / 6) + for candidate in _spatialFactorPairs(n): + cost = _nSubtiles(*candidate) + balance = abs(candidate[0] - candidate[1]) + if cost < bestCost or (cost == bestCost and balance < bestBalance): + best, bestCost, bestBalance = candidate, cost, balance + return best -def _findLowestNumberOfSubtilesReshapeOptions(dim: int) -> List[Tuple[int, int]]: - lowestNumberOfSubtiles = dim - bestOptions: List[Tuple[int, int]] = [(dim, 1)] - for option in _findAllReshapeOptions(dim): - nSubtiles = _nSubtiles(option) - if nSubtiles < lowestNumberOfSubtiles: - lowestNumberOfSubtiles = nSubtiles - bestOptions = [option] - elif nSubtiles == lowestNumberOfSubtiles: - bestOptions.append(option) - return bestOptions +def _extractSpatialDims(shape: List[int], channels_first: bool) -> List[int]: + return shape[-2:] if channels_first else shape[-3:-1] -def _bestReshapeOption(dim: int) -> Tuple[int, int]: - smallestDim = dim - biggestDim = 1 - for option in _findLowestNumberOfSubtilesReshapeOptions(dim): - if option[0] < smallestDim: - smallestDim = option[0] - biggestDim = option[1] - elif option[1] < smallestDim: - smallestDim = option[1] - biggestDim = option[0] - return biggestDim, smallestDim + +def _replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int], channels_first: bool) -> List[int]: + if channels_first: + return shape[:-2] + list(newSpatialDims) + return shape[:-3] + list(newSpatialDims) + shape[-1:] def _neureka_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, @@ -228,40 +210,32 @@ def _neureka_reshape_pointwise_convolution_fun(graph: gs.Graph, match: Match, na matched_nodes = list(match.nodes_map.values()) node = matched_nodes[0] - if not ("engine" in node.attrs and node.attrs["engine"] == neurekaEngineName): + if not all([ + node.attrs.get("engine") == neurekaEngineName, + node.attrs["kernel_shape"] == [1, 1], + ]): return graph - if not (node.attrs["kernel_shape"] == [1, 1]): - return graph + channels_first = bool(node.attrs.get("channels_first", default_channels_first)) - if "channels_first" in node.attrs: - channels_first = node.attrs["channels_first"] - else: - channels_first = default_channels_first - - def extractSpatialDims(shape: List[int]) -> List[int]: - if channels_first: - return shape[-2:] - else: - return shape[-3:-1] + _input = node.inputs[0] + output = node.outputs[0] - def replaceSpatialDims(shape: List[int], newSpatialDims: Tuple[int, int]) -> List[int]: - if channels_first: - return shape[:-2] + list(newSpatialDims) - else: - return shape[:-3] + list(newSpatialDims) + shape[-1:] + inputSpatialDims = _extractSpatialDims(_input.shape, channels_first) + outputSpatialDims = _extractSpatialDims(output.shape, channels_first) + if math.prod(inputSpatialDims) != math.prod(outputSpatialDims): + return graph - _input = node.inputs[0] - spatialDims = extractSpatialDims(_input.shape) - newSpatialDims = _bestReshapeOption(math.prod(spatialDims)) - newInputShape = replaceSpatialDims(_input.shape, newSpatialDims) + newSpatialDims = _bestSpatialReshape(math.prod(inputSpatialDims)) + if tuple(inputSpatialDims) == newSpatialDims: + return graph + newInputShape = _replaceSpatialDims(_input.shape, newSpatialDims, channels_first) inputReshapeNode, reshapedInput = _createReshape(_input, name, newInputShape) graph.nodes.append(inputReshapeNode) node.inputs[0] = reshapedInput - output = node.outputs[0] - newOutputShape = replaceSpatialDims(output.shape, newSpatialDims) + newOutputShape = _replaceSpatialDims(output.shape, newSpatialDims, channels_first) reshapedOutput = gs.Variable(output.name + "_Reshaped", dtype = output.dtype, shape = newOutputShape) outputReshapeNode, _ = _createReshape(reshapedOutput, name, output.shape, output) graph.nodes.append(outputReshapeNode) @@ -289,6 +263,56 @@ def __init__(self, default_channels_first: bool, neurekaEngineName: str): NonBranchingMatcher(regex_op = True)) +def _neureka_nchw_to_nhwc_dw_conv_fun(graph: gs.Graph, match: Match, name: str, default_channels_first: bool, + neurekaEngineName: str) -> gs.Graph: + node = next(iter(match.nodes_map.values())) + + if not _isDepthwise(node): + return graph + + # DW convs have different data layouts depending on the engine that executes them: + # - N-EUREKA reads the input channels-last (NHWC) and the weight with the filter dimension last, + # - the PULP cluster kernel reads the input channels-first (NCHW) and the weight with the filter + # dimension first (see PULPOpen DWConvTileConstraint). + # We dispatch on the engine the conv was colored with. This is authoritative here because the conv+requant + # merge preserves the convolution's engine color (see PULPConvRequantMergePass), so the coloring interleaved + # before this pass has already assigned the fused RequantizedConv to the correct engine. + if node.attrs.get("engine") == neurekaEngineName: + return _NCWHtoNHWC_dw_fun(graph, match, name, default_channels_first) + return _PULP_NCHWtoNHWC_dw_fun(graph, match, name, default_channels_first) + + +@contextagnostic +class NeurekaNCHWtoNHWCDwConvPass(ReplaceSequentialPatternPass): + + def __init__(self, default_channels_first: bool, neurekaEngineName: str): + graph = _singleNodePattern(op = "RequantizedConv|Conv") + name = "_NEUREKA_NCHW_TO_NHWC_DW_CONV_PASS" + super().__init__( + graph, + partial(_neureka_nchw_to_nhwc_dw_conv_fun, + default_channels_first = default_channels_first, + neurekaEngineName = neurekaEngineName), name, NonBranchingMatcher(regex_op = True)) + + +@contextagnostic +class NeurekaNCHWtoNHWCPass(SequentialPass): + """Channels-last lowering pass for the N-EUREKA pipeline. + + Behaves like PULPNCHWtoNHWCPass/NCHWtoNHWCPass for pads, maxpools and regular convolutions, but lowers each + depthwise convolution with the layout expected by the engine that will execute it (N-EUREKA or PULP cluster). + """ + + def __init__(self, default_channels_first: bool, neurekaEngineName: str): + passes = [ + NCHWtoNHWCPadPass(default_channels_first), + NCHWtoNHWCMaxPoolPass(default_channels_first), + NeurekaNCHWtoNHWCDwConvPass(default_channels_first, neurekaEngineName), + NCHWtoNHWCConvPass(default_channels_first), + ] + super().__init__(*passes) + + class ConvEngineDiscolorationPass(EngineDiscolorationPass): def __init__(self): diff --git a/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py b/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py index 43d490e80b..f79df85d02 100644 --- a/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py +++ b/Deeploy/Targets/PULPOpen/TopologyOptimizationPasses/Passes.py @@ -179,7 +179,14 @@ def _merge_conv_rq_fun(graph: gs.Graph, match: Match, name: str): _outputs = rqs.outputs - rqsConv = gs.Node(op = 'RequantizedConv', name = name, attrs = {**conv.attrs, **rqs.attrs, "shift": totalShift}) + # RequantizedConv must run on the same engine Conv (not RequantShift) was + # colored with. Hence, engine must be inherited from Conv or removed. + attrs = {**conv.attrs, **rqs.attrs, "shift": totalShift} + if "engine" in conv.attrs: + attrs["engine"] = conv.attrs["engine"] # engine inherited from Conv + else: + attrs.pop("engine", None) # engine removed from attrs + rqsConv = gs.Node(op = 'RequantizedConv', name = name, attrs = attrs) graph.replaceInsertNode(_inputs, _outputs, rqsConv) return graph diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz new file mode 100644 index 0000000000..1f4942f864 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx new file mode 100644 index 0000000000..7c538679db Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz new file mode 100644 index 0000000000..119a3d170c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz new file mode 100644 index 0000000000..e8926f0567 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx new file mode 100644 index 0000000000..86cefb53ff Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz new file mode 100644 index 0000000000..9b5e3ffe50 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/DW_3x3_RQ/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz new file mode 100644 index 0000000000..5a5a4b8433 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx new file mode 100644 index 0000000000..5923a9feee Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz new file mode 100644 index 0000000000..83302e5695 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3/outputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz new file mode 100644 index 0000000000..0d9fc0d791 Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/inputs.npz differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx new file mode 100644 index 0000000000..5c3e85a2dc Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/network.onnx differ diff --git a/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz new file mode 100644 index 0000000000..2585f5cf2c Binary files /dev/null and b/DeeployTest/Tests/Kernels/Integer/Conv/Regular_3x3_RQ/outputs.npz differ diff --git a/DeeployTest/testMVP.py b/DeeployTest/testMVP.py index 9678bc4e4f..b93e53a190 100644 --- a/DeeployTest/testMVP.py +++ b/DeeployTest/testMVP.py @@ -74,8 +74,6 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg platform, signProp = mapPlatform(args.platform) - if args.enable_3x3: - platform.engines[0].enable3x3 = True if args.enableStrides: platform.engines[0].enableStrides = True @@ -158,11 +156,6 @@ def setupDeployer(graph: gs.Graph, memoryHierarchy: MemoryHierarchy, defaultTarg action = "store_true", default = False, help = 'Adds weight memory and neureka engine color\n') - parser.add_argument('--enable-3x3', - dest = "enable_3x3", - action = "store_true", - default = False, - help = 'Adds EXPERIMENTAL support for 3x3 convolutions on N-EUREKA\n') parser.add_argument('--enableStrides', dest = "enableStrides", action = "store_true", diff --git a/DeeployTest/testUtils/deeployRunner.py b/DeeployTest/testUtils/deeployRunner.py index bad25ee7f5..00ec496ed9 100644 --- a/DeeployTest/testUtils/deeployRunner.py +++ b/DeeployTest/testUtils/deeployRunner.py @@ -246,6 +246,8 @@ def create_config_from_args(args: argparse.Namespace, gen_args_list.append(f"--searchStrategy={args.searchStrategy}") if hasattr(args, 'plotMemAlloc') and args.plotMemAlloc: gen_args_list.append("--plotMemAlloc") + if hasattr(args, 'neureka_wmem') and args.neureka_wmem: + gen_args_list.append("--neureka-wmem") if not tiling and getattr(args, 'profileUntiled', False): gen_args_list.append("--profileUntiled") diff --git a/DeeployTest/test_siracusa_neureka_tiled_config.py b/DeeployTest/test_siracusa_neureka_tiled_config.py index 68bd3dd96e..170e8eb45d 100644 --- a/DeeployTest/test_siracusa_neureka_tiled_config.py +++ b/DeeployTest/test_siracusa_neureka_tiled_config.py @@ -11,18 +11,28 @@ # L2 single-buffer kernel tests # Format: dict of {test_name: [L1_sizes]} L2_SINGLEBUFFER_KERNELS = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L2 double-buffer kernel tests L2_DOUBLEBUFFER_KERNELS = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L3 single-buffer model tests @@ -43,10 +53,15 @@ # L2 single-buffer kernel tests with weight memory (neureka-wmem) L2_SINGLEBUFFER_KERNELS_WMEM = { - "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], "Kernels/Integer/Conv/PW_2D": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Regular_RQ": [32000], "Kernels/Integer/Conv/PW_2D_RQ/Unsigned_RQ": [32000], + "Kernels/Integer/Conv/DW_3x3": [32000], + "Kernels/Integer/Conv/DW_3x3_RQ": [32000], + "Kernels/Integer/Conv/Regular_3x3": [32000], + "Kernels/Integer/Conv/Regular_3x3_RQ": [32000], + "Kernels/Integer/GEMM/Regular_RQPerColumn": [16000], + "Kernels/Integer/GEMM/Batch_RQ": [16000], } # L3 double-buffer model tests with weight memory (neureka-wmem)