From 496a88f5ceb7b6fcc3db8289519d16574d67fe9b Mon Sep 17 00:00:00 2001 From: Adrian Lundell Date: Fri, 27 Feb 2026 13:33:44 +0100 Subject: [PATCH 1/2] Cortex-M backend: Enable test linting Signed-off-by: Adrian Lundell Change-Id: I066358eb3033e07267593f7eb7b84905730cda02 --- .lintrunner.toml | 2 -- .../cortex_m/test/misc/test_portable_int8.py | 4 +-- .../cortex_m/test/misc/test_quantization.py | 28 +++++++++---------- .../cortex_m/test/models/test_mobilenet_v2.py | 6 ++-- .../cortex_m/test/models/test_mobilenet_v3.py | 6 ++-- backends/cortex_m/test/ops/test_add.py | 6 ++-- backends/cortex_m/test/ops/test_conv.py | 17 ++++++----- .../cortex_m/test/ops/test_conv_transpose.py | 7 +++-- backends/cortex_m/test/ops/test_lstm.py | 6 ++-- backends/cortex_m/test/ops/test_max_pool2d.py | 6 ++-- backends/cortex_m/test/ops/test_minimum.py | 6 ++-- backends/cortex_m/test/ops/test_mul.py | 6 ++-- backends/cortex_m/test/ops/test_softmax.py | 8 +++--- backends/cortex_m/test/ops/test_transpose.py | 4 +-- backends/cortex_m/test/tester.py | 22 ++++++++++----- 15 files changed, 70 insertions(+), 64 deletions(-) diff --git a/.lintrunner.toml b/.lintrunner.toml index 65e69a1f7c3..15cf0bec06b 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -376,7 +376,6 @@ exclude_patterns = [ 'scripts/check_binary_dependencies.py', 'profiler/test/test_profiler_e2e.py', 'backends/arm/test/ops/*.py', - 'backends/cortex_m/test/**/*.py', ] command = [ 'python', @@ -410,7 +409,6 @@ include_patterns = [ exclude_patterns = [ 'third-party/**', '**/third-party/**', - 'backends/cortex_m/test/**/*.py', ] command = [ 'python', diff --git a/backends/cortex_m/test/misc/test_portable_int8.py b/backends/cortex_m/test/misc/test_portable_int8.py index 8e08250bb37..82b719230eb 100644 --- a/backends/cortex_m/test/misc/test_portable_int8.py +++ b/backends/cortex_m/test/misc/test_portable_int8.py @@ -15,7 +15,7 @@ from executorch.backends.arm._passes import FoldAndAnnotateQParamsPass from executorch.backends.arm._passes.arm_pass_utils import get_first_fake_tensor from executorch.backends.arm.quantizer.arm_quantizer_utils import SharedQspecQuantizer -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.quantizer.quantizer import CortexMQuantizer from executorch.backends.cortex_m.test.tester import CortexMTester from executorch.backends.test.harness.stages import StageType @@ -660,7 +660,7 @@ def _quantize_and_export( ), } -xfails = { +xfails: dict[str, xfail_type] = { "contiguous": "MLETORCH-1863: Contiguos no-op is removed in to-edge, leading to unnecessary Q-DQ-Q-DQ chain.", "clamp": "MLETORCH-1864: Support non-fused clamp-type activations.", "clamp_tensor": "MLETORCH-1864: Support non-fused clamp-type activations.", diff --git a/backends/cortex_m/test/misc/test_quantization.py b/backends/cortex_m/test/misc/test_quantization.py index 1f92c6ffa3f..6f27d1a6c28 100644 --- a/backends/cortex_m/test/misc/test_quantization.py +++ b/backends/cortex_m/test/misc/test_quantization.py @@ -6,7 +6,7 @@ import torch from executorch.backends.arm._passes.arm_pass_utils import get_first_fake_tensor -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -141,8 +141,8 @@ def forward(self, x, y): class SharedQspecInputForkXConstant(torch.nn.Module): """Shared qspec cluster with an input fork with left input as global constant.""" - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} constant = torch.tensor(5.0) def forward(self, x): @@ -152,8 +152,8 @@ def forward(self, x): class SharedQspecInputForkYConstant(torch.nn.Module): """Shared qspec cluster with an input fork with left input as local constant.""" - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def forward(self, x): return torch.maximum(x, torch.tensor(5.0)) @@ -259,8 +259,8 @@ def forward(self, x): class SharedQspecSurroundedQuantizedOpConstant(torch.nn.Module): - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def forward(self, x): x1 = torch.clone(x) @@ -270,16 +270,16 @@ def forward(self, x): class SharedQspecSub(torch.nn.Module): - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def forward(self, x, y): return torch.clone(x - y) class SharedQspecCompetingQspecs(torch.nn.Module): - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def __init__(self): super().__init__() @@ -299,8 +299,8 @@ def forward(self, x): class SharedQspecNoQspecs(torch.nn.Module): - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def forward(self, x): z = torch.clone(x - x) @@ -358,7 +358,7 @@ def forward(self, x): ), } -xfails = { +xfails: dict[str, xfail_type] = { "surrounded_quantized_op_constant": "Numerical error since the add is forced to have non-correct qparams.", } diff --git a/backends/cortex_m/test/models/test_mobilenet_v2.py b/backends/cortex_m/test/models/test_mobilenet_v2.py index 1be721d5b71..5874a09d598 100644 --- a/backends/cortex_m/test/models/test_mobilenet_v2.py +++ b/backends/cortex_m/test/models/test_mobilenet_v2.py @@ -10,7 +10,7 @@ from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase from executorch.backends.test.harness.stages import StageType -from torchvision import models +from torchvision import models # type: ignore[import-untyped] ops_before_transforms: dict[str, int] = { @@ -56,7 +56,7 @@ @parametrize("test_case", test_cases) def test_dialect_mv2(test_case): - inputs = test_case.example_inputs() + inputs = test_case.get_example_inputs() tester = CortexMTester(test_case.model, inputs) tester.test_dialect( ops_before_transforms, @@ -78,7 +78,7 @@ def test_dialect_mv2(test_case): strict=False, ) def test_implementation_mv2(test_case): - inputs = test_case.example_inputs() + inputs = test_case.get_example_inputs() tester = CortexMTester(test_case.model, inputs) tester.test_implementation( qtol=10, diff --git a/backends/cortex_m/test/models/test_mobilenet_v3.py b/backends/cortex_m/test/models/test_mobilenet_v3.py index b778b94c3a7..08633d54dd6 100644 --- a/backends/cortex_m/test/models/test_mobilenet_v3.py +++ b/backends/cortex_m/test/models/test_mobilenet_v3.py @@ -9,7 +9,7 @@ from executorch.backends.cortex_m.test.tester import CortexMTester, McuTestCase from executorch.backends.test.harness.stages import StageType -from torchvision import models +from torchvision import models # type: ignore[import-untyped] ops_before_transforms: dict[str, int] = { @@ -65,7 +65,7 @@ strict=False, ) def test_dialect_mv3(test_case): - inputs = test_case.example_inputs() + inputs = test_case.get_example_inputs() tester = CortexMTester(test_case.model, inputs) tester.test_dialect( ops_before_transforms, @@ -89,7 +89,7 @@ def test_dialect_mv3(test_case): strict=False, ) def test_implementation_mv3(test_case): - inputs = test_case.example_inputs() + inputs = test_case.get_example_inputs() tester = CortexMTester(test_case.model, inputs) tester.test_implementation( qtol=20, diff --git a/backends/cortex_m/test/ops/test_add.py b/backends/cortex_m/test/ops/test_add.py index b776ac91002..5adc77ce4aa 100644 --- a/backends/cortex_m/test/ops/test_add.py +++ b/backends/cortex_m/test/ops/test_add.py @@ -5,7 +5,7 @@ import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -152,10 +152,10 @@ class CortexMAlphaAdd(ModelAlpha): } -xfails_implementation = { +xfails_implementation: dict[str, xfail_type] = { "alpha": "Expecting kwargs for aten op IR to be empty - alpha arg not supported.", } -xfails_dialect = xfails_implementation | { +xfails_dialect: dict[str, xfail_type] = xfails_implementation | { # Cortex-M quantizer will not quantize additions that require broadcasting # leading to the add op not being replaced by a cortex-m specific implementation "broadcast_1": "Broadcasting is not supported in Cortex-M backend", diff --git a/backends/cortex_m/test/ops/test_conv.py b/backends/cortex_m/test/ops/test_conv.py index 5c970b5512b..5750ccf3bdb 100644 --- a/backends/cortex_m/test/ops/test_conv.py +++ b/backends/cortex_m/test/ops/test_conv.py @@ -1,11 +1,11 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -14,8 +14,8 @@ class CortexMConv1D(torch.nn.Module): - ops_before_transforms = {} - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def __init__(self, *args, **kwargs): super().__init__() @@ -72,9 +72,8 @@ def forward(self, x): class CortexMConv3D(torch.nn.Module): - ops_before_transforms = {} - - ops_after_transforms = {} + ops_before_transforms: dict[str, int] = {} + ops_after_transforms: dict[str, int] = {} def __init__(self, *args, **kwargs): super().__init__() @@ -313,7 +312,7 @@ def forward(self, x): } -xfails_dialect = { +xfails_dialect: dict[str, xfail_type] = { "conv2d_dilation": "NotImplementedError: 'slow_conv_dilated<>' not implemented for 'Int'", "conv1d": "Currently not supported.", "conv2d_nchw": "Currently not supported.", @@ -330,7 +329,7 @@ def test_dialect_conv2d(test_case): ) -xfails_implementation = { +xfails_implementation: dict[str, xfail_type] = { "conv1d": "Currently not supported.", "conv3d": "Currently not supported.", } diff --git a/backends/cortex_m/test/ops/test_conv_transpose.py b/backends/cortex_m/test/ops/test_conv_transpose.py index 9e93076179b..7a91c5e1b6b 100644 --- a/backends/cortex_m/test/ops/test_conv_transpose.py +++ b/backends/cortex_m/test/ops/test_conv_transpose.py @@ -1,12 +1,13 @@ # Copyright (c) Meta Platforms, Inc. and affiliates. # All rights reserved. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -220,7 +221,7 @@ def forward(self, x): ), } -xfails_dialect = { +xfails_dialect: dict[str, xfail_type] = { # Grouped convolutions not supported by CMSIS-NN - rejected during quantization "conv_transpose2d_groups_2": "Grouped transpose conv not supported by CMSIS-NN", "conv_transpose2d_depthwise": "Depthwise transpose conv not supported by CMSIS-NN", @@ -248,7 +249,7 @@ def test_dialect_conv_transpose2d(test_case): # Implementation xfails: empty because unsupported configurations are now # rejected at AOT time by the quantizer filter, so they fall back to portable # ops and work correctly. Only xfails_dialect needs to track these. -xfails_implementation = {} +xfails_implementation: dict[str, xfail_type] = {} @parametrize("test_case", test_cases, xfails=xfails_implementation) diff --git a/backends/cortex_m/test/ops/test_lstm.py b/backends/cortex_m/test/ops/test_lstm.py index 60d7aba4271..ae9d17762f8 100644 --- a/backends/cortex_m/test/ops/test_lstm.py +++ b/backends/cortex_m/test/ops/test_lstm.py @@ -1,4 +1,4 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -30,7 +30,7 @@ class CortexMLSTM(torch.nn.Module): "executorch_exir_dialects_edge__ops_aten_cat_default": 1, } - ops_after_transforms = {} + ops_after_transforms: dict[str, int] = {} def __init__(self, input_size: int = 4, hidden_size: int = 3) -> None: super().__init__() @@ -59,7 +59,7 @@ class CortexMQuantizableLSTM(torch.nn.Module): "executorch_exir_dialects_edge__ops_quantized_decomposed_quantize_per_tensor_default": 27, } - ops_after_transforms = {} + ops_after_transforms: dict[str, int] = {} def __init__(self, input_size: int = 4, hidden_size: int = 3) -> None: super().__init__() diff --git a/backends/cortex_m/test/ops/test_max_pool2d.py b/backends/cortex_m/test/ops/test_max_pool2d.py index f27cec7a296..0ec3d721c22 100644 --- a/backends/cortex_m/test/ops/test_max_pool2d.py +++ b/backends/cortex_m/test/ops/test_max_pool2d.py @@ -5,7 +5,7 @@ import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -82,10 +82,10 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: ), } -xfails_max_pool2d = { +xfails_max_pool2d: dict[str, xfail_type] = { "maxpool_2x2_indices": ( "Indices output not supported; quantizer does not handle getitem on max_pool2d_with_indices.", - (NotImplementedError, AssertionError, RuntimeError, Exception), + Exception, ), } diff --git a/backends/cortex_m/test/ops/test_minimum.py b/backends/cortex_m/test/ops/test_minimum.py index 633ccdbf483..6a520194f46 100644 --- a/backends/cortex_m/test/ops/test_minimum.py +++ b/backends/cortex_m/test/ops/test_minimum.py @@ -1,11 +1,11 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -87,7 +87,7 @@ def forward(self, x, y): } -xfails = {} +xfails: dict[str, xfail_type] = {} @parametrize("test_case", test_cases, xfails=xfails) diff --git a/backends/cortex_m/test/ops/test_mul.py b/backends/cortex_m/test/ops/test_mul.py index 88dd904eb6e..47c5304d83a 100644 --- a/backends/cortex_m/test/ops/test_mul.py +++ b/backends/cortex_m/test/ops/test_mul.py @@ -1,11 +1,11 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -127,7 +127,7 @@ class CortexMTensorMul(Model): } -xfail_cases_dialect = { +xfail_cases_dialect: dict[str, xfail_type] = { # Cortex-M quantizer will not quantize multiplicaitons that require broadcasting # leading to the mul op not being replaced by a cortex-m specific implementation "broadcast_1": "Broadcasting is not supported in Cortex-M backend", diff --git a/backends/cortex_m/test/ops/test_softmax.py b/backends/cortex_m/test/ops/test_softmax.py index 49c8c3ecabc..ab490b3ff81 100644 --- a/backends/cortex_m/test/ops/test_softmax.py +++ b/backends/cortex_m/test/ops/test_softmax.py @@ -1,11 +1,11 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -54,13 +54,13 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: } -xfail_cases_dialect = { +xfail_cases_dialect: dict[str, xfail_type] = { "dim_not_last": ( "Softmax stays in ATen when dim isn’t the channel dimension, so dialect expectations fail", Exception, ), } -xfail_cases_impl = { +xfail_cases_impl: dict[str, xfail_type] = { "dim_not_last": ( "Softmax on Cortex-M currently supports only the last dimension", Exception, diff --git a/backends/cortex_m/test/ops/test_transpose.py b/backends/cortex_m/test/ops/test_transpose.py index de16c2f81ad..978cea1ec0d 100644 --- a/backends/cortex_m/test/ops/test_transpose.py +++ b/backends/cortex_m/test/ops/test_transpose.py @@ -1,4 +1,4 @@ -# Copyright 2025 Arm Limited and/or its affiliates. +# Copyright 2025-2026 Arm Limited and/or its affiliates. # # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. @@ -69,7 +69,7 @@ def forward(self, x): ), "permute_rank_1": McuTestCase( CortexMPermute((0,)), - (ramp_tensor(10, 100, (3)),), + (ramp_tensor(10, 100, (3,)),), ), "transpose_1_2": McuTestCase( CortexMTranspose(1, 2), diff --git a/backends/cortex_m/test/tester.py b/backends/cortex_m/test/tester.py index 1b7e623ed00..984e79d5b3a 100644 --- a/backends/cortex_m/test/tester.py +++ b/backends/cortex_m/test/tester.py @@ -5,7 +5,7 @@ from dataclasses import dataclass -from typing import Any +from typing import Any, Callable import torch from executorch.backends.arm.test.common import get_u55_compile_spec @@ -73,7 +73,11 @@ def __init__(self): class CortexMTester(TesterBase): def __init__(self, module, example_inputs): - super().__init__(module, example_inputs, cortex_m_stage_classes) + if callable(example_inputs): + resolved_example_inputs = example_inputs() + else: + resolved_example_inputs = example_inputs + super().__init__(module, resolved_example_inputs, cortex_m_stage_classes) def test_dialect( self, @@ -124,10 +128,14 @@ def test_implementation(self, qtol=0, calibration_samples=None): @dataclass class McuTestCase: model: torch.nn.Module - example_inputs: tuple[Any, ...] + example_inputs: tuple[Any, ...] | Callable[[], tuple[Any, ...]] + + def get_example_inputs(self) -> tuple[Any, ...]: + if callable(self.example_inputs): + return self.example_inputs() + return self.example_inputs -def ramp_tensor(start: int, end: int, shape: tuple[int, ...]) -> torch.Tensor: - return torch.linspace(start, end, steps=torch.prod(torch.tensor(shape))).reshape( - shape - ) +def ramp_tensor(start: float, end: float, shape: tuple[int, ...]) -> torch.Tensor: + steps = int(torch.prod(torch.tensor(shape)).item()) + return torch.linspace(start, end, steps=steps).reshape(shape) From a0fd147a7d3a86e1575bcf3f28b8531794254cc0 Mon Sep 17 00:00:00 2001 From: Adrian Lundell Date: Fri, 27 Mar 2026 09:49:32 +0100 Subject: [PATCH 2/2] Cortex-M backend: Add xfail type Signed-off-by: Adrian Lundell Change-Id: I6eaa632041cf566ee38b3b3ad06346ae5f48c7c1 --- backends/cortex_m/test/models/test_nn_modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/cortex_m/test/models/test_nn_modules.py b/backends/cortex_m/test/models/test_nn_modules.py index f016f94e7d0..77aa07e04c9 100644 --- a/backends/cortex_m/test/models/test_nn_modules.py +++ b/backends/cortex_m/test/models/test_nn_modules.py @@ -24,7 +24,7 @@ """ import torch -from executorch.backends.arm.test.common import parametrize +from executorch.backends.arm.test.common import parametrize, xfail_type from executorch.backends.cortex_m.test.tester import ( CortexMTester, McuTestCase, @@ -188,7 +188,7 @@ def forward(self, x): ), } -xfails = { +xfails: dict[str, xfail_type] = { "conv_add_relu": "Activation fusion does not support relu after add", }