diff --git a/backends/arm/scripts/docgen/ethos-u/backends-arm-ethos-u-overview.md.in b/backends/arm/scripts/docgen/ethos-u/backends-arm-ethos-u-overview.md.in index 555c61fd13b..4bb34fe6850 100644 --- a/backends/arm/scripts/docgen/ethos-u/backends-arm-ethos-u-overview.md.in +++ b/backends/arm/scripts/docgen/ethos-u/backends-arm-ethos-u-overview.md.in @@ -67,6 +67,11 @@ how to run `model_export/export_deit.py`, build the sample firmware, and convert test images into C arrays so the workflow described in this guide can be tried end to end. +[`examples/arm/mobilesam_prompt_segmentation_example_ethos_u`](https://github.com/pytorch/executorch/tree/main/examples/arm/mobilesam_prompt_segmentation_example_ethos_u) +contains a complete MobileSAM prompt segmentation workflow for Ethos-U85, +including fixed-prompt export, PT2E quantization, transformer lowering, debug +masks and overlays, a bare-metal Corstone-320 runtime app, and FVP execution. + ### Ethos-U memory modes The Ethos-U NPU provides two distinct memory interfaces: diff --git a/backends/arm/scripts/run_fvp.sh b/backends/arm/scripts/run_fvp.sh index 006f502d84c..7289f37484a 100755 --- a/backends/arm/scripts/run_fvp.sh +++ b/backends/arm/scripts/run_fvp.sh @@ -24,6 +24,8 @@ target="ethos-u55-128" timeout="600" etrecord_file="" trace_file="" +semihosting_cwd="" +ethosu_fast=0 help() { echo "Usage: $(basename $0) [options]" @@ -35,6 +37,8 @@ help() { echo " --timeout= Maximum target runtime, used to detect hanging, might need to be higer on large models Default: ${timeout}" echo " --etrecord= If ETDump is used you can supply a ETRecord file matching the PTE" echo " --trace_file= File to write PMU trace output to" + echo " --semihosting-cwd= Enable target semihosting with this host working directory" + echo " --fast Use fast Ethos-U model simulation for Ethos-U targets" exit 0 } @@ -48,6 +52,8 @@ for arg in "$@"; do --timeout=*) timeout="${arg#*=}";; --etrecord=*) etrecord_file="${arg#*=}";; --trace_file=*) trace_file="${arg#*=}";; + --semihosting-cwd=*) semihosting_cwd="${arg#*=}";; + --fast) ethosu_fast=1;; *) ;; esac @@ -104,9 +110,34 @@ log_file=$(mktemp) extra_args_u55=() extra_args_u85=() +ethosu_extra_args=() +if [[ "${ethosu_fast}" == 1 ]]; then + ethosu_extra_args+=("--fast") +fi if [[ -n "${trace_file}" ]]; then - extra_args_u55+=(-C "ethosu.extra_args=--pmu-trace ${trace_file}") - extra_args_u85+=(-C "mps4_board.subsystem.ethosu.extra_args=--pmu-trace ${trace_file}") + ethosu_extra_args+=("--pmu-trace" "${trace_file}") +fi +if [[ ${#ethosu_extra_args[@]} -gt 0 ]]; then + extra_args_u55+=(-C "ethosu.extra_args=${ethosu_extra_args[*]}") + extra_args_u85+=(-C "mps4_board.subsystem.ethosu.extra_args=${ethosu_extra_args[*]}") +fi + +semihosting_args_u55=() +semihosting_args_u85=() +if [[ -n "${semihosting_cwd}" ]]; then + semihosting_cwd=$(realpath "${semihosting_cwd}") + semihosting_args_u55+=( + -C cpu0.semihosting-enable=1 + -C cpu0.semihosting-stack_base=0 + -C cpu0.semihosting-heap_limit=0 + -C "cpu0.semihosting-cwd=${semihosting_cwd}" + ) + semihosting_args_u85+=( + -C mps4_board.subsystem.cpu0.semihosting-enable=1 + -C mps4_board.subsystem.cpu0.semihosting-stack_base=0 + -C mps4_board.subsystem.cpu0.semihosting-heap_limit=0 + -C "mps4_board.subsystem.cpu0.semihosting-cwd=${semihosting_cwd}" + ) fi if [[ ${target} == cortex-m* ]]; then @@ -154,6 +185,7 @@ elif [[ ${target} == *"ethos-u55"* || ${target} == *"ethos-u65"* ]]; then -C mps3_board.uart0.out_file='-' \ -C mps3_board.uart0.shutdown_on_eot=1 \ ${extra_args_u55[@]+"${extra_args_u55[@]}"} \ + ${semihosting_args_u55[@]+"${semihosting_args_u55[@]}"} \ -a "${elf_file}" \ ${data_file} \ --timelimit ${timeout} 2>&1 | sed 's/\r$//' | tee ${log_file} || true # seconds @@ -165,8 +197,10 @@ elif [[ ${target} == *"ethos-u85"* ]]; then -C vis_hdlcd.disable_visualisation=1 \ -C mps4_board.telnetterminal0.start_telnet=0 \ -C mps4_board.uart0.out_file='-' \ + -C mps4_board.uart0.unbuffered_output=1 \ -C mps4_board.uart0.shutdown_on_eot=1 \ ${extra_args_u85[@]+"${extra_args_u85[@]}"} \ + ${semihosting_args_u85[@]+"${semihosting_args_u85[@]}"} \ -a "${elf_file}" \ ${data_file} \ --timelimit ${timeout} 2>&1 | sed 's/\r$//' | tee ${log_file} || true # seconds @@ -200,11 +234,21 @@ if [ $? != 0 ]; then fi echo "Checking for problems in log:" -! grep -E "^(F|E|\\[critical\\]|Hard fault.|Info: Simulation is stopping. Reason: CPU time has been exceeded.).*$" ${log_file} -if [ $? != 0 ]; then +problem_log=$(mktemp) +grep -E "^(F|E|\\[critical\\]|Hard fault.|Info: Simulation is stopping. Reason: CPU time has been exceeded.).*$" "${log_file}" > "${problem_log}" || true +if [[ "${ethosu_fast}" == 1 ]]; then + filtered_problem_log=$(mktemp) + timing_adapter_fast_mode_regex="Failed to initialize timing-adapter #[0-9]+|Timing adapter has no effect if option --fast/-F is enabled" + grep -Ev "${timing_adapter_fast_mode_regex}" "${problem_log}" > "${filtered_problem_log}" || true + mv "${filtered_problem_log}" "${problem_log}" +fi +if [[ -s "${problem_log}" ]]; then + cat "${problem_log}" echo "Found ERROR" + rm "${problem_log}" rm "${log_file}" exit 1 fi echo "No problems found!" +rm "${problem_log}" rm "${log_file}" diff --git a/backends/arm/test/test_arm_ootb.sh b/backends/arm/test/test_arm_ootb.sh index a96f47a7dbe..c68f0728e0b 100755 --- a/backends/arm/test/test_arm_ootb.sh +++ b/backends/arm/test/test_arm_ootb.sh @@ -28,7 +28,7 @@ if [[ "$1" == "-h" || "$1" == "--help" ]]; then fi if [[ $# -eq 0 ]]; then - TEST_SUITES=(run_ootb_tests_ethos_u run_ootb_tests_tosa run_ootb_tests_vgf run_deit_e2e_ethos_u run_swin2sr_e2e_vgf) + TEST_SUITES=(run_ootb_tests_ethos_u run_ootb_tests_tosa run_ootb_tests_vgf run_deit_e2e_ethos_u run_mobilesam_e2e_ethos_u run_swin2sr_e2e_vgf) else TEST_SUITES=("$1") fi @@ -164,6 +164,133 @@ run_deit_e2e_ethos_u() { echo "${FUNCNAME}: PASS" } +run_mobilesam_e2e_ethos_u() { + echo "$FUNCNAME: Export, build, and run the MobileSAM e2e test" + + local example_dir="${et_root_dir}/examples/arm/mobilesam_prompt_segmentation_example_ethos_u" + local work_root="${et_root_dir}/arm_test/mobilesam_ootb_smoke" + local export_dir="${work_root}/export" + local artifact_dir="${work_root}/artifacts" + local debug_dir="${work_root}/debug" + local et_build_dir="${work_root}/cmake-out-arm" + local quantized_aot_build_dir="${work_root}/quantized_ops_aot" + local build_dir="${work_root}/runtime" + local mobile_sam_source="${work_root}/mobile_sam/source" + local image_path="${et_root_dir}/examples/models/dinov2/dog.jpg" + local pte_path="${export_dir}/mobilesam_prompt_smoke.pte" + local fvp_log="${work_root}/fvp.log" + local toolchain_file="${et_root_dir}/examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake" + local input_size=224 + local fvp_timelimit="${FVP_TIMELIMIT:-300}" + echo "${FUNCNAME}: Work directory: ${work_root}; existing artifacts will be reused if present" + + mkdir -p "${export_dir}" "${artifact_dir}" "${debug_dir}" "${build_dir}" + + setup_path_script=${et_root_dir}/examples/arm/arm-scratch/setup_path.sh + source ${setup_path_script} + + source ${et_root_dir}/backends/arm/scripts/utils.sh + local n_proc="$(get_parallel_jobs)" + + echo "${FUNCNAME}: Building ExecuTorch (if needed)" + cmake --preset arm-baremetal -B "${et_build_dir}" + cmake --build "${et_build_dir}" --target install -j"$n_proc" + + echo "${FUNCNAME}: Building host quantized AOT library" + local python_executable + python_executable="$(python3 -c 'import sys; print(sys.executable)')" + cmake \ + -S "${et_root_dir}" \ + -B "${quantized_aot_build_dir}" \ + -DCMAKE_BUILD_TYPE=Release \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \ + -DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON \ + -DEXECUTORCH_BUILD_XNNPACK=OFF \ + -DPYTHON_EXECUTABLE="${python_executable}" + cmake --build "${quantized_aot_build_dir}" --target quantized_ops_aot_lib -j"$n_proc" + + local quantized_ops_library + quantized_ops_library="$( + find "${quantized_aot_build_dir}/kernels/quantized" \ + -name 'libquantized_ops_aot_lib.*' \ + -type f \ + -print \ + -quit + )" + [[ -n "${quantized_ops_library}" ]] || { + echo "${FUNCNAME}: Missing quantized AOT library under ${quantized_aot_build_dir}" + return 1 + } + + echo "${FUNCNAME}: Installing example requirements" + pip install -r "${example_dir}/requirements.txt" + + echo "${FUNCNAME}: Preparing pinned MobileSAM source" + python3 "${example_dir}/model_export/prepare_mobilesam.py" \ + --source-dir "${mobile_sam_source}" + + echo "${FUNCNAME}: Exporting quantized MobileSAM PTE" + env EXECUTORCH_QUANTIZED_OPS_AOT_LIBRARY="${quantized_ops_library}" \ + python3 "${example_dir}/model_export/export_mobilesam.py" \ + --output-path "${pte_path}" \ + --calibration-image "${image_path}" \ + --eval-image "${image_path}" \ + --mobile-sam-source "${mobile_sam_source}" \ + --input-size "${input_size}" \ + --point 84 79 \ + --point 109 96 \ + --point 144 92 \ + --num-calibration-samples 1 \ + --num-eval-samples 1 \ + --num-debug-samples 1 \ + --artifact-dir "${artifact_dir}" \ + --debug-output-dir "${debug_dir}" + + for artifact in \ + "${pte_path}" \ + "${export_dir}/mobilesam_prompt_smoke.json" \ + "${export_dir}/mobilesam_prompt_smoke_delegation.txt" \ + "${export_dir}/mobilesam_prompt_smoke_metrics.json"; do + [[ -f "${artifact}" ]] || { + echo "${FUNCNAME}: Missing export artifact ${artifact}" + return 1 + } + done + + echo "${FUNCNAME}: Configuring the MobileSAM application" + cmake \ + -U "LIB_*" \ + -U executorch_DIR \ + -S "${example_dir}/runtime" \ + -B "${build_dir}" \ + -DCMAKE_TOOLCHAIN_FILE="${toolchain_file}" \ + -DET_PTE_FILE_PATH="${pte_path}" \ + -DIMAGE_PATH="${image_path}" \ + -DMASK_THRESHOLD=-5.0 \ + -DPYTHON_EXECUTABLE="${python_executable}" \ + -DET_BUILD_DIR_PATH="${et_build_dir}" + + echo "${FUNCNAME}: Building mobilesam_prompt_segmentation_example" + cmake --build "${build_dir}" -j"$n_proc" --target mobilesam_prompt_segmentation_example + + local elf="${build_dir}/mobilesam_prompt_segmentation_example" + + echo "${FUNCNAME}: Running on FVP" + backends/arm/scripts/run_fvp.sh \ + --elf="${elf}" \ + --target=ethos-u85-256 \ + --timeout="${fvp_timelimit}" \ + --semihosting-cwd="${build_dir}" \ + --fast | tee "${fvp_log}" + + grep -q "Model executed successfully." "${fvp_log}" || { + echo "${FUNCNAME}: FVP run did not report successful execution" + return 1 + } + + echo "${FUNCNAME}: PASS" +} + run_swin2sr_e2e_vgf() { echo "$FUNCNAME: Prepare demo assets, export FP/INT8, build, and run the Swin2SR VGF e2e test" diff --git a/docs/source/backends/arm-ethos-u/arm-ethos-u-overview.md b/docs/source/backends/arm-ethos-u/arm-ethos-u-overview.md index 15f2249e071..e502308c00b 100644 --- a/docs/source/backends/arm-ethos-u/arm-ethos-u-overview.md +++ b/docs/source/backends/arm-ethos-u/arm-ethos-u-overview.md @@ -121,6 +121,11 @@ how to run `model_export/export_deit.py`, build the sample firmware, and convert test images into C arrays so the workflow described in this guide can be tried end to end. +[`examples/arm/mobilesam_prompt_segmentation_example_ethos_u`](https://github.com/pytorch/executorch/tree/main/examples/arm/mobilesam_prompt_segmentation_example_ethos_u) +contains a complete MobileSAM prompt segmentation workflow for Ethos-U85, +including fixed-prompt export, PT2E quantization, transformer lowering, debug +masks and overlays, a bare-metal Corstone-320 runtime app, and FVP execution. + ### Ethos-U memory modes The Ethos-U NPU provides two distinct memory interfaces: diff --git a/examples/arm/README.md b/examples/arm/README.md index 07aecec51e2..434f9db9c9f 100644 --- a/examples/arm/README.md +++ b/examples/arm/README.md @@ -77,7 +77,11 @@ For Cortex-M testing, use a Cortex-M target and bundled I/O: - End-to-end DEiT-Tiny image classification flow for Ethos-U, including model fine-tuning, export, bare-metal runtime build, and Corstone-320 FVP execution. -- [image_classification_example_vgf](image_classification_example_vgf/) - + - [mobilesam_prompt_segmentation_example_ethos_u](mobilesam_prompt_segmentation_example_ethos_u/) + - End-to-end MobileSAM prompt segmentation flow for Ethos-U, including + export, quantization, debug mask generation, bare-metal runtime build, and + Corstone-320 FVP execution. + - [image_classification_example_vgf](image_classification_example_vgf/) - DEiT-Tiny image classification flow for VGF host execution. - [super_resolution_example_vgf](super_resolution_example_vgf) - Swin2SR image super-resolution. diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/README.md b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/README.md new file mode 100644 index 00000000000..842f5cff3eb --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/README.md @@ -0,0 +1,41 @@ +# MobileSAM Prompt Segmentation Example Application + +This end-to-end example shows how to use the Arm Ethos-U backend in +ExecuTorch for transformer-based prompt segmentation. MobileSAM predicts a +binary mask for fixed positive point prompts rather than semantic class IDs. +The host debug flow validates quantization by comparing FP32 and quantized +masks, with an optional binary reference mask when one is available. + +It covers: + +- Loading the MobileSAM `vit_t` checkpoint. +- Freezing one or more positive point prompts into the exported graph. +- Applying post-training quantization with the Ethos-U quantizer. +- Lowering the quantized model to an Ethos-U85-256 ExecuTorch program. +- Producing validation and debugging artifacts such as masks, overlays, + mismatch heatmaps, metrics, and delegation summaries. +- Building a bare-metal Corstone-320 runtime app and running it on FVP. + +The default export uses a reduced `512x512` image input and returns one +low-resolution `[1, 1, 128, 128]` mask-logit tensor. The example prepares the +official MobileSAM GitHub source at a pinned revision in an external checkout +and applies a small configurable-input patch there. Neither the MobileSAM +source nor checkpoint is redistributed in ExecuTorch. + +The export uses int8 activations and int8 weights globally, and A16W8 +quantization for TinyViT attention modules. This keeps the transformer +attention numerically stable while still producing one Ethos-U delegate. + +The exported graph intentionally uses `multimask_output=False` and leaves +mask thresholding outside the model. SAM-style candidate-mask selection can be +numerically sensitive after export and quantization, so this example keeps the +target graph focused on the fixed-prompt image encoder and mask decoder. + +## Layout + +- `model_export/prepare_mobilesam.py` - Prepares the pinned external MobileSAM + checkout and applies the configurable-input patch. +- `model_export/README.md` - Model loading, quantization, lowering, + validation, and debug artifact generation. +- `runtime/README.md` - Bare-metal runtime build, image header generation, and + Corstone-320 FVP execution. diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/README.md b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/README.md new file mode 100644 index 00000000000..af7d6743313 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/README.md @@ -0,0 +1,185 @@ +# MobileSAM Export, Quantization, and Debugging + +This directory exports MobileSAM `vit_t` for the ExecuTorch Ethos-U backend. +The exporter freezes one or more positive point prompts into the graph, so the +runtime app has one tensor input: the preprocessed image. The model returns one +mask-logit tensor for those prompts. The default export uses a `512x512` input, +which produces a `128x128` mask-logit tensor. + +Production SAM applications usually pass prompts dynamically from a UI or +tracking pipeline. This example freezes the prompt embeddings so the FVP +runtime stays small and demonstrates the Ethos-U flow with the same image +encoder and mask decoder used by MobileSAM. + +## Requirements + +- Python 3.10+ with `executorch`. +- Dependencies from + `examples/arm/mobilesam_prompt_segmentation_example_ethos_u/requirements.txt`. +- Git and internet access to prepare the pinned external MobileSAM checkout and + download the official checkpoint, unless both are already cached. +- Ethos-U dependencies from `examples/arm/setup.sh`. + +## Export + +Run from the ExecuTorch repo root: + +```bash +MOBILE_SAM_SOURCE="$HOME/.cache/executorch/mobilesam/f706ad9c4eb7f219c00d9050e46328518ffb65d2/source" +python examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/prepare_mobilesam.py \ + --source-dir "$MOBILE_SAM_SOURCE" + +python examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py \ + --output-path ./mobilesam_point_ethos_u85_512.pte \ + --mobile-sam-source "$MOBILE_SAM_SOURCE" \ + --calibration-image examples/models/dinov2/dog.jpg \ + --eval-image examples/models/dinov2/dog.jpg \ + --point 210 220 \ + --artifact-dir ./mobilesam_point_artifacts \ + --debug-output-dir ./mobilesam_point_debug +``` + +The default configuration is: + +- Model source: `https://github.com/ChaoningZhang/MobileSAM` +- MobileSAM source revision: `f706ad9c4eb7f219c00d9050e46328518ffb65d2` +- External source patch: `0001-Make-TinyViT-image-size-configurable.patch` +- Checkpoint URL: + `https://github.com/ChaoningZhang/MobileSAM/raw/f706ad9c4eb7f219c00d9050e46328518ffb65d2/weights/mobile_sam.pt` +- Checkpoint SHA256: + `6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f` +- MobileSAM source-code license: Apache-2.0 +- Static input shape: `[1, 3, 512, 512]` +- Output mask logits shape: `[1, 1, 128, 128]` +- Positive point prompt: `(256, 256)` in the padded `512x512` model input + frame +- Target: `ethos-u85-256` +- System config: `Ethos_U85_SYS_DRAM_Mid` +- Memory mode: `Dedicated_Sram_384KB` +- Calibration samples: `4` +- Validation samples: `4` + +The MobileSAM source and checkpoint are not redistributed by this example. The +preparation script clones the pinned official source into a managed external +cache and applies the configurable-input patch. The exporter downloads the +pinned official checkpoint and verifies its SHA256 unless `--checkpoint-path` +is provided. The source repository and checkpoint may have separate terms; the +export metadata records only the source-code license and does not assign a +license to the checkpoint. + +For a quick offline smoke test after caching the official checkpoint and source +checkout: + +```bash +python examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/prepare_mobilesam.py \ + --source-dir "$MOBILE_SAM_SOURCE" \ + --local-files-only + +python examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py \ + --output-path /tmp/mobilesam_smoke.pte \ + --local-files-only \ + --mobile-sam-source "$MOBILE_SAM_SOURCE" \ + --calibration-image examples/models/dinov2/dog.jpg \ + --eval-image examples/models/dinov2/dog.jpg \ + --point 210 220 \ + --num-calibration-samples 1 \ + --num-eval-samples 1 +``` + +Repeat `--point X Y` to freeze a multi-point prompt into the graph. Multiple +positive points can improve mask quality for reduced input sizes: + +```bash +python examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py \ + --output-path /tmp/mobilesam_multipoint_smoke.pte \ + --local-files-only \ + --mobile-sam-source "$MOBILE_SAM_SOURCE" \ + --calibration-image examples/models/dinov2/dog.jpg \ + --eval-image examples/models/dinov2/dog.jpg \ + --point 190 180 \ + --point 250 220 \ + --point 330 210 \ + --num-calibration-samples 1 \ + --num-eval-samples 1 \ + --debug-output-dir /tmp/mobilesam_multipoint_debug +``` + +Pass `--input-size 1024` to reproduce the original MobileSAM resolution. Smaller +inputs export and run faster, but very small inputs such as `224` or `256` +usually produce lower-quality masks because the checkpoint was trained for +`1024x1024` images. + +To validate against a known binary mask, pass one `--eval-mask` per +`--eval-image`. Non-zero mask pixels are treated as foreground. When no +reference mask is provided, validation reports FP32/quantized mask agreement. + +The export flow: + +1. Loads the MobileSAM `vit_t` checkpoint through the patched external API. +2. Builds a fixed-prompt wrapper containing the image encoder and mask decoder. +3. Calibrates PT2E quantization with `EthosUQuantizer`. +4. Uses stable softmax decomposition for transformer attention blocks. +5. Lowers the quantized graph with `EthosUPartitioner`. +6. Writes an ExecuTorch `.pte` program. + +The quantization recipe uses int8 activations and int8 weights globally, with +A16W8 quantization for the TinyViT attention modules. Static int8 activation +quantization collapses MobileSAM attention features, while the selective A16W8 +attention path preserves mask quality and still lowers as one Ethos-U delegate. + +## Outputs + +For `--output-path ./mobilesam_point_ethos_u85_512.pte`, the script writes: + +- `mobilesam_point_ethos_u85_512.pte` - Ethos-U-ready ExecuTorch program. +- `mobilesam_point_ethos_u85_512.json` - Export metadata. +- `mobilesam_point_ethos_u85_512_metrics.json` - FP32/quantized mask + agreement and optional reference-mask IoU. +- `mobilesam_point_ethos_u85_512_delegation.txt` - Operator delegation + summary. +- `mobilesam_point_artifacts/` - Optional TOSA/Vela intermediate artifacts. +- `mobilesam_point_debug/` - Optional per-sample masks, overlays, mismatch + heatmaps, and mask summaries. + +## Interpreting the Debug Artifacts + +Each debug sample contains: + +- `input.png` - The resized RGB input used by the exported model. +- `reference_mask.png` - Optional binary reference mask resized to the model + output-mask size. +- `fp32_mask.png` - Host-side FP32 model prediction. +- `fp32_overlay.png` - Colored FP32 prediction blended over the input image. +- `quantized_mask.png` - Host-side PT2E quantized prediction before lowering. +- `quantized_overlay.png` - Colored quantized prediction blended over the input + image. +- `mismatch_heatmap.png` - Green for FP32/quantized agreement and red for + mismatch. +- `mask_summary.json` - Foreground/background pixel counts and FP32/quantized + IoU. + +The runtime app thresholds mask logits on target, logs a mask hash and +foreground/background counts, and can dump the thresholded mask as RLE chunks. +Host-side debug masks are intentionally generated before lowering so users can +inspect quantization quality without needing target-side image output. + +## Limitations + +- The example freezes positive point prompts into the exported graph to keep + the target app to a single image input. Changing the prompt requires + re-exporting the `.pte`. +- The export uses `multimask_output=False` and does not include candidate-mask + argmax selection, upsampling, or thresholding in the graph. Keep those steps + in host-side or target-side post-processing when comparing mask quality. +- The runtime app logs a mask hash and foreground/background counts; it does + not render a color image on target. +- The first supported runtime target is Corstone-320/Ethos-U85-256. +- Reduced input sizes require the patch applied by `prepare_mobilesam.py`; the + MobileSAM source remains outside the ExecuTorch checkout. +- The default `512x512` input is a compromise between FVP runtime and mask + quality. `1024x1024` gives better masks, but is too slow for a practical FVP + smoke example. `224x224` and `256x256` are faster, but produced poor masks on + local demo images. +- MobileSAM PTQ is sensitive to the calibration set and the mask-logit + threshold. Inspect `*_metrics.json` and the debug overlays before treating + the quantized mask as an accuracy result. diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py new file mode 100644 index 00000000000..89dc254cf19 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/export_mobilesam.py @@ -0,0 +1,850 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +import argparse +import hashlib +import importlib +import inspect +import json +import os +import sys +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, cast + +import numpy as np +import torch +import tqdm # type: ignore[import] +from executorch.backends.arm.common.pipeline_config import ( + ArmPassPipelineConfig, + SoftmaxDecompositionConfig, +) +from executorch.backends.arm.ethosu import EthosUCompileSpec, EthosUPartitioner +from executorch.backends.arm.quantizer import ( + EthosUQuantizer, + get_symmetric_a16w8_quantization_config, + get_symmetric_quantization_config, +) +from executorch.devtools.backend_debug import get_delegation_info +from executorch.exir import ( + EdgeCompileConfig, + ExecutorchBackendConfig, + to_edge_transform_and_lower, +) +from executorch.extension.export_util.utils import save_pte_program +from PIL import Image +from torchao.quantization.pt2e.quantize_pt2e import ( # type: ignore[import] + convert_pt2e, + prepare_pt2e, +) + +MOBILE_SAM_SOURCE_URL = "https://github.com/ChaoningZhang/MobileSAM" +MOBILE_SAM_SOURCE_REVISION = "f706ad9c4eb7f219c00d9050e46328518ffb65d2" +MOBILE_SAM_PATCH = "0001-Make-TinyViT-image-size-configurable.patch" +DEFAULT_CHECKPOINT_FILENAME = "mobile_sam.pt" +DEFAULT_CHECKPOINT_URL = ( + f"{MOBILE_SAM_SOURCE_URL}/raw/{MOBILE_SAM_SOURCE_REVISION}/weights/" + f"{DEFAULT_CHECKPOINT_FILENAME}" +) +DEFAULT_CHECKPOINT_SHA256 = ( + "6dbb90523a35330fedd7f1d3dfc66f995213d81b29a5ca8108dbcdd4e37d6c2f" +) +MOBILE_SAM_SOURCE_LICENSE = "Apache-2.0" +DEFAULT_INPUT_SIZE = 512 +MOBILE_SAM_INPUT_ALIGNMENT = 16 + + +@dataclass +class PreparedSample: + name: str + image: Image.Image + pixel_values: torch.Tensor + labels: torch.Tensor | None + + +def load_mobile_sam( + checkpoint_path: str, + mobile_sam_source: str | None, + input_size: int, +) -> torch.nn.Module: + mobile_sam_module = import_mobile_sam_module(mobile_sam_source) + builder = mobile_sam_module.sam_model_registry["vit_t"] + if "image_size" not in inspect.signature(builder).parameters: + raise RuntimeError( + "The MobileSAM checkout does not provide configurable image sizes. " + "Run model_export/prepare_mobilesam.py and pass the prepared checkout " + "with --mobile-sam-source." + ) + return builder(checkpoint=checkpoint_path, image_size=input_size).eval() + + +class MobileSAMFixedPrompt(torch.nn.Module): + image_encoder: Any + mask_decoder: Any + + def __init__( + self, + sam: torch.nn.Module, + point_prompts: list[tuple[float, float]], + ) -> None: + super().__init__() + sam = cast(Any, sam) + + self.image_encoder = sam.image_encoder + self.mask_decoder = sam.mask_decoder + + with torch.no_grad(): + points = ( + torch.tensor([point_prompts], dtype=torch.float32), + torch.ones((1, len(point_prompts)), dtype=torch.int64), + ) + sparse_embeddings, dense_embeddings = sam.prompt_encoder( + points=points, + boxes=None, + masks=None, + ) + image_pe = sam.prompt_encoder.get_dense_pe() + + self.register_buffer("sparse_prompt_embeddings", sparse_embeddings) + self.register_buffer("dense_prompt_embeddings", dense_embeddings) + self.register_buffer("image_pe", image_pe) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + image_embeddings = self.image_encoder(pixel_values) + low_res_masks, _ = self.mask_decoder( + image_embeddings=image_embeddings, + image_pe=self.image_pe, + sparse_prompt_embeddings=self.sparse_prompt_embeddings, + dense_prompt_embeddings=self.dense_prompt_embeddings, + multimask_output=False, + ) + return low_res_masks + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Export fixed-prompt MobileSAM segmentation for Ethos-U." + ) + parser.add_argument( + "--checkpoint-path", + default=None, + help="Optional local MobileSAM checkpoint path.", + ) + parser.add_argument( + "--mobile-sam-source", + default=None, + help=( + "Optional MobileSAM checkout prepared by prepare_mobilesam.py. " + "Used before importing the mobile_sam package." + ), + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Load the checkpoint from the local cache only; do not download.", + ) + parser.add_argument( + "--calibration-image", + action="append", + default=[], + help="Local RGB image used for PTQ calibration. Can be repeated.", + ) + parser.add_argument( + "--eval-image", + action="append", + default=[], + help="Local RGB image used for validation/debugging. Can be repeated.", + ) + parser.add_argument( + "--eval-mask", + action="append", + default=[], + help=( + "Optional binary reference mask used for validation/debugging. " + "Can be repeated and must match --eval-image count when provided." + ), + ) + parser.add_argument( + "--point", + type=float, + nargs=2, + action="append", + default=[], + metavar=("X", "Y"), + help=( + "Positive point prompt in the resized square input frame. " + "Can be repeated for multi-point prompts." + ), + ) + parser.add_argument( + "--mask-threshold", + type=float, + default=0.0, + help="Mask-logit threshold used for metrics and debug masks.", + ) + parser.add_argument( + "--output-path", + type=str, + required=True, + help="Path to save the exported ExecuTorch program.", + ) + parser.add_argument( + "--input-size", + type=int, + default=DEFAULT_INPUT_SIZE, + help="Square MobileSAM input size. Must be divisible by 16.", + ) + parser.add_argument( + "--num-calibration-samples", + type=int, + default=4, + help="Number of local samples used for PTQ calibration.", + ) + parser.add_argument( + "--num-eval-samples", + type=int, + default=4, + help="Number of local samples used for host-side validation.", + ) + parser.add_argument( + "--num-debug-samples", + type=int, + default=4, + help="Number of validation samples written as visual debug artifacts.", + ) + parser.add_argument( + "--target", + default="ethos-u85-256", + help="Ethos-U target passed to Vela.", + ) + parser.add_argument( + "--system-config", + default="Ethos_U85_SYS_DRAM_Mid", + help="Vela system configuration.", + ) + parser.add_argument( + "--memory-mode", + default="Dedicated_Sram_384KB", + help="Vela memory mode.", + ) + parser.add_argument( + "--extra-vela-flag", + action="append", + default=[], + help="Additional Vela flag. Can be provided multiple times.", + ) + parser.add_argument( + "--artifact-dir", + default=None, + help="Optional directory for intermediate TOSA/Vela artifacts.", + ) + parser.add_argument( + "--debug-output-dir", + default=None, + help="Optional directory for masks, overlays, and validation summaries.", + ) + return parser.parse_args() + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +def import_mobile_sam_module(mobile_sam_source: str | None) -> Any: + if mobile_sam_source is not None: + sys.path.insert(0, str(Path(mobile_sam_source).expanduser().resolve())) + try: + return importlib.import_module("mobile_sam") + except ImportError as error: + raise ImportError( + "Could not import the patched mobile_sam package. Run " + "model_export/prepare_mobilesam.py and pass its checkout with " + "--mobile-sam-source." + ) from error + + +def find_module_type(module: torch.nn.Module, class_name: str) -> type[torch.nn.Module]: + for child in module.modules(): + if child.__class__.__name__ == class_name: + return child.__class__ + raise ValueError(f"Could not find module type {class_name} in {module.__class__}.") + + +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for chunk in iter(lambda: file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def default_checkpoint_cache_dir() -> Path: + return ( + Path.home() / ".cache" / "executorch" / "mobilesam" / MOBILE_SAM_SOURCE_REVISION + ) + + +def verify_checkpoint(path: Path, expected_sha256: str) -> None: + actual_sha256 = file_sha256(path) + if actual_sha256 != expected_sha256: + raise RuntimeError( + f"Checkpoint SHA256 mismatch for {path}: expected {expected_sha256}, " + f"got {actual_sha256}." + ) + + +def download_checkpoint(url: str, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temp_path = path.with_suffix(path.suffix + ".tmp") + try: + with ( + urllib.request.urlopen(url, timeout=60) as response, # nosec B310 + temp_path.open("wb") as file, + ): + for chunk in iter(lambda: response.read(1024 * 1024), b""): + file.write(chunk) + temp_path.replace(path) + except (OSError, urllib.error.URLError) as error: + temp_path.unlink(missing_ok=True) + raise RuntimeError( + f"Failed to download MobileSAM checkpoint from {url}." + ) from error + + +def resolve_checkpoint( + args: argparse.Namespace, +) -> tuple[str, str | None, str | None]: + if args.checkpoint_path is not None: + checkpoint_path = Path(args.checkpoint_path).expanduser().resolve() + if not checkpoint_path.is_file(): + raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}") + return str(checkpoint_path), None, None + + checkpoint_path = default_checkpoint_cache_dir() / DEFAULT_CHECKPOINT_FILENAME + if not checkpoint_path.is_file(): + if args.local_files_only: + raise FileNotFoundError( + f"Checkpoint not found in local cache: {checkpoint_path}" + ) + download_checkpoint(DEFAULT_CHECKPOINT_URL, checkpoint_path) + + verify_checkpoint(checkpoint_path, DEFAULT_CHECKPOINT_SHA256) + return ( + str(checkpoint_path.resolve()), + DEFAULT_CHECKPOINT_URL, + DEFAULT_CHECKPOINT_SHA256, + ) + + +def preprocess_image( + image: Image.Image, + segmentation_map: Image.Image | None, + *, + name: str, + input_size: int, + pixel_mean: np.ndarray, + pixel_std: np.ndarray, + output_mask_size: tuple[int, int] | None, +) -> PreparedSample: + rgb_image = image.convert("RGB") + width, height = rgb_image.size + scale = input_size / max(height, width) + resized_size = (round(width * scale), round(height * scale)) + resized_image = rgb_image.resize(resized_size, Image.Resampling.BILINEAR) + + padded_image = Image.new("RGB", (input_size, input_size)) + padded_image.paste(resized_image, (0, 0)) + + image_np = np.asarray(resized_image, dtype=np.float32) + image_np = (image_np - pixel_mean) / pixel_std + pixel_values = torch.from_numpy(image_np).permute(2, 0, 1).unsqueeze(0) + pixel_values = torch.nn.functional.pad( + pixel_values, + ( + 0, + input_size - resized_size[0], + 0, + input_size - resized_size[1], + ), + ) + pixel_values = pixel_values.contiguous() + + labels = None + if segmentation_map is not None: + if output_mask_size is None: + raise ValueError("An output mask size is required for reference masks.") + resized_mask = segmentation_map.convert("L").resize( + resized_size, + Image.Resampling.NEAREST, + ) + padded_mask = Image.new("L", (input_size, input_size)) + padded_mask.paste(resized_mask, (0, 0)) + resized_mask = padded_mask.resize(output_mask_size, Image.Resampling.NEAREST) + mask_np = np.asarray(resized_mask, dtype=np.uint8) + labels = torch.from_numpy((mask_np > 0).astype(np.uint8)).to(torch.long) + + return PreparedSample( + name=name, + image=padded_image, + pixel_values=pixel_values, + labels=labels, + ) + + +def load_local_samples( + image_paths: list[str], + mask_paths: list[str], + limit: int, + *, + include_labels: bool, + input_size: int, + pixel_mean: np.ndarray, + pixel_std: np.ndarray, + output_mask_size: tuple[int, int] | None = None, +) -> list[PreparedSample]: + if include_labels and len(mask_paths) not in (0, len(image_paths)): + raise ValueError("--eval-mask must be omitted or match --eval-image count.") + + samples: list[PreparedSample] = [] + for index, image_path in enumerate(image_paths): + if len(samples) >= limit: + break + image = Image.open(image_path) + mask = None + if include_labels and len(mask_paths) > 0: + mask = Image.open(mask_paths[index]) + samples.append( + preprocess_image( + image, + mask, + name=Path(image_path).stem or f"local_sample_{index:04d}", + input_size=input_size, + pixel_mean=pixel_mean, + pixel_std=pixel_std, + output_mask_size=output_mask_size, + ) + ) + if len(samples) == 0: + raise ValueError("No local samples were loaded.") + return samples + + +def run_mask_logits(model: torch.nn.Module, input_tensor: torch.Tensor) -> torch.Tensor: + output = model(input_tensor) + if isinstance(output, (tuple, list)): + output = output[0] + if not isinstance(output, torch.Tensor): + raise TypeError(f"Expected tensor mask logits, got {type(output)}") + return output + + +def predict_mask(logits: torch.Tensor, threshold: float) -> np.ndarray: + mask = logits.detach().cpu().squeeze(0).squeeze(0).numpy() > threshold + return mask.astype(np.uint8) + + +def binary_iou(mask_a: np.ndarray, mask_b: np.ndarray) -> float: + intersection = np.logical_and(mask_a, mask_b).sum() + union = np.logical_or(mask_a, mask_b).sum() + if union == 0: + return 1.0 + return float(intersection / union) + + +def save_binary_mask(path: Path, mask: np.ndarray) -> Image.Image: + image = Image.fromarray((mask.astype(np.uint8) * 255), mode="L") + image.save(path) + return image.convert("RGB") + + +def save_mask_overlay( + path: Path, + image: Image.Image, + mask: np.ndarray, + color: tuple[int, int, int], +) -> None: + rgb_image = image.convert("RGB") + resized_mask = Image.fromarray(mask.astype(np.uint8) * 255, mode="L").resize( + rgb_image.size, + Image.Resampling.NEAREST, + ) + mask_np = np.asarray(resized_mask, dtype=np.uint8) > 0 + overlay_np = np.asarray(rgb_image, dtype=np.float32) + color_np = np.asarray(color, dtype=np.float32) + overlay_np[mask_np] = overlay_np[mask_np] * 0.55 + color_np * 0.45 + Image.fromarray(np.clip(overlay_np, 0, 255).astype(np.uint8), mode="RGB").save(path) + + +def write_debug_artifacts( + debug_dir: Path, + sample: PreparedSample, + fp32_mask: np.ndarray, + quantized_mask: np.ndarray, +) -> None: + sample_dir = debug_dir / sample.name + sample_dir.mkdir(parents=True, exist_ok=True) + + sample.image.save(sample_dir / "input.png") + save_binary_mask(sample_dir / "fp32_mask.png", fp32_mask) + save_binary_mask(sample_dir / "quantized_mask.png", quantized_mask) + save_mask_overlay( + sample_dir / "fp32_overlay.png", + sample.image, + fp32_mask, + (0, 220, 120), + ) + save_mask_overlay( + sample_dir / "quantized_overlay.png", + sample.image, + quantized_mask, + (0, 170, 255), + ) + if sample.labels is not None: + save_binary_mask( + sample_dir / "reference_mask.png", + sample.labels.detach().cpu().numpy().astype(np.uint8), + ) + + mismatch = fp32_mask != quantized_mask + heatmap = np.zeros((*fp32_mask.shape, 3), dtype=np.uint8) + heatmap[~mismatch] = [0, 128, 0] + heatmap[mismatch] = [255, 0, 0] + Image.fromarray(heatmap, mode="RGB").save(sample_dir / "mismatch_heatmap.png") + + write_json( + sample_dir / "mask_summary.json", + { + "foreground_pixels": int(quantized_mask.sum()), + "background_pixels": int(quantized_mask.size - quantized_mask.sum()), + "fp32_quantized_iou": binary_iou(fp32_mask, quantized_mask), + }, + ) + + +def evaluate_and_debug( + fp32_model: torch.nn.Module, + quantized_model: torch.nn.Module, + eval_samples: list[PreparedSample], + debug_dir: Path | None, + num_debug_samples: int, + threshold: float, +) -> dict[str, float]: + fp32_quantized_ious: list[float] = [] + fp32_quantized_pixel_agreements: list[float] = [] + reference_ious: list[float] = [] + + if debug_dir is not None: + debug_dir.mkdir(parents=True, exist_ok=True) + + print("\nEvaluating quantized MobileSAM on validation samples...") + for index, sample in enumerate(tqdm.tqdm(eval_samples)): + fp32_logits = run_mask_logits(fp32_model, sample.pixel_values) + quantized_logits = run_mask_logits(quantized_model, sample.pixel_values) + fp32_mask = predict_mask(fp32_logits, threshold) + quantized_mask = predict_mask(quantized_logits, threshold) + + fp32_quantized_ious.append(binary_iou(fp32_mask, quantized_mask)) + fp32_quantized_pixel_agreements.append( + float(np.mean(fp32_mask == quantized_mask)) + ) + if sample.labels is not None: + labels = sample.labels.detach().cpu().numpy().astype(np.uint8) + reference_ious.append(binary_iou(quantized_mask, labels)) + + if debug_dir is not None and index < num_debug_samples: + write_debug_artifacts(debug_dir, sample, fp32_mask, quantized_mask) + + metrics = { + "num_samples": float(len(eval_samples)), + "fp32_quantized_mean_iou": float(np.mean(fp32_quantized_ious)), + "fp32_quantized_pixel_agreement": float( + np.mean(fp32_quantized_pixel_agreements) + ), + } + if len(reference_ious) > 0: + metrics["reference_mean_iou"] = float(np.mean(reference_ious)) + return metrics + + +def quantize_model( + model: torch.nn.Module, + quantizer: EthosUQuantizer, + example_inputs: tuple[torch.Tensor], + calibration_samples: list[PreparedSample], +) -> torch.export.ExportedProgram: + exported = torch.export.export(model, example_inputs) + prepared = prepare_pt2e(exported.module(), quantizer) + + print("\nCalibrating MobileSAM...") + for sample in tqdm.tqdm(calibration_samples): + prepared(sample.pixel_values) + + quantized = convert_pt2e(prepared) + return torch.export.export(quantized, example_inputs) + + +def has_quantized_out_variants() -> bool: + try: + _ = torch.ops.quantized_decomposed.quantize_per_tensor.out + _ = torch.ops.quantized_decomposed.dequantize_per_tensor.out + return True + except AttributeError: + return False + + +def load_quantized_ops_library(library_path: Path) -> Path: + if not library_path.is_file(): + raise FileNotFoundError(f"Quantized ops library not found: {library_path}") + torch.ops.load_library(str(library_path)) + if has_quantized_out_variants(): + return library_path + raise RuntimeError( + f"Quantized ops library did not register required out variants: {library_path}" + ) + + +def ensure_quantized_ops_loaded() -> Path | None: + if has_quantized_out_variants(): + return None + + quantized_ops_library = os.environ.get("EXECUTORCH_QUANTIZED_OPS_AOT_LIBRARY") + if quantized_ops_library: + return load_quantized_ops_library( + Path(quantized_ops_library).expanduser().resolve() + ) + + try: + import executorch.kernels.quantized # noqa: F401 + except ImportError: + pass + else: + if has_quantized_out_variants(): + return None + + repo_root = Path(__file__).resolve().parents[4] + search_patterns = ( + "cmake-out/kernels/quantized/libquantized_ops_aot_lib.*", + "arm_test/*/kernels/quantized/libquantized_ops_aot_lib.*", + "arm_test/**/kernels/quantized/libquantized_ops_aot_lib.*", + ) + for pattern in search_patterns: + for candidate in sorted(repo_root.glob(pattern)): + if not candidate.is_file(): + continue + return load_quantized_ops_library(candidate) + + raise RuntimeError( + "MobileSAM int8 export requires the quantized ops out-variant library. " + "Build or install ExecuTorch quantized kernels so that " + "`quantized_decomposed::quantize_per_tensor.out` and " + "`quantized_decomposed::dequantize_per_tensor.out` are available." + ) + + +def write_delegation_report(edge_program_manager: Any, report_path: Path) -> None: + delegation_info = get_delegation_info( + edge_program_manager.exported_program().graph_module + ) + report_path.write_text(delegation_info.get_summary() + "\n") + + +def resolve_point_prompts(args: argparse.Namespace) -> list[tuple[float, float]]: + if len(args.point) > 0: + point_prompts = [(float(x), float(y)) for x, y in args.point] + else: + point_prompts = [(args.input_size / 2, args.input_size / 2)] + + for point_x, point_y in point_prompts: + if not (0 <= point_x <= args.input_size and 0 <= point_y <= args.input_size): + raise ValueError("Point prompts must be inside the square input.") + return point_prompts + + +def main() -> None: + args = parse_args() + if args.input_size < 224 or args.input_size % MOBILE_SAM_INPUT_ALIGNMENT != 0: + raise ValueError("--input-size must be at least 224 and divisible by 16.") + if args.num_calibration_samples <= 0: + raise ValueError("--num-calibration-samples must be positive.") + if args.num_eval_samples <= 0: + raise ValueError("--num-eval-samples must be positive.") + if len(args.calibration_image) == 0: + raise ValueError("At least one --calibration-image is required.") + if len(args.eval_image) == 0: + args.eval_image = list(args.calibration_image) + if len(args.eval_mask) not in (0, len(args.eval_image)): + raise ValueError("--eval-mask must be omitted or match --eval-image count.") + point_prompts = resolve_point_prompts(args) + quantized_ops_library = ensure_quantized_ops_loaded() + if quantized_ops_library is not None: + print(f"Loaded quantized ops library from {quantized_ops_library}") + + output_path = Path(args.output_path).resolve() + output_path.parent.mkdir(parents=True, exist_ok=True) + metadata_path = output_path.with_suffix(".json") + metrics_path = output_path.with_name(f"{output_path.stem}_metrics.json") + delegation_path = output_path.with_name(f"{output_path.stem}_delegation.txt") + debug_dir = Path(args.debug_output_dir).resolve() if args.debug_output_dir else None + + checkpoint_path, checkpoint_url, checkpoint_sha256 = resolve_checkpoint(args) + mobile_sam = load_mobile_sam( + checkpoint_path, + args.mobile_sam_source, + args.input_size, + ) + pixel_mean = cast(Any, mobile_sam).pixel_mean.detach().cpu().reshape(-1).numpy() + pixel_std = cast(Any, mobile_sam).pixel_std.detach().cpu().reshape(-1).numpy() + if pixel_mean.shape != (3,) or pixel_std.shape != (3,): + raise ValueError("MobileSAM preprocessing must provide three RGB values.") + wrapped_model = MobileSAMFixedPrompt(mobile_sam, point_prompts).eval() + + calibration_samples = load_local_samples( + args.calibration_image, + [], + args.num_calibration_samples, + include_labels=False, + input_size=args.input_size, + pixel_mean=pixel_mean, + pixel_std=pixel_std, + ) + example_inputs = (calibration_samples[0].pixel_values,) + with torch.no_grad(): + output_shape = list(run_mask_logits(wrapped_model, example_inputs[0]).shape) + if len(output_shape) != 4 or output_shape[:2] != [1, 1]: + raise ValueError( + f"Expected MobileSAM output shape [1, 1, height, width], got {output_shape}." + ) + output_mask_size = (output_shape[3], output_shape[2]) + + eval_samples = load_local_samples( + args.eval_image, + args.eval_mask, + args.num_eval_samples, + include_labels=True, + input_size=args.input_size, + pixel_mean=pixel_mean, + pixel_std=pixel_std, + output_mask_size=output_mask_size, + ) + + compile_spec = EthosUCompileSpec( + target=args.target, + system_config=args.system_config, + memory_mode=args.memory_mode, + extra_flags=args.extra_vela_flag, + ) + compile_spec.set_pass_pipeline_config( + ArmPassPipelineConfig(softmax=SoftmaxDecompositionConfig.STABLE) + ) + if args.artifact_dir is not None: + artifact_dir = Path(args.artifact_dir).resolve() + artifact_dir.mkdir(parents=True, exist_ok=True) + compile_spec.dump_intermediate_artifacts_to(str(artifact_dir)) + + quantizer = EthosUQuantizer(compile_spec) + quantizer.set_global(get_symmetric_quantization_config()) + attention_module_type = find_module_type(wrapped_model.image_encoder, "Attention") + quantizer.set_module_type( + attention_module_type, + get_symmetric_a16w8_quantization_config(), + ) + + with torch.no_grad(): + quantized_program = quantize_model( + wrapped_model, + quantizer, + example_inputs, + calibration_samples, + ) + quantized_module = quantized_program.module() + metrics = evaluate_and_debug( + wrapped_model, + quantized_module, + eval_samples, + debug_dir, + args.num_debug_samples, + args.mask_threshold, + ) + write_json(metrics_path, metrics) + print( + "Validation metrics: " + f"fp32_quantized_mean_iou={metrics['fp32_quantized_mean_iou']:.4f} " + "fp32_quantized_pixel_agreement=" + f"{metrics['fp32_quantized_pixel_agreement']:.4f}" + ) + + edge_program_manager = to_edge_transform_and_lower( + programs=quantized_program, + partitioner=[EthosUPartitioner(compile_spec)], + compile_config=EdgeCompileConfig(_check_ir_validity=False), + ) + write_delegation_report(edge_program_manager, delegation_path) + + executorch_program_manager = edge_program_manager.to_executorch( + config=ExecutorchBackendConfig(extract_delegate_segments=False) + ) + save_pte_program( + executorch_program_manager, + str(output_path), + output_dir=str(output_path.parent), + ) + + write_json( + metadata_path, + { + "model_name": "MobileSAM vit_t", + "checkpoint_filename": DEFAULT_CHECKPOINT_FILENAME, + "checkpoint_path": checkpoint_path, + "checkpoint_url": checkpoint_url, + "checkpoint_sha256": checkpoint_sha256, + "mobile_sam_source_license": MOBILE_SAM_SOURCE_LICENSE, + "mobile_sam_source_url": MOBILE_SAM_SOURCE_URL, + "mobile_sam_source_revision": MOBILE_SAM_SOURCE_REVISION, + "mobile_sam_patch": MOBILE_SAM_PATCH, + "input_shape": list(example_inputs[0].shape), + "output_shape": output_shape, + "input_size": args.input_size, + "preprocessing": { + "pixel_mean": pixel_mean.tolist(), + "pixel_std": pixel_std.tolist(), + "resize": "longest_side_then_zero_pad", + }, + "point_prompts_xy": point_prompts, + "mask_threshold": args.mask_threshold, + "target": args.target, + "system_config": args.system_config, + "memory_mode": args.memory_mode, + "extra_vela_flags": args.extra_vela_flag, + "quantization": { + "global": "int8 activations and int8 weights", + "tinyvit_attention": "int16 activations and int8 weights", + }, + "num_calibration_samples": len(calibration_samples), + "num_eval_samples": len(eval_samples), + "calibration_images": args.calibration_image, + "eval_images": args.eval_image, + "eval_masks": args.eval_mask, + "output_path": str(output_path), + "metrics_path": str(metrics_path), + "delegation_path": str(delegation_path), + "debug_output_dir": str(debug_dir) if debug_dir is not None else None, + }, + ) + + print(f"\nExported model saved to {output_path}") + print(f"Metadata saved to {metadata_path}") + print(f"Metrics saved to {metrics_path}") + print(f"Delegation summary saved to {delegation_path}") + if debug_dir is not None: + print(f"Debug artifacts saved to {debug_dir}") + + +if __name__ == "__main__": + main() diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/patches/mobile_sam/0001-Make-TinyViT-image-size-configurable.patch b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/patches/mobile_sam/0001-Make-TinyViT-image-size-configurable.patch new file mode 100644 index 00000000000..177ae0fbedb --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/patches/mobile_sam/0001-Make-TinyViT-image-size-configurable.patch @@ -0,0 +1,34 @@ +diff --git a/mobile_sam/build_sam.py b/mobile_sam/build_sam.py +index 9a52c50..d657eff 100644 +--- a/mobile_sam/build_sam.py ++++ b/mobile_sam/build_sam.py +@@ -47,11 +47,12 @@ def build_sam_vit_b(checkpoint=None): +-def build_sam_vit_t(checkpoint=None): ++def build_sam_vit_t(checkpoint=None, image_size=1024): ++ if image_size < 16 or image_size % 16 != 0: ++ raise ValueError("image_size must be at least 16 and divisible by 16") + prompt_embed_dim = 256 +- image_size = 1024 + vit_patch_size = 16 + image_embedding_size = image_size // vit_patch_size + mobile_sam = Sam( +- image_encoder=TinyViT(img_size=1024, in_chans=3, num_classes=1000, ++ image_encoder=TinyViT(img_size=image_size, in_chans=3, num_classes=1000, + embed_dims=[64, 128, 160, 320], + depths=[2, 2, 6, 2], + num_heads=[2, 4, 5, 10], + window_sizes=[7, 7, 14, 7], +diff --git a/mobile_sam/modeling/tiny_vit_sam.py b/mobile_sam/modeling/tiny_vit_sam.py +index 93fa9f5..6e1f37b 100644 +--- a/mobile_sam/modeling/tiny_vit_sam.py ++++ b/mobile_sam/modeling/tiny_vit_sam.py +@@ -607,7 +607,8 @@ class TinyViT(nn.Module): + layer = self.layers[i] + x = layer(x) + B,_,C=x.size() +- x = x.view(B, 64, 64, C) ++ height, width = self.layers[-1].input_resolution ++ x = x.view(B, height, width, C) + x=x.permute(0, 3, 1, 2) + x=self.neck(x) + return x diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/prepare_mobilesam.py b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/prepare_mobilesam.py new file mode 100644 index 00000000000..42b7f961019 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/model_export/prepare_mobilesam.py @@ -0,0 +1,122 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +import argparse +import subprocess # nosec B404 +from pathlib import Path + + +MOBILE_SAM_SOURCE_URL = "https://github.com/ChaoningZhang/MobileSAM.git" +MOBILE_SAM_SOURCE_REVISION = "f706ad9c4eb7f219c00d9050e46328518ffb65d2" +PATCH_DIR = Path(__file__).resolve().parent / "patches" / "mobile_sam" + + +def run(command: list[str], *, cwd: Path | None = None) -> None: + subprocess.run(command, cwd=cwd, check=True) # nosec B603 + + +def default_source_dir() -> Path: + return ( + Path.home() + / ".cache" + / "executorch" + / "mobilesam" + / MOBILE_SAM_SOURCE_REVISION + / "source" + ) + + +def prepare_source(source_dir: Path, *, local_files_only: bool) -> None: + source_dir = source_dir.expanduser().resolve() + marker = source_dir.parent / f".{source_dir.name}.executorch-managed" + + if source_dir.exists() and not marker.is_file(): + raise RuntimeError( + f"Refusing to modify unmanaged MobileSAM directory: {source_dir}" + ) + + if not source_dir.exists(): + if local_files_only: + raise FileNotFoundError( + f"Managed MobileSAM checkout not found: {source_dir}" + ) + source_dir.parent.mkdir(parents=True, exist_ok=True) + run( + [ + "git", + "clone", + "--filter=blob:none", + "--no-checkout", + MOBILE_SAM_SOURCE_URL, + str(source_dir), + ] + ) + marker.write_text(MOBILE_SAM_SOURCE_REVISION + "\n") + run( + [ + "git", + "sparse-checkout", + "set", + "mobile_sam", + "LICENSE", + ], + cwd=source_dir, + ) + + if not local_files_only: + run( + ["git", "fetch", "--quiet", "origin", MOBILE_SAM_SOURCE_REVISION], + cwd=source_dir, + ) + + try: + run( + ["git", "cat-file", "-e", f"{MOBILE_SAM_SOURCE_REVISION}^{{commit}}"], + cwd=source_dir, + ) + except subprocess.CalledProcessError as error: + raise RuntimeError( + f"MobileSAM revision {MOBILE_SAM_SOURCE_REVISION} is unavailable locally." + ) from error + + run( + ["git", "checkout", "--detach", "--force", MOBILE_SAM_SOURCE_REVISION], + cwd=source_dir, + ) + run(["git", "reset", "--hard", MOBILE_SAM_SOURCE_REVISION], cwd=source_dir) + + patches = sorted(PATCH_DIR.glob("*.patch")) + if not patches: + raise FileNotFoundError(f"No MobileSAM patches found in {PATCH_DIR}") + for patch in patches: + run(["git", "apply", "--check", str(patch)], cwd=source_dir) + run(["git", "apply", str(patch)], cwd=source_dir) + + print(f"Prepared patched MobileSAM source at {source_dir}") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Prepare the pinned MobileSAM source with ExecuTorch patches." + ) + parser.add_argument( + "--source-dir", + type=Path, + default=default_source_dir(), + help="Managed checkout destination.", + ) + parser.add_argument( + "--local-files-only", + action="store_true", + help="Reuse an existing managed checkout without network access.", + ) + args = parser.parse_args() + prepare_source(args.source_dir, local_files_only=args.local_files_only) + + +if __name__ == "__main__": + main() diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/requirements.txt b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/requirements.txt new file mode 100644 index 00000000000..6eb309f00de --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/requirements.txt @@ -0,0 +1,6 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +tqdm == 4.67.1 diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/CMakeLists.txt b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/CMakeLists.txt new file mode 100644 index 00000000000..aeee9de49f8 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/CMakeLists.txt @@ -0,0 +1,209 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +cmake_minimum_required(VERSION 3.20) + +project(mobilesam_prompt_segmentation_ethos_u_application) + +set(ET_DIR_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/../../../.." + CACHE PATH "Path to ExecuTorch dir" +) +set(ET_BUILD_DIR_PATH + "${ET_DIR_PATH}/cmake-out-arm" + CACHE PATH "Path to ExecuTorch build/install dir" +) +set(ET_INCLUDE_PATH + "${ET_DIR_PATH}/.." + CACHE PATH "Path to ExecuTorch headers" +) +set(ET_PTE_FILE_PATH + "" + CACHE PATH "Path to ExecuTorch model pte" +) +set(MODEL_METADATA_PATH + "" + CACHE PATH "Path to MobileSAM exporter metadata" +) +set(IMAGE_PATH + "" + CACHE PATH "Path to an RGB image to use for the application" +) +set(MASK_THRESHOLD + "-5.0" + CACHE STRING "MobileSAM mask logit threshold" +) +set(SYSTEM_CONFIG + "Ethos_U85_SYS_DRAM_Mid" + CACHE STRING "Vela system configuration" +) +set(MEMORY_MODE + "Dedicated_Sram_384KB" + CACHE STRING "Vela memory mode" +) +set(ETHOS_SDK_PATH + "${ET_DIR_PATH}/examples/arm/arm-scratch/ethos-u" + CACHE PATH "Path to Ethos-U bare metal driver/env" +) +set(PYTHON_EXECUTABLE + "python" + CACHE PATH "Define to override python executable used" +) +option(ET_SEGMENTATION_DUMP_MASK + "Dump the predicted segmentation mask as run-length encoded log chunks" + OFF +) +option(ET_SEGMENTATION_SEMIHOSTING_OUTPUT + "Emit validation logs through Arm semihosting for FVP smoke runs" ON +) + +if(NOT EXISTS "${IMAGE_PATH}") + message( + FATAL_ERROR + "Image not provided. Please provide -DIMAGE_PATH= and retry." + ) +endif() +if(NOT EXISTS "${ET_PTE_FILE_PATH}") + message( + FATAL_ERROR + "PTE file not provided. Please provide -DET_PTE_FILE_PATH= and retry." + ) +endif() +if(NOT MODEL_METADATA_PATH) + get_filename_component(model_directory "${ET_PTE_FILE_PATH}" DIRECTORY) + get_filename_component(model_name "${ET_PTE_FILE_PATH}" NAME_WE) + set(MODEL_METADATA_PATH "${model_directory}/${model_name}.json") +endif() +if(NOT EXISTS "${MODEL_METADATA_PATH}") + message( + FATAL_ERROR + "Model metadata not found. Provide -DMODEL_METADATA_PATH=." + ) +endif() +if(NOT SYSTEM_CONFIG MATCHES "Ethos_U85") + message(FATAL_ERROR "This example currently supports Corstone-320/Ethos-U85.") +endif() + +include(${ET_DIR_PATH}/backends/arm/scripts/corstone_utils.cmake) +fetch_ethos_u_content(${ETHOS_SDK_PATH} ${ET_DIR_PATH}) + +if(NOT EXISTS "${ETHOS_SDK_PATH}") + message( + FATAL_ERROR + "The ${ETHOS_SDK_PATH} directory does not exist. Please run examples/arm/setup.sh and retry." + ) +endif() + +find_package( + executorch REQUIRED HINTS "${ET_BUILD_DIR_PATH}/lib/cmake/ExecuTorch" +) + +add_corstone_subdirectory(${SYSTEM_CONFIG} ${ETHOS_SDK_PATH}) +configure_timing_adapters(${SYSTEM_CONFIG} ${MEMORY_MODE}) + +add_executable(mobilesam_prompt_segmentation_example main.cpp) +target_sources( + mobilesam_prompt_segmentation_example + PRIVATE main.cpp + ${CMAKE_SOURCE_DIR}/../../executor_runner/arm_memory_allocator.cpp +) +target_link_libraries( + mobilesam_prompt_segmentation_example + PUBLIC executorch + ethosu_target_init + extension_runner_util + quantized_ops_lib + portable_kernels + cortex_m_kernels + cortex_m_ops_lib +) + +include(${ET_DIR_PATH}/tools/cmake/Utils.cmake) +executorch_target_link_options_shared_lib(executorch_delegate_ethos_u) +target_link_libraries( + mobilesam_prompt_segmentation_example PUBLIC executorch_delegate_ethos_u +) + +if(MEMORY_MODE MATCHES "Dedicated_Sram") + set(ETHOSU_ARENA "1") + if(NOT DEFINED ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE) + set(ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE 0x1000000) + endif() + if(NOT DEFINED ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE) + set(ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE 0x60000) + endif() +else() + set(ETHOSU_ARENA "0") + if(NOT DEFINED ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE) + set(ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE 0x400000) + endif() +endif() +if(NOT DEFINED ET_SEGMENTATION_METHOD_ALLOCATOR_POOL_SIZE) + set(ET_SEGMENTATION_METHOD_ALLOCATOR_POOL_SIZE 0x4000000) +endif() + +target_compile_definitions( + mobilesam_prompt_segmentation_example + PRIVATE + ET_SEGMENTATION_METHOD_ALLOCATOR_POOL_SIZE=${ET_SEGMENTATION_METHOD_ALLOCATOR_POOL_SIZE} + ET_SEGMENTATION_MASK_THRESHOLD=${MASK_THRESHOLD} + ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE=${ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE} +) +if(DEFINED ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE) + target_compile_definitions( + mobilesam_prompt_segmentation_example + PRIVATE + ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE=${ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE} + ) +endif() +if(ET_SEGMENTATION_DUMP_MASK) + target_compile_definitions( + mobilesam_prompt_segmentation_example PRIVATE ET_SEGMENTATION_DUMP_MASK + ) +endif() +if(ET_SEGMENTATION_SEMIHOSTING_OUTPUT) + target_compile_definitions( + mobilesam_prompt_segmentation_example + PRIVATE ET_SEGMENTATION_SEMIHOSTING_OUTPUT + ) +endif() + +if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set(LINK_FILE_EXT ld) + set(COMPILER_PREPROCESSOR_OPTIONS -E -x c -P) +endif() + +set(LINK_FILE_OUT_BASE "platform_linker_script") +set(LINK_FILE_IN "${CMAKE_SOURCE_DIR}/../../executor_runner/Corstone-320.ld") +set(LINK_FILE_OUT + ${CMAKE_CURRENT_BINARY_DIR}/${LINK_FILE_OUT_BASE}.${LINK_FILE_EXT} +) +execute_process( + COMMAND ${CMAKE_C_COMPILER} ${COMPILER_PREPROCESSOR_OPTIONS} -DETHOSU_MODEL=1 + -DETHOSU_ARENA=${ETHOSU_ARENA} -o ${LINK_FILE_OUT} ${LINK_FILE_IN} +) +target_link_options( + mobilesam_prompt_segmentation_example PRIVATE "-T" "${LINK_FILE_OUT}" +) + +execute_process( + COMMAND + ${PYTHON_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/../../executor_runner/pte_to_header.py --pte + ${ET_PTE_FILE_PATH} --outdir ${CMAKE_CURRENT_BINARY_DIR} + COMMAND_ERROR_IS_FATAL ANY +) +execute_process( + COMMAND + ${PYTHON_EXECUTABLE} ${CMAKE_SOURCE_DIR}/image_to_array.py --image + ${IMAGE_PATH} --metadata ${MODEL_METADATA_PATH} --output + ${CMAKE_CURRENT_BINARY_DIR}/image.h COMMAND_ERROR_IS_FATAL ANY +) + +target_include_directories( + mobilesam_prompt_segmentation_example + PRIVATE ${ET_INCLUDE_PATH} ${ET_DIR_PATH}/runtime/core/portable_type/c10 + ${CMAKE_SOURCE_DIR}/../../executor_runner ${CMAKE_CURRENT_BINARY_DIR} +) diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/README.md b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/README.md new file mode 100644 index 00000000000..66a4a5a2f60 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/README.md @@ -0,0 +1,96 @@ +# MobileSAM Runtime Example + +This directory builds a bare-metal Corstone-320 application for a MobileSAM +fixed-prompt `.pte` generated by `model_export/export_mobilesam.py`. + +## Build ExecuTorch for Arm + +Run the Arm setup first if it has not already been run: + +```bash +./examples/arm/setup.sh --i-agree-to-the-contained-eula +source examples/arm/arm-scratch/setup_path.sh +``` + +Build and install the Arm bare-metal ExecuTorch libraries from +`examples/arm`: + +```bash +cmake --preset arm-baremetal \ + -DCMAKE_BUILD_TYPE=Release \ + -B../../cmake-out-arm ../.. +cmake --build ../../cmake-out-arm --target install -j$(nproc) +``` + +The Arm bare-metal preset installs into the build directory. If you use a +different `-B` path, pass the same path as `-DET_BUILD_DIR_PATH` when +configuring the runtime app. + +## Configure the Runtime App + +Use the `.pte` from the export step and any RGB image: + +```bash +cmake \ + -DCMAKE_TOOLCHAIN_FILE=$(pwd)/ethos-u-setup/arm-none-eabi-gcc.cmake \ + -DET_BUILD_DIR_PATH=../../cmake-out-arm \ + -DET_PTE_FILE_PATH= \ + -DMODEL_METADATA_PATH= \ + -DIMAGE_PATH= \ + -DMASK_THRESHOLD=-5.0 \ + -DSYSTEM_CONFIG=Ethos_U85_SYS_DRAM_Mid \ + -DMEMORY_MODE=Dedicated_Sram_384KB \ + -Bmobilesam_point_runtime \ + mobilesam_prompt_segmentation_example_ethos_u/runtime +``` + +The generated image header reads the input shape and normalization values from +the exporter metadata, resizes the longest side, and pads the shorter side with +zeros. `MODEL_METADATA_PATH` defaults to the `.json` file beside the `.pte`. +The generated `512x512` float input is placed in the Corstone-320 DDR input +section so it does not consume BRAM. + +The `MEMORY_MODE` value must match the value used during export. The default +`Dedicated_Sram_384KB` places the Ethos-U tensor arena in DDR and the fast +scratch buffer in SRAM, matching the Corstone-320 linker script. The default +method allocator pool is sized for MobileSAM's reduced `512x512` input tensor. + +The runtime emits validation logs through Arm semihosting by default so the FVP +smoke run below prints the segmentation summary. Configure with +`-DET_SEGMENTATION_SEMIHOSTING_OUTPUT=OFF` when targeting an environment +without semihosting support. + +The default target-side mask threshold is `-5.0`, selected from FVP threshold +sweeps on the reduced `512x512` export. The runtime logs foreground counts for +several thresholds so users can inspect and retune post-processing without +re-exporting the graph. + +## Compile + +```bash +cmake --build mobilesam_point_runtime -j$(nproc) -- mobilesam_prompt_segmentation_example +``` + +## Run on Corstone-320 FVP + +```bash +../../backends/arm/scripts/run_fvp.sh \ + --elf=mobilesam_point_runtime/mobilesam_prompt_segmentation_example \ + --target=ethos-u85-256 \ + --timeout=300 \ + --semihosting-cwd=$(pwd)/mobilesam_point_runtime \ + --fast +``` + +Expected logs include: + +- `MobileSAM Ethos-U example started`. +- Input and output tensor shapes. +- `Mask threshold`. +- `Segmentation mask hash`. +- Foreground/background pixel counts. +- `Model executed successfully.` + +To dump the predicted binary mask as run-length encoded log chunks, configure +with `-DET_SEGMENTATION_DUMP_MASK=ON`. This is useful for debugging but makes +the UART output larger. diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/image_to_array.py b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/image_to_array.py new file mode 100644 index 00000000000..a3d169907ec --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/image_to_array.py @@ -0,0 +1,123 @@ +# Copyright 2026 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +# +# Copyright (c) Meta Platforms, Inc. and affiliates. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +from __future__ import annotations + +import json +import os +from argparse import ArgumentParser +from pathlib import Path + +import numpy as np +from PIL import Image + + +def convert_image_to_c_array( + image_path: str, + output_path: str, + image_size: tuple[int, int], + pixel_mean: tuple[float, float, float], + pixel_std: tuple[float, float, float], + array_name: str = "image_data", +) -> None: + image = Image.open(image_path).convert("RGB") + width, height = image.size + target_width, target_height = image_size + if target_width != target_height: + raise ValueError("MobileSAM runtime preprocessing expects a square input.") + + scale = target_width / max(height, width) + resized_size = (round(width * scale), round(height * scale)) + image = image.resize(resized_size, resample=Image.Resampling.BILINEAR) + + data = np.asarray(image, dtype=np.float32) + data = (data - np.asarray(pixel_mean, dtype=np.float32)) / np.asarray( + pixel_std, dtype=np.float32 + ) + padded_data = np.zeros((target_height, target_width, 3), dtype=np.float32) + padded_data[: resized_size[1], : resized_size[0], :] = data + data = np.transpose(padded_data, (2, 0, 1)).flatten() + + array_lines = [] + for i in range(0, len(data), 12): + line = ", ".join(f"{value:.8f}" for value in data[i : i + 12]) + array_lines.append(" " + line + ",") + + c_array = f"""#include +#include + +const size_t image_width = {image_size[0]}; +const size_t image_height = {image_size[1]}; +const size_t image_channels = 3; +__attribute__((section("input_data_sec"), aligned(16))) float {array_name}[{len(data)}] = {{ +{os.linesep.join(array_lines)} +}}; +""" + with open(output_path, "w") as output_file: + output_file.write(c_array) + print(f"Converted '{image_path}' to '{output_path}' ({len(data)} floats)") + + +def load_model_metadata( + metadata_path: str, +) -> tuple[tuple[int, int], tuple[float, float, float], tuple[float, float, float]]: + metadata = json.loads(Path(metadata_path).read_text()) + input_shape = metadata.get("input_shape") + if not isinstance(input_shape, list) or len(input_shape) != 4: + raise ValueError("Model metadata must contain a four-dimensional input_shape.") + if input_shape[0] != 1 or input_shape[1] != 3: + raise ValueError("MobileSAM runtime expects input shape [1, 3, H, W].") + + preprocessing = metadata.get("preprocessing") + if not isinstance(preprocessing, dict): + raise ValueError("Model metadata does not contain preprocessing values.") + pixel_mean_values = preprocessing.get("pixel_mean") + pixel_std_values = preprocessing.get("pixel_std") + if not isinstance(pixel_mean_values, list) or len(pixel_mean_values) != 3: + raise ValueError("MobileSAM pixel_mean must contain three RGB values.") + if not isinstance(pixel_std_values, list) or len(pixel_std_values) != 3: + raise ValueError("MobileSAM pixel_std must contain three RGB values.") + + image_size = (int(input_shape[3]), int(input_shape[2])) + pixel_mean = tuple(float(value) for value in pixel_mean_values) + pixel_std = tuple(float(value) for value in pixel_std_values) + return ( + image_size, + (pixel_mean[0], pixel_mean[1], pixel_mean[2]), + (pixel_std[0], pixel_std[1], pixel_std[2]), + ) + + +def main() -> None: + parser = ArgumentParser() + parser.add_argument("--image", required=True, help="Path to an RGB image.") + parser.add_argument( + "--output", required=True, help="Output path for the generated C array." + ) + parser.add_argument( + "--metadata", + required=True, + help="Exporter metadata containing input shape and preprocessing values.", + ) + args = parser.parse_args() + + image_size, pixel_mean, pixel_std = load_model_metadata(args.metadata) + convert_image_to_c_array( + args.image, + args.output, + image_size, + pixel_mean, + pixel_std, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/main.cpp b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/main.cpp new file mode 100644 index 00000000000..caa28085045 --- /dev/null +++ b/examples/arm/mobilesam_prompt_segmentation_example_ethos_u/runtime/main.cpp @@ -0,0 +1,413 @@ +/* + * Copyright 2026 Arm Limited and/or its affiliates. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include "arm_memory_allocator.h" +#include "image.h" +#include "model_pte.h" + +using executorch::aten::ScalarType; +using executorch::aten::Tensor; +using executorch::extension::BufferDataLoader; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; + +const size_t method_allocation_pool_size = + ET_SEGMENTATION_METHOD_ALLOCATOR_POOL_SIZE; +unsigned char __attribute__(( + section("input_data_sec"), + aligned(16))) method_allocation_pool[method_allocation_pool_size]; + +const size_t temp_allocation_pool_size = + ET_SEGMENTATION_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE; +unsigned char __attribute__(( + section(".bss.tensor_arena"), + aligned(16))) temp_allocation_pool[temp_allocation_pool_size]; + +#if defined(ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE) +extern "C" { +size_t ethosu_fast_scratch_size = + ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE; +unsigned char __attribute__((section(".bss.ethosu_scratch"), aligned(16))) +dedicated_sram[ET_SEGMENTATION_FAST_SCRATCH_TEMP_ALLOCATOR_POOL_SIZE]; +unsigned char* ethosu_fast_scratch = dedicated_sram; +} +#endif + +namespace { + +#if defined(ET_SEGMENTATION_MASK_THRESHOLD) +constexpr float kMaskThreshold = ET_SEGMENTATION_MASK_THRESHOLD; +#else +constexpr float kMaskThreshold = 0.0f; +#endif + +#if defined(ET_SEGMENTATION_SEMIHOSTING_OUTPUT) && \ + (defined(__arm__) || defined(__thumb__)) +constexpr uint32_t kSemihostingSysWrite0 = 0x04; +constexpr uint32_t kSemihostingSysExitExtended = 0x20; +constexpr uint32_t kAdpStoppedApplicationExit = 0x20026; + +uint32_t semihosting_call(uint32_t operation_code, const void* argument) { + uint32_t result; + asm volatile( + "mov r0, %[operation]\n" + "mov r1, %[argument]\n" + "bkpt 0xab\n" + "mov %[result], r0\n" + : [result] "=r"(result) + : [operation] "r"(operation_code), [argument] "r"(argument) + : "r0", "r1", "memory"); + return result; +} + +void semihosting_write0(const char* message) { + semihosting_call(kSemihostingSysWrite0, message); +} +#endif + +void write_runtime_line(const char* message) { +#if defined(ET_SEGMENTATION_SEMIHOSTING_OUTPUT) && \ + (defined(__arm__) || defined(__thumb__)) + semihosting_write0(message); + semihosting_write0("\n"); +#else + (void)message; +#endif +} + +void write_runtime_format(const char* format, ...) { + char line[768]; + va_list args; + va_start(args, format); + vsnprintf(line, sizeof(line), format, args); + va_end(args); + write_runtime_line(line); +} + +void request_runtime_exit(int code) { +#if defined(ET_SEGMENTATION_SEMIHOSTING_OUTPUT) && \ + (defined(__arm__) || defined(__thumb__)) + const uint32_t exit_block[2] = { + kAdpStoppedApplicationExit, + static_cast(code), + }; + semihosting_call(kSemihostingSysExitExtended, exit_block); +#else + (void)code; +#endif +} + +uint32_t update_hash(uint32_t hash, uint8_t value) { + hash ^= value; + hash *= 16777619u; + return hash; +} + +#if defined(ET_SEGMENTATION_DUMP_MASK) +void dump_mask_rle(const std::vector& mask) { + ET_LOG(Info, "Segmentation mask RLE begin"); + write_runtime_line("Segmentation mask RLE begin"); + char line[512]; + size_t line_len = 0; + line[0] = '\0'; + + size_t index = 0; + while (index < mask.size()) { + const uint8_t class_id = mask[index]; + size_t run_len = 1; + while (index + run_len < mask.size() && mask[index + run_len] == class_id) { + ++run_len; + } + + char entry[32]; + const int entry_len = snprintf( + entry, + sizeof(entry), + "%u:%zu,", + static_cast(class_id), + run_len); + if (entry_len <= 0) { + break; + } + if (line_len + static_cast(entry_len) >= sizeof(line)) { + ET_LOG(Info, "Segmentation mask RLE chunk %s", line); + write_runtime_format("Segmentation mask RLE chunk %s", line); + line_len = 0; + line[0] = '\0'; + } + line_len += static_cast( + snprintf(line + line_len, sizeof(line) - line_len, "%s", entry)); + index += run_len; + } + if (line_len > 0) { + ET_LOG(Info, "Segmentation mask RLE chunk %s", line); + write_runtime_format("Segmentation mask RLE chunk %s", line); + } + ET_LOG(Info, "Segmentation mask RLE end"); + write_runtime_line("Segmentation mask RLE end"); +} +#endif + +void summarize_segmentation_output(const Tensor& out) { + ET_CHECK_MSG( + out.dim() == 4, + "Expected mask logits with shape [1, 1, height, width], got rank %zd", + out.dim()); + ET_CHECK_MSG( + out.scalar_type() == ScalarType::Float, + "Expected float mask logits, got dtype %d", + out.scalar_type()); + ET_CHECK_MSG(out.size(0) == 1, "Only batch size 1 is supported."); + ET_CHECK_MSG( + out.size(1) == 1, + "MobileSAM fixed-prompt export expects one mask channel, got %zd", + out.size(1)); + + const size_t height = static_cast(out.size(2)); + const size_t width = static_cast(out.size(3)); + const auto strides = out.strides(); + const float* data = out.const_data_ptr(); + + size_t foreground_pixels = 0; +#if defined(ET_SEGMENTATION_DUMP_MASK) + std::vector mask(height * width, 0); +#endif + uint32_t mask_hash = 2166136261u; + float min_score = data[0]; + float max_score = data[0]; + double score_sum = 0.0; + const float threshold_sweep[] = { + 0.0f, + -2.0f, + -4.0f, + -5.0f, + -6.0f, + -7.0f, + -8.0f, + -10.0f, + -12.0f, + -14.0f, + }; + size_t threshold_sweep_counts + [sizeof(threshold_sweep) / sizeof(threshold_sweep[0])] = {}; + + for (size_t y = 0; y < height; ++y) { + for (size_t x = 0; x < width; ++x) { + const float score = data[y * strides[2] + x * strides[3]]; + min_score = std::min(min_score, score); + max_score = std::max(max_score, score); + score_sum += score; + for (size_t i = 0; + i < sizeof(threshold_sweep) / sizeof(threshold_sweep[0]); + ++i) { + threshold_sweep_counts[i] += score > threshold_sweep[i] ? 1 : 0; + } + const uint8_t mask_value = score > kMaskThreshold ? 1 : 0; + foreground_pixels += mask_value; +#if defined(ET_SEGMENTATION_DUMP_MASK) + mask[y * width + x] = mask_value; +#endif + mask_hash = update_hash(mask_hash, mask_value); + } + } + + ET_LOG(Info, "Output mask logits shape = [1, 1, %zu, %zu]", height, width); + write_runtime_format( + "Output mask logits shape = [1, 1, %zu, %zu]", height, width); + ET_LOG( + Info, + "Segmentation input image = %zu x %zu x %zu", + image_width, + image_height, + image_channels); + write_runtime_format( + "Segmentation input image = %zu x %zu x %zu", + image_width, + image_height, + image_channels); + ET_LOG(Info, "Mask threshold = %.4f", static_cast(kMaskThreshold)); + write_runtime_format( + "Mask threshold = %.4f", static_cast(kMaskThreshold)); + ET_LOG( + Info, + "Mask logits min/max/mean = %.6f / %.6f / %.6f", + static_cast(min_score), + static_cast(max_score), + score_sum / static_cast(height * width)); + write_runtime_format( + "Mask logits min/max/mean = %.6f / %.6f / %.6f", + static_cast(min_score), + static_cast(max_score), + score_sum / static_cast(height * width)); + for (size_t i = 0; i < sizeof(threshold_sweep) / sizeof(threshold_sweep[0]); + ++i) { + write_runtime_format( + "Threshold %.1f foreground pixels = %zu", + static_cast(threshold_sweep[i]), + threshold_sweep_counts[i]); + } + ET_LOG(Info, "Segmentation mask hash = 0x%08" PRIx32, mask_hash); + write_runtime_format("Segmentation mask hash = 0x%08" PRIx32, mask_hash); + ET_LOG(Info, "Mask foreground pixels = %zu", foreground_pixels); + write_runtime_format("Mask foreground pixels = %zu", foreground_pixels); + ET_LOG( + Info, "Mask background pixels = %zu", height * width - foreground_pixels); + write_runtime_format( + "Mask background pixels = %zu", height * width - foreground_pixels); + +#if defined(ET_SEGMENTATION_DUMP_MASK) + dump_mask_rle(mask); +#endif +} + +} // namespace + +int main() { + executorch::runtime::runtime_init(); + ET_LOG(Info, "Runtime initialized"); + write_runtime_line("MobileSAM Ethos-U example started"); + BufferDataLoader loader(model_pte, sizeof(model_pte)); + ET_LOG(Info, "Size of the model = %zu", sizeof(model_pte)); + write_runtime_format("Model size = %zu bytes", sizeof(model_pte)); + write_runtime_line("Loading ExecuTorch program"); + Result program = Program::load(&loader); + ET_CHECK_MSG(program.ok(), "Program::load failed: 0x%x", program.error()); + write_runtime_line("Program loaded"); + + const auto method_name_result = program->get_method_name(0); + ET_CHECK_MSG(method_name_result.ok(), "Program has no methods"); + const char* method_name = *method_name_result; + ET_LOG(Info, "Running method %s", method_name); + write_runtime_format("Running method %s", method_name); + + Result method_meta_result = program->method_meta(method_name); + ET_CHECK_MSG( + method_meta_result.ok(), + "method_meta lookup failed: 0x%x", + method_meta_result.error()); + + ArmMemoryAllocator method_allocator( + method_allocation_pool_size, method_allocation_pool); + ArmMemoryAllocator temp_allocator( + temp_allocation_pool_size, temp_allocation_pool); + + std::vector planned_buffers; + std::vector> planned_spans; + const size_t num_memory_planned_buffers = + method_meta_result->num_memory_planned_buffers(); + ET_LOG(Info, "num_memory_planned_buffers = %zu", num_memory_planned_buffers); + for (size_t id = 0; id < num_memory_planned_buffers; ++id) { + const size_t buffer_size = + method_meta_result->memory_planned_buffer_size(id).get(); + ET_LOG(Info, "Planned memory buffer_size %zu %zu bytes", id, buffer_size); + + uint8_t* buffer = reinterpret_cast( + method_allocator.allocate(buffer_size, 16UL)); + ET_CHECK_MSG( + buffer != nullptr, + "Could not allocate memory for memory planned buffer size %zu", + buffer_size); + planned_buffers.push_back(buffer); + planned_spans.push_back({planned_buffers.back(), buffer_size}); + } + HierarchicalAllocator planned_memory( + {planned_spans.data(), planned_spans.size()}); + + MemoryManager memory_manager( + &method_allocator, &planned_memory, &temp_allocator); + write_runtime_line("Loading method"); + Result method = program->load_method(method_name, &memory_manager); + ET_CHECK_MSG(method.ok(), "load_method failed: 0x%x", method.error()); + write_runtime_line("Method loaded"); + + const size_t num_inputs = method->inputs_size(); + ET_LOG(Info, "Number of input tensors = %zu", num_inputs); + ET_CHECK_MSG( + num_inputs == 1, + "The segmentation model has a single input tensor, but the provided model has %zu input tensors", + num_inputs); + + EValue* input_evalues = method_allocator.allocateList(num_inputs); + Error err = method->get_inputs(input_evalues, num_inputs); + ET_CHECK_MSG(err == Error::Ok, "get_inputs failed"); + Tensor& input_tensor = input_evalues[0].toTensor(); + const size_t expected_elems = input_tensor.numel(); + const size_t image_elements = sizeof(image_data) / sizeof(image_data[0]); + ET_CHECK_MSG( + expected_elems == image_elements, + "Input tensor expects %zu elements, but image_data has %zu elements", + expected_elems, + image_elements); + ET_CHECK_MSG( + input_tensor.scalar_type() == ScalarType::Float, + "Expected float input tensor, got dtype %d", + input_tensor.scalar_type()); + + float* input_data = input_tensor.mutable_data_ptr(); + write_runtime_format("Copying %zu input elements", expected_elems); + for (size_t i = 0; i < expected_elems; ++i) { + input_data[i] = image_data[i]; + } + + write_runtime_line("Running model execution"); + Error status_inference = method->execute(); + ET_CHECK_MSG( + status_inference == Error::Ok, + "Inference failed 0x%" PRIx32, + status_inference); + write_runtime_line("Inference finished"); + + const size_t num_outputs = method->outputs_size(); + std::vector outputs(num_outputs); + Error status_outputs = method->get_outputs(outputs.data(), outputs.size()); + ET_CHECK_MSG( + status_outputs == Error::Ok, + "get_outputs failed 0x%" PRIx32, + status_outputs); + + for (size_t i = 0; i < outputs.size(); ++i) { + if (outputs[i].isTensor()) { + summarize_segmentation_output(outputs[i].toTensor()); + ET_LOG(Info, "Model executed successfully."); + write_runtime_line("Model executed successfully."); + request_runtime_exit(0); + return 0; + } + } + + ET_CHECK_MSG(false, "No tensor output found."); + request_runtime_exit(1); + return 1; +}