From 9242e3a681a81502fa9ea83a24014b0ba7ef61f5 Mon Sep 17 00:00:00 2001 From: lucasdonordeste Date: Fri, 8 May 2026 23:41:14 -0300 Subject: [PATCH 1/3] fix: enable Google ML Kit on Apple Silicon iOS 26+ simulators Apple removed Rosetta 2 from the default iOS 26 simulator runtime, which breaks `flutter run` for any project depending on Google ML Kit on Apple Silicon Macs. The published GoogleMLKit/* CocoaPods only ship arm64-iphoneos and x86_64-iphonesimulator slices and pin EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64, so the simulator build no longer finds a matching destination and dies with "Unable to find a destination matching the provided destination specifier". Until Google publishes proper arm64-iphonesimulator slices (https://issuetracker.google.com/issues/178965151), this change ships an opt-in Podfile helper under google_mlkit_commons that: 1. Re-labels the existing arm64 device slice of every ML Kit framework binary as iOS Simulator. Only the 4-byte LC_BUILD_VERSION.platform field is changed (2 -> 7), the same approach the well-known arm64-to-sim tool uses on closed-source SDKs. Idempotent. 2. Strips EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64 from the xcconfigs CocoaPods generates from the pod's user_target_xcconfig, so the user's app target is allowed to build for arm64-iphonesimulator. The example/ios Podfile is wired to call the helper from post_install, which lets the example app build, install and run on an Apple Silicon iOS 26.3 simulator. End-to-end validated: Text Recognition on the Text-From-Widget view returned the exact widget text. Device builds and release builds are not affected. Closes #825 --- packages/example/ios/Podfile | 9 + packages/example/ios/Podfile.lock | 2 +- packages/google_mlkit_commons/README.md | 27 +++ .../ios/scripts/apple_silicon_simulator.rb | 66 ++++++ .../ios/scripts/patch_arm64_simulator.py | 208 ++++++++++++++++++ 5 files changed, 311 insertions(+), 1 deletion(-) create mode 100644 packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb create mode 100644 packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py diff --git a/packages/example/ios/Podfile b/packages/example/ios/Podfile index 9f0d4515..cfb1a912 100644 --- a/packages/example/ios/Podfile +++ b/packages/example/ios/Podfile @@ -24,6 +24,12 @@ end require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) +# Apple Silicon iOS Simulator support for Google ML Kit pods. +# See packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb +# and the iOS section of packages/google_mlkit_commons/README.md +# for the rationale and the upstream Google issue. +require File.expand_path('../../google_mlkit_commons/ios/scripts/apple_silicon_simulator', __dir__) + flutter_ios_podfile_setup target 'Runner' do @@ -56,4 +62,7 @@ post_install do |installer| end end end + + # Lets the example build on Apple Silicon iOS 26+ simulators. + mlkit_apple_silicon_simulator_patch(installer) end diff --git a/packages/example/ios/Podfile.lock b/packages/example/ios/Podfile.lock index 3d814d2b..de28065a 100644 --- a/packages/example/ios/Podfile.lock +++ b/packages/example/ios/Podfile.lock @@ -483,6 +483,6 @@ SPEC CHECKSUMS: PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 SSZipArchive: 8a6ee5677c8e304bebc109e39cf0da91ccef22ea -PODFILE CHECKSUM: 810ea711de6d4d578877638350f293e7020676b9 +PODFILE CHECKSUM: e40fd05d6a8f17b70c24ecea80ce69cb70722338 COCOAPODS: 1.16.2 diff --git a/packages/google_mlkit_commons/README.md b/packages/google_mlkit_commons/README.md index 46e77f5a..94bf6aeb 100644 --- a/packages/google_mlkit_commons/README.md +++ b/packages/google_mlkit_commons/README.md @@ -77,6 +77,33 @@ end Notice that the minimum `IPHONEOS_DEPLOYMENT_TARGET` is 15.5, you can set it to something newer but not older. +#### Apple Silicon iOS Simulator (iOS 26+) + +Google's `GoogleMLKit/*` pods only ship `arm64-iphoneos` and `x86_64-iphonesimulator` slices and exclude `arm64` from simulator builds. On Apple Silicon Macs running iOS 26+ simulators (where Rosetta 2 is no longer the default for the iOS Simulator) this makes `flutter run` fail with `Unable to find a destination matching the provided destination specifier`. Issue tracked upstream by Google: https://issuetracker.google.com/issues/178965151. + +Until proper `arm64-iphonesimulator` slices are published, this plugin ships an **opt-in** Podfile helper that re-labels the existing arm64 device slice as iOS Simulator (the same `LC_BUILD_VERSION.platform` swap used by the well-known [`arm64-to-sim`](https://github.com/bogo/arm64-to-sim) tool) and strips the `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64` from the generated xcconfigs. + +To enable it, add two lines to your iOS `Podfile`: + +```ruby +# Near the top, after `require ... podhelper ...`: +require File.expand_path( + '.symlinks/plugins/google_mlkit_commons/ios/scripts/apple_silicon_simulator', + __dir__, +) + +post_install do |installer| + # ...your existing post_install code... + + # Add this line at the end: + mlkit_apple_silicon_simulator_patch(installer) +end +``` + +Then re-run `pod install`. The example app under `packages/example` is wired up this way. + +The helper only changes vendored binaries inside `Pods/` and the `EXCLUDED_ARCHS` line in pod-generated xcconfigs. Device builds and release builds are unaffected. Remove the two lines to revert. + ### Android - minSdkVersion: 21 diff --git a/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb new file mode 100644 index 00000000..53f07dd9 --- /dev/null +++ b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb @@ -0,0 +1,66 @@ +# Helper to make Google ML Kit pods build for Apple Silicon iOS Simulators. +# +# Why this exists +# --------------- +# The frameworks Google publishes under the `GoogleMLKit/*` CocoaPods only ship +# `arm64-iphoneos` and `x86_64-iphonesimulator` slices. Their podspecs set +# `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64`, which on Apple Silicon Macs +# running iOS 26+ simulators (where Rosetta is not available by default) +# breaks `flutter run` with: +# +# Unable to find a destination matching the provided destination specifier +# +# Until Google publishes proper `arm64-iphonesimulator` slices (tracked in +# https://issuetracker.google.com/issues/178965151), this helper applies a +# well-known workaround at `pod install` time: +# +# 1. Re-labels the `arm64` device slice of every ML Kit framework binary +# as iOS Simulator (only the 4-byte `LC_BUILD_VERSION.platform` field is +# modified — same approach the `arm64-to-sim` tool uses). +# 2. Strips `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64` from the +# generated xcconfig files so the user's app target is allowed to build +# for `arm64-iphonesimulator`. +# +# This is opt-in. Add a single line inside your existing `post_install` block: +# +# require File.expand_path( +# '.symlinks/plugins/google_mlkit_commons/ios/scripts/apple_silicon_simulator', +# __dir__, +# ) +# post_install do |installer| +# # ...your existing post_install code... +# mlkit_apple_silicon_simulator_patch(installer) +# end +# +# Notes +# ----- +# * Idempotent: running `pod install` multiple times is safe (the patcher +# skips slices that already report platform=iOS Simulator). +# * Affects only the simulator build. Device builds are untouched. +# * Modifies vendored binaries inside `Pods/` only; nothing in your app or in +# pub.dev caches is altered. + +def mlkit_apple_silicon_simulator_patch(installer) + pods_dir = File.expand_path(installer.sandbox.root.to_s) + patcher = File.expand_path('patch_arm64_simulator.py', __dir__) + + framework_dirs = Dir.glob(File.join(pods_dir, '{MLKit*,MLImage*}')) + .select { |d| File.directory?(d) } + unless framework_dirs.empty? + Pod::UI.puts '' + Pod::UI.puts "[ml_kit] Patching #{framework_dirs.size} ML Kit " \ + 'framework(s) for Apple Silicon iOS Simulator...' + unless system('python3', patcher, *framework_dirs) + Pod::UI.warn '[ml_kit] arm64 simulator patcher failed; ' \ + 'simulator build may still require Rosetta.' + end + end + + excluded = 'EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64' + Dir.glob(File.join(pods_dir, 'Target Support Files', '**', '*.xcconfig')) + .each do |xcconfig| + text = File.read(xcconfig) + new_text = text.lines.reject { |l| l.strip == excluded }.join + File.write(xcconfig, new_text) if text != new_text + end +end diff --git a/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py new file mode 100644 index 00000000..d543ac05 --- /dev/null +++ b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +""" +Re-label the arm64 device slice of Google ML Kit static frameworks as +iOS Simulator, so they link on Apple Silicon iOS 26+ simulators without +requiring Rosetta 2. + +Background +---------- +The frameworks Google ships under the GoogleMLKit/* CocoaPods only contain +two slices: ``arm64`` (built for iOS device, platform=2) and ``x86_64`` +(iOS Simulator, platform=7). Their podspecs therefore set +``EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64`` so Xcode does not try to +link the device slice into a simulator build. + +On Apple Silicon Macs running iOS 26+ simulators, Apple no longer +auto-translates ``x86_64`` simulator binaries through Rosetta 2 by default, +which makes ``flutter run`` fail with:: + + Unable to find a destination matching the provided destination specifier + +Until Google publishes proper ``arm64-iphonesimulator`` slices (tracked in +https://issuetracker.google.com/issues/178965151), this script applies the +same in-place modification that the well-known ``arm64-to-sim`` tool uses: +it walks every ``.o`` member of the arm64 static archive and changes the +``LC_BUILD_VERSION.platform`` field from ``2`` (iOS) to ``7`` +(iOS Simulator). No instructions, symbols or metadata other than the +single 4-byte platform field are touched. + +Usage +----- +:: + + python3 patch_arm64_simulator.py [ ...] + +Each path must be the directory that contains +``Frameworks/.framework/``. + +Idempotent: running it twice is a no-op (after the first pass nothing in +the arm64 slice still claims platform=iOS). +""" + +import os +import struct +import subprocess +import sys +import tempfile + +FAT_MAGIC = 0xCAFEBABE +FAT_MAGIC_64 = 0xCAFEBABF +MH_MAGIC_64 = 0xFEEDFACF +LC_BUILD_VERSION = 0x32 +PLATFORM_IOS = 2 +PLATFORM_IOS_SIMULATOR = 7 +CPU_TYPE_ARM64 = 0x0100000c + + +def _patch_macho_object(buf): + """Patch LC_BUILD_VERSION.platform in a single Mach-O 64 object/dylib. + Returns ``(new_buf, was_patched)``.""" + if len(buf) < 32: + return buf, False + magic = struct.unpack_from('\n': + return 0 + out = bytearray(data[:8]) + pos = 8 + n_patched = 0 + while pos + 60 <= len(data): + header = data[pos:pos + 60] + name = header[:16].rstrip().decode('ascii', errors='replace') + try: + size = int(header[48:58].rstrip().decode('ascii', errors='replace')) + except ValueError: + break + body_start = pos + 60 + body_end = body_start + size + body = data[body_start:body_end] + if name.startswith('#1/'): # BSD long-name extension + try: + name_len = int(name[3:]) + except ValueError: + name_len = 0 + obj_buf = data[body_start + name_len:body_end] + new_obj, patched = _patch_macho_object(obj_buf) + new_body = body[:name_len] + new_obj + else: + new_obj, patched = _patch_macho_object(body) + new_body = new_obj + if patched: + n_patched += 1 + out += header + new_body + pos = body_end + (body_end & 1) # 2-byte alignment + if n_patched > 0: + with open(archive_path, 'wb') as f: + f.write(bytes(out)) + return n_patched + + +def _patch_thin(path): + """Patch a non-fat file: either a Mach-O 64 binary or an ``ar`` archive.""" + with open(path, 'rb') as f: + head = f.read(8) + if head[:8] == b'!\n': + return _patch_static_archive(path) + if len(head) >= 4 and struct.unpack('I', head[:4])[0] + if magic in (FAT_MAGIC, FAT_MAGIC_64): + archs = subprocess.run( + ['lipo', '-archs', fat_path], + capture_output=True, text=True, check=True, + ).stdout.strip().split() + if 'arm64' not in archs: + return 0 + with tempfile.TemporaryDirectory() as td: + arm64_thin = os.path.join(td, 'arm64.bin') + subprocess.run( + ['lipo', fat_path, '-thin', 'arm64', '-output', arm64_thin], + check=True, + ) + n = _patch_thin(arm64_thin) + if n == 0: + return 0 + subprocess.run( + ['lipo', fat_path, '-replace', 'arm64', arm64_thin, + '-output', fat_path], + check=True, + ) + return n + return _patch_thin(fat_path) + + +def _find_framework_binary(pod_dir): + """For ``Pods//``, return ``.framework/``.""" + fw_dir = os.path.join(pod_dir, 'Frameworks') + if not os.path.isdir(fw_dir): + return None + for name in os.listdir(fw_dir): + if name.endswith('.framework'): + base = name[:-len('.framework')] + binary = os.path.join(fw_dir, name, base) + if os.path.isfile(binary): + return binary + return None + + +def main(args): + if not args: + print(__doc__, file=sys.stderr) + return 1 + total = 0 + for path in args: + binary = _find_framework_binary(path) + if not binary: + continue + n = _patch_fat_binary(binary) + if n > 0: + print(f' patched {os.path.basename(binary)}: ' + f'{n} object(s) relabeled to iOS Simulator') + total += n + if total > 0: + print(f'[ml_kit] Total Mach-O objects relabeled: {total}') + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) From 13952f2e60fa097ed266040b1d2aa0729a544f9c Mon Sep 17 00:00:00 2001 From: lucasdonordeste Date: Fri, 8 May 2026 23:48:56 -0300 Subject: [PATCH 2/3] chore: drop redundant comments in arm64 simulator helper The Ruby and Python entry points already have a brief header pointing at the README. Per-function docstrings were restating what the function name already said, and the Podfile inline comments duplicated the helper's self-documenting name. --- packages/example/ios/Podfile | 6 --- packages/example/ios/Podfile.lock | 2 +- .../ios/scripts/apple_silicon_simulator.rb | 45 ++-------------- .../ios/scripts/patch_arm64_simulator.py | 52 +++---------------- 4 files changed, 12 insertions(+), 93 deletions(-) diff --git a/packages/example/ios/Podfile b/packages/example/ios/Podfile index cfb1a912..2f65a326 100644 --- a/packages/example/ios/Podfile +++ b/packages/example/ios/Podfile @@ -23,11 +23,6 @@ def flutter_root end require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) - -# Apple Silicon iOS Simulator support for Google ML Kit pods. -# See packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb -# and the iOS section of packages/google_mlkit_commons/README.md -# for the rationale and the upstream Google issue. require File.expand_path('../../google_mlkit_commons/ios/scripts/apple_silicon_simulator', __dir__) flutter_ios_podfile_setup @@ -63,6 +58,5 @@ post_install do |installer| end end - # Lets the example build on Apple Silicon iOS 26+ simulators. mlkit_apple_silicon_simulator_patch(installer) end diff --git a/packages/example/ios/Podfile.lock b/packages/example/ios/Podfile.lock index de28065a..c90136b2 100644 --- a/packages/example/ios/Podfile.lock +++ b/packages/example/ios/Podfile.lock @@ -483,6 +483,6 @@ SPEC CHECKSUMS: PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 SSZipArchive: 8a6ee5677c8e304bebc109e39cf0da91ccef22ea -PODFILE CHECKSUM: e40fd05d6a8f17b70c24ecea80ce69cb70722338 +PODFILE CHECKSUM: df61d3916884bb4fa8c7ec2fdffdf0dd0d3cec36 COCOAPODS: 1.16.2 diff --git a/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb index 53f07dd9..c483d5ca 100644 --- a/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb +++ b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb @@ -1,44 +1,7 @@ -# Helper to make Google ML Kit pods build for Apple Silicon iOS Simulators. -# -# Why this exists -# --------------- -# The frameworks Google publishes under the `GoogleMLKit/*` CocoaPods only ship -# `arm64-iphoneos` and `x86_64-iphonesimulator` slices. Their podspecs set -# `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64`, which on Apple Silicon Macs -# running iOS 26+ simulators (where Rosetta is not available by default) -# breaks `flutter run` with: -# -# Unable to find a destination matching the provided destination specifier -# -# Until Google publishes proper `arm64-iphonesimulator` slices (tracked in -# https://issuetracker.google.com/issues/178965151), this helper applies a -# well-known workaround at `pod install` time: -# -# 1. Re-labels the `arm64` device slice of every ML Kit framework binary -# as iOS Simulator (only the 4-byte `LC_BUILD_VERSION.platform` field is -# modified — same approach the `arm64-to-sim` tool uses). -# 2. Strips `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64` from the -# generated xcconfig files so the user's app target is allowed to build -# for `arm64-iphonesimulator`. -# -# This is opt-in. Add a single line inside your existing `post_install` block: -# -# require File.expand_path( -# '.symlinks/plugins/google_mlkit_commons/ios/scripts/apple_silicon_simulator', -# __dir__, -# ) -# post_install do |installer| -# # ...your existing post_install code... -# mlkit_apple_silicon_simulator_patch(installer) -# end -# -# Notes -# ----- -# * Idempotent: running `pod install` multiple times is safe (the patcher -# skips slices that already report platform=iOS Simulator). -# * Affects only the simulator build. Device builds are untouched. -# * Modifies vendored binaries inside `Pods/` only; nothing in your app or in -# pub.dev caches is altered. +# Opt-in Podfile helper that lets Google ML Kit pods build for Apple Silicon +# iOS 26+ simulators. See packages/google_mlkit_commons/README.md (iOS +# section) for rationale and usage. Upstream Google bug: +# https://issuetracker.google.com/issues/178965151 def mlkit_apple_silicon_simulator_patch(installer) pods_dir = File.expand_path(installer.sandbox.root.to_s) diff --git a/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py index d543ac05..f7507eda 100644 --- a/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py +++ b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py @@ -1,42 +1,11 @@ #!/usr/bin/env python3 -""" -Re-label the arm64 device slice of Google ML Kit static frameworks as -iOS Simulator, so they link on Apple Silicon iOS 26+ simulators without -requiring Rosetta 2. - -Background ----------- -The frameworks Google ships under the GoogleMLKit/* CocoaPods only contain -two slices: ``arm64`` (built for iOS device, platform=2) and ``x86_64`` -(iOS Simulator, platform=7). Their podspecs therefore set -``EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64`` so Xcode does not try to -link the device slice into a simulator build. - -On Apple Silicon Macs running iOS 26+ simulators, Apple no longer -auto-translates ``x86_64`` simulator binaries through Rosetta 2 by default, -which makes ``flutter run`` fail with:: - - Unable to find a destination matching the provided destination specifier - -Until Google publishes proper ``arm64-iphonesimulator`` slices (tracked in -https://issuetracker.google.com/issues/178965151), this script applies the -same in-place modification that the well-known ``arm64-to-sim`` tool uses: -it walks every ``.o`` member of the arm64 static archive and changes the -``LC_BUILD_VERSION.platform`` field from ``2`` (iOS) to ``7`` -(iOS Simulator). No instructions, symbols or metadata other than the -single 4-byte platform field are touched. - -Usage ------ -:: - - python3 patch_arm64_simulator.py [ ...] - -Each path must be the directory that contains -``Frameworks/.framework/``. - -Idempotent: running it twice is a no-op (after the first pass nothing in -the arm64 slice still claims platform=iOS). +"""Re-label the arm64 device slice of Google ML Kit static frameworks as +iOS Simulator. Walks every .o member of the arm64 archive and flips +LC_BUILD_VERSION.platform from 2 (iOS) to 7 (iOS Simulator); no +instructions or symbols are touched. Same approach as arm64-to-sim. +Idempotent. See packages/google_mlkit_commons/README.md (iOS section). + +Usage: python3 patch_arm64_simulator.py [ ...] """ import os @@ -55,8 +24,6 @@ def _patch_macho_object(buf): - """Patch LC_BUILD_VERSION.platform in a single Mach-O 64 object/dylib. - Returns ``(new_buf, was_patched)``.""" if len(buf) < 32: return buf, False magic = struct.unpack_from('\n': @@ -123,7 +88,6 @@ def _patch_static_archive(archive_path): def _patch_thin(path): - """Patch a non-fat file: either a Mach-O 64 binary or an ``ar`` archive.""" with open(path, 'rb') as f: head = f.read(8) if head[:8] == b'!\n': @@ -140,7 +104,6 @@ def _patch_thin(path): def _patch_fat_binary(fat_path): - """Detect file type and patch the arm64 slice. Returns total patched count.""" with open(fat_path, 'rb') as f: head = f.read(4) if len(head) < 4: @@ -172,7 +135,6 @@ def _patch_fat_binary(fat_path): def _find_framework_binary(pod_dir): - """For ``Pods//``, return ``.framework/``.""" fw_dir = os.path.join(pod_dir, 'Frameworks') if not os.path.isdir(fw_dir): return None From 7367089fd79cf71fb2b5f8c7f4b4439ab8e0a0bb Mon Sep 17 00:00:00 2001 From: Utku <98432874+utkuvrs@users.noreply.github.com> Date: Wed, 20 May 2026 11:39:08 +0300 Subject: [PATCH 3/3] Add Vision-based confidence and angle extraction Integrate Apple Vision to derive confidence scores and text rotation angles alongside MLKit results. Runs VNRecognizeTextRequest on the same image (converted to CGImage for parity) and maps observations to pixel coordinates; matches observations to MLKit element/line frames by IoU (threshold 0.3) to pick a best confidence. Populates per-element confidences (and per-line averages or fallback matches), computes line/element angles from corner points, and adds CGImage extraction helpers for file/bytes/bitmap imageData (including CVPixelBuffer/CIImage conversion and safe memory handling). Falls back to nil confidences if Vision processing fails. --- .../GoogleMlKitTextRecognitionPlugin.swift | 192 +++++++++++++++++- 1 file changed, 182 insertions(+), 10 deletions(-) diff --git a/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift b/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift index fb7e74f6..31598544 100644 --- a/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift +++ b/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift @@ -3,6 +3,11 @@ import MLKitVision import MLKitTextRecognition import MLKitTextRecognitionCommon import google_mlkit_commons +import Vision +import UIKit +import CoreGraphics +import CoreImage +import CoreVideo #if canImport(MLKitTextRecognitionChinese) import MLKitTextRecognitionChinese #endif @@ -97,6 +102,8 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { instances[uid] = recognizer } + let visionConfidences = self.computeVisionConfidences(from: imageData) + recognizer.process(image) { visionText, error in if let error = error as NSError? { result(FlutterError(code: "Error \(error.code)", message: error.domain, details: error.localizedDescription)) @@ -118,27 +125,36 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { ) var textLines: [[String: Any]] = [] for line in block.lines { - var lineData = self.addData( - cornerPoints: line.cornerPoints, - frame: line.frame, - languages: line.recognizedLanguages, - text: line.text, - confidence: nil, - angle: nil - ) var elementsData: [[String: Any]] = [] + var elementConfidences: [Float] = [] for element in line.elements { + let elementConfidence = self.matchConfidence( + for: element.frame, + in: visionConfidences + ) var elementData = self.addData( cornerPoints: element.cornerPoints, frame: element.frame, languages: element.recognizedLanguages, text: element.text, - confidence: nil, - angle: nil + confidence: elementConfidence.map { NSNumber(value: $0) }, + angle: self.angle(from: element.cornerPoints) ) elementData["symbols"] = [] as [[String: Any]] elementsData.append(elementData) + if let c = elementConfidence { elementConfidences.append(c) } } + let lineConfidence: NSNumber? = elementConfidences.isEmpty + ? self.matchConfidence(for: line.frame, in: visionConfidences).map { NSNumber(value: $0) } + : NSNumber(value: elementConfidences.reduce(0, +) / Float(elementConfidences.count)) + var lineData = self.addData( + cornerPoints: line.cornerPoints, + frame: line.frame, + languages: line.recognizedLanguages, + text: line.text, + confidence: lineConfidence, + angle: self.angle(from: line.cornerPoints) + ) lineData["elements"] = elementsData textLines.append(lineData) } @@ -176,4 +192,160 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { "angle": angle ?? NSNull() ] } + + /// Derives rotation angle in degrees from the top-left → top-right corner vector. + /// MLKit returns cornerPoints clockwise starting top-left, so index 0→1 spans the top edge. + private func angle(from cornerPoints: [NSValue]) -> NSNumber? { + guard cornerPoints.count >= 2 else { return nil } + let p0 = cornerPoints[0].cgPointValue + let p1 = cornerPoints[1].cgPointValue + let radians = atan2(p1.y - p0.y, p1.x - p0.x) + return NSNumber(value: Double(radians) * 180.0 / .pi) + } + + // MARK: - Apple Vision confidence pass + + private struct VisionConfidenceRect { + let rect: CGRect + let confidence: Float + } + + /// Runs Apple's VNRecognizeTextRequest on the same image and returns observations + /// in MLKit's pixel/top-left coordinate space so they can be matched by IoU. + /// Returns empty array on failure — confidence stays nil, behavior degrades to pre-fork state. + private func computeVisionConfidences(from imageData: [String: Any]) -> [VisionConfidenceRect] { + guard let cgImage = Self.cgImage(from: imageData) else { return [] } + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.usesLanguageCorrection = false + let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:]) + do { + try handler.perform([request]) + } catch { + return [] + } + guard let observations = request.results else { return [] } + let width = CGFloat(cgImage.width) + let height = CGFloat(cgImage.height) + return observations.compactMap { obs in + let bbox = obs.boundingBox // normalized, bottom-left origin + let pixelRect = CGRect( + x: bbox.minX * width, + y: (1.0 - bbox.maxY) * height, + width: bbox.width * width, + height: bbox.height * height + ) + let confidence = obs.topCandidates(1).first?.confidence ?? obs.confidence + return VisionConfidenceRect(rect: pixelRect, confidence: confidence) + } + } + + /// Finds the Vision observation with highest IoU against the MLKit element/line frame. + /// Returns nil if no observation reaches the overlap threshold. + private func matchConfidence(for frame: CGRect, in rects: [VisionConfidenceRect]) -> Float? { + guard !rects.isEmpty, frame.width > 0, frame.height > 0 else { return nil } + var bestIoU: CGFloat = 0 + var bestConfidence: Float? = nil + for entry in rects { + let intersection = frame.intersection(entry.rect) + if intersection.isNull || intersection.isEmpty { continue } + let interArea = intersection.width * intersection.height + let unionArea = frame.width * frame.height + entry.rect.width * entry.rect.height - interArea + guard unionArea > 0 else { continue } + let iou = interArea / unionArea + if iou > bestIoU { + bestIoU = iou + bestConfidence = entry.confidence + } + } + return bestIoU >= 0.3 ? bestConfidence : nil + } + + // MARK: - CGImage extraction from imageData + + /// Mirrors VisionImage.visionImage(from:) but exposes the underlying CGImage + /// so Apple Vision can process the same pixels MLKit consumed. + private static func cgImage(from imageData: [String: Any]) -> CGImage? { + guard let imageType = imageData["type"] as? String else { return nil } + switch imageType { + case "file": + guard let path = imageData["path"] as? String, + let ui = UIImage(contentsOfFile: path) else { return nil } + return ui.cgImage + case "bytes": + return cgImageFromBytes(imageData) + case "bitmap": + return cgImageFromBitmap(imageData) + default: + return nil + } + } + + private static func cgImageFromBytes(_ imageData: [String: Any]) -> CGImage? { + guard let byteData = imageData["bytes"] as? FlutterStandardTypedData, + let metadata = imageData["metadata"] as? [String: Any], + let width = metadata["width"] as? NSNumber, + let height = metadata["height"] as? NSNumber, + let rawFormat = metadata["image_format"] as? NSNumber, + let bytesPerRow = metadata["bytes_per_row"] as? NSNumber else { + return nil + } + let w = Int(truncating: width) + let h = Int(truncating: height) + let bpr = Int(truncating: bytesPerRow) + let format = OSType(truncating: rawFormat) + let bufferSize = bpr * h + guard bufferSize > 0, byteData.data.count >= bufferSize else { return nil } + + let copy = UnsafeMutableRawPointer.allocate(byteCount: bufferSize, alignment: 1) + let buf = UnsafeMutableBufferPointer(start: copy.assumingMemoryBound(to: UInt8.self), count: bufferSize) + byteData.data.copyBytes(to: buf) + + var pxBuffer: CVPixelBuffer? + let status = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, + w, h, format, + copy, bpr, + { _, baseAddress in + guard let baseAddress = baseAddress else { return } + UnsafeMutableRawPointer(mutating: baseAddress).deallocate() + }, + nil, nil, &pxBuffer + ) + guard status == kCVReturnSuccess, let buffer = pxBuffer else { + copy.deallocate() + return nil + } + let ciImage = CIImage(cvPixelBuffer: buffer) + let context = CIContext(options: nil) + return context.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: w, height: h)) + } + + private static func cgImageFromBitmap(_ imageDict: [String: Any]) -> CGImage? { + guard let bitmapData = imageDict["bitmapData"] as? FlutterStandardTypedData else { return nil } + if let metadata = imageDict["metadata"] as? [String: Any], + let width = metadata["width"] as? NSNumber, + let height = metadata["height"] as? NSNumber { + let colorSpace = CGColorSpaceCreateDeviceRGB() + let bytesPerPixel = 4 + let bpr = bytesPerPixel * width.intValue + var result: CGImage? + bitmapData.data.withUnsafeBytes { raw in + guard let base = raw.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } + guard let ctx = CGContext( + data: UnsafeMutableRawPointer(mutating: base), + width: width.intValue, + height: height.intValue, + bitsPerComponent: 8, + bytesPerRow: bpr, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Big.rawValue + ) else { return } + result = ctx.makeImage() + } + if let result = result { return result } + } + return UIImage(data: bitmapData.data)?.cgImage + } } +