diff --git a/packages/example/ios/Podfile b/packages/example/ios/Podfile index 9f0d4515..2f65a326 100644 --- a/packages/example/ios/Podfile +++ b/packages/example/ios/Podfile @@ -23,6 +23,7 @@ def flutter_root end require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) +require File.expand_path('../../google_mlkit_commons/ios/scripts/apple_silicon_simulator', __dir__) flutter_ios_podfile_setup @@ -56,4 +57,6 @@ post_install do |installer| end end end + + mlkit_apple_silicon_simulator_patch(installer) end diff --git a/packages/example/ios/Podfile.lock b/packages/example/ios/Podfile.lock index 3d814d2b..c90136b2 100644 --- a/packages/example/ios/Podfile.lock +++ b/packages/example/ios/Podfile.lock @@ -483,6 +483,6 @@ SPEC CHECKSUMS: PromisesObjC: f5707f49cb48b9636751c5b2e7d227e43fba9f47 SSZipArchive: 8a6ee5677c8e304bebc109e39cf0da91ccef22ea -PODFILE CHECKSUM: 810ea711de6d4d578877638350f293e7020676b9 +PODFILE CHECKSUM: df61d3916884bb4fa8c7ec2fdffdf0dd0d3cec36 COCOAPODS: 1.16.2 diff --git a/packages/google_mlkit_commons/README.md b/packages/google_mlkit_commons/README.md index 46e77f5a..94bf6aeb 100644 --- a/packages/google_mlkit_commons/README.md +++ b/packages/google_mlkit_commons/README.md @@ -77,6 +77,33 @@ end Notice that the minimum `IPHONEOS_DEPLOYMENT_TARGET` is 15.5, you can set it to something newer but not older. +#### Apple Silicon iOS Simulator (iOS 26+) + +Google's `GoogleMLKit/*` pods only ship `arm64-iphoneos` and `x86_64-iphonesimulator` slices and exclude `arm64` from simulator builds. On Apple Silicon Macs running iOS 26+ simulators (where Rosetta 2 is no longer the default for the iOS Simulator) this makes `flutter run` fail with `Unable to find a destination matching the provided destination specifier`. Issue tracked upstream by Google: https://issuetracker.google.com/issues/178965151. + +Until proper `arm64-iphonesimulator` slices are published, this plugin ships an **opt-in** Podfile helper that re-labels the existing arm64 device slice as iOS Simulator (the same `LC_BUILD_VERSION.platform` swap used by the well-known [`arm64-to-sim`](https://github.com/bogo/arm64-to-sim) tool) and strips the `EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64` from the generated xcconfigs. + +To enable it, add two lines to your iOS `Podfile`: + +```ruby +# Near the top, after `require ... podhelper ...`: +require File.expand_path( + '.symlinks/plugins/google_mlkit_commons/ios/scripts/apple_silicon_simulator', + __dir__, +) + +post_install do |installer| + # ...your existing post_install code... + + # Add this line at the end: + mlkit_apple_silicon_simulator_patch(installer) +end +``` + +Then re-run `pod install`. The example app under `packages/example` is wired up this way. + +The helper only changes vendored binaries inside `Pods/` and the `EXCLUDED_ARCHS` line in pod-generated xcconfigs. Device builds and release builds are unaffected. Remove the two lines to revert. + ### Android - minSdkVersion: 21 diff --git a/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb new file mode 100644 index 00000000..c483d5ca --- /dev/null +++ b/packages/google_mlkit_commons/ios/scripts/apple_silicon_simulator.rb @@ -0,0 +1,29 @@ +# Opt-in Podfile helper that lets Google ML Kit pods build for Apple Silicon +# iOS 26+ simulators. See packages/google_mlkit_commons/README.md (iOS +# section) for rationale and usage. Upstream Google bug: +# https://issuetracker.google.com/issues/178965151 + +def mlkit_apple_silicon_simulator_patch(installer) + pods_dir = File.expand_path(installer.sandbox.root.to_s) + patcher = File.expand_path('patch_arm64_simulator.py', __dir__) + + framework_dirs = Dir.glob(File.join(pods_dir, '{MLKit*,MLImage*}')) + .select { |d| File.directory?(d) } + unless framework_dirs.empty? + Pod::UI.puts '' + Pod::UI.puts "[ml_kit] Patching #{framework_dirs.size} ML Kit " \ + 'framework(s) for Apple Silicon iOS Simulator...' + unless system('python3', patcher, *framework_dirs) + Pod::UI.warn '[ml_kit] arm64 simulator patcher failed; ' \ + 'simulator build may still require Rosetta.' + end + end + + excluded = 'EXCLUDED_ARCHS[sdk=iphonesimulator*] = arm64' + Dir.glob(File.join(pods_dir, 'Target Support Files', '**', '*.xcconfig')) + .each do |xcconfig| + text = File.read(xcconfig) + new_text = text.lines.reject { |l| l.strip == excluded }.join + File.write(xcconfig, new_text) if text != new_text + end +end diff --git a/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py new file mode 100644 index 00000000..f7507eda --- /dev/null +++ b/packages/google_mlkit_commons/ios/scripts/patch_arm64_simulator.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Re-label the arm64 device slice of Google ML Kit static frameworks as +iOS Simulator. Walks every .o member of the arm64 archive and flips +LC_BUILD_VERSION.platform from 2 (iOS) to 7 (iOS Simulator); no +instructions or symbols are touched. Same approach as arm64-to-sim. +Idempotent. See packages/google_mlkit_commons/README.md (iOS section). + +Usage: python3 patch_arm64_simulator.py [ ...] +""" + +import os +import struct +import subprocess +import sys +import tempfile + +FAT_MAGIC = 0xCAFEBABE +FAT_MAGIC_64 = 0xCAFEBABF +MH_MAGIC_64 = 0xFEEDFACF +LC_BUILD_VERSION = 0x32 +PLATFORM_IOS = 2 +PLATFORM_IOS_SIMULATOR = 7 +CPU_TYPE_ARM64 = 0x0100000c + + +def _patch_macho_object(buf): + if len(buf) < 32: + return buf, False + magic = struct.unpack_from('\n': + return 0 + out = bytearray(data[:8]) + pos = 8 + n_patched = 0 + while pos + 60 <= len(data): + header = data[pos:pos + 60] + name = header[:16].rstrip().decode('ascii', errors='replace') + try: + size = int(header[48:58].rstrip().decode('ascii', errors='replace')) + except ValueError: + break + body_start = pos + 60 + body_end = body_start + size + body = data[body_start:body_end] + if name.startswith('#1/'): # BSD long-name extension + try: + name_len = int(name[3:]) + except ValueError: + name_len = 0 + obj_buf = data[body_start + name_len:body_end] + new_obj, patched = _patch_macho_object(obj_buf) + new_body = body[:name_len] + new_obj + else: + new_obj, patched = _patch_macho_object(body) + new_body = new_obj + if patched: + n_patched += 1 + out += header + new_body + pos = body_end + (body_end & 1) # 2-byte alignment + if n_patched > 0: + with open(archive_path, 'wb') as f: + f.write(bytes(out)) + return n_patched + + +def _patch_thin(path): + with open(path, 'rb') as f: + head = f.read(8) + if head[:8] == b'!\n': + return _patch_static_archive(path) + if len(head) >= 4 and struct.unpack('I', head[:4])[0] + if magic in (FAT_MAGIC, FAT_MAGIC_64): + archs = subprocess.run( + ['lipo', '-archs', fat_path], + capture_output=True, text=True, check=True, + ).stdout.strip().split() + if 'arm64' not in archs: + return 0 + with tempfile.TemporaryDirectory() as td: + arm64_thin = os.path.join(td, 'arm64.bin') + subprocess.run( + ['lipo', fat_path, '-thin', 'arm64', '-output', arm64_thin], + check=True, + ) + n = _patch_thin(arm64_thin) + if n == 0: + return 0 + subprocess.run( + ['lipo', fat_path, '-replace', 'arm64', arm64_thin, + '-output', fat_path], + check=True, + ) + return n + return _patch_thin(fat_path) + + +def _find_framework_binary(pod_dir): + fw_dir = os.path.join(pod_dir, 'Frameworks') + if not os.path.isdir(fw_dir): + return None + for name in os.listdir(fw_dir): + if name.endswith('.framework'): + base = name[:-len('.framework')] + binary = os.path.join(fw_dir, name, base) + if os.path.isfile(binary): + return binary + return None + + +def main(args): + if not args: + print(__doc__, file=sys.stderr) + return 1 + total = 0 + for path in args: + binary = _find_framework_binary(path) + if not binary: + continue + n = _patch_fat_binary(binary) + if n > 0: + print(f' patched {os.path.basename(binary)}: ' + f'{n} object(s) relabeled to iOS Simulator') + total += n + if total > 0: + print(f'[ml_kit] Total Mach-O objects relabeled: {total}') + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift b/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift index fb7e74f6..31598544 100644 --- a/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift +++ b/packages/google_mlkit_text_recognition/ios/Classes/GoogleMlKitTextRecognitionPlugin.swift @@ -3,6 +3,11 @@ import MLKitVision import MLKitTextRecognition import MLKitTextRecognitionCommon import google_mlkit_commons +import Vision +import UIKit +import CoreGraphics +import CoreImage +import CoreVideo #if canImport(MLKitTextRecognitionChinese) import MLKitTextRecognitionChinese #endif @@ -97,6 +102,8 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { instances[uid] = recognizer } + let visionConfidences = self.computeVisionConfidences(from: imageData) + recognizer.process(image) { visionText, error in if let error = error as NSError? { result(FlutterError(code: "Error \(error.code)", message: error.domain, details: error.localizedDescription)) @@ -118,27 +125,36 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { ) var textLines: [[String: Any]] = [] for line in block.lines { - var lineData = self.addData( - cornerPoints: line.cornerPoints, - frame: line.frame, - languages: line.recognizedLanguages, - text: line.text, - confidence: nil, - angle: nil - ) var elementsData: [[String: Any]] = [] + var elementConfidences: [Float] = [] for element in line.elements { + let elementConfidence = self.matchConfidence( + for: element.frame, + in: visionConfidences + ) var elementData = self.addData( cornerPoints: element.cornerPoints, frame: element.frame, languages: element.recognizedLanguages, text: element.text, - confidence: nil, - angle: nil + confidence: elementConfidence.map { NSNumber(value: $0) }, + angle: self.angle(from: element.cornerPoints) ) elementData["symbols"] = [] as [[String: Any]] elementsData.append(elementData) + if let c = elementConfidence { elementConfidences.append(c) } } + let lineConfidence: NSNumber? = elementConfidences.isEmpty + ? self.matchConfidence(for: line.frame, in: visionConfidences).map { NSNumber(value: $0) } + : NSNumber(value: elementConfidences.reduce(0, +) / Float(elementConfidences.count)) + var lineData = self.addData( + cornerPoints: line.cornerPoints, + frame: line.frame, + languages: line.recognizedLanguages, + text: line.text, + confidence: lineConfidence, + angle: self.angle(from: line.cornerPoints) + ) lineData["elements"] = elementsData textLines.append(lineData) } @@ -176,4 +192,160 @@ public class GoogleMlKitTextRecognitionPlugin: NSObject, FlutterPlugin { "angle": angle ?? NSNull() ] } + + /// Derives rotation angle in degrees from the top-left → top-right corner vector. + /// MLKit returns cornerPoints clockwise starting top-left, so index 0→1 spans the top edge. + private func angle(from cornerPoints: [NSValue]) -> NSNumber? { + guard cornerPoints.count >= 2 else { return nil } + let p0 = cornerPoints[0].cgPointValue + let p1 = cornerPoints[1].cgPointValue + let radians = atan2(p1.y - p0.y, p1.x - p0.x) + return NSNumber(value: Double(radians) * 180.0 / .pi) + } + + // MARK: - Apple Vision confidence pass + + private struct VisionConfidenceRect { + let rect: CGRect + let confidence: Float + } + + /// Runs Apple's VNRecognizeTextRequest on the same image and returns observations + /// in MLKit's pixel/top-left coordinate space so they can be matched by IoU. + /// Returns empty array on failure — confidence stays nil, behavior degrades to pre-fork state. + private func computeVisionConfidences(from imageData: [String: Any]) -> [VisionConfidenceRect] { + guard let cgImage = Self.cgImage(from: imageData) else { return [] } + let request = VNRecognizeTextRequest() + request.recognitionLevel = .accurate + request.usesLanguageCorrection = false + let handler = VNImageRequestHandler(cgImage: cgImage, orientation: .up, options: [:]) + do { + try handler.perform([request]) + } catch { + return [] + } + guard let observations = request.results else { return [] } + let width = CGFloat(cgImage.width) + let height = CGFloat(cgImage.height) + return observations.compactMap { obs in + let bbox = obs.boundingBox // normalized, bottom-left origin + let pixelRect = CGRect( + x: bbox.minX * width, + y: (1.0 - bbox.maxY) * height, + width: bbox.width * width, + height: bbox.height * height + ) + let confidence = obs.topCandidates(1).first?.confidence ?? obs.confidence + return VisionConfidenceRect(rect: pixelRect, confidence: confidence) + } + } + + /// Finds the Vision observation with highest IoU against the MLKit element/line frame. + /// Returns nil if no observation reaches the overlap threshold. + private func matchConfidence(for frame: CGRect, in rects: [VisionConfidenceRect]) -> Float? { + guard !rects.isEmpty, frame.width > 0, frame.height > 0 else { return nil } + var bestIoU: CGFloat = 0 + var bestConfidence: Float? = nil + for entry in rects { + let intersection = frame.intersection(entry.rect) + if intersection.isNull || intersection.isEmpty { continue } + let interArea = intersection.width * intersection.height + let unionArea = frame.width * frame.height + entry.rect.width * entry.rect.height - interArea + guard unionArea > 0 else { continue } + let iou = interArea / unionArea + if iou > bestIoU { + bestIoU = iou + bestConfidence = entry.confidence + } + } + return bestIoU >= 0.3 ? bestConfidence : nil + } + + // MARK: - CGImage extraction from imageData + + /// Mirrors VisionImage.visionImage(from:) but exposes the underlying CGImage + /// so Apple Vision can process the same pixels MLKit consumed. + private static func cgImage(from imageData: [String: Any]) -> CGImage? { + guard let imageType = imageData["type"] as? String else { return nil } + switch imageType { + case "file": + guard let path = imageData["path"] as? String, + let ui = UIImage(contentsOfFile: path) else { return nil } + return ui.cgImage + case "bytes": + return cgImageFromBytes(imageData) + case "bitmap": + return cgImageFromBitmap(imageData) + default: + return nil + } + } + + private static func cgImageFromBytes(_ imageData: [String: Any]) -> CGImage? { + guard let byteData = imageData["bytes"] as? FlutterStandardTypedData, + let metadata = imageData["metadata"] as? [String: Any], + let width = metadata["width"] as? NSNumber, + let height = metadata["height"] as? NSNumber, + let rawFormat = metadata["image_format"] as? NSNumber, + let bytesPerRow = metadata["bytes_per_row"] as? NSNumber else { + return nil + } + let w = Int(truncating: width) + let h = Int(truncating: height) + let bpr = Int(truncating: bytesPerRow) + let format = OSType(truncating: rawFormat) + let bufferSize = bpr * h + guard bufferSize > 0, byteData.data.count >= bufferSize else { return nil } + + let copy = UnsafeMutableRawPointer.allocate(byteCount: bufferSize, alignment: 1) + let buf = UnsafeMutableBufferPointer(start: copy.assumingMemoryBound(to: UInt8.self), count: bufferSize) + byteData.data.copyBytes(to: buf) + + var pxBuffer: CVPixelBuffer? + let status = CVPixelBufferCreateWithBytes( + kCFAllocatorDefault, + w, h, format, + copy, bpr, + { _, baseAddress in + guard let baseAddress = baseAddress else { return } + UnsafeMutableRawPointer(mutating: baseAddress).deallocate() + }, + nil, nil, &pxBuffer + ) + guard status == kCVReturnSuccess, let buffer = pxBuffer else { + copy.deallocate() + return nil + } + let ciImage = CIImage(cvPixelBuffer: buffer) + let context = CIContext(options: nil) + return context.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: w, height: h)) + } + + private static func cgImageFromBitmap(_ imageDict: [String: Any]) -> CGImage? { + guard let bitmapData = imageDict["bitmapData"] as? FlutterStandardTypedData else { return nil } + if let metadata = imageDict["metadata"] as? [String: Any], + let width = metadata["width"] as? NSNumber, + let height = metadata["height"] as? NSNumber { + let colorSpace = CGColorSpaceCreateDeviceRGB() + let bytesPerPixel = 4 + let bpr = bytesPerPixel * width.intValue + var result: CGImage? + bitmapData.data.withUnsafeBytes { raw in + guard let base = raw.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return } + guard let ctx = CGContext( + data: UnsafeMutableRawPointer(mutating: base), + width: width.intValue, + height: height.intValue, + bitsPerComponent: 8, + bytesPerRow: bpr, + space: colorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue | CGBitmapInfo.byteOrder32Big.rawValue + ) else { return } + result = ctx.makeImage() + } + if let result = result { return result } + } + return UIImage(data: bitmapData.data)?.cgImage + } } +