From d32784d69a3f4f3112cc0453dea1211e12a7e77e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 22 Jul 2026 05:57:07 +0200 Subject: [PATCH 01/31] initial new schema implementation --- .../src/core/schema.ts | 203 ++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 packages/react-native-executorch/src/core/schema.ts diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts new file mode 100644 index 0000000000..ff990f75a8 --- /dev/null +++ b/packages/react-native-executorch/src/core/schema.ts @@ -0,0 +1,203 @@ +import { type DType } from './tensor'; + +export type Range = { min: number; max: number; step?: number }; + +export type TypedDim = + | { readonly kind: 'static'; readonly symbol: string } + | { readonly kind: 'dynamic'; readonly symbol: string; readonly range?: Range } + | { readonly kind: 'constant'; readonly value: number }; + +export type TypedShape = readonly TypedDim[]; + +export type TensorConstraint = { + readonly kind: 'Tensor'; + readonly dtype?: DType; + readonly shapes?: readonly TypedShape[]; +}; + +export type ValueConstraint = + | TensorConstraint + | { readonly kind: 'None' } + | { readonly kind: 'Int' } + | { readonly kind: 'Double' } + | { readonly kind: 'Bool' } + | { readonly kind: 'String' }; + +export type MethodSchema = { + inputs: ValueConstraint[]; + outputs: ValueConstraint[]; +}; + +export type SymbolicShape = readonly (number | string | TypedDim)[]; + +export const SymbolicTensor = (dtype?: DType, ...shapes: readonly SymbolicShape[]) => { + const toTypedShape = (shape: SymbolicShape): TypedShape => { + return shape.map((dim) => { + if (typeof dim === 'number') return { kind: 'constant', value: dim } as TypedDim; + if (typeof dim === 'string') return { kind: 'static', symbol: dim } as TypedDim; + return dim; + }); + }; + return { kind: 'Tensor', dtype, shapes: shapes.map(toTypedShape) }; +}; + +type SymbolMap = Map; + +function isRangeCovered(expRange?: Range, actRange?: Range): boolean { + // Expected takes whatever range actual offers + if (!expRange) return true; + // Expected requires a specific range, but actual range is unknown + if (!actRange) return false; + // Range containment check + if (actRange.min > expRange.min || actRange.max < expRange.max) return false; + // Step and grid alignment + if (actRange.step !== undefined) { + if (expRange.step === undefined) return false; + if (expRange.step % actRange.step !== 0) return false; + if (Math.abs(expRange.min - actRange.min) % actRange.step !== 0) return false; + } + return true; +} + +export function matchDim(exp: TypedDim, act: TypedDim, env: SymbolMap): SymbolMap | null { + const bindSymbol = (key: string, value: string | number) => { + if (env.has(key)) return env.get(key) === value ? env : null; + return new Map(env).set(key, value); + }; + + if (exp.kind === 'constant' && act.kind === 'constant') { + return exp.value === act.value ? env : null; + } + + if (exp.kind === 'constant' && act.kind === 'dynamic') { + if (!act.range) return null; // Unknown model range cannot guarantee exp.value + + if (exp.value < act.range.min || exp.value > act.range.max) return null; + + if (act.range.step !== undefined) { + if (Math.abs(exp.value - act.range.min) % act.range.step !== 0) return null; + } + + return env; + } + + if (exp.kind === 'static' && act.kind === 'constant') { + return bindSymbol(exp.symbol, act.value); + } + + if (exp.kind === 'static' && act.kind === 'static') { + return bindSymbol(exp.symbol, act.symbol); + } + + if (exp.kind === 'dynamic' && act.kind === 'dynamic') { + if (!isRangeCovered(exp.range, act.range)) return null; + return bindSymbol(exp.symbol, act.symbol); + } + + // All remaining combinations fail + return null; +} + +function constraintToString(constraint: ValueConstraint): string { + if (constraint.kind !== 'Tensor') { + return constraint.kind; + } + + const dimToString = (dim: TypedDim) => { + if (dim.kind === 'constant') return `${dim.value}`; + if (dim.kind === 'static') return dim.symbol; + if (dim.range) { + const stepStr = dim.range.step !== undefined ? `:${dim.range.step}` : ''; + return `${dim.symbol}[${dim.range.min}..${dim.range.max}${stepStr}]`; + } + return `Dynamic(${dim.symbol})`; + }; + + const parts: string[] = []; + if (constraint.dtype !== undefined) { + parts.push(`dtype=${constraint.dtype}`); + } + if (constraint.shapes && constraint.shapes.length > 0) { + const shapesStr = constraint.shapes + .map((shape) => `[${shape.map(dimToString).join(', ')}]`) + .join(' | '); + parts.push(`shapes=${shapesStr}`); + } + + return parts.length > 0 ? `Tensor(${parts.join(', ')})` : 'Tensor'; +} + +export function schemaToString(schema: MethodSchema): string { + const inputsStr = schema.inputs.map(constraintToString).join(', '); + const outputsStr = schema.outputs.map(constraintToString).join(', '); + return `(${inputsStr}) -> (${outputsStr})`; +} + +export function validateSchema(expected: MethodSchema, actual: MethodSchema) { + const formatError = (msg: string) => + `${msg}\n` + + `Expected schema: ${schemaToString(expected)}\n` + + `Actual schema : ${schemaToString(actual)}`; + + if (expected.inputs.length !== actual.inputs.length) { + throw new Error(formatError('Number of inputs does not match!')); + } + if (expected.outputs.length !== actual.outputs.length) { + throw new Error(formatError('Number of outputs does not match!')); + } + + const expList = [...expected.inputs, ...expected.outputs]; + const actList = [...actual.inputs, ...actual.outputs]; + + const match = (index: number, map: SymbolMap): boolean => { + if (index === expList.length) return true; + + const exp = expList[index]!; + const act = actList[index]!; + + if (exp.kind !== act.kind) return false; + if (exp.kind !== 'Tensor') return match(index + 1, map); + + const expTensor = exp as TensorConstraint; + const actTensor = act as TensorConstraint; + + // DType Sub-typing + if (expTensor.dtype !== undefined && actTensor.dtype !== expTensor.dtype) { + return false; + } + + // Shape Sub-typing + if (expTensor.shapes && expTensor.shapes.length > 0) { + if (!actTensor.shapes || actTensor.shapes.length === 0) { + return false; + } + + for (const expShape of expTensor.shapes) { + for (const actShape of actTensor.shapes) { + if (expShape.length !== actShape.length) continue; + + let currentMap: SymbolMap | null = map; + for (let dim = 0; dim < expShape.length; dim++) { + currentMap = matchDim(expShape[dim]!, actShape[dim]!, currentMap); + if (!currentMap) break; + } + + if (currentMap && match(index + 1, currentMap)) { + return true; // Found a globally consistent path! + } + } + } + + return false; // No shape overload satisfied constraints + } + + // Expected doesn't restrict shapes; any actual shape is valid + return match(index + 1, map); + }; + + if (!match(0, new Map())) { + throw new Error(formatError('No possible consistent matching!')); + } + + return actual; +} From 9eee392a81757968c874a2f2c84b493a9a3b4c94 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 23 Jul 2026 18:13:58 +0200 Subject: [PATCH 02/31] wip --- .../src/core/schema.ts | 359 +++++++++++------- 1 file changed, 231 insertions(+), 128 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index ff990f75a8..9eb71e9089 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -1,18 +1,16 @@ import { type DType } from './tensor'; -export type Range = { min: number; max: number; step?: number }; +export type Range = { min: number; max: number; step: number }; export type TypedDim = | { readonly kind: 'static'; readonly symbol: string } | { readonly kind: 'dynamic'; readonly symbol: string; readonly range?: Range } | { readonly kind: 'constant'; readonly value: number }; -export type TypedShape = readonly TypedDim[]; - export type TensorConstraint = { readonly kind: 'Tensor'; readonly dtype?: DType; - readonly shapes?: readonly TypedShape[]; + readonly shape?: readonly TypedDim[]; }; export type ValueConstraint = @@ -23,181 +21,286 @@ export type ValueConstraint = | { readonly kind: 'Bool' } | { readonly kind: 'String' }; -export type MethodSchema = { - inputs: ValueConstraint[]; - outputs: ValueConstraint[]; -}; +export type MethodSignature = { inputs: ValueConstraint[]; outputs: ValueConstraint[] }; + +export type MethodOverloads = MethodSignature[]; + +export type ModelSchema = Record; export type SymbolicShape = readonly (number | string | TypedDim)[]; -export const SymbolicTensor = (dtype?: DType, ...shapes: readonly SymbolicShape[]) => { - const toTypedShape = (shape: SymbolicShape): TypedShape => { - return shape.map((dim) => { - if (typeof dim === 'number') return { kind: 'constant', value: dim } as TypedDim; - if (typeof dim === 'string') return { kind: 'static', symbol: dim } as TypedDim; - return dim; - }); - }; - return { kind: 'Tensor', dtype, shapes: shapes.map(toTypedShape) }; +export type SymbolBinding = { + readonly exp: Exclude; + readonly act: TypedDim; }; -type SymbolMap = Map; - -function isRangeCovered(expRange?: Range, actRange?: Range): boolean { - // Expected takes whatever range actual offers - if (!expRange) return true; - // Expected requires a specific range, but actual range is unknown - if (!actRange) return false; - // Range containment check - if (actRange.min > expRange.min || actRange.max < expRange.max) return false; - // Step and grid alignment - if (actRange.step !== undefined) { - if (expRange.step === undefined) return false; - if (expRange.step % actRange.step !== 0) return false; - if (Math.abs(expRange.min - actRange.min) % actRange.step !== 0) return false; - } - return true; +export type SymbolBindings = readonly SymbolBinding[]; + +export type MatchResult = + | { readonly ok: true; readonly bindings: SymbolBindings } + | { readonly ok: false; readonly error: string }; + +const ok = (bindings: SymbolBindings): MatchResult => ({ ok: true, bindings }); +const err = (error: string): MatchResult => ({ ok: false, error }); + +function validateRange(range: Range): void { + if (range.min < 0) { + throw new Error(`Invalid range min (${range.min}): dimension minimum cannot be negative.`); + } + if (range.max < range.min) { + throw new Error(`Invalid range [${range.min}, ${range.max}]: max cannot be less than min.`); + } + if (range.step <= 0) { + throw new Error(`Invalid range step (${range.step}): step must be a positive integer.`); + } } -export function matchDim(exp: TypedDim, act: TypedDim, env: SymbolMap): SymbolMap | null { - const bindSymbol = (key: string, value: string | number) => { - if (env.has(key)) return env.get(key) === value ? env : null; - return new Map(env).set(key, value); - }; +export const StaticDim = (symbol: string) => { + return { kind: 'static', symbol } as TypedDim; +}; - if (exp.kind === 'constant' && act.kind === 'constant') { - return exp.value === act.value ? env : null; +export const DynamicDim = (symbol: string, range?: Range) => { + if (range) validateRange(range); + return { kind: 'dynamic', symbol, range } as TypedDim; +}; + +export const ConstantDim = (value: number) => { + if (value <= 0 || !Number.isInteger(value)) { + throw new Error(`Invalid value (${value}): must be a positive integer.`); } + return { kind: 'constant', value } as TypedDim; +}; - if (exp.kind === 'constant' && act.kind === 'dynamic') { - if (!act.range) return null; // Unknown model range cannot guarantee exp.value +export const SymbolicTensor = (dtype?: DType, shape?: SymbolicShape) => { + const typedShape = shape?.map((dim) => { + if (typeof dim === 'string') return StaticDim(dim); + if (typeof dim === 'number') return ConstantDim(dim); + return dim; + }); + return { kind: 'Tensor', dtype, shape: typedShape } as TensorConstraint; +}; - if (exp.value < act.range.min || exp.value > act.range.max) return null; +function rangesEqual(range1?: Range, range2?: Range): boolean { + if (range1 === range2) return true; + if (!range1 || !range2) return false; + return range1.min === range2.min && range1.max === range2.max && range1.step === range2.step; +} + +function dimsEqual(dim1: TypedDim, dim2: TypedDim): boolean { + if (dim1.kind === 'static' && dim2.kind === 'static') { + return dim1.symbol === dim2.symbol; + } + if (dim1.kind === 'dynamic' && dim2.kind === 'dynamic') { + return dim1.symbol === dim2.symbol && rangesEqual(dim1.range, dim2.range); + } + if (dim1.kind === 'constant' && dim2.kind === 'constant') { + return dim1.value === dim2.value; + } + return false; +} - if (act.range.step !== undefined) { - if (Math.abs(exp.value - act.range.min) % act.range.step !== 0) return null; +function matchDim(exp: TypedDim, act: TypedDim, bindings: SymbolBindings, ctx: string) { + if ('symbol' in act) { + for (const b of bindings) { + if ('symbol' in b.act && act.symbol === b.act.symbol && !dimsEqual(act, b.act)) { + return err(`${ctx}: Actual '${act.symbol}' is used inconsistently across signatures.`); + } } + } - return env; + if ('symbol' in exp) { + for (const b of bindings) { + if (exp.symbol === b.exp.symbol && !dimsEqual(exp, b.exp)) { + return err(`${ctx}: Expected '${exp.symbol}' is used inconsistently across signatures.`); + } + } } - if (exp.kind === 'static' && act.kind === 'constant') { - return bindSymbol(exp.symbol, act.value); + // [constant, constant] + if (exp.kind === 'constant' && act.kind === 'constant') { + if (exp.value !== act.value) { + return err(`${ctx}: Constant dimension value mismatch.`); + } + return ok(bindings); } - if (exp.kind === 'static' && act.kind === 'static') { - return bindSymbol(exp.symbol, act.symbol); + // [static, *] + if (exp.kind === 'static') { + const existing = bindings.find((b) => b.exp.symbol === exp.symbol); + if (existing) { + if (!dimsEqual(existing.act, act)) { + return err(`${ctx}: Static '${exp.symbol}' has inconsistent bindings.`); + } + return ok(bindings); + } + return ok([...bindings, { exp, act }]); } - if (exp.kind === 'dynamic' && act.kind === 'dynamic') { - if (!isRangeCovered(exp.range, act.range)) return null; - return bindSymbol(exp.symbol, act.symbol); + // [constant, dynamic] + if (exp.kind === 'constant' && act.kind === 'dynamic') { + if (!act.range) { + return err(`${ctx}: Actual dynamic dimension '${act.symbol}' has an unspecified range.`); + } + if (exp.value < act.range.min || act.range.max < exp.value) { + return err(`${ctx}: Constant ${exp.value} falls outside dynamic range.`); + } + if ((exp.value - act.range.min) % act.range.step !== 0) { + return err(`${ctx}: Constant ${exp.value} does not align with range step.`); + } + return ok(bindings); } - // All remaining combinations fail - return null; -} + // [dynamic, dynamic] + if (exp.kind === 'dynamic' && act.kind === 'dynamic') { + const existing = bindings.find((b) => b.exp.symbol === exp.symbol); + if (existing) { + if (!dimsEqual(existing.act, act)) { + return err(`${ctx}: Dynamic '${exp.symbol}' has inconsistent bindings.`); + } + return ok(bindings); + } + + if (exp.range) { + if (!act.range) { + return err(`${ctx}: Actual dynamic '${act.symbol}' has an unspecified range.`); + } + if (exp.range.min < act.range.min || exp.range.max > act.range.max) { + return err(`${ctx}: Expected range exceeds accepted actual range.`); + } + if ((exp.range.min - act.range.min) % act.range.step !== 0) { + return err(`${ctx}: Expected min does not align with actual range step.`); + } + if (exp.range.step % act.range.step !== 0) { + return err(`${ctx}: Expected step is not a multiple of accepted step.`); + } + } -function constraintToString(constraint: ValueConstraint): string { - if (constraint.kind !== 'Tensor') { - return constraint.kind; + return ok([...bindings, { exp, act }]); } - const dimToString = (dim: TypedDim) => { - if (dim.kind === 'constant') return `${dim.value}`; - if (dim.kind === 'static') return dim.symbol; - if (dim.range) { - const stepStr = dim.range.step !== undefined ? `:${dim.range.step}` : ''; - return `${dim.symbol}[${dim.range.min}..${dim.range.max}${stepStr}]`; - } - return `Dynamic(${dim.symbol})`; - }; + return err(`${ctx}: cannot match expected ${exp.kind} with actual ${act.kind}.`); +} - const parts: string[] = []; - if (constraint.dtype !== undefined) { - parts.push(`dtype=${constraint.dtype}`); +function matchSignature( + expSgn: MethodSignature, + actSgn: MethodSignature, + initialBindings: SymbolBindings = [], + ctx: string = '' +): MatchResult { + if (expSgn.inputs.length !== actSgn.inputs.length) { + return err(`${ctx}: Input count mismatch.`); } - if (constraint.shapes && constraint.shapes.length > 0) { - const shapesStr = constraint.shapes - .map((shape) => `[${shape.map(dimToString).join(', ')}]`) - .join(' | '); - parts.push(`shapes=${shapesStr}`); + if (expSgn.outputs.length !== actSgn.outputs.length) { + return err(`${ctx}: Output count mismatch.`); } - return parts.length > 0 ? `Tensor(${parts.join(', ')})` : 'Tensor'; -} + const expConstraints = [...expSgn.inputs, ...expSgn.outputs]; + const actConstraints = [...actSgn.inputs, ...actSgn.outputs]; + const numConstraints = expConstraints.length; -export function schemaToString(schema: MethodSchema): string { - const inputsStr = schema.inputs.map(constraintToString).join(', '); - const outputsStr = schema.outputs.map(constraintToString).join(', '); - return `(${inputsStr}) -> (${outputsStr})`; -} + let currentBindings = initialBindings; -export function validateSchema(expected: MethodSchema, actual: MethodSchema) { - const formatError = (msg: string) => - `${msg}\n` + - `Expected schema: ${schemaToString(expected)}\n` + - `Actual schema : ${schemaToString(actual)}`; + for (let c = 0; c < numConstraints; ++c) { + const isInput = c < expSgn.inputs.length; + const constraintIdx = isInput ? c : c - expSgn.inputs.length; + const constraintCtx = `${ctx} ${isInput ? 'input' : 'output'} #${constraintIdx}`; - if (expected.inputs.length !== actual.inputs.length) { - throw new Error(formatError('Number of inputs does not match!')); - } - if (expected.outputs.length !== actual.outputs.length) { - throw new Error(formatError('Number of outputs does not match!')); - } + const expConstr = expConstraints[c]!; + const actConstr = actConstraints[c]!; - const expList = [...expected.inputs, ...expected.outputs]; - const actList = [...actual.inputs, ...actual.outputs]; + if (expConstr.kind !== actConstr.kind) { + return err(`${constraintCtx}: Constraint kind mismatch.`); + } - const match = (index: number, map: SymbolMap): boolean => { - if (index === expList.length) return true; + if (expConstr.kind !== 'Tensor' && actConstr.kind !== 'Tensor') continue; - const exp = expList[index]!; - const act = actList[index]!; + const expTensor = expConstr as TensorConstraint; + const actTensor = actConstr as TensorConstraint; - if (exp.kind !== act.kind) return false; - if (exp.kind !== 'Tensor') return match(index + 1, map); + // Unspecified expTensor.dtype means "whatever dtype actual gives me" + if (expTensor.dtype && expTensor.dtype !== actTensor.dtype) { + return err(`${constraintCtx}: DType mismatch.`); + } - const expTensor = exp as TensorConstraint; - const actTensor = act as TensorConstraint; + // Unspecified expTensor.shape means "whatever shape actual gives me" + if (!expTensor.shape) continue; - // DType Sub-typing - if (expTensor.dtype !== undefined && actTensor.dtype !== expTensor.dtype) { - return false; + // Unspecified actTensor.shape means "we don't know", throws because there is no way to check it + if (!actTensor.shape) { + return err(`${constraintCtx}: Actual tensor shape is missing.`); } - // Shape Sub-typing - if (expTensor.shapes && expTensor.shapes.length > 0) { - if (!actTensor.shapes || actTensor.shapes.length === 0) { - return false; + if (expTensor.shape.length !== actTensor.shape.length) { + return err(`${constraintCtx}: Rank mismatch.`); + } + + const rank = expTensor.shape.length; + for (let d = 0; d < rank; ++d) { + const expDim = expTensor.shape[d]!; + const actDim = actTensor.shape[d]!; + const dimCtx = `${constraintCtx} Tensor dim #${d}`; + const dimResult = matchDim(expDim, actDim, currentBindings, dimCtx); + if (!dimResult.ok) { + return dimResult; } + currentBindings = dimResult.bindings; + } + } - for (const expShape of expTensor.shapes) { - for (const actShape of actTensor.shapes) { - if (expShape.length !== actShape.length) continue; + return ok(currentBindings); +} - let currentMap: SymbolMap | null = map; - for (let dim = 0; dim < expShape.length; dim++) { - currentMap = matchDim(expShape[dim]!, actShape[dim]!, currentMap); - if (!currentMap) break; - } +export function matchSchema(expectedSchema: ModelSchema, actualSchema: ModelSchema): MatchResult { + for (const methodName in expectedSchema) { + if (!actualSchema[methodName]) { + return err(`Method '${methodName}' not found in actual schema.`); + } + } + + const methodNames = Object.keys(expectedSchema); - if (currentMap && match(index + 1, currentMap)) { - return true; // Found a globally consistent path! + const match = (idx: number, bindings: SymbolBindings): MatchResult => { + if (idx >= methodNames.length) return ok(bindings); + + const methodName = methodNames[idx]!; + const expOverloads = expectedSchema[methodName]!; + const actOverloads = actualSchema[methodName]!; + + const errors: string[] = []; + + for (let e = 0; e < expOverloads.length; ++e) { + for (let a = 0; a < actOverloads.length; ++a) { + const sigCtx = `Method '${methodName}' (exp overload #${e}, act overload #${a})`; + const sigResult = matchSignature(expOverloads[e]!, actOverloads[a]!, bindings, sigCtx); + + if (sigResult.ok) { + const nextResult = match(idx + 1, sigResult.bindings); + if (nextResult.ok) { + return nextResult; } + errors.push(nextResult.error); + } else { + errors.push(sigResult.error); } } - - return false; // No shape overload satisfied constraints } - // Expected doesn't restrict shapes; any actual shape is valid - return match(index + 1, map); + return err(`Method '${methodName}' failed overload matching:\n - ` + errors.join('\n - ')); }; - if (!match(0, new Map())) { - throw new Error(formatError('No possible consistent matching!')); + return match(0, []); +} + +export function validateSchema(expectedVariants: ModelSchema[], actualSchema: ModelSchema) { + const errors: string[] = []; + + for (let i = 0; i < expectedVariants.length; ++i) { + const result = matchSchema(expectedVariants[i]!, actualSchema); + if (result.ok) { + return actualSchema; + } + errors.push(`Variant ${i}:\n${result.error}`); } - return actual; + throw new Error(`Model schema doesn't match any of the provided variants:\n` + errors.join('\n')); } From 36eb008187501837fffec726b3d1362064c981fa Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 24 Jul 2026 02:46:11 +0200 Subject: [PATCH 03/31] ts side kinda finished --- .../src/core/schema.ts | 365 ++++++++---------- 1 file changed, 169 insertions(+), 196 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 9eb71e9089..9213e79b7d 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -1,305 +1,278 @@ -import { type DType } from './tensor'; +import type { DType } from './tensor'; +import type { ExecuTorchTag } from './model'; export type Range = { min: number; max: number; step: number }; -export type TypedDim = +export type ConcreteDim = + | { readonly kind: 'constant'; readonly value: number } + | { readonly kind: 'range'; readonly range: Range }; + +export type SymbolicDim = + | ConcreteDim | { readonly kind: 'static'; readonly symbol: string } - | { readonly kind: 'dynamic'; readonly symbol: string; readonly range?: Range } - | { readonly kind: 'constant'; readonly value: number }; + | { readonly kind: 'dynamic'; readonly symbol: string }; -export type TensorConstraint = { +export type TensorConstraint = { readonly kind: 'Tensor'; - readonly dtype?: DType; - readonly shape?: readonly TypedDim[]; + readonly dtype: DType; + readonly shape: readonly D[]; }; -export type ValueConstraint = - | TensorConstraint - | { readonly kind: 'None' } - | { readonly kind: 'Int' } - | { readonly kind: 'Double' } - | { readonly kind: 'Bool' } - | { readonly kind: 'String' }; +export type ValueConstraint = + | TensorConstraint + | { readonly kind: Exclude }; -export type MethodSignature = { inputs: ValueConstraint[]; outputs: ValueConstraint[] }; +export type MethodSignature = { + inputs: ValueConstraint[]; + outputs: ValueConstraint[]; +}; -export type MethodOverloads = MethodSignature[]; +export type MethodOverloads = readonly MethodSignature[]; -export type ModelSchema = Record; +export type ModelSchema = Record>; -export type SymbolicShape = readonly (number | string | TypedDim)[]; +export type SymbolicShape = readonly (number | string | SymbolicDim)[]; -export type SymbolBinding = { - readonly exp: Exclude; - readonly act: TypedDim; +export const S = (symbol: string): SymbolicDim => { + return { kind: 'static', symbol }; }; -export type SymbolBindings = readonly SymbolBinding[]; - -export type MatchResult = - | { readonly ok: true; readonly bindings: SymbolBindings } - | { readonly ok: false; readonly error: string }; +export const D = (symbol: string): SymbolicDim => { + return { kind: 'dynamic', symbol }; +}; -const ok = (bindings: SymbolBindings): MatchResult => ({ ok: true, bindings }); -const err = (error: string): MatchResult => ({ ok: false, error }); +export const C = (value: number): ConcreteDim => { + if (value <= 0 || !Number.isInteger(value)) { + throw new Error(`Invalid value (${value}): must be a positive integer.`); + } + return { kind: 'constant', value }; +}; -function validateRange(range: Range): void { +export const R = (range: Range): ConcreteDim => { if (range.min < 0) { - throw new Error(`Invalid range min (${range.min}): dimension minimum cannot be negative.`); + throw new Error(`Invalid range: dimension minimum cannot be negative.`); } if (range.max < range.min) { throw new Error(`Invalid range [${range.min}, ${range.max}]: max cannot be less than min.`); } - if (range.step <= 0) { - throw new Error(`Invalid range step (${range.step}): step must be a positive integer.`); + if (!Number.isInteger(range.min)) { + throw new Error(`Invalid range min (${range.min}): must be a non-negative integer.`); } -} - -export const StaticDim = (symbol: string) => { - return { kind: 'static', symbol } as TypedDim; -}; - -export const DynamicDim = (symbol: string, range?: Range) => { - if (range) validateRange(range); - return { kind: 'dynamic', symbol, range } as TypedDim; -}; - -export const ConstantDim = (value: number) => { - if (value <= 0 || !Number.isInteger(value)) { - throw new Error(`Invalid value (${value}): must be a positive integer.`); + if (!Number.isInteger(range.max)) { + throw new Error(`Invalid range max (${range.max}): must be a non-negative integer.`); + } + if (range.step <= 0 || !Number.isInteger(range.step)) { + throw new Error(`Invalid range step (${range.step}): must be a positive integer.`); } - return { kind: 'constant', value } as TypedDim; + return { kind: 'range', range }; }; -export const SymbolicTensor = (dtype?: DType, shape?: SymbolicShape) => { - const typedShape = shape?.map((dim) => { - if (typeof dim === 'string') return StaticDim(dim); - if (typeof dim === 'number') return ConstantDim(dim); +export const SymbolicTensor = (dtype: DType, shape: SymbolicShape) => { + const typedShape = shape.map((dim) => { + if (typeof dim === 'string') return S(dim); + if (typeof dim === 'number') return C(dim); return dim; }); - return { kind: 'Tensor', dtype, shape: typedShape } as TensorConstraint; + return { kind: 'Tensor', dtype, shape: typedShape } as TensorConstraint; }; -function rangesEqual(range1?: Range, range2?: Range): boolean { - if (range1 === range2) return true; - if (!range1 || !range2) return false; - return range1.min === range2.min && range1.max === range2.max && range1.step === range2.step; -} +type SymbolBinding = { symbolic: Exclude; concrete: ConcreteDim }; +type SymbolBindings = readonly SymbolBinding[]; +type MatchResult = + | { readonly ok: true; readonly bindings: SymbolBindings } + | { readonly ok: false; readonly error: string }; -function dimsEqual(dim1: TypedDim, dim2: TypedDim): boolean { - if (dim1.kind === 'static' && dim2.kind === 'static') { - return dim1.symbol === dim2.symbol; - } - if (dim1.kind === 'dynamic' && dim2.kind === 'dynamic') { - return dim1.symbol === dim2.symbol && rangesEqual(dim1.range, dim2.range); - } - if (dim1.kind === 'constant' && dim2.kind === 'constant') { - return dim1.value === dim2.value; - } - return false; -} +const ok = (bindings: SymbolBindings): MatchResult => ({ ok: true, bindings }); +const err = (error: string): MatchResult => ({ ok: false, error }); +const rangesEqual = (r1: Range, r2: Range) => { + return r1.min === r2.min && r1.max === r2.max && r1.step === r2.step; +}; -function matchDim(exp: TypedDim, act: TypedDim, bindings: SymbolBindings, ctx: string) { - if ('symbol' in act) { - for (const b of bindings) { - if ('symbol' in b.act && act.symbol === b.act.symbol && !dimsEqual(act, b.act)) { - return err(`${ctx}: Actual '${act.symbol}' is used inconsistently across signatures.`); - } +function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, bindings: SymbolBindings, ctx: string) { + if ('symbol' in sDim) { + if (bindings.some((b) => b.symbolic.symbol === sDim.symbol && b.symbolic.kind !== sDim.kind)) { + // returning err(...) here would treat a self-contradictory schema as a + // simple match failure and move on to the next variant. + throw new Error( + `Invalid schema ${ctx}: Symbol ${sDim.symbol} is used as both 'static' and 'dynamic'` + ); } } - if ('symbol' in exp) { - for (const b of bindings) { - if (exp.symbol === b.exp.symbol && !dimsEqual(exp, b.exp)) { - return err(`${ctx}: Expected '${exp.symbol}' is used inconsistently across signatures.`); - } + if (sDim.kind === 'constant' && cDim.kind === 'constant') { + if (sDim.value !== cDim.value) { + return err(`${ctx}: Constant dimension value mismatch.`); } + return ok(bindings); } - // [constant, constant] - if (exp.kind === 'constant' && act.kind === 'constant') { - if (exp.value !== act.value) { - return err(`${ctx}: Constant dimension value mismatch.`); + if (sDim.kind === 'range' && cDim.kind === 'range') { + if (!rangesEqual(sDim.range, cDim.range)) { + return err(`${ctx}: Range dimension mismatch`); } return ok(bindings); } - // [static, *] - if (exp.kind === 'static') { - const existing = bindings.find((b) => b.exp.symbol === exp.symbol); + if (sDim.kind === 'static' && cDim.kind === 'constant') { + const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); if (existing) { - if (!dimsEqual(existing.act, act)) { - return err(`${ctx}: Static '${exp.symbol}' has inconsistent bindings.`); + if (existing.concrete.kind === 'constant' && cDim.value !== existing.concrete.value) { + return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); } return ok(bindings); } - return ok([...bindings, { exp, act }]); - } - - // [constant, dynamic] - if (exp.kind === 'constant' && act.kind === 'dynamic') { - if (!act.range) { - return err(`${ctx}: Actual dynamic dimension '${act.symbol}' has an unspecified range.`); - } - if (exp.value < act.range.min || act.range.max < exp.value) { - return err(`${ctx}: Constant ${exp.value} falls outside dynamic range.`); - } - if ((exp.value - act.range.min) % act.range.step !== 0) { - return err(`${ctx}: Constant ${exp.value} does not align with range step.`); - } - return ok(bindings); + return ok([...bindings, { symbolic: sDim, concrete: cDim }]); } - // [dynamic, dynamic] - if (exp.kind === 'dynamic' && act.kind === 'dynamic') { - const existing = bindings.find((b) => b.exp.symbol === exp.symbol); + if (sDim.kind === 'dynamic' && cDim.kind === 'range') { + const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); if (existing) { - if (!dimsEqual(existing.act, act)) { - return err(`${ctx}: Dynamic '${exp.symbol}' has inconsistent bindings.`); + if (existing.concrete.kind === 'range' && !rangesEqual(cDim.range, existing.concrete.range)) { + return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); } return ok(bindings); } - - if (exp.range) { - if (!act.range) { - return err(`${ctx}: Actual dynamic '${act.symbol}' has an unspecified range.`); - } - if (exp.range.min < act.range.min || exp.range.max > act.range.max) { - return err(`${ctx}: Expected range exceeds accepted actual range.`); - } - if ((exp.range.min - act.range.min) % act.range.step !== 0) { - return err(`${ctx}: Expected min does not align with actual range step.`); - } - if (exp.range.step % act.range.step !== 0) { - return err(`${ctx}: Expected step is not a multiple of accepted step.`); - } - } - - return ok([...bindings, { exp, act }]); + return ok([...bindings, { symbolic: sDim, concrete: cDim }]); } - return err(`${ctx}: cannot match expected ${exp.kind} with actual ${act.kind}.`); + return err(`${ctx}: Cannot match symbolic ${sDim.kind} with concrete ${cDim.kind}.`); } function matchSignature( - expSgn: MethodSignature, - actSgn: MethodSignature, - initialBindings: SymbolBindings = [], - ctx: string = '' + symbolicSgn: MethodSignature, + concreteSgn: MethodSignature, + bindings: SymbolBindings, + ctx: string ): MatchResult { - if (expSgn.inputs.length !== actSgn.inputs.length) { + if (symbolicSgn.inputs.length !== concreteSgn.inputs.length) { return err(`${ctx}: Input count mismatch.`); } - if (expSgn.outputs.length !== actSgn.outputs.length) { + if (symbolicSgn.outputs.length !== concreteSgn.outputs.length) { return err(`${ctx}: Output count mismatch.`); } - const expConstraints = [...expSgn.inputs, ...expSgn.outputs]; - const actConstraints = [...actSgn.inputs, ...actSgn.outputs]; - const numConstraints = expConstraints.length; + const symbolicConstraints = [...symbolicSgn.inputs, ...symbolicSgn.outputs]; + const concreteConstraints = [...concreteSgn.inputs, ...concreteSgn.outputs]; + const numConstraints = symbolicConstraints.length; - let currentBindings = initialBindings; + let currentBindings = bindings; for (let c = 0; c < numConstraints; ++c) { - const isInput = c < expSgn.inputs.length; - const constraintIdx = isInput ? c : c - expSgn.inputs.length; + const isInput = c < symbolicSgn.inputs.length; + const constraintIdx = isInput ? c : c - concreteSgn.inputs.length; const constraintCtx = `${ctx} ${isInput ? 'input' : 'output'} #${constraintIdx}`; - const expConstr = expConstraints[c]!; - const actConstr = actConstraints[c]!; + const symbolicConstr = symbolicConstraints[c]!; + const concreteConstr = concreteConstraints[c]!; - if (expConstr.kind !== actConstr.kind) { + if (symbolicConstr.kind !== concreteConstr.kind) { return err(`${constraintCtx}: Constraint kind mismatch.`); } - if (expConstr.kind !== 'Tensor' && actConstr.kind !== 'Tensor') continue; + if (symbolicConstr.kind !== 'Tensor' && concreteConstr.kind !== 'Tensor') continue; - const expTensor = expConstr as TensorConstraint; - const actTensor = actConstr as TensorConstraint; + const symbolicTensor = symbolicConstr as TensorConstraint; + const concreteTensor = concreteConstr as TensorConstraint; - // Unspecified expTensor.dtype means "whatever dtype actual gives me" - if (expTensor.dtype && expTensor.dtype !== actTensor.dtype) { + if (symbolicTensor.dtype !== concreteTensor.dtype) { return err(`${constraintCtx}: DType mismatch.`); } - // Unspecified expTensor.shape means "whatever shape actual gives me" - if (!expTensor.shape) continue; - - // Unspecified actTensor.shape means "we don't know", throws because there is no way to check it - if (!actTensor.shape) { - return err(`${constraintCtx}: Actual tensor shape is missing.`); - } - - if (expTensor.shape.length !== actTensor.shape.length) { + if (symbolicTensor.shape.length !== concreteTensor.shape.length) { return err(`${constraintCtx}: Rank mismatch.`); } - const rank = expTensor.shape.length; + const rank = symbolicTensor.shape.length; + for (let d = 0; d < rank; ++d) { - const expDim = expTensor.shape[d]!; - const actDim = actTensor.shape[d]!; + const sDim = symbolicTensor.shape[d]!; + const cDim = concreteTensor.shape[d]!; const dimCtx = `${constraintCtx} Tensor dim #${d}`; - const dimResult = matchDim(expDim, actDim, currentBindings, dimCtx); - if (!dimResult.ok) { - return dimResult; - } - currentBindings = dimResult.bindings; + + const result = matchDim(sDim, cDim, currentBindings, dimCtx); + if (!result.ok) return result; + currentBindings = result.bindings; } } return ok(currentBindings); } -export function matchSchema(expectedSchema: ModelSchema, actualSchema: ModelSchema): MatchResult { - for (const methodName in expectedSchema) { - if (!actualSchema[methodName]) { - return err(`Method '${methodName}' not found in actual schema.`); +export function matchSchema( + symbolicSchema: ModelSchema, + concreteSchema: ModelSchema, + ctx: string +): MatchResult { + const methodNames = Object.keys(symbolicSchema); + for (const methodName of methodNames) { + if (!concreteSchema[methodName]) { + return err(`${ctx}: Method '${methodName}' not found in actual schema.`); } } - const methodNames = Object.keys(expectedSchema); + const match = (nameIdx: number, sOverloadIdx: number, bindings: SymbolBindings): MatchResult => { + if (nameIdx >= methodNames.length) { + return ok(bindings); + } + + const methodName = methodNames[nameIdx]!; + const sOverloads = symbolicSchema[methodName]!; - const match = (idx: number, bindings: SymbolBindings): MatchResult => { - if (idx >= methodNames.length) return ok(bindings); + if (sOverloadIdx >= sOverloads.length) { + return match(nameIdx + 1, 0, bindings); + } - const methodName = methodNames[idx]!; - const expOverloads = expectedSchema[methodName]!; - const actOverloads = actualSchema[methodName]!; + const cOverloads = concreteSchema[methodName]!; + const sOverload = sOverloads[sOverloadIdx]!; const errors: string[] = []; - for (let e = 0; e < expOverloads.length; ++e) { - for (let a = 0; a < actOverloads.length; ++a) { - const sigCtx = `Method '${methodName}' (exp overload #${e}, act overload #${a})`; - const sigResult = matchSignature(expOverloads[e]!, actOverloads[a]!, bindings, sigCtx); - - if (sigResult.ok) { - const nextResult = match(idx + 1, sigResult.bindings); - if (nextResult.ok) { - return nextResult; - } - errors.push(nextResult.error); - } else { - errors.push(sigResult.error); - } + for (const [idx, cOverload] of cOverloads.entries()) { + const overloadCtx = `${ctx}: Method '${methodName}' (symbolic overload #${sOverloadIdx}, concrete overload #${idx})`; + + const sgnResult = matchSignature(sOverload, cOverload, bindings, overloadCtx); + if (!sgnResult.ok) { + errors.push(sgnResult.error); + continue; } + + const matchResult = match(nameIdx, sOverloadIdx + 1, sgnResult.bindings); + if (!matchResult.ok) { + errors.push(matchResult.error); + continue; + } + + return matchResult; } - return err(`Method '${methodName}' failed overload matching:\n - ` + errors.join('\n - ')); + return err(`${ctx}: Method '${methodName}' failed matching:\n - ` + errors.join('\n - ')); }; - return match(0, []); + return match(0, 0, []); } -export function validateSchema(expectedVariants: ModelSchema[], actualSchema: ModelSchema) { +export function validateSchema( + symbolicVariants: ModelSchema[], + concreteSchema: ModelSchema +) { const errors: string[] = []; - for (let i = 0; i < expectedVariants.length; ++i) { - const result = matchSchema(expectedVariants[i]!, actualSchema); - if (result.ok) { - return actualSchema; + for (const [idx, symbolicSchema] of symbolicVariants.entries()) { + const result = matchSchema(symbolicSchema, concreteSchema, `Variant ${idx}`); + if (!result.ok) { + errors.push(result.error); + continue; } - errors.push(`Variant ${i}:\n${result.error}`); + return new Map( + result.bindings.map((binding) => { + switch (binding.concrete.kind) { + case 'constant': + return [binding.symbolic.symbol, binding.concrete.value]; + case 'range': + return [binding.symbolic.symbol, binding.concrete.range]; + } + }) + ); } throw new Error(`Model schema doesn't match any of the provided variants:\n` + errors.join('\n')); From 2f4ca6e12847efa44cf5cc6441b02cdd6ab370bf Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Fri, 24 Jul 2026 03:12:39 +0200 Subject: [PATCH 04/31] small fix --- .../src/core/schema.ts | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 9213e79b7d..8c0794323d 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -20,7 +20,7 @@ export type TensorConstraint = { export type ValueConstraint = | TensorConstraint - | { readonly kind: Exclude }; + | { readonly kind: Exclude }; export type MethodSignature = { inputs: ValueConstraint[]; @@ -84,17 +84,22 @@ type MatchResult = const ok = (bindings: SymbolBindings): MatchResult => ({ ok: true, bindings }); const err = (error: string): MatchResult => ({ ok: false, error }); -const rangesEqual = (r1: Range, r2: Range) => { +const rangesEqual = (r1: Range, r2: Range): boolean => { return r1.min === r2.min && r1.max === r2.max && r1.step === r2.step; }; -function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, bindings: SymbolBindings, ctx: string) { +function matchDim( + sDim: SymbolicDim, + cDim: ConcreteDim, + bindings: SymbolBindings, + ctx: string +): MatchResult { if ('symbol' in sDim) { if (bindings.some((b) => b.symbolic.symbol === sDim.symbol && b.symbolic.kind !== sDim.kind)) { // returning err(...) here would treat a self-contradictory schema as a // simple match failure and move on to the next variant. throw new Error( - `Invalid schema ${ctx}: Symbol ${sDim.symbol} is used as both 'static' and 'dynamic'` + `Invalid schema ${ctx}: ${sDim.symbol} is used as both 'static' and 'dynamic'` ); } } @@ -116,7 +121,10 @@ function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, bindings: SymbolBindings if (sDim.kind === 'static' && cDim.kind === 'constant') { const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); if (existing) { - if (existing.concrete.kind === 'constant' && cDim.value !== existing.concrete.value) { + if (existing.concrete.kind !== 'constant') { + throw new Error(`Matcher error ${ctx}: 'static' was matched to 'range'`); + } + if (cDim.value !== existing.concrete.value) { return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); } return ok(bindings); @@ -127,7 +135,10 @@ function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, bindings: SymbolBindings if (sDim.kind === 'dynamic' && cDim.kind === 'range') { const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); if (existing) { - if (existing.concrete.kind === 'range' && !rangesEqual(cDim.range, existing.concrete.range)) { + if (existing.concrete.kind !== 'range') { + throw new Error(`Matcher error ${ctx}: 'dynamic' was matched to 'constant'`); + } + if (!rangesEqual(cDim.range, existing.concrete.range)) { return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); } return ok(bindings); @@ -198,7 +209,7 @@ function matchSignature( return ok(currentBindings); } -export function matchSchema( +function matchSchema( symbolicSchema: ModelSchema, concreteSchema: ModelSchema, ctx: string @@ -254,7 +265,7 @@ export function matchSchema( export function validateSchema( symbolicVariants: ModelSchema[], concreteSchema: ModelSchema -) { +): Map { const errors: string[] = []; for (const [idx, symbolicSchema] of symbolicVariants.entries()) { From aec8fe74f9fe1c62bb6f8d34a8cbc20edd163e08 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Sat, 25 Jul 2026 01:55:00 +0200 Subject: [PATCH 05/31] refactor ts side --- .../react-native-executorch/src/core/model.ts | 3 + .../src/core/schema.ts | 619 +++++++++++++----- 2 files changed, 457 insertions(+), 165 deletions(-) diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index e539afecdd..f2cc0280f2 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -1,5 +1,6 @@ import { rnexecutorchJsi } from '../native/bridge'; import type { DType, Tensor } from './tensor'; +import type { ModelSpec, ConcreteDim } from './schema'; declare const modelBrand: unique symbol; @@ -89,6 +90,8 @@ export type ModelMethodMeta = { export interface Model { /** The local filesystem path of the `.pte` model file. */ readonly path: string; + /** The exported spec of this model. */ + readonly spec: ModelSpec; /** * Returns the list of exported method names available on this model (e.g. diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 8c0794323d..30e96b9159 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -1,290 +1,579 @@ +/** + * Model specs and spec validation. + * + * A model spec is a structural contract describing a model's methods: the + * parameter specs of every input and output (primitive tags, tensor data + * types, and per-dimension domains) and the runtime constraints the method + * declares over its tensor dimensions. A spec is either: + * - **allowed** (`SymbolicDim`) — written by a pipeline to state which models + * it can work with. Dimensions may be named symbols: `static` symbols bind + * to constant dimensions, `dynamic` symbols to ranges or enums, and reusing + * a symbol requires every occurrence to bind to the same domain (nothing + * more — symbol reuse implies no runtime relation). Several allowed specs + * can be passed as variants; matching any one of them is enough. + * - **exported** (`ConcreteDim`) — derived from an exported model's metadata, + * stating what the model actually provides. + * + * Two different validations must not be confused: + * - **Spec validation** (this module, {@link validateSpec}) — a static, + * load-time check that an exported spec satisfies an allowed spec. Dimension + * domains must match exactly (symbol binding), and the exported spec must + * declare exactly the allowed spec's runtime constraints. Constraints are + * matched as declarations only: whether they actually hold is not, and + * cannot be, checked here. + * - **Runtime validation** (the native runtime, not this module) — at + * execution time the runtime checks the shapes of concrete tensors against + * the dimensions and enforces the declared runtime constraints. + * + * Note that only equality constraints on static dimensions are fully decided + * by spec validation: a constant dimension has exactly one possible value, + * so equal constants are equal at runtime — no enforcement left. Linear + * constraints, in contrast, are never evaluated against dimensions here — + * they are matched as declarations even between constants, so checking them + * always remains with the runtime. + * @packageDocumentation + */ import type { DType } from './tensor'; import type { ExecuTorchTag } from './model'; +// ======================================================== +// Parameter specs +// ======================================================== + +/** + * Inclusive integer domain of a single dynamic dimension — values from `min` + * to `max` in increments of `step`. + * @category Types + */ export type Range = { min: number; max: number; step: number }; +/** + * A single dimension with a fully known domain: + * - `constant` — exactly `value`. + * - `range` — any value of a {@link Range}. + * - `enum` — one of the listed `choices`. + * @category Types + */ export type ConcreteDim = | { readonly kind: 'constant'; readonly value: number } - | { readonly kind: 'range'; readonly range: Range }; - + | { readonly kind: 'range'; readonly range: Range } + | { readonly kind: 'enum'; readonly choices: readonly number[] }; + +/** + * A single dimension of an allowed model spec. On top of {@link ConcreteDim} + * domains, a named symbol binds to the exported spec's dimension at + * validation: `static` symbols bind to constants, `dynamic` symbols to ranges + * or enums. Reusing a symbol requires every occurrence to bind to the same + * domain — it does NOT imply any runtime relation between the dimensions. + * @category Types + */ export type SymbolicDim = | ConcreteDim | { readonly kind: 'static'; readonly symbol: string } | { readonly kind: 'dynamic'; readonly symbol: string }; -export type TensorConstraint = { +/** + * Spec of a tensor parameter: the expected element `dtype` and one + * dimension spec per axis. + * @category Types + */ +export type TensorSpec = { readonly kind: 'Tensor'; readonly dtype: DType; - readonly shape: readonly D[]; + readonly shape: readonly Dim[]; }; -export type ValueConstraint = - | TensorConstraint +/** + * Spec of a single input or output parameter of a method — either a + * {@link TensorSpec} or a primitive ExecuTorch value tag (`Int`, `Bool`, ...). + * @category Types + */ +export type ParamSpec = + | TensorSpec | { readonly kind: Exclude }; -export type MethodSignature = { - inputs: ValueConstraint[]; - outputs: ValueConstraint[]; +// ======================================================== +// Runtime constraints +// ======================================================== + +/** + * Reference to a single tensor dimension of a method's input or output. + * `tensorIdx` counts only tensor parameters (skipping primitives), consistent + * with ExecuTorch's `inputTensorMeta` / `outputTensorMeta` ordering. + * @category Types + */ +export type DimRef = { + readonly io: 'input' | 'output'; + readonly tensorIdx: number; + readonly dimIdx: number; +}; + +/** + * Runtime constraint declaring that all referenced dimensions must be equal + * to each other in any given execution of the method. + * @category Types + */ +export type EqualityConstraint = { + readonly kind: 'equality'; + readonly dims: readonly DimRef[]; }; -export type MethodOverloads = readonly MethodSignature[]; +/** + * Runtime constraint declaring that two dimensions must satisfy + * `dimLhs = coefficients[0] * dimRhs + coefficients[1]` (integer + * coefficients) in any given execution of the method. + * @category Types + */ +export type LinearConstraint = { + readonly kind: 'linear'; + readonly dimLhs: DimRef; + readonly dimRhs: DimRef; + readonly coefficients: [number, number]; +}; -export type ModelSchema = Record>; +/** + * A requirement on the runtime values of a method's tensor dimensions: the + * concrete tensors passed to and produced by the method must satisfy it in + * any given execution. Matched as a declaration during spec validation. + * @category Types + */ +export type RuntimeConstraint = LinearConstraint | EqualityConstraint; + +// ======================================================== +// Model specs +// ======================================================== + +/** + * Spec of a single model method: the ordered input and output parameter specs + * and the runtime constraints the method declares over its tensor dimensions. + * @category Types + */ +export type MethodSpec = { + inputs: readonly ParamSpec[]; + outputs: readonly ParamSpec[]; + runtimeConstraints: readonly RuntimeConstraint[]; +}; +/** + * Spec of a whole model, mapping method names to their {@link MethodSpec}. + * A `SymbolicDim` spec describes allowed models; a `ConcreteDim` spec + * describes an exported model. + * @category Types + */ +export type ModelSpec = Record>; + +/** + * Shape notation accepted by {@link SymbolicTensor}: numbers become + * {@link ConstantDim}, strings become {@link StaticDim}, and + * {@link SymbolicDim} values are used as-is. + * @category Types + */ export type SymbolicShape = readonly (number | string | SymbolicDim)[]; -export const S = (symbol: string): SymbolicDim => { +// ======================================================== +// Dim helper functions +// ======================================================== + +/** + * Creates a static symbolic dimension. Static symbols bind to constant + * dimensions of the exported spec; repeated uses must bind to the same value. + * @category Typescript API + * @param symbol The symbol name. + * @returns The symbolic dimension. + */ +export const StaticDim = (symbol: string): SymbolicDim => { return { kind: 'static', symbol }; }; -export const D = (symbol: string): SymbolicDim => { +/** + * Creates a dynamic symbolic dimension. Dynamic symbols bind to range or enum + * dimensions of the exported spec; repeated uses must bind to the same domain. + * @category Typescript API + * @param symbol The symbol name. + * @returns The symbolic dimension. + */ +export const DynamicDim = (symbol: string): SymbolicDim => { return { kind: 'dynamic', symbol }; }; -export const C = (value: number): ConcreteDim => { +/** + * Creates a constant dimension matching exactly `value`. + * @category Typescript API + * @param value The required dimension size. + * @returns The concrete dimension. + * @throws {Error} If `value` is not a positive integer. + */ +export const ConstantDim = (value: number): ConcreteDim => { if (value <= 0 || !Number.isInteger(value)) { throw new Error(`Invalid value (${value}): must be a positive integer.`); } return { kind: 'constant', value }; }; -export const R = (range: Range): ConcreteDim => { - if (range.min < 0) { - throw new Error(`Invalid range: dimension minimum cannot be negative.`); +/** + * Creates an enumerated dimension matching one of `choices`. + * @category Typescript API + * @param choices The allowed dimension sizes. + * @returns The concrete dimension. + * @throws {Error} If any choice is not a positive integer. + */ +export const EnumDim = (choices: readonly number[]): ConcreteDim => { + if (choices.some((dim) => dim <= 0 || !Number.isInteger(dim))) { + throw new Error(`Invalid enum choice: must be a positive integer`); } - if (range.max < range.min) { - throw new Error(`Invalid range [${range.min}, ${range.max}]: max cannot be less than min.`); + return { kind: 'enum', choices }; +}; + +/** + * Creates a range dimension matching values from `min` to `max` in increments + * of `step`. + * @category Typescript API + * @param min The smallest allowed dimension size. + * @param max The largest allowed dimension size. + * @param step The increment between allowed sizes. Defaults to 1. + * @returns The concrete dimension. + * @throws {Error} If the range bounds or step are not valid positive integers. + */ +export const RangeDim = (min: number, max: number, step?: number): ConcreteDim => { + if (min < 0 || !Number.isInteger(min)) { + throw new Error(`Invalid range min (${min}): must be a non-negative integer.`); } - if (!Number.isInteger(range.min)) { - throw new Error(`Invalid range min (${range.min}): must be a non-negative integer.`); + if (max < min) { + throw new Error(`Invalid range [${min}, ${max}]: max cannot be less than min.`); } - if (!Number.isInteger(range.max)) { - throw new Error(`Invalid range max (${range.max}): must be a non-negative integer.`); + if (!Number.isInteger(max)) { + throw new Error(`Invalid range max (${max}): must be a non-negative integer.`); } - if (range.step <= 0 || !Number.isInteger(range.step)) { - throw new Error(`Invalid range step (${range.step}): must be a positive integer.`); + if (step && (step <= 0 || !Number.isInteger(step))) { + throw new Error(`Invalid range step (${step}): must be a positive integer.`); } - return { kind: 'range', range }; + return { kind: 'range', range: { min, max, step: step ?? 1 } }; }; +/** + * Creates a {@link TensorSpec} from a dtype and a {@link SymbolicShape}: + * numbers become {@link ConstantDim}, strings become {@link StaticDim}. + * @category Typescript API + * @param dtype The expected element data type. + * @param shape The per-dimension specs. + * @returns The tensor spec. + */ export const SymbolicTensor = (dtype: DType, shape: SymbolicShape) => { const typedShape = shape.map((dim) => { - if (typeof dim === 'string') return S(dim); - if (typeof dim === 'number') return C(dim); + if (typeof dim === 'string') return StaticDim(dim); + if (typeof dim === 'number') return ConstantDim(dim); return dim; }); - return { kind: 'Tensor', dtype, shape: typedShape } as TensorConstraint; + return { kind: 'Tensor', dtype, shape: typedShape } as TensorSpec; }; -type SymbolBinding = { symbolic: Exclude; concrete: ConcreteDim }; -type SymbolBindings = readonly SymbolBinding[]; -type MatchResult = - | { readonly ok: true; readonly bindings: SymbolBindings } - | { readonly ok: false; readonly error: string }; +// ======================================================== +// Parameter spec validation +// ======================================================== -const ok = (bindings: SymbolBindings): MatchResult => ({ ok: true, bindings }); -const err = (error: string): MatchResult => ({ ok: false, error }); -const rangesEqual = (r1: Range, r2: Range): boolean => { +type SymbolBindings = Map; + +function rangesEqual(r1: Range, r2: Range): boolean { return r1.min === r2.min && r1.max === r2.max && r1.step === r2.step; -}; +} + +function choicesEqual(c1: readonly number[], c2: readonly number[]): boolean { + const s1 = new Set(c1); + const s2 = new Set(c2); + return s1.size === s2.size && [...s1].every((elem) => s2.has(elem)); +} function matchDim( sDim: SymbolicDim, cDim: ConcreteDim, bindings: SymbolBindings, ctx: string -): MatchResult { - if ('symbol' in sDim) { - if (bindings.some((b) => b.symbolic.symbol === sDim.symbol && b.symbolic.kind !== sDim.kind)) { - // returning err(...) here would treat a self-contradictory schema as a - // simple match failure and move on to the next variant. - throw new Error( - `Invalid schema ${ctx}: ${sDim.symbol} is used as both 'static' and 'dynamic'` - ); - } - } - +): void { if (sDim.kind === 'constant' && cDim.kind === 'constant') { if (sDim.value !== cDim.value) { - return err(`${ctx}: Constant dimension value mismatch.`); + throw new Error(`${ctx}: Constant dimension mismatch.`); } - return ok(bindings); + return; } if (sDim.kind === 'range' && cDim.kind === 'range') { if (!rangesEqual(sDim.range, cDim.range)) { - return err(`${ctx}: Range dimension mismatch`); + throw new Error(`${ctx}: Range dimension mismatch.`); } - return ok(bindings); + return; + } + + if (sDim.kind === 'enum' && cDim.kind === 'enum') { + if (!choicesEqual(sDim.choices, cDim.choices)) { + throw new Error(`${ctx}: Enum dimension mismatch.`); + } + return; } if (sDim.kind === 'static' && cDim.kind === 'constant') { - const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); - if (existing) { - if (existing.concrete.kind !== 'constant') { - throw new Error(`Matcher error ${ctx}: 'static' was matched to 'range'`); - } - if (cDim.value !== existing.concrete.value) { - return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); + const bind = bindings.get(sDim.symbol); + if (bind) { + if (bind.kind !== 'constant' || bind.value !== cDim.value) { + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); } - return ok(bindings); + return; } - return ok([...bindings, { symbolic: sDim, concrete: cDim }]); + bindings.set(sDim.symbol, cDim); + return; } - if (sDim.kind === 'dynamic' && cDim.kind === 'range') { - const existing = bindings.find((binding) => binding.symbolic.symbol === sDim.symbol); - if (existing) { - if (existing.concrete.kind !== 'range') { - throw new Error(`Matcher error ${ctx}: 'dynamic' was matched to 'constant'`); + if (sDim.kind === 'dynamic' && (cDim.kind === 'range' || cDim.kind === 'enum')) { + const bind = bindings.get(sDim.symbol); + if (bind) { + const consistentRange = + bind.kind === 'range' && cDim.kind === 'range' && rangesEqual(bind.range, cDim.range); + const consistentEnum = + bind.kind === 'enum' && cDim.kind === 'enum' && choicesEqual(bind.choices, cDim.choices); + if (!consistentRange && !consistentEnum) { + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); } - if (!rangesEqual(cDim.range, existing.concrete.range)) { - return err(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings`); - } - return ok(bindings); + return; } - return ok([...bindings, { symbolic: sDim, concrete: cDim }]); + bindings.set(sDim.symbol, cDim); + return; } - return err(`${ctx}: Cannot match symbolic ${sDim.kind} with concrete ${cDim.kind}.`); + throw new Error(`${ctx}: Cannot match symbolic '${sDim.kind}' with concrete '${cDim.kind}'.`); } -function matchSignature( - symbolicSgn: MethodSignature, - concreteSgn: MethodSignature, +function matchMethodSpecs( + allowedMethodSpec: MethodSpec, + exportedMethodSpec: MethodSpec, bindings: SymbolBindings, ctx: string -): MatchResult { - if (symbolicSgn.inputs.length !== concreteSgn.inputs.length) { - return err(`${ctx}: Input count mismatch.`); +): void { + if (allowedMethodSpec.inputs.length !== exportedMethodSpec.inputs.length) { + throw new Error(`${ctx}: Input count mismatch.`); } - if (symbolicSgn.outputs.length !== concreteSgn.outputs.length) { - return err(`${ctx}: Output count mismatch.`); + if (allowedMethodSpec.outputs.length !== exportedMethodSpec.outputs.length) { + throw new Error(`${ctx}: Output count mismatch.`); } - const symbolicConstraints = [...symbolicSgn.inputs, ...symbolicSgn.outputs]; - const concreteConstraints = [...concreteSgn.inputs, ...concreteSgn.outputs]; - const numConstraints = symbolicConstraints.length; - - let currentBindings = bindings; + const allowedParamSpecs = [...allowedMethodSpec.inputs, ...allowedMethodSpec.outputs]; + const exportedParamSpecs = [...exportedMethodSpec.inputs, ...exportedMethodSpec.outputs]; - for (let c = 0; c < numConstraints; ++c) { - const isInput = c < symbolicSgn.inputs.length; - const constraintIdx = isInput ? c : c - concreteSgn.inputs.length; - const constraintCtx = `${ctx} ${isInput ? 'input' : 'output'} #${constraintIdx}`; + for (let p = 0; p < allowedParamSpecs.length; ++p) { + const isInput = p < allowedMethodSpec.inputs.length; + const paramSpecIdx = isInput ? p : p - allowedMethodSpec.inputs.length; + const paramSpecCtx = `${ctx} ${isInput ? 'input' : 'output'} #${paramSpecIdx}`; - const symbolicConstr = symbolicConstraints[c]!; - const concreteConstr = concreteConstraints[c]!; + const allowedParamSpec = allowedParamSpecs[p]!; + const exportedParamSpec = exportedParamSpecs[p]!; - if (symbolicConstr.kind !== concreteConstr.kind) { - return err(`${constraintCtx}: Constraint kind mismatch.`); + if (allowedParamSpec.kind !== exportedParamSpec.kind) { + throw new Error(`${paramSpecCtx}: Param spec kind mismatch.`); } - if (symbolicConstr.kind !== 'Tensor' && concreteConstr.kind !== 'Tensor') continue; + if (allowedParamSpec.kind !== 'Tensor') continue; - const symbolicTensor = symbolicConstr as TensorConstraint; - const concreteTensor = concreteConstr as TensorConstraint; + const allowedTensorSpec = allowedParamSpec as TensorSpec; + const exportedTensorSpec = exportedParamSpec as TensorSpec; - if (symbolicTensor.dtype !== concreteTensor.dtype) { - return err(`${constraintCtx}: DType mismatch.`); + if (allowedTensorSpec.dtype !== exportedTensorSpec.dtype) { + throw new Error(`${paramSpecCtx}: DType mismatch.`); } - if (symbolicTensor.shape.length !== concreteTensor.shape.length) { - return err(`${constraintCtx}: Rank mismatch.`); + if (allowedTensorSpec.shape.length !== exportedTensorSpec.shape.length) { + throw new Error(`${paramSpecCtx}: Rank mismatch.`); } - const rank = symbolicTensor.shape.length; - - for (let d = 0; d < rank; ++d) { - const sDim = symbolicTensor.shape[d]!; - const cDim = concreteTensor.shape[d]!; - const dimCtx = `${constraintCtx} Tensor dim #${d}`; + for (let d = 0; d < allowedTensorSpec.shape.length; ++d) { + const dimCtx = `${paramSpecCtx} Tensor dim #${d}`; + matchDim(allowedTensorSpec.shape[d]!, exportedTensorSpec.shape[d]!, bindings, dimCtx); + } + } +} - const result = matchDim(sDim, cDim, currentBindings, dimCtx); - if (!result.ok) return result; - currentBindings = result.bindings; +function matchModelSpecsSymbols( + allowedModelSpec: ModelSpec, + exportedModelSpec: ModelSpec, + bindings: SymbolBindings +): void { + for (const [methodName, allowedMethodSpec] of Object.entries(allowedModelSpec)) { + const exportedMethodSpec = exportedModelSpec[methodName]; + if (!exportedMethodSpec) { + throw new Error(`Method '${methodName}' not found in exported model spec.`); } + matchMethodSpecs(allowedMethodSpec, exportedMethodSpec, bindings, `Method '${methodName}'`); } +} + +// ======================================================== +// Runtime constraints validation +// ======================================================== - return ok(currentBindings); +function refsEqual(r1: DimRef, r2: DimRef): boolean { + return r1.io === r2.io && r1.tensorIdx === r2.tensorIdx && r1.dimIdx === r2.dimIdx; } -function matchSchema( - symbolicSchema: ModelSchema, - concreteSchema: ModelSchema, - ctx: string -): MatchResult { - const methodNames = Object.keys(symbolicSchema); - for (const methodName of methodNames) { - if (!concreteSchema[methodName]) { - return err(`${ctx}: Method '${methodName}' not found in actual schema.`); +function constraintsEqual(c1: RuntimeConstraint, c2: RuntimeConstraint): boolean { + if (c1.kind === 'linear' && c2.kind === 'linear') { + return ( + refsEqual(c1.dimLhs, c2.dimLhs) && + refsEqual(c1.dimRhs, c2.dimRhs) && + c1.coefficients[0] === c2.coefficients[0] && + c1.coefficients[1] === c2.coefficients[1] + ); + } + + if (c1.kind === 'equality' && c2.kind === 'equality') { + if (c1.dims.length !== c2.dims.length) { + return false; } + + const unclaimed = [...c2.dims]; + for (const ref of c1.dims) { + const idx = unclaimed.findIndex((r) => refsEqual(r, ref)); + if (idx === -1) return false; + unclaimed.splice(idx, 1); + } + return true; + } + + return false; +} + +function resolveDim(methodSpec: MethodSpec, ref: DimRef): D { + let tensorSpecs: TensorSpec[]; + switch (ref.io) { + case 'input': + tensorSpecs = methodSpec.inputs.filter((v): v is TensorSpec => v.kind === 'Tensor'); + break; + case 'output': + tensorSpecs = methodSpec.outputs.filter((v): v is TensorSpec => v.kind === 'Tensor'); + break; + } + + const tensorSpec = tensorSpecs[ref.tensorIdx]; + if (!tensorSpec) { + throw new Error(`Invalid DimRef (${JSON.stringify(ref)}): tensor index out of range.`); } - const match = (nameIdx: number, sOverloadIdx: number, bindings: SymbolBindings): MatchResult => { - if (nameIdx >= methodNames.length) { - return ok(bindings); + const dim = tensorSpec.shape[ref.dimIdx]; + if (!dim) { + throw new Error(`Invalid DimRef (${JSON.stringify(ref)}): dimension index out of range.`); + } + + return dim; +} + +function matchRuntimeConstraints( + allowedModelSpec: ModelSpec, + exportedModelSpec: ModelSpec +): void { + for (const [methodName, allowedMethodSpec] of Object.entries(allowedModelSpec)) { + const exportedMethodSpec = exportedModelSpec[methodName]; + if (!exportedMethodSpec) { + throw new Error(`Method '${methodName}' not found in exported model spec.`); } - const methodName = methodNames[nameIdx]!; - const sOverloads = symbolicSchema[methodName]!; + const unclaimed = [...exportedMethodSpec.runtimeConstraints]; - if (sOverloadIdx >= sOverloads.length) { - return match(nameIdx + 1, 0, bindings); + for (const [idx, constraint] of allowedMethodSpec.runtimeConstraints.entries()) { + const find = unclaimed.findIndex((c) => constraintsEqual(c, constraint)); + if (find === -1) { + throw new Error(`Constraint ${idx}: Not declared by the exported model spec.`); + } + unclaimed.splice(find, 1); + } + + if (unclaimed.length > 0) { + throw new Error(`'${methodName}': Exported spec declares unexpected runtime constraints`); } + } +} - const cOverloads = concreteSchema[methodName]!; - const sOverload = sOverloads[sOverloadIdx]!; +// ======================================================== +// Spec validation +// ======================================================== - const errors: string[] = []; +function verifySymbolKindConsistency(modelSpec: ModelSpec): void { + const symbolKinds = new Map(); - for (const [idx, cOverload] of cOverloads.entries()) { - const overloadCtx = `${ctx}: Method '${methodName}' (symbolic overload #${sOverloadIdx}, concrete overload #${idx})`; + for (const methodSpec of Object.values(modelSpec)) { + for (const paramSpec of [...methodSpec.inputs, ...methodSpec.outputs]) { + if (paramSpec.kind !== 'Tensor') continue; - const sgnResult = matchSignature(sOverload, cOverload, bindings, overloadCtx); - if (!sgnResult.ok) { - errors.push(sgnResult.error); - continue; - } + for (const dim of paramSpec.shape) { + if (!('symbol' in dim)) continue; - const matchResult = match(nameIdx, sOverloadIdx + 1, sgnResult.bindings); - if (!matchResult.ok) { - errors.push(matchResult.error); - continue; + const existing = symbolKinds.get(dim.symbol); + if (existing && existing !== dim.kind) { + throw new Error(`Invalid spec: '${dim.symbol}' is used as both 'static' and 'dynamic'.`); + } + symbolKinds.set(dim.symbol, dim.kind); } - - return matchResult; } + } +} - return err(`${ctx}: Method '${methodName}' failed matching:\n - ` + errors.join('\n - ')); - }; +function verifyConstraintCorrectness(modelSpec: ModelSpec): void { + for (const [methodName, methodSpec] of Object.entries(modelSpec)) { + for (const [idx, constraint] of methodSpec.runtimeConstraints.entries()) { + const ctx = `Method '${methodName}' constraint ${idx}`; + + if (constraint.kind === 'linear') { + const [A, B] = constraint.coefficients; + if (!Number.isInteger(A) || !Number.isInteger(B)) { + throw new Error(`${ctx}: Coefficients must be integers.`); + } + resolveDim(methodSpec, constraint.dimLhs); + resolveDim(methodSpec, constraint.dimRhs); + } - return match(0, 0, []); + if (constraint.kind === 'equality') { + if (constraint.dims.length < 2) { + throw new Error(`${ctx}: Equality requires at least two dimensions.`); + } + constraint.dims.forEach((ref) => resolveDim(methodSpec, ref)); + } + } + } } -export function validateSchema( - symbolicVariants: ModelSchema[], - concreteSchema: ModelSchema -): Map { +/** + * Validates that an exported (concrete) model spec satisfies at least one of + * the allowed (symbolic) model specs — variants are tried in order and the + * first match wins. For a variant to match: + * - Every method exists in the exported spec and its signature matches, + * binding each symbol to a constant value (static) or a range/enum + * (dynamic). Repeated symbols must bind consistently across the whole spec. + * - The exported spec declares exactly the same runtime constraints per + * method (1-to-1, no missing, no extras). Constraints are matched as + * declarations only; whether they hold at runtime is the model's guarantee. + * + * Authoring bugs in an allowed spec (conflicting symbol kinds, invalid + * constraint coefficients or references) throw immediately, before matching. + * @param allowedModelSpecs The allowed model spec variants to try in order. + * @param exportedModelSpec The exported model spec to validate against. + * @returns The symbol bindings of the first matching variant. + * @throws {Error} A human-readable description of why every variant failed. + */ +export function validateSpec( + allowedModelSpecs: readonly ModelSpec[], + exportedModelSpec: ModelSpec +): SymbolBindings { + allowedModelSpecs.forEach(verifySymbolKindConsistency); + allowedModelSpecs.forEach(verifyConstraintCorrectness); + const errors: string[] = []; - for (const [idx, symbolicSchema] of symbolicVariants.entries()) { - const result = matchSchema(symbolicSchema, concreteSchema, `Variant ${idx}`); - if (!result.ok) { - errors.push(result.error); + for (const [idx, allowedModelSpec] of allowedModelSpecs.entries()) { + try { + const bindings: SymbolBindings = new Map(); + matchModelSpecsSymbols(allowedModelSpec, exportedModelSpec, bindings); + matchRuntimeConstraints(allowedModelSpec, exportedModelSpec); + return bindings; + } catch (e: any) { + errors.push(`Variant ${idx}: ${e.message}`); continue; } - return new Map( - result.bindings.map((binding) => { - switch (binding.concrete.kind) { - case 'constant': - return [binding.symbolic.symbol, binding.concrete.value]; - case 'range': - return [binding.symbolic.symbol, binding.concrete.range]; - } - }) - ); } - throw new Error(`Model schema doesn't match any of the provided variants:\n` + errors.join('\n')); + throw new Error(`Spec doesn't match any of the provided variants:\n - ${errors.join('\n - ')}`); } From 1beef4735df434583525f04a42498658dda58977 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 27 Jul 2026 13:06:32 +0200 Subject: [PATCH 06/31] update --- .../react-native-executorch/src/core/model.ts | 47 +++++-------- .../src/core/schema.ts | 66 ++++++++++++++----- 2 files changed, 63 insertions(+), 50 deletions(-) diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index f2cc0280f2..71857521ac 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -1,6 +1,6 @@ import { rnexecutorchJsi } from '../native/bridge'; import type { DType, Tensor } from './tensor'; -import type { ModelSpec, ConcreteDim } from './schema'; +import type { ExecuTorchTag, ModelSpec, ConcreteDim } from './schema'; declare const modelBrand: unique symbol; @@ -22,58 +22,41 @@ export type ModelOutput = Tensor | number | boolean | null; */ export type TensorMeta = { /** The name associated with this tensor slot (may be empty). */ - name: string; + readonly name: string; /** The number of dimensions. */ - ndim: number; + readonly ndim: number; /** The total byte size of the tensor buffer. */ - nbytes: number; + readonly nbytes: number; /** The element data type. */ - dtype: DType; + readonly dtype: DType; /** The concrete size of each dimension (e.g. `[1, 3, 224, 224]`). */ - shape: number[]; + readonly shape: number[]; }; -/** - * The ExecuTorch value-tag that classifies the runtime type of a model input or - * output slot. - * @category Types - */ -export type ExecuTorchTag = - | 'None' - | 'Tensor' - | 'Int' - | 'Double' - | 'Bool' - | 'String' - | 'ListBool' - | 'ListDouble' - | 'ListInt' - | 'ListTensor'; - /** * Metadata describing a single exported method of an ExecuTorch model. * @category Types */ export type ModelMethodMeta = { /** The exported method name (e.g. `'forward'`). */ - name: string; + readonly name: string; /** The total number of input arguments the method accepts. */ - numInputs: number; + readonly numInputs: number; /** The total number of output values the method returns. */ - numOutputs: number; + readonly numOutputs: number; /** Runtime value-tags for each input slot, in order. */ - inputTags: ExecuTorchTag[]; + readonly inputTags: readonly ExecuTorchTag[]; /** Runtime value-tags for each output slot, in order. */ - outputTags: ExecuTorchTag[]; + readonly outputTags: readonly ExecuTorchTag[]; /** * A map from backend name to a boolean indicating whether this method * delegates to that backend. */ - usesBackend: Record; + readonly usesBackend: Record; /** Detailed tensor metadata for every input tensor slot, in order. */ - inputTensorMeta: TensorMeta[]; + readonly inputTensorMeta: readonly TensorMeta[]; /** Detailed tensor metadata for every output tensor slot, in order. */ - outputTensorMeta: TensorMeta[]; + readonly outputTensorMeta: readonly TensorMeta[]; }; /** @@ -97,7 +80,7 @@ export interface Model { * Returns the list of exported method names available on this model (e.g. * `['forward']`). */ - getMethodNames(): string[]; + getMethodNames(): readonly string[]; /** * Returns detailed metadata for the specified exported method, including diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 30e96b9159..8415f10ae0 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -8,33 +8,46 @@ * - **allowed** (`SymbolicDim`) — written by a pipeline to state which models * it can work with. Dimensions may be named symbols: `static` symbols bind * to constant dimensions, `dynamic` symbols to ranges or enums, and reusing - * a symbol requires every occurrence to bind to the same domain (nothing - * more — symbol reuse implies no runtime relation). Several allowed specs - * can be passed as variants; matching any one of them is enough. + * a symbol requires every occurrence to bind to the same domain. Several + * allowed specs can be passed as variants; matching any one of them is + * enough. * - **exported** (`ConcreteDim`) — derived from an exported model's metadata, * stating what the model actually provides. * - * Two different validations must not be confused: + * **Dimension domains vs runtime values.** The central distinction of this + * module is between a dimension's *domain* and its *runtime value*. The + * domain is the set of values the dimension may take: `constant` is a + * singleton (the value is fully known statically), while `range` and `enum` + * are proper sets that only narrow the possibilities. The runtime value is + * the actual size of the dimension for a concrete tensor in one given + * execution — a single element drawn from the domain. + * + * Accordingly, two different validations must not be confused: * - **Spec validation** (this module, {@link validateSpec}) — a static, - * load-time check that an exported spec satisfies an allowed spec. Dimension - * domains must match exactly (symbol binding), and the exported spec must - * declare exactly the allowed spec's runtime constraints. Constraints are - * matched as declarations only: whether they actually hold is not, and - * cannot be, checked here. + * load-time check that an exported spec satisfies an allowed spec. It only + * ever compares domains: symbols bind to domains (repeated symbols to the + * same one), and constraints are matched as declarations. Domain equality + * says nothing about runtime values — two dimensions with the same domain + * may still take different values in an execution. * - **Runtime validation** (the native runtime, not this module) — at - * execution time the runtime checks the shapes of concrete tensors against - * the dimensions and enforces the declared runtime constraints. + * execution time the runtime checks that every concrete tensor shape lies + * within the declared domains and enforces the declared runtime + * constraints. * - * Note that only equality constraints on static dimensions are fully decided - * by spec validation: a constant dimension has exactly one possible value, - * so equal constants are equal at runtime — no enforcement left. Linear - * constraints, in contrast, are never evaluated against dimensions here — - * they are matched as declarations even between constants, so checking them - * always remains with the runtime. + * Runtime constraints are statements about runtime values, not domains: an + * equality constraint requires its dimensions' values to coincide in any + * given execution, and a linear constraint requires them to satisfy + * `lhs = a * rhs + b`. Since spec validation only sees domains, it cannot + * decide whether such a relation holds — the exported spec must simply + * declare exactly the allowed spec's constraints (1-to-1, no missing, no + * extras). The only exception is degenerate: a constant domain has exactly + * one possible value, so an equality constraint between constants is fully + * decided statically — equal constants are equal at runtime. Linear + * constraints, in contrast, are never evaluated against domains here, even + * between constants. * @packageDocumentation */ import type { DType } from './tensor'; -import type { ExecuTorchTag } from './model'; // ======================================================== // Parameter specs @@ -83,6 +96,23 @@ export type TensorSpec = { readonly shape: readonly Dim[]; }; +/** + * The ExecuTorch value-tag that classifies the runtime type of a model input or + * output slot. + * @category Types + */ +export type ExecuTorchTag = + | 'None' + | 'Tensor' + | 'Int' + | 'Double' + | 'Bool' + | 'String' + | 'ListBool' + | 'ListDouble' + | 'ListInt' + | 'ListTensor'; + /** * Spec of a single input or output parameter of a method — either a * {@link TensorSpec} or a primitive ExecuTorch value tag (`Int`, `Bool`, ...). From 7ef1c8b9c77a6ec2ebd06ea0fee46380dadedcf8 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 27 Jul 2026 23:44:53 +0200 Subject: [PATCH 07/31] update --- .../cpp/core/model.cpp | 357 +++---------- .../react-native-executorch/cpp/core/model.h | 6 +- .../cpp/core/schema.cpp | 477 ++++++++++++++++++ .../react-native-executorch/cpp/core/schema.h | 217 ++++++++ .../cpp/core/tensor_helpers.cpp | 109 ++-- .../cpp/core/tensor_helpers.h | 67 ++- .../react-native-executorch/src/core/model.ts | 67 +-- .../src/core/schema.ts | 20 +- 8 files changed, 892 insertions(+), 428 deletions(-) create mode 100644 packages/react-native-executorch/cpp/core/schema.cpp create mode 100644 packages/react-native-executorch/cpp/core/schema.h diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 5b9503cb82..3c1c7f02ee 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -10,6 +10,7 @@ #include #include "dtype.h" +#include "schema.h" #include "tensor_helpers.h" #include @@ -20,8 +21,6 @@ namespace { namespace jsi = facebook::jsi; -namespace types = rnexecutorch::core::types; -namespace tensor = rnexecutorch::core::tensor; template T unwrap(const std::string &ctx, executorch::runtime::Result result) { @@ -39,155 +38,6 @@ T unwrap(jsi::Runtime &rt, const std::string &ctx, executorch::runtime::Result(tensorMeta.sizes().size())); - jsTensorMeta.setProperty(rt, "nbytes", static_cast(tensorMeta.nbytes())); - - try { - auto dtypeStr = types::toString(types::fromScalarType(tensorMeta.scalar_type())); - jsTensorMeta.setProperty(rt, "dtype", jsi::String::createFromUtf8(rt, dtypeStr)); - } catch (const std::exception &) { - jsTensorMeta.setProperty(rt, "dtype", jsi::String::createFromUtf8(rt, "not supported")); - } - - auto jsShapeArray = jsi::Array(rt, tensorMeta.sizes().size()); - for (size_t i = 0; i < tensorMeta.sizes().size(); ++i) { - jsShapeArray.setValueAtIndex(rt, i, static_cast(tensorMeta.sizes()[i])); - } - jsTensorMeta.setProperty(rt, "shape", jsShapeArray); - - return jsTensorMeta; -} - -/** - * Parses compile-time dynamic dimension constraints for the inputs of a module - * method. - * - * @note This is a temporary workaround. ExecuTorch (.pte) model metadata - * natively serializes only the static upper-bound limits of dynamic/symbolic - * dimensions, and does not expose the active dynamic range (min, max, step) at - * runtime. - * - * To use this feature, the .pte model must be exported with a companion method - * named "get_dynamic_dims_" (e.g., "get_dynamic_dims_forward"). - * - * Python export requirements: - * 1. The companion method must take no arguments. - * 2. It must return a list of outputs matching the number of Tag::Tensor inputs - * of the target method (scalar inputs are excluded). - * 3. Each output must be a 2D int32 tensor of shape [rank, 3], where each row - * represents [min, max, step] constraints for the corresponding dimension of - * that input tensor. - * - * @param module The ExecuTorch extension module to query. - * @param methodName The name of the target module method (e.g. "forward"). - * @return A vector of SymbolicShape objects, or std::nullopt if the companion - * method is not defined. If returned, the vector's size is guaranteed - * to equal methodMeta.num_inputs(), containing parsed SymbolicShapes - * for tensor inputs and empty SymbolicShapes for non-tensor inputs. - */ -std::optional> -parseDynamicInputShapes(executorch::extension::Module &module, const std::string &methodName) { - using executorch::aten::ScalarType; - - const auto getDynamicShapesMethodName = std::format("get_dynamic_dims_{}", methodName); - const auto ctx = getDynamicShapesMethodName + ": "; - - auto methodNames = unwrap(ctx + "failed to get method names", module.method_names()); - if (methodName == getDynamicShapesMethodName || - !methodNames.contains(getDynamicShapesMethodName)) { - return std::nullopt; - } - - auto methodMeta = unwrap(std::format("{}failed to get meta for method '{}'", ctx, methodName), - module.method_meta(methodName)); - - size_t expectedTensorInputs = 0; - for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { - auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i), methodMeta.input_tag(i)); - if (tag == executorch::runtime::Tag::Tensor) { - expectedTensorInputs++; - } - } - - auto result = unwrap(ctx + "failed to execute", module.execute(getDynamicShapesMethodName)); - if (result.size() != expectedTensorInputs) { - throw std::runtime_error(std::format("{}number of outputs returned ({}) does not match the number of " - "tensor inputs declared by method '{}' ({})", - ctx, result.size(), methodName, expectedTensorInputs)); - } - - std::vector dynamicShapes; - dynamicShapes.reserve(methodMeta.num_inputs()); - size_t tensorIndex = 0; - - for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { - auto tag = unwrap(std::format("{}failed to get tag for input [{}]", ctx, i), - methodMeta.input_tag(i)); - - if (tag != executorch::runtime::Tag::Tensor) { - // Emplace an empty shape for non-tensor inputs to maintain a 1-to-1 - // index alignment between dynamicShapes and the model's inputs vector. - dynamicShapes.emplace_back(); - continue; - } - - const auto &out = result.at(tensorIndex); - if (!out.isTensor()) { - throw std::runtime_error(std::format("{}output[{}] is not a tensor", ctx, tensorIndex)); - } - - auto inputMeta = unwrap(std::format("{}failed to get tensor meta for input [{}]", ctx, i), - methodMeta.input_tensor_meta(i)); - const auto rank = inputMeta.sizes().size(); - const auto shapeTensor = out.toTensor(); - - if (shapeTensor.dim() != 2 || shapeTensor.size(1) != 3 || - shapeTensor.size(0) != static_cast(rank) || - shapeTensor.scalar_type() != ScalarType::Int) { - throw std::runtime_error(std::format("{}output[{}] expected to be a 2D int32_t tensor of shape [{}, 3]", - ctx, tensorIndex, rank)); - } - - const auto *shape = shapeTensor.const_data_ptr(); - tensor::SymbolicShape symbolicShape; - symbolicShape.reserve(rank); - - for (size_t axis = 0; axis < rank; ++axis) { - const auto minDim = shape[axis * 3 + 0]; - const auto maxDim = shape[axis * 3 + 1]; - const auto step = shape[axis * 3 + 2]; - if (minDim < 0 || maxDim < minDim || step < 1) { - throw std::runtime_error(std::format("{}output[{}], axis {} is invalid: " - "expected 0 <= min <= max and step >= 1 but got [{}, {}, {}]", - ctx, tensorIndex, axis, minDim, maxDim, step)); - } - if (maxDim > inputMeta.sizes()[axis]) { - throw std::runtime_error(std::format("{}output[{}], axis {} max dimension ({}) " - "exceeds model metadata upper limit ({})", - ctx, tensorIndex, axis, maxDim, inputMeta.sizes()[axis])); - } - - symbolicShape.emplace_back(tensor::RangeDim{.min = minDim, .max = maxDim, .step = step}); - } - - dynamicShapes.push_back(std::move(symbolicShape)); - ++tensorIndex; - } - - return dynamicShapes; -} } // namespace namespace rnexecutorch::core::model { @@ -196,21 +46,43 @@ namespace conversions = rnexecutorch::core::conversions; using rnexecutorch::core::tensor::TensorHostObject; +constexpr auto kSchemaMethod = "get_model_schema"; + ModelHostObject::ModelHostObject(const std::string &modelPath) : modelPath_(modelPath), etModule_(std::make_unique(modelPath)) { + auto error = etModule_->load(); if (!etModule_->is_loaded()) { const std::string errorMsg = executorch::runtime::to_string(error); throw std::runtime_error(std::format("Failed to load model: {}", errorMsg)); } - auto methodNames = unwrap("Failed to get method names", etModule_->method_names()); + const auto methodNames = unwrap("loadModel", etModule_->method_names()); + schema::ModelSpec overrideSpec; + + if (methodNames.contains(kSchemaMethod)) { + auto ctx = std::format("loadModel: {}", kSchemaMethod); + auto result = unwrap(ctx, etModule_->execute(kSchemaMethod)); + + if (result.empty() || result[0].tag != executorch::runtime::Tag::String) { + throw std::runtime_error(std::format("{} must return a single string value", ctx)); + } + + auto jsonStr = std::string(result[0].toString()); + overrideSpec = schema::parseModelSpecJson(ctx, jsonStr); + } + for (const auto &methodName : methodNames) { - auto dynamicShapes = parseDynamicInputShapes(*etModule_, methodName); - if (dynamicShapes) { - dynamicInputShapes_.emplace(methodName, std::move(*dynamicShapes)); + auto methodMeta = unwrap("loadModel", etModule_->method_meta(methodName)); + spec_[methodName] = schema::methodSpecFromMetadata(methodMeta); + backends_[methodName] = schema::getUsedBackends(methodMeta); + + if (overrideSpec.contains(methodName)) { + spec_[methodName] = std::move(overrideSpec[methodName]); } + + schema::validateSpecAgainstMeta(spec_[methodName], methodMeta, methodName); } } @@ -221,105 +93,12 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { return jsi::String::createFromUtf8(rt, modelPath_); } - if (nameStr == "getMethodNames") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value * /*args*/, size_t count) -> jsi::Value { - if (count != 0) { - throw jsi::JSError(rt, "getMethodNames: Usage: getMethodNames()"); - } - - std::unique_lock lock(self->mutex_, std::try_to_lock); - if (!lock.owns_lock()) { - throw jsi::JSError(rt, "getMethodNames: Model is currently in use"); - } - - if (!self->etModule_) { - throw jsi::JSError(rt, "getMethodNames: Model has been disposed"); - } - - auto methodNames = unwrap(rt, "getMethodNames", self->etModule_->method_names()); - - auto jsArray = jsi::Array(rt, methodNames.size()); - size_t index = 0; - for (const auto &methodName : methodNames) { - jsArray.setValueAtIndex(rt, index, jsi::String::createFromUtf8(rt, methodName)); - ++index; - } - - return jsArray; - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getMethodNames"), 0, fnBody); + if (nameStr == "schema") { + return schema::modelSpecToJs(rt, spec_); } - if (nameStr == "getMethodMeta") { - auto self = shared_from_this(); - auto fnBody = [self](jsi::Runtime &rt, const jsi::Value & /*thisVal*/, const jsi::Value *args, size_t count) -> jsi::Value { - using executorch::runtime::tag_to_string; - - if (count != 1) { - throw jsi::JSError(rt, "getMethodMeta: Usage: getMethodMeta(methodName)"); - } - - std::unique_lock lock(self->mutex_, std::try_to_lock); - if (!lock.owns_lock()) { - throw jsi::JSError(rt, "getMethodMeta: Model is currently in use"); - } - - if (!self->etModule_) { - throw jsi::JSError(rt, "getMethodMeta: Model has been disposed"); - } - - auto methodName = conversions::asType(rt, "getMethodMeta: methodName", args[0]); - auto methodMeta = unwrap(rt, "getMethodMeta", self->etModule_->method_meta(methodName)); - - auto inputTagsArray = jsi::Array(rt, methodMeta.num_inputs()); - for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { - auto ctx = std::format("getMethodMeta: input tag [{}]", i); - auto tag = unwrap(rt, ctx, methodMeta.input_tag(i)); - inputTagsArray.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, tag_to_string(tag))); - } - - auto outputTagsArray = jsi::Array(rt, methodMeta.num_outputs()); - for (size_t i = 0; i < methodMeta.num_outputs(); ++i) { - auto ctx = std::format("getMethodMeta: output tag [{}]", i); - auto tag = unwrap(rt, ctx, methodMeta.output_tag(i)); - outputTagsArray.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, tag_to_string(tag))); - } - - auto usesBackendMap = jsi::Object(rt); - for (size_t i = 0; i < methodMeta.num_backends(); ++i) { - auto ctx = std::format("getMethodMeta: backend name [{}]", i); - const auto *backendName = unwrap(rt, ctx, methodMeta.get_backend_name(i)); - usesBackendMap.setProperty(rt, backendName, methodMeta.uses_backend(backendName)); - } - - auto inputTensorMetaArray = jsi::Array(rt, methodMeta.num_inputs()); - for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { - auto ctx = std::format("getMethodMeta: input tensor meta [{}]", i); - auto tensorMeta = unwrap(rt, ctx, methodMeta.input_tensor_meta(i)); - inputTensorMetaArray.setValueAtIndex(rt, i, tensorMetaToJs(rt, tensorMeta)); - } - - auto outputTensorMetaArray = jsi::Array(rt, methodMeta.num_outputs()); - for (size_t i = 0; i < methodMeta.num_outputs(); ++i) { - auto ctx = std::format("getMethodMeta: output tensor meta [{}]", i); - auto tensorMeta = unwrap(rt, ctx, methodMeta.output_tensor_meta(i)); - outputTensorMetaArray.setValueAtIndex(rt, i, tensorMetaToJs(rt, tensorMeta)); - } - - auto jsMeta = jsi::Object(rt); - jsMeta.setProperty(rt, "name", jsi::String::createFromUtf8(rt, methodMeta.name())); - jsMeta.setProperty(rt, "numInputs", static_cast(methodMeta.num_inputs())); - jsMeta.setProperty(rt, "numOutputs", static_cast(methodMeta.num_outputs())); - jsMeta.setProperty(rt, "inputTags", inputTagsArray); - jsMeta.setProperty(rt, "outputTags", outputTagsArray); - jsMeta.setProperty(rt, "usesBackend", usesBackendMap); - jsMeta.setProperty(rt, "inputTensorMeta", inputTensorMetaArray); - jsMeta.setProperty(rt, "outputTensorMeta", outputTensorMetaArray); - - return jsMeta; - }; - return jsi::Function::createFromHostFunction(rt, jsi::PropNameID::forAscii(rt, "getMethodMeta"), 1, fnBody); + if (nameStr == "backends") { + return schema::backendsToJs(rt, backends_); } if (nameStr == "execute") { @@ -339,44 +118,39 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } auto methodName = conversions::asType(rt, "execute: methodName", args[0]); - auto methodMeta = unwrap(rt, "execute", self->etModule_->method_meta(methodName)); + if (!self->spec_.contains(methodName)) { + throw jsi::JSError(rt, std::format("execute: Unknown method '{}'", methodName)); + } + const auto &methodSpec = self->spec_.at(methodName); auto inputsArray = conversions::asType(rt, "execute: inputs", args[1]); auto outputTensorsArray = conversions::asType(rt, "execute: outputTensors", args[2]); - if (inputsArray.size(rt) != methodMeta.num_inputs()) { - throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs: got {}, expected {}", - inputsArray.size(rt), methodMeta.num_inputs())); + if (inputsArray.size(rt) != methodSpec.inputs.size()) { + throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs")); } - std::vector inputs(methodMeta.num_inputs()); + std::vector inputs(methodSpec.inputs.size()); std::vector> tensorLocks; std::unordered_set lockedTensors; + std::vector> inputShapes; - for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + for (size_t i = 0; i < methodSpec.inputs.size(); ++i) { auto ctx = std::format("execute: inputs[{}]", i); - auto tag = unwrap(rt, ctx, methodMeta.input_tag(i)); + auto tag = methodSpec.inputs[i].tag; auto val = inputsArray.getValueAtIndex(rt, i); switch (tag) { case executorch::runtime::Tag::Tensor: { - auto tensorMeta = unwrap(rt, ctx + ": tensor meta", methodMeta.input_tensor_meta(i)); - auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type()); - - std::shared_ptr tensorHostObject; - if (self->dynamicInputShapes_.contains(methodName)) { - auto expectedShape = self->dynamicInputShapes_[methodName][i]; - tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, expectedShape); - } else { - tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, tensorMeta.sizes()); - } + const auto &tSpec = methodSpec.inputs[i]; + auto tensorHostObject = tensor::fromJs(rt, ctx, val, tSpec.dtype, tSpec.shape); if (!lockedTensors.insert(tensorHostObject.get()).second) { throw jsi::JSError(rt, "execute: Tensor aliasing detected. " "The same tensor was passed multiple times."); } tensorLocks.emplace_back(tensor::tryLockUnique(rt, ctx, tensorHostObject)); - inputs[i] = tensorHostObject->tensor_; + inputShapes.push_back(tensorHostObject->shape_); break; } case executorch::runtime::Tag::Double: @@ -397,6 +171,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } } + schema::validateRuntimeConstraints(rt, methodSpec.runtimeConstraints, inputShapes, + std::format("execute: method '{}'", methodName)); + auto startTime = std::chrono::high_resolution_clock::now(); auto executeResult = self->etModule_->execute(methodName, inputs); auto finishTime = std::chrono::high_resolution_clock::now(); @@ -409,15 +186,19 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { logFn.callWithThis(rt, consoleObj, {jsi::String::createFromUtf8(rt, info)}); #endif - auto result = unwrap(rt, - std::format("execute: Method '{}' failed (check getMethodMeta() " - "for required backends and getRegisteredBackends() " - "for registered ones)", - methodName), - std::move(executeResult)); + const auto *error = "execute: Method '{}' failed. Check the following common pitfalls:" + "\n- Make sure that the backends used by the model (`model.backends`)" + "\n are registered in ExecuTorch runtime (`getRegisteredBackends`)." + "\n- If your model requires specific runtime constraints or dynamic shapes" + "\n (e.g. equality of certain dynamic dimensions), make sure you exported" + "\n it with '{}' companion method that returns JSON string with dynamic" + "\n model spec that overrides the default one constructed from ExecuTorch" + "\n metadata which only contain the static upper bounds of tensor dimension" + "\n and do not contain information about runtime constraints."; + auto result = unwrap(rt, std::format(error, methodName, kSchemaMethod), std::move(executeResult)); auto jsOutputArray = jsi::Array(rt, result.size()); - size_t index = 0; + size_t outputIdx = 0; size_t tensorOutputIdx = 0; for (const auto &output : result) { @@ -430,9 +211,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto ctx = std::format("execute: outputTensors[{}]", tensorOutputIdx); auto val = outputTensorsArray.getValueAtIndex(rt, tensorOutputIdx); - auto tensorMeta = unwrap(rt, ctx + ": tensor meta", methodMeta.output_tensor_meta(index)); - auto expectedDtype = fromScalarType(rt, ctx, tensorMeta.scalar_type()); - auto tensorHostObject = tensor::fromJs(rt, ctx, val, expectedDtype, output.toTensor().sizes()); + auto dtype = types::fromScalarType(output.toTensor().dtype()); + auto shape = output.toTensor().sizes(); + auto tensorHostObject = tensor::fromJs(rt, ctx, val, dtype, shape); if (!lockedTensors.insert(tensorHostObject.get()).second) { throw jsi::JSError(rt, "execute: Tensor aliasing detected. " @@ -443,28 +224,28 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { output.toTensor().const_data_ptr(), output.toTensor().nbytes()); - jsOutputArray.setValueAtIndex(rt, index, val); + jsOutputArray.setValueAtIndex(rt, outputIdx, val); ++tensorOutputIdx; break; } case executorch::runtime::Tag::Double: - jsOutputArray.setValueAtIndex(rt, index, output.toDouble()); + jsOutputArray.setValueAtIndex(rt, outputIdx, output.toDouble()); break; case executorch::runtime::Tag::Int: - jsOutputArray.setValueAtIndex(rt, index, static_cast(output.toInt())); + jsOutputArray.setValueAtIndex(rt, outputIdx, static_cast(output.toInt())); break; case executorch::runtime::Tag::Bool: - jsOutputArray.setValueAtIndex(rt, index, output.toBool()); + jsOutputArray.setValueAtIndex(rt, outputIdx, output.toBool()); break; case executorch::runtime::Tag::None: - jsOutputArray.setValueAtIndex(rt, index, jsi::Value::null()); + jsOutputArray.setValueAtIndex(rt, outputIdx, jsi::Value::null()); break; default: throw jsi::JSError(rt, std::format("execute: Unsupported return type: {}", executorch::runtime::tag_to_string(output.tag))); } - ++index; + ++outputIdx; } return jsOutputArray; @@ -498,8 +279,8 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { std::vector ModelHostObject::getPropertyNames(jsi::Runtime &rt) { std::vector properties; properties.push_back(jsi::PropNameID::forAscii(rt, "path")); - properties.push_back(jsi::PropNameID::forAscii(rt, "getMethodNames")); - properties.push_back(jsi::PropNameID::forAscii(rt, "getMethodMeta")); + properties.push_back(jsi::PropNameID::forAscii(rt, "schema")); + properties.push_back(jsi::PropNameID::forAscii(rt, "backends")); properties.push_back(jsi::PropNameID::forAscii(rt, "execute")); properties.push_back(jsi::PropNameID::forAscii(rt, "dispose")); return properties; diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index b196d48664..d8087b2729 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -6,6 +6,7 @@ #include #include +#include "schema.h" #include "tensor_helpers.h" #include @@ -32,10 +33,11 @@ class ModelHostObject : public jsi::HostObject, private: std::string modelPath_; + schema::ModelSpec spec_; + std::unordered_map> backends_; + std::unique_ptr etModule_; std::mutex mutex_; - - std::unordered_map> dynamicInputShapes_; }; void install_loadModel(jsi::Runtime &rt, jsi::Object &module); diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp new file mode 100644 index 0000000000..95a3a2d3d3 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -0,0 +1,477 @@ +#include "schema.h" + +#include +#include + +#include +#include + +#include "dtype.h" + +namespace nlohmann { +template <> +// Tag lives in executorch::runtime; adl_serializer is the +// nlohmann-blessed extension point for types we don't own. +// See: https://github.com/nlohmann/json#how-do-i-convert-third-party-types +struct adl_serializer { + static executorch::runtime::Tag from_json(const json &j) { + auto s = j.get(); + if (s == "Tensor") { + return executorch::runtime::Tag::Tensor; + } + if (s == "Int") { + return executorch::runtime::Tag::Int; + } + if (s == "None") { + return executorch::runtime::Tag::None; + } + if (s == "Bool") { + return executorch::runtime::Tag::Bool; + } + if (s == "Double") { + return executorch::runtime::Tag::Double; + } + if (s == "String") { + return executorch::runtime::Tag::String; + } + if (s == "ListInt") { + return executorch::runtime::Tag::ListInt; + } + if (s == "ListBool") { + return executorch::runtime::Tag::ListBool; + } + if (s == "ListDouble") { + return executorch::runtime::Tag::ListDouble; + } + if (s == "ListTensor") { + return executorch::runtime::Tag::ListTensor; + } + throw std::runtime_error(std::format("unknown param kind '{}'", s)); + } + static void to_json(json &j, executorch::runtime::Tag t) { + j = executorch::runtime::tag_to_string(t); + } +}; +} // namespace nlohmann + +namespace rnexecutorch::core::schema { +namespace types = rnexecutorch::core::types; + +using executorch::runtime::Tag; +using nlohmann::json; + +namespace { + +template +T unwrap(const std::string &ctx, executorch::runtime::Result result) { + if (!result.ok()) { + throw std::runtime_error(std::format("{}: {}", ctx, executorch::runtime::to_string(result.error()))); + } + return std::move(result.get()); +} + +} // namespace + +// ======================================================== +// Nlohmann JSON conversions +// ======================================================== + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RangeDim, min, max, step) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EnumDim, choices) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(DimRef, side, tensorIdx, dimIdx) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EqualityConstraint, dims) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(LinearConstraint, dimLhs, dimRhs, coefficients) +NLOHMANN_JSON_SERIALIZE_ENUM(ParameterSide, { + {ParameterSide::input, "input"}, + {ParameterSide::output, "output"}, + }) +NLOHMANN_JSON_SERIALIZE_ENUM(types::DType, { + {types::DType::uint8, "uint8"}, + {types::DType::int32, "int32"}, + {types::DType::int64, "int64"}, + {types::DType::float32, "float32"}, + }) + +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void from_json(const json &j, ConcreteDim &d) { + auto kind = j.at("kind").get(); + if (kind == "constant") { + d = j.at("value").get(); + } else if (kind == "range") { + d = j.at("range").get(); + } else if (kind == "enum") { + d = j.at("choices").get(); + } else { + throw std::runtime_error(std::format("unsupported dim kind '{}'", kind)); + } +} +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void to_json(json &j, const ConcreteDim &d) { + if (const auto *c = std::get_if(&d)) { + j = json::object({{"kind", "constant"}, {"value", *c}}); + } + if (const auto *r = std::get_if(&d)) { + j = json::object({{"kind", "range"}, {"range", *r}}); + } + if (const auto *e = std::get_if(&d)) { + j = json::object({{"kind", "enum"}, {"choices", *e}}); + } +} + +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void from_json(const json &j, ParamSpec &p) { + p.tag = j.at("kind").get(); + if (p.tag == Tag::Tensor) { + p.dtype = j.at("dtype").get(); + p.shape = j.at("shape").get>(); + } +} +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void to_json(json &j, const ParamSpec &p) { + if (p.tag == Tag::Tensor) { + j = json::object({{"kind", "Tensor"}, {"dtype", p.dtype}, {"shape", p.shape}}); + } else { + j = json::object({{"kind", p.tag}}); + } +} + +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void from_json(const json &j, RuntimeConstraint &c) { + auto kind = j.at("kind").get(); + if (kind == "equality") { + c = j.get(); + } else if (kind == "linear") { + c = j.get(); + } else { + throw std::runtime_error(std::format("unknown constraint kind '{}'", kind)); + } +} +// NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. +void to_json(json &j, const RuntimeConstraint &c) { + if (const auto *eq = std::get_if(&c)) { + j = json::object({{"kind", "equality"}, {"dims", eq->dims}}); + } + if (const auto *lin = std::get_if(&c)) { + j = json::object({{"kind", "linear"}, + {"dimLhs", lin->dimLhs}, + {"dimRhs", lin->dimRhs}, + {"coefficients", lin->coefficients}}); + } +} + +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MethodSpec, inputs, outputs, runtimeConstraints) + +// ======================================================== +// Serialization +// ======================================================== + +ModelSpec parseModelSpecJson(const std::string &ctx, const std::string &jsonStr) { + try { + return json::parse(jsonStr).get(); + } catch (const std::exception &e) { + throw std::runtime_error(std::format("{}: {}", ctx, e.what())); + } +} + +namespace { + +// NOLINTNEXTLINE(misc-no-recursion): recursion is bounded by JSON nesting depth. +jsi::Value jsonToJs(jsi::Runtime &rt, const json &j) { + switch (j.type()) { + case json::value_t::null: + return jsi::Value::null(); + case json::value_t::boolean: + return jsi::Value(j.get()); + case json::value_t::number_integer: + case json::value_t::number_unsigned: + return jsi::Value(static_cast(j.get())); + case json::value_t::number_float: + return jsi::Value(j.get()); + case json::value_t::string: + return jsi::String::createFromUtf8(rt, j.get()); + case json::value_t::array: { + auto arr = jsi::Array(rt, j.size()); + size_t i = 0; + for (const auto &el : j) { + arr.setValueAtIndex(rt, i++, jsonToJs(rt, el)); + } + return arr; + } + case json::value_t::object: { + auto obj = jsi::Object(rt); + for (const auto &[k, v] : j.items()) { + obj.setProperty(rt, k.c_str(), jsonToJs(rt, v)); + } + return obj; + } + default: + return jsi::Value::undefined(); + } +} + +} // namespace + +jsi::Value modelSpecToJs(jsi::Runtime &rt, const ModelSpec &spec) { + return jsonToJs(rt, spec); +} + +jsi::Object backendsToJs(jsi::Runtime &rt, + const std::unordered_map> &backends) { + jsi::Object obj(rt); + for (const auto &[methodName, backendList] : backends) { + auto arr = jsi::Array(rt, backendList.size()); + for (size_t i = 0; i < backendList.size(); ++i) { + arr.setValueAtIndex(rt, i, jsi::String::createFromUtf8(rt, backendList[i])); + } + obj.setProperty(rt, methodName.c_str(), arr); + } + return obj; +} + +// ======================================================== +// Metadata reflection +// ======================================================== + +namespace { + +ParamSpec tensorMetaToParamSpec(const executorch::runtime::TensorInfo &tensorMeta) { + ParamSpec p{.tag = Tag::Tensor, .dtype = types::fromScalarType(tensorMeta.scalar_type())}; + const auto sizes = tensorMeta.sizes(); + p.shape.reserve(sizes.size()); + for (const auto size : sizes) { + p.shape.emplace_back(size); + } + return p; +} + +} // namespace + +MethodSpec methodSpecFromMetadata(const executorch::runtime::MethodMeta &methodMeta) { + MethodSpec spec; + + spec.inputs.reserve(methodMeta.num_inputs()); + for (size_t i = 0; i < methodMeta.num_inputs(); ++i) { + auto ctx = std::format("buildFromMetadata input[{}]", i); + auto tag = unwrap(ctx, methodMeta.input_tag(i)); + if (tag == Tag::Tensor) { + auto tensorMeta = unwrap(ctx, methodMeta.input_tensor_meta(i)); + spec.inputs.emplace_back(tensorMetaToParamSpec(tensorMeta)); + } else { + spec.inputs.emplace_back(ParamSpec{.tag = tag}); + } + } + + spec.outputs.reserve(methodMeta.num_outputs()); + for (size_t i = 0; i < methodMeta.num_outputs(); ++i) { + auto ctx = std::format("buildFromMetadata output[{}]", i); + auto tag = unwrap(ctx, methodMeta.output_tag(i)); + if (tag == Tag::Tensor) { + auto tensorMeta = unwrap(ctx, methodMeta.output_tensor_meta(i)); + spec.outputs.emplace_back(tensorMetaToParamSpec(tensorMeta)); + } else { + spec.outputs.emplace_back(ParamSpec{.tag = tag}); + } + } + + return spec; +} + +std::vector getUsedBackends(const executorch::runtime::MethodMeta &methodMeta) { + std::vector backends; + for (size_t i = 0; i < methodMeta.num_backends(); ++i) { + auto ctx = std::format("getUsedBackends: backend [{}]", i); + const auto *name = unwrap(ctx, methodMeta.get_backend_name(i)); + if (methodMeta.uses_backend(name)) { + backends.emplace_back(name); + } + } + return backends; +} + +// ======================================================== +// Validation +// ======================================================== + +namespace { + +void validateParamAgainstMeta(const ParamSpec ¶m, + const executorch::runtime::TensorInfo &tensorMeta, + const std::string &ctx) { + auto metaDtype = types::fromScalarType(tensorMeta.scalar_type()); + if (param.dtype != metaDtype) { + throw std::runtime_error(std::format("{}: dtype mismatch: schema has '{}', metadata has '{}", + ctx, types::toString(param.dtype), types::toString(metaDtype))); + } + auto metaShape = tensorMeta.sizes(); + if (param.shape.size() != metaShape.size()) { + throw std::runtime_error(std::format("{}: rank mismatch: schema has {}, metadata has {}", + ctx, param.shape.size(), metaShape.size())); + } + for (size_t d = 0; d < param.shape.size(); ++d) { + if (const auto *c = std::get_if(¶m.shape[d])) { + if (*c != metaShape[d]) { + throw std::runtime_error(std::format("{}: shape[{}] mismatch: schema has {}, metadata has {}", + ctx, d, *c, metaShape[d])); + } + } + } +} + +void validateDimRefAgainstSpec(const DimRef &ref, + size_t numTensorInputs, size_t numTensorOutputs, + const std::vector &inputRanks, + const std::vector &outputRanks, + const std::string &ctx) { + bool isInput = (ref.side == ParameterSide::input); + size_t numTensors = isInput ? numTensorInputs : numTensorOutputs; + const auto &ranks = isInput ? inputRanks : outputRanks; + if (std::cmp_greater_equal(ref.tensorIdx, numTensors)) { + throw std::runtime_error(std::format("{}: DimRef tensorIdx {} out of range (have {} {} tensors)", + ctx, ref.tensorIdx, numTensors, isInput ? "input" : "output")); + } + if (std::cmp_greater_equal(ref.dimIdx, ranks[static_cast(ref.tensorIdx)])) { + throw std::runtime_error(std::format("{}: DimRef dimIdx {} out of range for tensor (rank {})", + ctx, ref.dimIdx, ranks[static_cast(ref.tensorIdx)])); + } +} + +void validateConstraintsAgainstSpec(const MethodSpec &spec, const std::string &methodName) { + size_t numTensorInputs = 0; + std::vector inputRanks; + for (const auto &p : spec.inputs) { + if (p.tag == Tag::Tensor) { + inputRanks.push_back(p.shape.size()); + ++numTensorInputs; + } + } + size_t numTensorOutputs = 0; + std::vector outputRanks; + for (const auto &p : spec.outputs) { + if (p.tag == Tag::Tensor) { + outputRanks.push_back(p.shape.size()); + ++numTensorOutputs; + } + } + + for (size_t i = 0; i < spec.runtimeConstraints.size(); ++i) { + auto ctx = std::format("loadModel: method '{}' constraint[{}]", methodName, i); + const auto &constraint = spec.runtimeConstraints[i]; + + if (const auto *eq = std::get_if(&constraint)) { + if (eq->dims.size() < 2) { + throw std::runtime_error(std::format("{}: equality constraint requires at least two dims", ctx)); + } + for (const auto &dim : eq->dims) { + validateDimRefAgainstSpec(dim, numTensorInputs, numTensorOutputs, + inputRanks, outputRanks, ctx); + } + } else if (const auto *lin = std::get_if(&constraint)) { + validateDimRefAgainstSpec(lin->dimLhs, numTensorInputs, numTensorOutputs, + inputRanks, outputRanks, ctx + " dimLhs"); + validateDimRefAgainstSpec(lin->dimRhs, numTensorInputs, numTensorOutputs, + inputRanks, outputRanks, ctx + " dimRhs"); + } + } +} + +} // namespace + +void validateSpecAgainstMeta(const MethodSpec &spec, + const executorch::runtime::MethodMeta &meta, + const std::string &methodName) { + if (spec.inputs.size() != meta.num_inputs()) { + throw std::runtime_error(std::format("loadModel: method '{}' input count mismatch: " + "schema has {}, metadata has {}", + methodName, spec.inputs.size(), meta.num_inputs())); + } + if (spec.outputs.size() != meta.num_outputs()) { + throw std::runtime_error(std::format("loadModel: method '{}' output count mismatch: " + "schema has {}, metadata has {}", + methodName, spec.outputs.size(), meta.num_outputs())); + } + + for (size_t i = 0; i < spec.inputs.size(); ++i) { + auto ctx = std::format("loadModel: method '{}' input[{}]", methodName, i); + auto metaTag = unwrap(ctx, meta.input_tag(i)); + if (spec.inputs[i].tag != metaTag) { + throw std::runtime_error(std::format("{}: tag mismatch: schema has '{}', metadata has '{}", + ctx, executorch::runtime::tag_to_string(spec.inputs[i].tag), + executorch::runtime::tag_to_string(metaTag))); + } + if (metaTag == Tag::Tensor) { + auto tensorMeta = unwrap(ctx, meta.input_tensor_meta(i)); + validateParamAgainstMeta(spec.inputs[i], tensorMeta, ctx); + } + } + + for (size_t i = 0; i < spec.outputs.size(); ++i) { + auto ctx = std::format("loadModel: method '{}' output[{}]", methodName, i); + auto metaTag = unwrap(ctx, meta.output_tag(i)); + if (spec.outputs[i].tag != metaTag) { + throw std::runtime_error(std::format("{}: tag mismatch: schema has '{}', metadata has '{}", + ctx, executorch::runtime::tag_to_string(spec.outputs[i].tag), + executorch::runtime::tag_to_string(metaTag))); + } + if (metaTag == Tag::Tensor) { + auto tensorMeta = unwrap(ctx, meta.output_tensor_meta(i)); + validateParamAgainstMeta(spec.outputs[i], tensorMeta, ctx); + } + } + + validateConstraintsAgainstSpec(spec, methodName); +} + +namespace { + +int32_t getInputDimValue(const DimRef &ref, + const std::vector> &inputShapes) { + return inputShapes[static_cast(ref.tensorIdx)][static_cast(ref.dimIdx)]; +} + +} // namespace + +void validateRuntimeConstraints(jsi::Runtime &rt, + const std::vector &constraints, + const std::vector> &inputShapes, + const std::string &ctx) { + for (size_t i = 0; i < constraints.size(); ++i) { + const auto &constraint = constraints[i]; + auto cctx = std::format("{} constraint[{}]", ctx, i); + + if (const auto *eq = std::get_if(&constraint)) { + if (eq->dims.size() < 2) { + continue; + } + // Skip if any dim references an output — ExecuTorch validates output shapes + if (std::ranges::any_of(eq->dims, + [](const auto &d) { return d.side == ParameterSide::output; })) { + continue; + } + int32_t first = getInputDimValue(eq->dims[0], inputShapes); + for (size_t j = 1; j < eq->dims.size(); ++j) { + int32_t val = getInputDimValue(eq->dims[j], inputShapes); + if (val != first) { + throw jsi::JSError(rt, std::format("{}: equality constraint violated: " + "expected all dims to be {}, got {}", + cctx, first, val)); + } + } + } else if (const auto *lin = std::get_if(&constraint)) { + if (lin->dimLhs.side == ParameterSide::output || + lin->dimRhs.side == ParameterSide::output) { + continue; + } + int32_t lhs = getInputDimValue(lin->dimLhs, inputShapes); + int32_t rhs = getInputDimValue(lin->dimRhs, inputShapes); + int32_t expected = lin->coefficients[0] * rhs + lin->coefficients[1]; + if (lhs != expected) { + throw jsi::JSError(rt, std::format("{}: linear constraint violated: " + "expected {} = {} * {} + {}", + cctx, lhs, lin->coefficients[0], rhs, + lin->coefficients[1])); + } + } + } +} + +} // namespace rnexecutorch::core::schema diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h new file mode 100644 index 0000000000..291a742f08 --- /dev/null +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -0,0 +1,217 @@ +#pragma once + +#include "core/dtype.h" +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +namespace rnexecutorch::core::schema { +namespace jsi = facebook::jsi; + +// ======================================================== +// Parameter specs +// ======================================================== + +/** + * Inclusive integer domain of a single dynamic dimension — values from `min` + * to `max` in increments of `step`. + */ +struct RangeDim { + int32_t min = 0; + int32_t max = 0; + int32_t step = 1; +}; + +/** + * A single dimension matching one of the listed `choices`. + */ +struct EnumDim { + std::vector choices; +}; + +/** + * A dimension of an exported (concrete) model spec. + */ +using ConcreteDim = std::variant; + +/** + * A single input or output parameter of a method. `tag` is the discriminator: + * `executorch::runtime::Tag::Tensor` for tensor params (in which case + * `dtype`/`shape` describe the tensor), or any primitive tag for non-tensor + * params (in which case `dtype`/`shape` are unused). + */ +struct ParamSpec { + executorch::runtime::Tag tag = executorch::runtime::Tag::None; + types::DType dtype{}; ///< Valid iff `tag == Tag::Tensor`. + std::vector shape; ///< Valid iff `tag == Tag::Tensor`. +}; + +// ======================================================== +// Runtime constraints +// ======================================================== + +/// Whether the referenced tensor dimension belongs to an input or output. +enum class ParameterSide { input, + output }; + +/** + * Reference to a single tensor dimension of a method's input or output. + * `tensorIdx` counts only tensor parameters (skipping primitives), consistent + * with ExecuTorch's `inputTensorMeta` / `outputTensorMeta` ordering. + */ +struct DimRef { + ParameterSide side = ParameterSide::input; + int32_t tensorIdx = 0; + int32_t dimIdx = 0; +}; + +/** + * Runtime constraint declaring that all referenced dimensions must be equal + * to each other in any given execution of the method. + */ +struct EqualityConstraint { + std::vector dims; +}; + +/** + * Runtime constraint declaring that two dimensions must satisfy + * `dimLhs = coefficients[0] * dimRhs + coefficients[1]` (integer coefficients) + * in any given execution of the method. + */ +struct LinearConstraint { + DimRef dimLhs{}; + DimRef dimRhs{}; + std::array coefficients{}; +}; + +/** + * A requirement on the runtime values of a method's tensor dimensions: the + * concrete tensors passed to and produced by the method must satisfy it in + * any given execution. Matched as a declaration during spec validation. + */ +using RuntimeConstraint = std::variant; + +// ======================================================== +// Model specs +// ======================================================== + +/** + * Spec of a single model method: the ordered input and output parameter + * specs and the runtime constraints the method declares over its tensor + * dimensions. + */ +struct MethodSpec { + std::vector inputs; + std::vector outputs; + std::vector runtimeConstraints; +}; + +/** + * Spec of a whole model, mapping method names to their MethodSpec. + */ +using ModelSpec = std::unordered_map; + +// ======================================================== +// Serialization +// ======================================================== + +/** + * Parses a JSON-encoded ModelSpec using nlohmann/json, validating + * every value (positive constants, ordered range bounds, known kinds and tags, + * in-range indices). Throws std::runtime_error with `ctx` context on invalid + * JSON or a malformed spec. + * + * @param ctx Context description used for error messages. + * @param json The JSON string to parse. + * @return The parsed model spec. + */ +ModelSpec parseModelSpecJson(const std::string &ctx, const std::string &json); + +/** + * Serializes a ModelSpec to a fresh JS object matching ModelSpec + * from src/core/schema.ts. + * + * @param rt The JSI runtime instance. + * @param spec The model spec to serialize. + * @return The spec as a JS value. + */ +jsi::Value modelSpecToJs(jsi::Runtime &rt, const ModelSpec &spec); + +/** + * Serializes a backends map to a JSI object mapping method names to arrays of + * backend name strings. + * + * @param rt The JSI runtime instance. + * @param backends Map of method name -> backend name list. + * @return The backends as a JSI object. + */ +jsi::Object backendsToJs(jsi::Runtime &rt, + const std::unordered_map> &backends); + +// ======================================================== +// Metadata reflection +// ======================================================== + +/** + * Builds a MethodSpec directly from ExecuTorch MethodMeta, without requiring + * a JSON companion. All shapes become ConcreteDim constants since MethodMeta + * only exposes the static shape from export. + * + * @param methodMeta The method metadata from the .pte program. + * @return A MethodSpec derived from the metadata. + */ +MethodSpec methodSpecFromMetadata(const executorch::runtime::MethodMeta &methodMeta); + +/** + * Collects the names of backends declared as used by a method. + * + * @param methodMeta The method metadata from the .pte program. + * @return Vector of backend names for which uses_backend returns true. + */ +std::vector getUsedBackends(const executorch::runtime::MethodMeta &methodMeta); + +// ======================================================== +// Validation +// ======================================================== + +/** + * Validates a MethodSpec against ExecuTorch MethodMeta at load time. + * Checks input/output counts, tags, tensor dtypes, and static shape dimensions. + * Dynamic dimensions are skipped. + * + * @param spec The method spec to validate. + * @param meta The method metadata from the .pte program. + * @param methodName Method name for error messages. + * @throws std::runtime_error on any mismatch. + */ +void validateSpecAgainstMeta(const MethodSpec &spec, + const executorch::runtime::MethodMeta &meta, + const std::string &methodName); + +/** + * Validates runtime constraints on input tensors before execution. + * Only input-only constraints are checked; constraints referencing outputs + * are skipped (ExecuTorch validates output shapes internally). + * + * @param rt The JSI runtime instance (for error reporting). + * @param constraints The constraints to validate. + * @param inputShapes Per-tensor input shapes (indexed by DimRef.tensorIdx). + * @param ctx Context string for error messages (e.g. method name). + * @throws jsi::JSError on constraint violation. + */ +void validateRuntimeConstraints(jsi::Runtime &rt, + const std::vector &constraints, + const std::vector> &inputShapes, + const std::string &ctx); + +} // namespace rnexecutorch::core::schema diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp index 54c9e23064..0826340ba1 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp @@ -1,8 +1,14 @@ #include "tensor_helpers.h" +#include +#include #include +#include +#include #include +#include +#include "core/schema.h" #include "dtype.h" namespace rnexecutorch::core::tensor { @@ -10,34 +16,34 @@ namespace types = rnexecutorch::core::types; namespace conversions = rnexecutorch::core::conversions; std::shared_lock -tryLockShared(jsi::Runtime &rt, const std::string &name, const std::shared_ptr &tensor) { +tryLockShared(jsi::Runtime &rt, const std::string &ctx, const std::shared_ptr &tensor) { std::shared_lock lock(tensor->mutex_, std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, std::format("{} tensor is currently in use", name)); + throw jsi::JSError(rt, std::format("{} tensor is currently in use", ctx)); } if (!tensor->data_) { - throw jsi::JSError(rt, std::format("{} tensor has been disposed", name)); + throw jsi::JSError(rt, std::format("{} tensor has been disposed", ctx)); } return lock; } std::unique_lock -tryLockUnique(jsi::Runtime &rt, const std::string &name, const std::shared_ptr &tensor) { +tryLockUnique(jsi::Runtime &rt, const std::string &ctx, const std::shared_ptr &tensor) { std::unique_lock lock(tensor->mutex_, std::try_to_lock); if (!lock.owns_lock()) { - throw jsi::JSError(rt, std::format("{} tensor is currently in use", name)); + throw jsi::JSError(rt, std::format("{} tensor is currently in use", ctx)); } if (!tensor->data_) { - throw jsi::JSError(rt, std::format("{} tensor has been disposed", name)); + throw jsi::JSError(rt, std::format("{} tensor has been disposed", ctx)); } return lock; } void checkNotSameTensor(jsi::Runtime &rt, - const std::string &name1, const std::shared_ptr &t1, - const std::string &name2, const std::shared_ptr &t2) { + const std::string &ctx1, const std::shared_ptr &t1, + const std::string &ctx2, const std::shared_ptr &t2) { if (t1 == t2) { - throw jsi::JSError(rt, std::format("{} and {} cannot be the same tensor", name1, name2)); + throw jsi::JSError(rt, std::format("{} and {} cannot be the same tensor", ctx1, ctx2)); } } @@ -45,34 +51,43 @@ namespace { std::string shapeToString(const SymbolicShape &shape) { std::string s; for (const auto &dim : shape) { + if (std::holds_alternative(dim)) { + s += std::get(dim); + } if (std::holds_alternative(dim)) { s += std::to_string(std::get(dim)); - } else if (std::holds_alternative(dim)) { - s += std::get(dim); - } else { - const auto &rangeDim = std::get(dim); - s += std::format("[{}..{}{}]", - rangeDim.min, - rangeDim.max, - rangeDim.step ? std::format(" step {}", *rangeDim.step) : ""); } - s += ", "; + if (std::holds_alternative(dim)) { + auto range = std::get(dim); + s += std::format("[{}..{}:{}]", range.min, range.max, range.step); + } + if (std::holds_alternative(dim)) { + auto enumeration = std::get(dim); + s += "{"; + for (const auto choice : enumeration.choices) { + s += std::to_string(choice) + ","; + } + if (!enumeration.choices.empty()) { + s.pop_back(); + } + s += "}"; + } + s += ","; } if (!shape.empty()) { s.pop_back(); - s.pop_back(); } return "[" + s + "]"; } } // namespace std::shared_ptr -fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, +fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, std::optional expectedDtype, const std::optional &expectedShape) { - auto obj = conversions::asType(rt, name, value); + auto obj = conversions::asType(rt, ctx, value); if (!obj.isHostObject(rt)) { - throw jsi::JSError(rt, name + " must be a Tensor"); + throw jsi::JSError(rt, ctx + " must be a Tensor"); } auto tensor = obj.getHostObject(rt); @@ -80,7 +95,7 @@ fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, const auto &shape = tensor->shape_; if (expectedDtype && dtype != *expectedDtype) { - throw jsi::JSError(rt, std::format("{} must be of type {}", name, types::toString(*expectedDtype))); + throw jsi::JSError(rt, std::format("{} must be of type {}", ctx, types::toString(*expectedDtype))); } if (!expectedShape) { @@ -89,40 +104,46 @@ fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, if (shape.size() != expectedShape->size()) { throw jsi::JSError(rt, std::format("{} must have shape {} (expected {} dimensions, got {})", - name, shapeToString(*expectedShape), expectedShape->size(), shape.size())); + ctx, shapeToString(*expectedShape), expectedShape->size(), shape.size())); } - std::unordered_map symbolToConcrete; + std::unordered_map symbolBinding; for (size_t i = 0; i < expectedShape->size(); ++i) { const auto &dim = expectedShape->at(i); - if (std::holds_alternative(dim)) { - const auto expected = std::get(dim); - if (shape[i] != expected) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} mismatch: expected {}, got {})", - name, shapeToString(*expectedShape), i, expected, shape[i])); - } - } else if (std::holds_alternative(dim)) { + if (std::holds_alternative(dim)) { const auto &symbol = std::get(dim); - if (symbolToConcrete.contains(symbol) && shape[i] != symbolToConcrete[symbol]) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} mismatch: expected {}, got {})", - name, shapeToString(*expectedShape), i, symbolToConcrete[symbol], shape[i])); + if (symbolBinding.contains(symbol) && symbolBinding[symbol] != shape[i]) { + throw jsi::JSError(rt, ""); + } + symbolBinding[symbol] = shape[i]; + } + if (std::holds_alternative(dim)) { + if (shape[i] != std::get(dim)) { + throw jsi::JSError(rt, ""); } - symbolToConcrete[symbol] = shape[i]; - } else { - const auto &rangeDim = std::get(dim); - if (shape[i] < rangeDim.min) { + } + if (std::holds_alternative(dim)) { + auto range = std::get(dim); + if (shape[i] < range.min) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} < min {})", - name, shapeToString(*expectedShape), i, shape[i], rangeDim.min)); + ctx, shapeToString(*expectedShape), i, shape[i], range.min)); } - if (shape[i] > rangeDim.max) { + if (shape[i] > range.max) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} > max {})", - name, shapeToString(*expectedShape), i, shape[i], rangeDim.max)); + ctx, shapeToString(*expectedShape), i, shape[i], range.max)); } - if (rangeDim.step && (shape[i] - rangeDim.min) % *rangeDim.step != 0) { + if ((shape[i] - range.min) % range.step != 0) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} must be min({}) + k*step({}), got {})", - name, shapeToString(*expectedShape), i, rangeDim.min, *rangeDim.step, shape[i])); + ctx, shapeToString(*expectedShape), i, range.min, range.step, shape[i])); + } + } + if (std::holds_alternative(dim)) { + auto enumeration = std::get(dim); + if (std::ranges::find(enumeration.choices, shape[i]) == enumeration.choices.end()) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} not allowed: got {})", + ctx, shapeToString(*expectedShape), i, shape[i])); } } } diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.h b/packages/react-native-executorch/cpp/core/tensor_helpers.h index 9004e6f262..2c616d7276 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.h +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.h @@ -14,6 +14,7 @@ #include "conversions.h" #include "dtype.h" +#include "schema.h" #include "tensor.h" #include @@ -22,14 +23,11 @@ namespace rnexecutorch::core::tensor { namespace jsi = facebook::jsi; using rnexecutorch::core::types::DType; +using schema::EnumDim; +using schema::RangeDim; -struct RangeDim { - int32_t min = 0; - int32_t max = 0; - std::optional step; -}; - -using SymbolicShape = std::vector>; +using SymbolicDim = std::variant; +using SymbolicShape = std::vector; /** * Tries to acquire a shared (read) lock on the underlying tensor resource. @@ -42,7 +40,7 @@ using SymbolicShape = std::vector>; * @return A shared lock protecting the tensor data. */ [[nodiscard]] std::shared_lock -tryLockShared(jsi::Runtime &rt, const std::string &name, const std::shared_ptr &tensor); +tryLockShared(jsi::Runtime &rt, const std::string &ctx, const std::shared_ptr &tensor); /** * Tries to acquire a unique (write) lock on the underlying tensor resource. @@ -50,12 +48,12 @@ tryLockShared(jsi::Runtime &rt, const std::string &name, const std::shared_ptr -tryLockUnique(jsi::Runtime &rt, const std::string &name, const std::shared_ptr &tensor); +tryLockUnique(jsi::Runtime &rt, const std::string &ctx, const std::shared_ptr &tensor); /** * Validates that two JSI Tensor parameters do not point to the exact same @@ -63,14 +61,14 @@ tryLockUnique(jsi::Runtime &rt, const std::string &name, const std::shared_ptr &t1, - const std::string &name2, const std::shared_ptr &t2); + const std::string &ctx1, const std::shared_ptr &t1, + const std::string &ctx2, const std::shared_ptr &t2); /** * Extracts, type-checks, and shape-validates a TensorHostObject from a JSI @@ -78,16 +76,30 @@ void checkNotSameTensor(jsi::Runtime &rt, * checking or shape validation fails. * * @param rt The JSI runtime instance. - * @param name Parameter name for contextual error messages. + * @param ctx Parameter name for contextual error messages. * @param value The JSI value to extract the Tensor from. * @param expectedDtype Optional expected DType constraint. * @param expectedShape Optional expected shape (SymbolicShape) constraint. * @return Shared pointer to the validated TensorHostObject. */ std::shared_ptr -fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, +fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, std::optional expectedDtype, const std::optional &expectedShape); +/** + * @overload + * + * Convenience wrapper that accepts an initializer list of symbolic shape + * elements. Allows passing shape constraints like `{"H", "W", 1}` directly + * without typing SymbolicShape explicitly. + */ +inline std::shared_ptr +fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, + std::optional expectedDtype, + std::initializer_list expectedShape) { + return fromJs(rt, ctx, value, expectedDtype, std::optional(expectedShape)); +} + /** * @overload * @@ -100,23 +112,28 @@ template requires std::ranges::input_range && std::convertible_to, int32_t> inline std::shared_ptr -fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, +fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, std::optional expectedDtype, const Range &expectedShape) { SymbolicShape convertedShape(expectedShape.begin(), expectedShape.end()); - return fromJs(rt, name, value, expectedDtype, std::move(convertedShape)); + return fromJs(rt, ctx, value, expectedDtype, std::move(convertedShape)); } /** * @overload * - * Convenience wrapper that accepts an initializer list of symbolic shape - * elements. Allows passing shape constraints like `{"H", "W", 1}` directly - * without typing SymbolicShape explicitly. + * Convenience wrapper that accepts a vector of ConcreteDim values as the + * expected shape. Useful when building shapes programmatically from + * ConcreteDim without manually wrapping each element in SymbolicDim. */ inline std::shared_ptr -fromJs(jsi::Runtime &rt, const std::string &name, const jsi::Value &value, - std::optional expectedDtype, - std::initializer_list> expectedShape) { - return fromJs(rt, name, value, expectedDtype, std::optional(expectedShape)); +fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, + std::optional expectedDtype, const std::vector &expectedShape) { + SymbolicShape convertedShape; + convertedShape.reserve(expectedShape.size()); + for (const auto &dim : expectedShape) { + convertedShape.push_back(std::visit([](const auto &d) -> SymbolicDim { return d; }, dim)); + } + return fromJs(rt, ctx, value, expectedDtype, std::move(convertedShape)); } + } // namespace rnexecutorch::core::tensor diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index 71857521ac..eacd936907 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -1,6 +1,6 @@ import { rnexecutorchJsi } from '../native/bridge'; -import type { DType, Tensor } from './tensor'; -import type { ExecuTorchTag, ModelSpec, ConcreteDim } from './schema'; +import type { Tensor } from './tensor'; +import type { ModelSpec, ConcreteDim } from './schema'; declare const modelBrand: unique symbol; @@ -16,49 +16,6 @@ export type ModelInput = Tensor | number | boolean | null; */ export type ModelOutput = Tensor | number | boolean | null; -/** - * Metadata describing a single tensor slot (input or output) of a model method. - * @category Types - */ -export type TensorMeta = { - /** The name associated with this tensor slot (may be empty). */ - readonly name: string; - /** The number of dimensions. */ - readonly ndim: number; - /** The total byte size of the tensor buffer. */ - readonly nbytes: number; - /** The element data type. */ - readonly dtype: DType; - /** The concrete size of each dimension (e.g. `[1, 3, 224, 224]`). */ - readonly shape: number[]; -}; - -/** - * Metadata describing a single exported method of an ExecuTorch model. - * @category Types - */ -export type ModelMethodMeta = { - /** The exported method name (e.g. `'forward'`). */ - readonly name: string; - /** The total number of input arguments the method accepts. */ - readonly numInputs: number; - /** The total number of output values the method returns. */ - readonly numOutputs: number; - /** Runtime value-tags for each input slot, in order. */ - readonly inputTags: readonly ExecuTorchTag[]; - /** Runtime value-tags for each output slot, in order. */ - readonly outputTags: readonly ExecuTorchTag[]; - /** - * A map from backend name to a boolean indicating whether this method - * delegates to that backend. - */ - readonly usesBackend: Record; - /** Detailed tensor metadata for every input tensor slot, in order. */ - readonly inputTensorMeta: readonly TensorMeta[]; - /** Detailed tensor metadata for every output tensor slot, in order. */ - readonly outputTensorMeta: readonly TensorMeta[]; -}; - /** * A compiled, ready-to-run ExecuTorch model loaded into native memory. * @@ -73,22 +30,10 @@ export type ModelMethodMeta = { export interface Model { /** The local filesystem path of the `.pte` model file. */ readonly path: string; - /** The exported spec of this model. */ - readonly spec: ModelSpec; - - /** - * Returns the list of exported method names available on this model (e.g. - * `['forward']`). - */ - getMethodNames(): readonly string[]; - - /** - * Returns detailed metadata for the specified exported method, including - * input/output tags, tensor shapes, dtype, and backend delegation info. - * @param methodName The name of the exported method to inspect. - * @returns The {@link ModelMethodMeta} for the requested method. - */ - getMethodMeta(methodName: string): ModelMethodMeta; + /** The exported schema of this model. */ + readonly schema: ModelSpec; + /** ExecuTorch backends used by a given method. */ + readonly backends: Record; /** * Executes a named model method synchronously. diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 8415f10ae0..8cbc55c93b 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -58,7 +58,7 @@ import type { DType } from './tensor'; * to `max` in increments of `step`. * @category Types */ -export type Range = { min: number; max: number; step: number }; +export type Range = { readonly min: number; readonly max: number; readonly step: number }; /** * A single dimension with a fully known domain: @@ -133,7 +133,7 @@ export type ParamSpec = * @category Types */ export type DimRef = { - readonly io: 'input' | 'output'; + readonly parameterSide: 'input' | 'output'; readonly tensorIdx: number; readonly dimIdx: number; }; @@ -192,6 +192,10 @@ export type MethodSpec = { */ export type ModelSpec = Record>; +// ======================================================== +// Dim helper functions +// ======================================================== + /** * Shape notation accepted by {@link SymbolicTensor}: numbers become * {@link ConstantDim}, strings become {@link StaticDim}, and @@ -200,10 +204,6 @@ export type ModelSpec = Record> */ export type SymbolicShape = readonly (number | string | SymbolicDim)[]; -// ======================================================== -// Dim helper functions -// ======================================================== - /** * Creates a static symbolic dimension. Static symbols bind to constant * dimensions of the exported spec; repeated uses must bind to the same value. @@ -438,7 +438,11 @@ function matchModelSpecsSymbols( // ======================================================== function refsEqual(r1: DimRef, r2: DimRef): boolean { - return r1.io === r2.io && r1.tensorIdx === r2.tensorIdx && r1.dimIdx === r2.dimIdx; + return ( + r1.parameterSide === r2.parameterSide && + r1.tensorIdx === r2.tensorIdx && + r1.dimIdx === r2.dimIdx + ); } function constraintsEqual(c1: RuntimeConstraint, c2: RuntimeConstraint): boolean { @@ -470,7 +474,7 @@ function constraintsEqual(c1: RuntimeConstraint, c2: RuntimeConstraint): boolean function resolveDim(methodSpec: MethodSpec, ref: DimRef): D { let tensorSpecs: TensorSpec[]; - switch (ref.io) { + switch (ref.parameterSide) { case 'input': tensorSpecs = methodSpec.inputs.filter((v): v is TensorSpec => v.kind === 'Tensor'); break; From cf58a2c1db541ee3c312c313c6cdc5c7c8cebf59 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Mon, 27 Jul 2026 23:47:18 +0200 Subject: [PATCH 08/31] update --- packages/react-native-executorch/cpp/core/model.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 3c1c7f02ee..a13260d6ba 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -151,6 +151,7 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { } tensorLocks.emplace_back(tensor::tryLockUnique(rt, ctx, tensorHostObject)); inputShapes.push_back(tensorHostObject->shape_); + inputs[i] = tensorHostObject->tensor_; break; } case executorch::runtime::Tag::Double: From de8c713c63f4e9dd661464aef6910106d601a982 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 05:12:23 +0200 Subject: [PATCH 09/31] refactor schema validation to use validateSpec across all tasks --- apps/computer-vision/app/inspect/index.tsx | 77 +++--- .../cpp/core/model.cpp | 35 ++- .../react-native-executorch/cpp/core/model.h | 2 +- .../cpp/core/schema.cpp | 259 +++++++++--------- .../react-native-executorch/cpp/core/schema.h | 41 ++- .../src/core/modelSchema.ts | 223 --------------- .../src/core/schema.ts | 180 +++++++++--- .../src/extensions/cv/tasks/classification.ts | 39 +-- .../src/extensions/cv/tasks/imageEmbedding.ts | 26 +- .../cv/tasks/instanceSegmentation.ts | 52 ++-- .../extensions/cv/tasks/keypointDetection.ts | 35 +-- .../extensions/cv/tasks/objectDetection.ts | 44 ++- .../src/extensions/cv/tasks/preprocessing.ts | 13 +- .../extensions/cv/tasks/sdxsTextToImage.ts | 47 ++-- .../cv/tasks/semanticSegmentation.ts | 38 +-- .../src/extensions/cv/tasks/styleTransfer.ts | 35 ++- .../src/extensions/nlp/tasks/textEmbedding.ts | 30 +- .../tasks/fsmnVoiceActivityDetection.ts | 24 +- packages/react-native-executorch/src/index.ts | 21 +- packages/react-native-executorch/src/utils.ts | 20 +- 20 files changed, 594 insertions(+), 647 deletions(-) delete mode 100644 packages/react-native-executorch/src/core/modelSchema.ts diff --git a/apps/computer-vision/app/inspect/index.tsx b/apps/computer-vision/app/inspect/index.tsx index 5085ec0d31..b1505028bd 100644 --- a/apps/computer-vision/app/inspect/index.tsx +++ b/apps/computer-vision/app/inspect/index.tsx @@ -9,12 +9,25 @@ import { ActivityIndicator, Alert, } from 'react-native'; -import { inspectModel, type TensorMeta } from 'react-native-executorch'; +import { inspectModel, type ConcreteDim, type ParamSpec } from 'react-native-executorch'; import ScreenWrapper from '../../components/ScreenWrapper'; import { ColorPalette } from '../../theme'; type InspectionResult = Awaited>; +const formatDim = (dim: ConcreteDim): string => { + switch (dim.kind) { + case 'constant': + return `${dim.value}`; + case 'range': + return dim.range.step !== 1 + ? `${dim.range.min}..${dim.range.max} (step ${dim.range.step})` + : `${dim.range.min}..${dim.range.max}`; + case 'enum': + return dim.choices.join(' | '); + } +}; + function InspectContent() { const [url, setUrl] = useState(''); const [loading, setLoading] = useState(false); @@ -39,27 +52,32 @@ function InspectContent() { } }; - const renderTensorList = (tensors: TensorMeta[] | undefined, title: string) => { - if (!tensors || tensors.length === 0) return null; + const renderParamList = ( + params: readonly ParamSpec[] | undefined, + title: string + ) => { + if (!params || params.length === 0) return null; return ( {title} - {tensors.map((tensor, idx) => ( + {params.map((param, idx) => ( - - {tensor.name || `${title.slice(0, -1)} #${idx}`} - - {tensor.dtype} - - - - Shape: [{tensor.shape.join(', ')}] - - - Bytes: {tensor.nbytes} + #{idx} + + {param.kind === 'Tensor' ? param.dtype : param.kind} + {param.kind === 'Tensor' && ( + + + Shape:{' '} + + [{param.shape.map(formatDim).join(', ')}] + + + + )} ))} @@ -113,12 +131,12 @@ function InspectContent() { Source URL: {result.source} - Methods ({result.methods.length}) + Methods ({Object.keys(result.schema).length}) - {result.methods.map((method, mIdx) => ( - + {Object.entries(result.schema).map(([methodName, spec]) => ( + - {method.name} + {methodName} Method @@ -126,35 +144,30 @@ function InspectContent() { - {method.meta.numInputs} + {spec.inputs.length} Inputs - {method.meta.numOutputs} + {spec.outputs.length} Outputs - {method.meta.usesBackend && Object.keys(method.meta.usesBackend).length > 0 && ( + {(result.backends[methodName]?.length ?? 0) > 0 && ( Backends Used: - {Object.entries(method.meta.usesBackend).map(([backend, used]) => ( - - - {backend}: {used ? 'Yes' : 'No'} - + {result.backends[methodName]?.map((backend) => ( + + {backend} ))} )} - {renderTensorList(method.meta.inputTensorMeta, 'Input Tensors')} - {renderTensorList(method.meta.outputTensorMeta, 'Output Tensors')} + {renderParamList(spec.inputs, 'Inputs')} + {renderParamList(spec.outputs, 'Outputs')} ))} diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index a13260d6ba..ce33d9fa14 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -62,7 +62,7 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) schema::ModelSpec overrideSpec; if (methodNames.contains(kSchemaMethod)) { - auto ctx = std::format("loadModel: {}", kSchemaMethod); + auto ctx = std::format("loadModel: '{}'", kSchemaMethod); auto result = unwrap(ctx, etModule_->execute(kSchemaMethod)); if (result.empty() || result[0].tag != executorch::runtime::Tag::String) { @@ -82,7 +82,8 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) spec_[methodName] = std::move(overrideSpec[methodName]); } - schema::validateSpecAgainstMeta(spec_[methodName], methodMeta, methodName); + auto ctx = std::format("loadModel: method '{}'", methodName); + schema::validateSpec(spec_[methodName], methodMeta, ctx); } } @@ -187,16 +188,23 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { logFn.callWithThis(rt, consoleObj, {jsi::String::createFromUtf8(rt, info)}); #endif - const auto *error = "execute: Method '{}' failed. Check the following common pitfalls:" - "\n- Make sure that the backends used by the model (`model.backends`)" - "\n are registered in ExecuTorch runtime (`getRegisteredBackends`)." - "\n- If your model requires specific runtime constraints or dynamic shapes" - "\n (e.g. equality of certain dynamic dimensions), make sure you exported" - "\n it with '{}' companion method that returns JSON string with dynamic" - "\n model spec that overrides the default one constructed from ExecuTorch" - "\n metadata which only contain the static upper bounds of tensor dimension" - "\n and do not contain information about runtime constraints."; - auto result = unwrap(rt, std::format(error, methodName, kSchemaMethod), std::move(executeResult)); + const auto *error = "execute: Method '{}' failed.\n" + "\n" + "Common causes:\n" + " 1. Backend not registered\n" + " Ensure backends from `model.backends` are registered\n" + " in the ExecuTorch runtime.\n" + "\n" + " 2. Shape/constraint mismatch\n" + " If the model uses dynamic shapes or runtime constraints\n" + " (e.g. equality between dimensions), export a companion\n" + " '{}' method returning a JSON model spec\n" + " (see `src/core/schema.ts` for the JSON structure).\n" + "\n" + " Without it, validation falls back to static metadata\n" + " from ExecuTorch which only contains upper bounds and\n" + " does not capture runtime constraints."; + auto result = unwrap(rt, std::vformat(error, std::make_format_args(methodName, kSchemaMethod)), std::move(executeResult)); auto jsOutputArray = jsi::Array(rt, result.size()); size_t outputIdx = 0; @@ -241,6 +249,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { case executorch::runtime::Tag::None: jsOutputArray.setValueAtIndex(rt, outputIdx, jsi::Value::null()); break; + case executorch::runtime::Tag::String: + jsOutputArray.setValueAtIndex(rt, outputIdx, jsi::String::createFromUtf8(rt, std::string(output.toString()))); + break; default: throw jsi::JSError(rt, std::format("execute: Unsupported return type: {}", executorch::runtime::tag_to_string(output.tag))); diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index d8087b2729..34ace946f9 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -36,8 +36,8 @@ class ModelHostObject : public jsi::HostObject, schema::ModelSpec spec_; std::unordered_map> backends_; - std::unique_ptr etModule_; std::mutex mutex_; + std::unique_ptr etModule_; }; void install_loadModel(jsi::Runtime &rt, jsi::Object &module); diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp index 95a3a2d3d3..c8943feaed 100644 --- a/packages/react-native-executorch/cpp/core/schema.cpp +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -1,5 +1,6 @@ #include "schema.h" +#include #include #include @@ -81,16 +82,7 @@ NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EnumDim, choices) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(DimRef, side, tensorIdx, dimIdx) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EqualityConstraint, dims) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(LinearConstraint, dimLhs, dimRhs, coefficients) -NLOHMANN_JSON_SERIALIZE_ENUM(ParameterSide, { - {ParameterSide::input, "input"}, - {ParameterSide::output, "output"}, - }) -NLOHMANN_JSON_SERIALIZE_ENUM(types::DType, { - {types::DType::uint8, "uint8"}, - {types::DType::int32, "int32"}, - {types::DType::int64, "int64"}, - {types::DType::float32, "float32"}, - }) +NLOHMANN_JSON_SERIALIZE_ENUM(ParamSide, {{ParamSide::input, "input"}, {ParamSide::output, "output"}}) // NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. void from_json(const json &j, ConcreteDim &d) { @@ -122,14 +114,16 @@ void to_json(json &j, const ConcreteDim &d) { void from_json(const json &j, ParamSpec &p) { p.tag = j.at("kind").get(); if (p.tag == Tag::Tensor) { - p.dtype = j.at("dtype").get(); + p.dtype = types::parseDType(j.at("dtype").get()); p.shape = j.at("shape").get>(); } } // NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. void to_json(json &j, const ParamSpec &p) { if (p.tag == Tag::Tensor) { - j = json::object({{"kind", "Tensor"}, {"dtype", p.dtype}, {"shape", p.shape}}); + // DType is (de)serialized via its string helpers — a JSON macro for it + // would have to live in namespace `types` for ADL to find it. + j = json::object({{"kind", "Tensor"}, {"dtype", types::toString(p.dtype)}, {"shape", p.shape}}); } else { j = json::object({{"kind", p.tag}}); } @@ -188,7 +182,7 @@ jsi::Value jsonToJs(jsi::Runtime &rt, const json &j) { case json::value_t::number_float: return jsi::Value(j.get()); case json::value_t::string: - return jsi::String::createFromUtf8(rt, j.get()); + return jsi::Value(jsi::String::createFromUtf8(rt, j.get())); case json::value_t::array: { auto arr = jsi::Array(rt, j.size()); size_t i = 0; @@ -281,7 +275,7 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & for (size_t i = 0; i < methodMeta.num_backends(); ++i) { auto ctx = std::format("getUsedBackends: backend [{}]", i); const auto *name = unwrap(ctx, methodMeta.get_backend_name(i)); - if (methodMeta.uses_backend(name)) { + if (methodMeta.uses_backend(name) && std::ranges::find(backends, name) == backends.end()) { backends.emplace_back(name); } } @@ -294,131 +288,158 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & namespace { -void validateParamAgainstMeta(const ParamSpec ¶m, - const executorch::runtime::TensorInfo &tensorMeta, - const std::string &ctx) { +void validateConcreteDim(const ConcreteDim &dim, const std::string &ctx) { + if (const auto *c = std::get_if(&dim)) { + if (*c <= 0) { + throw std::runtime_error(std::format("{}: constant dim must be positive", ctx)); + } + } + if (const auto *r = std::get_if(&dim)) { + if (r->min < 0) { + throw std::runtime_error(std::format("{}: range min must be non-negative", ctx)); + } + if (r->max < r->min) { + throw std::runtime_error(std::format("{}: range max must be >= min", ctx)); + } + if (r->step <= 0) { + throw std::runtime_error(std::format("{}: range step must be positive", ctx)); + } + } + if (const auto *e = std::get_if(&dim)) { + if (e->choices.empty()) { + throw std::runtime_error(std::format("{}: enum must have at least one choice", ctx)); + } + for (const auto &choice : e->choices) { + if (choice <= 0) { + throw std::runtime_error(std::format("{}: enum choices must be positive", ctx)); + } + } + } +} + +void validateSpecDimDomains(const MethodSpec &spec, const std::string &ctx) { + auto validateParams = [&](const std::vector ¶ms, const char *label) { + for (size_t i = 0; i < params.size(); ++i) { + if (params[i].tag != Tag::Tensor) { + continue; + } + for (size_t d = 0; d < params[i].shape.size(); ++d) { + auto dctx = std::format("{} {}[{}] dim[{}]", ctx, label, i, d); + validateConcreteDim(params[i].shape[d], dctx); + } + } + }; + validateParams(spec.inputs, "input"); + validateParams(spec.outputs, "output"); +} + +void validateTensorParam(const ParamSpec ¶m, + const executorch::runtime::TensorInfo &tensorMeta, + const std::string &ctx) { auto metaDtype = types::fromScalarType(tensorMeta.scalar_type()); if (param.dtype != metaDtype) { - throw std::runtime_error(std::format("{}: dtype mismatch: schema has '{}', metadata has '{}", - ctx, types::toString(param.dtype), types::toString(metaDtype))); + throw std::runtime_error(std::format("{}: dtype mismatch", ctx)); } + auto metaShape = tensorMeta.sizes(); if (param.shape.size() != metaShape.size()) { - throw std::runtime_error(std::format("{}: rank mismatch: schema has {}, metadata has {}", - ctx, param.shape.size(), metaShape.size())); + throw std::runtime_error(std::format("{}: rank mismatch", ctx)); } + for (size_t d = 0; d < param.shape.size(); ++d) { - if (const auto *c = std::get_if(¶m.shape[d])) { - if (*c != metaShape[d]) { - throw std::runtime_error(std::format("{}: shape[{}] mismatch: schema has {}, metadata has {}", - ctx, d, *c, metaShape[d])); - } + if (std::holds_alternative(param.shape[d]) && + std::get(param.shape[d]) != metaShape[d]) { + throw std::runtime_error(std::format("{}: shape[{}] mismatch", ctx, d)); } } } -void validateDimRefAgainstSpec(const DimRef &ref, - size_t numTensorInputs, size_t numTensorOutputs, - const std::vector &inputRanks, - const std::vector &outputRanks, - const std::string &ctx) { - bool isInput = (ref.side == ParameterSide::input); - size_t numTensors = isInput ? numTensorInputs : numTensorOutputs; +void validateDimRef(const DimRef &ref, + const std::vector &inputRanks, + const std::vector &outputRanks, + const std::string &ctx) { + bool isInput = (ref.side == ParamSide::input); const auto &ranks = isInput ? inputRanks : outputRanks; - if (std::cmp_greater_equal(ref.tensorIdx, numTensors)) { - throw std::runtime_error(std::format("{}: DimRef tensorIdx {} out of range (have {} {} tensors)", - ctx, ref.tensorIdx, numTensors, isInput ? "input" : "output")); + if (std::cmp_greater_equal(ref.tensorIdx, ranks.size())) { + throw std::runtime_error(std::format("{}: tensorIdx {} out of range", ctx, ref.tensorIdx)); } if (std::cmp_greater_equal(ref.dimIdx, ranks[static_cast(ref.tensorIdx)])) { - throw std::runtime_error(std::format("{}: DimRef dimIdx {} out of range for tensor (rank {})", - ctx, ref.dimIdx, ranks[static_cast(ref.tensorIdx)])); + throw std::runtime_error(std::format("{}: dimIdx {} out of range", ctx, ref.dimIdx)); } } -void validateConstraintsAgainstSpec(const MethodSpec &spec, const std::string &methodName) { - size_t numTensorInputs = 0; - std::vector inputRanks; - for (const auto &p : spec.inputs) { - if (p.tag == Tag::Tensor) { - inputRanks.push_back(p.shape.size()); - ++numTensorInputs; - } - } - size_t numTensorOutputs = 0; - std::vector outputRanks; - for (const auto &p : spec.outputs) { +std::vector gatherTensorRanks(const std::vector ¶ms) { + std::vector ranks; + for (const auto &p : params) { if (p.tag == Tag::Tensor) { - outputRanks.push_back(p.shape.size()); - ++numTensorOutputs; + ranks.push_back(p.shape.size()); } } + return ranks; +} + +void validateConstraintSpecs(const MethodSpec &spec, const std::string &ctx) { + auto inputRanks = gatherTensorRanks(spec.inputs); + auto outputRanks = gatherTensorRanks(spec.outputs); for (size_t i = 0; i < spec.runtimeConstraints.size(); ++i) { - auto ctx = std::format("loadModel: method '{}' constraint[{}]", methodName, i); + auto cctx = std::format("{} constraint[{}]", ctx, i); const auto &constraint = spec.runtimeConstraints[i]; if (const auto *eq = std::get_if(&constraint)) { if (eq->dims.size() < 2) { - throw std::runtime_error(std::format("{}: equality constraint requires at least two dims", ctx)); + throw std::runtime_error(std::format("{}: equality needs at least two dims", cctx)); } for (const auto &dim : eq->dims) { - validateDimRefAgainstSpec(dim, numTensorInputs, numTensorOutputs, - inputRanks, outputRanks, ctx); + validateDimRef(dim, inputRanks, outputRanks, cctx); } - } else if (const auto *lin = std::get_if(&constraint)) { - validateDimRefAgainstSpec(lin->dimLhs, numTensorInputs, numTensorOutputs, - inputRanks, outputRanks, ctx + " dimLhs"); - validateDimRefAgainstSpec(lin->dimRhs, numTensorInputs, numTensorOutputs, - inputRanks, outputRanks, ctx + " dimRhs"); + } + + if (const auto *lin = std::get_if(&constraint)) { + validateDimRef(lin->dimLhs, inputRanks, outputRanks, cctx); + validateDimRef(lin->dimRhs, inputRanks, outputRanks, cctx); } } } -} // namespace +void validateParamsAgainstMeta(const std::vector ¶ms, + bool isInput, + const executorch::runtime::MethodMeta &meta, + const std::string &ctx) { + for (size_t i = 0; i < params.size(); ++i) { + auto pctx = std::format("{} {}[{}]", ctx, isInput ? "input" : "output", i); + auto tagResult = isInput ? unwrap(pctx, meta.input_tag(i)) + : unwrap(pctx, meta.output_tag(i)); -void validateSpecAgainstMeta(const MethodSpec &spec, - const executorch::runtime::MethodMeta &meta, - const std::string &methodName) { - if (spec.inputs.size() != meta.num_inputs()) { - throw std::runtime_error(std::format("loadModel: method '{}' input count mismatch: " - "schema has {}, metadata has {}", - methodName, spec.inputs.size(), meta.num_inputs())); - } - if (spec.outputs.size() != meta.num_outputs()) { - throw std::runtime_error(std::format("loadModel: method '{}' output count mismatch: " - "schema has {}, metadata has {}", - methodName, spec.outputs.size(), meta.num_outputs())); - } - - for (size_t i = 0; i < spec.inputs.size(); ++i) { - auto ctx = std::format("loadModel: method '{}' input[{}]", methodName, i); - auto metaTag = unwrap(ctx, meta.input_tag(i)); - if (spec.inputs[i].tag != metaTag) { - throw std::runtime_error(std::format("{}: tag mismatch: schema has '{}', metadata has '{}", - ctx, executorch::runtime::tag_to_string(spec.inputs[i].tag), - executorch::runtime::tag_to_string(metaTag))); + if (params[i].tag != tagResult) { + throw std::runtime_error(std::format("{}: tag mismatch", pctx)); } - if (metaTag == Tag::Tensor) { - auto tensorMeta = unwrap(ctx, meta.input_tensor_meta(i)); - validateParamAgainstMeta(spec.inputs[i], tensorMeta, ctx); + + if (tagResult == Tag::Tensor) { + auto tensorMeta = isInput ? unwrap(pctx, meta.input_tensor_meta(i)) + : unwrap(pctx, meta.output_tensor_meta(i)); + validateTensorParam(params[i], tensorMeta, pctx); } } +} - for (size_t i = 0; i < spec.outputs.size(); ++i) { - auto ctx = std::format("loadModel: method '{}' output[{}]", methodName, i); - auto metaTag = unwrap(ctx, meta.output_tag(i)); - if (spec.outputs[i].tag != metaTag) { - throw std::runtime_error(std::format("{}: tag mismatch: schema has '{}', metadata has '{}", - ctx, executorch::runtime::tag_to_string(spec.outputs[i].tag), - executorch::runtime::tag_to_string(metaTag))); - } - if (metaTag == Tag::Tensor) { - auto tensorMeta = unwrap(ctx, meta.output_tensor_meta(i)); - validateParamAgainstMeta(spec.outputs[i], tensorMeta, ctx); - } +} // namespace + +void validateSpec(const MethodSpec &spec, + const executorch::runtime::MethodMeta &meta, + const std::string &ctx) { + + if (spec.inputs.size() != meta.num_inputs()) { + throw std::runtime_error(std::format("{}: input count mismatch", ctx)); + } + if (spec.outputs.size() != meta.num_outputs()) { + throw std::runtime_error(std::format("{}: output count mismatch", ctx)); } - validateConstraintsAgainstSpec(spec, methodName); + validateSpecDimDomains(spec, ctx); + validateParamsAgainstMeta(spec.inputs, /*isInput=*/true, meta, ctx); + validateParamsAgainstMeta(spec.outputs, /*isInput=*/false, meta, ctx); + validateConstraintSpecs(spec, ctx); } namespace { @@ -435,40 +456,34 @@ void validateRuntimeConstraints(jsi::Runtime &rt, const std::vector> &inputShapes, const std::string &ctx) { for (size_t i = 0; i < constraints.size(); ++i) { - const auto &constraint = constraints[i]; auto cctx = std::format("{} constraint[{}]", ctx, i); - if (const auto *eq = std::get_if(&constraint)) { - if (eq->dims.size() < 2) { - continue; + if (const auto *eq = std::get_if(&constraints[i])) { + std::vector inputVals; + for (const auto &d : eq->dims) { + if (d.side == ParamSide::input) { + inputVals.push_back(getInputDimValue(d, inputShapes)); + } } - // Skip if any dim references an output — ExecuTorch validates output shapes - if (std::ranges::any_of(eq->dims, - [](const auto &d) { return d.side == ParameterSide::output; })) { + if (inputVals.size() < 2) { continue; } - int32_t first = getInputDimValue(eq->dims[0], inputShapes); - for (size_t j = 1; j < eq->dims.size(); ++j) { - int32_t val = getInputDimValue(eq->dims[j], inputShapes); - if (val != first) { - throw jsi::JSError(rt, std::format("{}: equality constraint violated: " - "expected all dims to be {}, got {}", - cctx, first, val)); + for (size_t j = 1; j < inputVals.size(); ++j) { + if (inputVals[j] != inputVals[0]) { + throw jsi::JSError(rt, std::format("{}: equality constraint violated", cctx)); } } - } else if (const auto *lin = std::get_if(&constraint)) { - if (lin->dimLhs.side == ParameterSide::output || - lin->dimRhs.side == ParameterSide::output) { + } + + if (const auto *lin = std::get_if(&constraints[i])) { + if (lin->dimLhs.side == ParamSide::output || + lin->dimRhs.side == ParamSide::output) { continue; } int32_t lhs = getInputDimValue(lin->dimLhs, inputShapes); int32_t rhs = getInputDimValue(lin->dimRhs, inputShapes); - int32_t expected = lin->coefficients[0] * rhs + lin->coefficients[1]; - if (lhs != expected) { - throw jsi::JSError(rt, std::format("{}: linear constraint violated: " - "expected {} = {} * {} + {}", - cctx, lhs, lin->coefficients[0], rhs, - lin->coefficients[1])); + if (lhs != lin->coefficients[0] * rhs + lin->coefficients[1]) { + throw jsi::JSError(rt, std::format("{}: linear constraint violated", cctx)); } } } diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h index 291a742f08..c1f7047b5a 100644 --- a/packages/react-native-executorch/cpp/core/schema.h +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -15,6 +15,28 @@ #include #include +/** + * Model spec types and validation. + * + * This module is the C++ counterpart of `src/core/schema.ts`. Both files + * define the same data model (MethodSpec, ParamSpec, ConcreteDim, etc.) + * and validation logic, but serve different roles: + * + * - **schema.ts** — validates *allowed* (symbolic) specs written by pipeline + * authors against *exported* (concrete) specs. Handles symbol binding, + * dim-domain matching, and 1-to-1 constraint matching. Runs in JS. + * + * - **schema.cpp / schema.h** — validates exported specs at load time against + * ExecuTorch MethodMeta (dtype, rank, static shape). Also validates runtime + * constraint values before execution. Runs in native C++. + * + * The exported spec originates as JSON from the companion `get_model_schema` + * method (or is derived from MethodMeta when absent). It is parsed by + * `parseModelSpecJson` (C++) and also passed to `validateSpec` (TS) for + * allowed-spec matching. + * + * @see src/core/schema.ts for the TypeScript validation layer. + */ namespace rnexecutorch::core::schema { namespace jsi = facebook::jsi; @@ -61,8 +83,8 @@ struct ParamSpec { // ======================================================== /// Whether the referenced tensor dimension belongs to an input or output. -enum class ParameterSide { input, - output }; +enum class ParamSide { input, + output }; /** * Reference to a single tensor dimension of a method's input or output. @@ -70,7 +92,7 @@ enum class ParameterSide { input, * with ExecuTorch's `inputTensorMeta` / `outputTensorMeta` ordering. */ struct DimRef { - ParameterSide side = ParameterSide::input; + ParamSide side = ParamSide::input; int32_t tensorIdx = 0; int32_t dimIdx = 0; }; @@ -194,14 +216,17 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & * @param methodName Method name for error messages. * @throws std::runtime_error on any mismatch. */ -void validateSpecAgainstMeta(const MethodSpec &spec, - const executorch::runtime::MethodMeta &meta, - const std::string &methodName); +void validateSpec(const MethodSpec &spec, + const executorch::runtime::MethodMeta &meta, + const std::string &ctx); /** * Validates runtime constraints on input tensors before execution. - * Only input-only constraints are checked; constraints referencing outputs - * are skipped (ExecuTorch validates output shapes internally). + * Only input-only constraints are checked. Constraints referencing outputs + * are skipped because ExecuTorch constructs the output tensors internally, + * guaranteeing they satisfy the declared constraints by construction. + * After execution, the output shapes are validated separately against the + * shapes returned by ExecuTorch when copying data to the user-provided buffers. * * @param rt The JSI runtime instance (for error reporting). * @param constraints The constraints to validate. diff --git a/packages/react-native-executorch/src/core/modelSchema.ts b/packages/react-native-executorch/src/core/modelSchema.ts deleted file mode 100644 index 0fcdc579ef..0000000000 --- a/packages/react-native-executorch/src/core/modelSchema.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { type DType } from './tensor'; -import { type Model, type ExecuTorchTag, type ModelMethodMeta, type TensorMeta } from './model'; - -/** - * A single dimension in a symbolic tensor shape. - * - * - A **number** matches only that exact dimension size (e.g. `1`, `3`). - * - A **string** is a symbolic variable that can match any integer and is - * resolved consistently within the same tensor shape (e.g. `'N'`, `'H'`, - * `'W'`). - * @category Types - */ -export type SymbolicShape = readonly (number | string)[]; - -/** - * A constraint on the dtype and/or shape of a tensor input or output slot. - * - * Both fields are optional; omitting `dtype` skips dtype checking, and omitting - * `shapes` (or providing an empty array) skips shape checking. - * @category Types - */ -export type TensorConstraint = { - /** If set, the tensor's dtype must match exactly. */ - readonly dtype?: DType; - /** - * One or more acceptable symbolic shapes. The tensor passes validation if - * its concrete shape matches **at least one** of the listed shapes. - */ - readonly shapes?: readonly SymbolicShape[]; -}; - -/** - * Convenience constructor for a {@link TensorConstraint}. - * - * ```ts - * // Accepts a float32 tensor with shape [1, 3, H, W] or [3, H, W] - * SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W']) - * - * // Accepts any tensor of shape [1, N] - * SymbolicTensor(undefined, [1, 'N']) - * ``` - * @category Typescript API - * @param dtype Optional dtype requirement. Pass `undefined` to skip dtype - * checking. - * @param shapes Zero or more acceptable symbolic shapes. An empty list skips - * shape checking. - * @returns A {@link TensorConstraint} object. - */ -export function SymbolicTensor( - dtype?: DType, - ...shapes: readonly SymbolicShape[] -): TensorConstraint { - return { dtype, shapes }; -} - -const primitiveTagMap = { - number: ['Int', 'Double'] as ExecuTorchTag[], - boolean: ['Bool'] as ExecuTorchTag[], - null: ['None'] as ExecuTorchTag[], -} as const; - -/** - * A constraint describing an expected input or output slot of a model method. - * - * - `'number'` — the slot must carry an integer or double primitive value. - * - `'boolean'` — the slot must carry a boolean primitive value. - * - `'null'` — the slot must carry a `None` value. - * - {@link TensorConstraint} — the slot must carry a tensor, optionally with a - * constrained dtype and/or shape. - * @category Types - */ -export type ValueConstraint = keyof typeof primitiveTagMap | TensorConstraint; - -/** - * Checks whether a concrete tensor shape matches at least one of the provided - * symbolic shapes. - * - * Symbolic dimensions (strings) act as named wildcards: within a single shape - * candidate they must resolve to the same integer value every time they appear, - * but they are not enforced consistently across different tensor slots. - * @category Typescript API - * @param actual The concrete shape array to test (e.g. `[1, 3, 224, 224]`). - * @param expected One or more symbolic shapes to match against. - * @returns `true` if `actual` matches at least one of the `expected` shapes. - */ -export function matchShape(actual: number[], ...expected: readonly SymbolicShape[]): boolean { - return expected.some((shape) => { - if (actual.length !== shape.length) return false; - const symbolMap = new Map(); - return shape.every((dim, i) => { - const act = actual[i]!; - if (typeof dim === 'number') return act === dim; - if (symbolMap.has(dim)) return symbolMap.get(dim) === act; - symbolMap.set(dim, act); - return true; - }); - }); -} - -// TODO: Implement cross-tensor symbol validation (e.g. enforcing that 'N' or -// 'H' has the same value across different input/output tensors). Note that a -// proper implementation will require backtracking/solving when multiple tensors -// have multiple alternative shapes. For now, we just check that each tensor -// individually matches at least one of its expected shapes, without enforcing -// consistency of symbolic dimensions across tensors, only within each tensor. -function validateTags( - side: 'input' | 'output', - expected: readonly ValueConstraint[], - actualTags: ExecuTorchTag[], - tensorMetas: TensorMeta[] -) { - const numTensors = expected.filter((t) => typeof t === 'object').length; - if (tensorMetas.length !== numTensors) - throw new Error( - `${side} tensor count mismatch: expected ${numTensors}, got ${tensorMetas.length}` - ); - - let tIdx = 0; - expected.forEach((exp, i) => { - const act = actualTags[i]!; - if (typeof exp === 'string') { - if (!primitiveTagMap[exp].includes(act)) { - throw new Error(`${side}[${i}]: expected primitive '${exp}', got '${act}'`); - } - } else { - if (act !== 'Tensor') { - throw new Error(`${side}[${i}]: expected Tensor, got primitive '${act}'`); - } - const tMeta = tensorMetas[tIdx++]!; - if (exp.dtype && tMeta.dtype !== exp.dtype) { - throw new Error( - `${side}[${i}]: dtype mismatch: expected '${exp.dtype}', got '${tMeta.dtype}'` - ); - } - if (exp.shapes?.length && !matchShape(tMeta.shape, ...exp.shapes)) { - const expectedShapesStr = exp.shapes.map((s) => `[${s.join(',')}]`).join('|'); - throw new Error( - `${side}[${i}]: shape mismatch: expected shape matching ${expectedShapesStr}, got [${tMeta.shape.join(',')}]` - ); - } - } - }); -} - -/** - * Validates that a compiled model's method signature matches the declared - * input and output constraints, throwing a descriptive error on mismatch. - * - * The function checks: - * - That the method exists on the model. - * - That the number of input and output slots matches the expected counts. - * - That the value-tag of each slot (tensor, int, bool, etc.) is compatible - * with its declared {@link ValueConstraint}. - * - For tensor slots: that the dtype and shape (if specified) satisfy the - * - * {@link TensorConstraint}. - * - * On success it returns the method's {@link ModelMethodMeta}, which can be used - * to read concrete input/output tensor shapes for pre-allocating scratch - * tensors. - * @remarks - * A symbolic (string) input dimension here only relaxes *validation*. For a - * dimension that must genuinely vary at runtime (e.g. sequence length), the - * `.pte` must additionally be exported with a companion method - * `get_dynamic_dims_` (e.g. `get_dynamic_dims_forward`) that - * returns, per tensor input, an `int32` `[rank, 3]` tensor of `[min, max, step]` - * bounds — ExecuTorch metadata only serializes the static upper bound, so the - * runtime reads the active range from this companion. Without it, the method - * only accepts the exact shape it was exported with. The reported upper bound is - * still read from `meta.inputTensorMeta`. - * @category Typescript API - * @param model The compiled model to validate. - * @param methodName The exported method name to validate (e.g. `'forward'`). - * @param expectedInputs Ordered list of {@link ValueConstraint}s for each input - * slot. - * @param expectedOutputs Ordered list of {@link ValueConstraint}s for each - * output slot. - * @returns The {@link ModelMethodMeta} for the validated method. - * @throws {Error} A human-readable description of which constraint failed. - */ -export function validateModelSchema( - model: Model, - methodName: string, - expectedInputs: readonly ValueConstraint[], - expectedOutputs: readonly ValueConstraint[] -): ModelMethodMeta { - if (!model.getMethodNames().includes(methodName)) - throw new Error(`signature validation: '${methodName}' method not found`); - - const meta = model.getMethodMeta(methodName); - - const formatError = (errorMsg: string) => { - return ( - `signature validation failed for '${methodName}': ${errorMsg}\n` + - ` Expected: ${JSON.stringify(expectedInputs)} -> ${JSON.stringify(expectedOutputs)}\n` + - ` Actual: [${meta.inputTags.join(', ')}] -> [${meta.outputTags.join(', ')}] (metas: ${JSON.stringify(meta.inputTensorMeta)} -> ${JSON.stringify(meta.outputTensorMeta)})` - ); - }; - - if (meta.inputTags.length !== expectedInputs.length) { - throw new Error( - formatError( - `input count mismatch: expected ${expectedInputs.length}, got ${meta.inputTags.length}` - ) - ); - } - if (meta.outputTags.length !== expectedOutputs.length) { - throw new Error( - formatError( - `output count mismatch: expected ${expectedOutputs.length}, got ${meta.outputTags.length}` - ) - ); - } - - try { - validateTags('input', expectedInputs, meta.inputTags, meta.inputTensorMeta); - validateTags('output', expectedOutputs, meta.outputTags, meta.outputTensorMeta); - } catch (e: any) { - throw new Error(formatError(e.message)); - } - - return meta; -} diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 8cbc55c93b..eaabef4603 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -193,7 +193,7 @@ export type MethodSpec = { export type ModelSpec = Record>; // ======================================================== -// Dim helper functions +// Helper functions // ======================================================== /** @@ -297,6 +297,20 @@ export const SymbolicTensor = (dtype: DType, shape: SymbolicShape) => { return { kind: 'Tensor', dtype, shape: typedShape } as TensorSpec; }; +export const f32 = (...shape: SymbolicShape) => SymbolicTensor('float32', shape); +export const i64 = (...shape: SymbolicShape) => SymbolicTensor('int64', shape); +export const i32 = (...shape: SymbolicShape) => SymbolicTensor('int32', shape); +export const ui8 = (...shape: SymbolicShape) => SymbolicTensor('uint8', shape); + +export function method( + name: string, + inputs: ParamSpec[], + outputs: ParamSpec[], + constraints?: RuntimeConstraint[] +): Record> { + return { [name]: { inputs, outputs, runtimeConstraints: constraints ?? [] } }; +} + // ======================================================== // Parameter spec validation // ======================================================== @@ -313,12 +327,7 @@ function choicesEqual(c1: readonly number[], c2: readonly number[]): boolean { return s1.size === s2.size && [...s1].every((elem) => s2.has(elem)); } -function matchDim( - sDim: SymbolicDim, - cDim: ConcreteDim, - bindings: SymbolBindings, - ctx: string -): void { +function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, dims: SymbolBindings, ctx: string): void { if (sDim.kind === 'constant' && cDim.kind === 'constant') { if (sDim.value !== cDim.value) { throw new Error(`${ctx}: Constant dimension mismatch.`); @@ -341,30 +350,30 @@ function matchDim( } if (sDim.kind === 'static' && cDim.kind === 'constant') { - const bind = bindings.get(sDim.symbol); + const bind = dims.get(sDim.symbol); if (bind) { if (bind.kind !== 'constant' || bind.value !== cDim.value) { - throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent dims.`); } return; } - bindings.set(sDim.symbol, cDim); + dims.set(sDim.symbol, cDim); return; } if (sDim.kind === 'dynamic' && (cDim.kind === 'range' || cDim.kind === 'enum')) { - const bind = bindings.get(sDim.symbol); + const bind = dims.get(sDim.symbol); if (bind) { const consistentRange = bind.kind === 'range' && cDim.kind === 'range' && rangesEqual(bind.range, cDim.range); const consistentEnum = bind.kind === 'enum' && cDim.kind === 'enum' && choicesEqual(bind.choices, cDim.choices); if (!consistentRange && !consistentEnum) { - throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent dims.`); } return; } - bindings.set(sDim.symbol, cDim); + dims.set(sDim.symbol, cDim); return; } @@ -374,7 +383,7 @@ function matchDim( function matchMethodSpecs( allowedMethodSpec: MethodSpec, exportedMethodSpec: MethodSpec, - bindings: SymbolBindings, + dims: SymbolBindings, ctx: string ): void { if (allowedMethodSpec.inputs.length !== exportedMethodSpec.inputs.length) { @@ -414,7 +423,7 @@ function matchMethodSpecs( for (let d = 0; d < allowedTensorSpec.shape.length; ++d) { const dimCtx = `${paramSpecCtx} Tensor dim #${d}`; - matchDim(allowedTensorSpec.shape[d]!, exportedTensorSpec.shape[d]!, bindings, dimCtx); + matchDim(allowedTensorSpec.shape[d]!, exportedTensorSpec.shape[d]!, dims, dimCtx); } } } @@ -422,14 +431,14 @@ function matchMethodSpecs( function matchModelSpecsSymbols( allowedModelSpec: ModelSpec, exportedModelSpec: ModelSpec, - bindings: SymbolBindings + dims: SymbolBindings ): void { for (const [methodName, allowedMethodSpec] of Object.entries(allowedModelSpec)) { const exportedMethodSpec = exportedModelSpec[methodName]; if (!exportedMethodSpec) { throw new Error(`Method '${methodName}' not found in exported model spec.`); } - matchMethodSpecs(allowedMethodSpec, exportedMethodSpec, bindings, `Method '${methodName}'`); + matchMethodSpecs(allowedMethodSpec, exportedMethodSpec, dims, `Method '${methodName}'`); } } @@ -526,7 +535,7 @@ function matchRuntimeConstraints( // Spec validation // ======================================================== -function verifySymbolKindConsistency(modelSpec: ModelSpec): void { +function validateSymbolKindConsistency(modelSpec: ModelSpec): void { const symbolKinds = new Map(); for (const methodSpec of Object.values(modelSpec)) { @@ -546,7 +555,7 @@ function verifySymbolKindConsistency(modelSpec: ModelSpec): void { } } -function verifyConstraintCorrectness(modelSpec: ModelSpec): void { +function validateConstraintCorrectness(modelSpec: ModelSpec): void { for (const [methodName, methodSpec] of Object.entries(modelSpec)) { for (const [idx, constraint] of methodSpec.runtimeConstraints.entries()) { const ctx = `Method '${methodName}' constraint ${idx}`; @@ -570,12 +579,81 @@ function verifyConstraintCorrectness(modelSpec: ModelSpec): void { } } +function validateDimDomains(modelSpec: ModelSpec): void { + for (const [methodName, methodSpec] of Object.entries(modelSpec)) { + const allParams = [...methodSpec.inputs, ...methodSpec.outputs]; + for (const [p, param] of allParams.entries()) { + if (param.kind !== 'Tensor') continue; + + const isInput = p < methodSpec.inputs.length; + const label = isInput ? 'input' : 'output'; + const paramIdx = isInput ? p : p - methodSpec.inputs.length; + + for (const [d, dim] of param.shape.entries()) { + const ctx = `Method '${methodName}' ${label} #${paramIdx} dim #${d}`; + + if (dim.kind === 'constant') { + if (dim.value <= 0 || !Number.isInteger(dim.value)) { + throw new Error(`${ctx}: constant dim must be a positive integer.`); + } + } + if (dim.kind === 'range') { + if (dim.range.min < 0 || !Number.isInteger(dim.range.min)) { + throw new Error(`${ctx}: range min must be a non-negative integer.`); + } + if (dim.range.max < dim.range.min || !Number.isInteger(dim.range.max)) { + throw new Error(`${ctx}: range max must be >= min.`); + } + if (dim.range.step <= 0 || !Number.isInteger(dim.range.step)) { + throw new Error(`${ctx}: range step must be a positive integer.`); + } + } + if (dim.kind === 'enum') { + if (dim.choices.length === 0) { + throw new Error(`${ctx}: enum must have at least one choice.`); + } + if (dim.choices.some((c) => c <= 0 || !Number.isInteger(c))) { + throw new Error(`${ctx}: enum choices must be positive integers.`); + } + } + } + } + } +} + +/** + * Result of validating an exported model spec against allowed variants. + * @typeParam V The variant key type. + */ +export type SpecMatch = { + /** Key of the matched variant. */ + readonly variant: K; + /** + * Returns the concrete value for a symbol. + * @param name The symbol name. + * @param kind Expected dimension kind — determines the return type. Omit to + * get the raw {@link ConcreteDim} when the kind is not known upfront. + * @throws {Error} If the symbol is not found or has a different kind. + */ + dim(name: string, kind: 'constant'): number; + dim(name: string, kind: 'range'): Range; + dim(name: string, kind: 'enum'): readonly number[]; + dim(name: string): ConcreteDim; + + dims: { + any(...names: S): { [I in keyof S]: ConcreteDim }; + enum(...names: S): { [I in keyof S]: readonly number[] }; + range(...names: S): { [I in keyof S]: Range }; + constant(...names: S): { [I in keyof S]: number }; + }; +}; + /** * Validates that an exported (concrete) model spec satisfies at least one of * the allowed (symbolic) model specs — variants are tried in order and the * first match wins. For a variant to match: * - Every method exists in the exported spec and its signature matches, - * binding each symbol to a constant value (static) or a range/enum + * dim each symbol to a constant value (static) or a range/enum * (dynamic). Repeated symbols must bind consistently across the whole spec. * - The exported spec declares exactly the same runtime constraints per * method (1-to-1, no missing, no extras). Constraints are matched as @@ -583,28 +661,64 @@ function verifyConstraintCorrectness(modelSpec: ModelSpec): void { * * Authoring bugs in an allowed spec (conflicting symbol kinds, invalid * constraint coefficients or references) throw immediately, before matching. - * @param allowedModelSpecs The allowed model spec variants to try in order. * @param exportedModelSpec The exported model spec to validate against. - * @returns The symbol bindings of the first matching variant. + * @param allowedModelSpecs The allowed model spec variants keyed by name. + * @returns A {@link SpecMatch} with the matched variant key and dim + * accessors. * @throws {Error} A human-readable description of why every variant failed. */ -export function validateSpec( - allowedModelSpecs: readonly ModelSpec[], - exportedModelSpec: ModelSpec -): SymbolBindings { - allowedModelSpecs.forEach(verifySymbolKindConsistency); - allowedModelSpecs.forEach(verifyConstraintCorrectness); +export function validateSpec>>( + exportedModelSpec: ModelSpec, + allowedModelSpecs: T +): SpecMatch { + const entries = Object.entries(allowedModelSpecs); + + for (const spec of Object.values(allowedModelSpecs)) { + validateSymbolKindConsistency(spec); + validateConstraintCorrectness(spec); + validateDimDomains(spec); + } + validateDimDomains(exportedModelSpec); + validateConstraintCorrectness(exportedModelSpec); const errors: string[] = []; - for (const [idx, allowedModelSpec] of allowedModelSpecs.entries()) { + for (const [key, allowedModelSpec] of entries) { try { - const bindings: SymbolBindings = new Map(); - matchModelSpecsSymbols(allowedModelSpec, exportedModelSpec, bindings); + const dims: SymbolBindings = new Map(); + matchModelSpecsSymbols(allowedModelSpec, exportedModelSpec, dims); matchRuntimeConstraints(allowedModelSpec, exportedModelSpec); - return bindings; + + const dimFn = (name: string, kind?: string): any => { + const dim = dims.get(name); + if (!dim) { + throw new Error(`Symbol '${name}' not found in dims.`); + } + if (kind && dim.kind !== kind) { + throw new Error(`Symbol '${name}' is '${dim.kind}', expected '${kind}'.`); + } + if (dim.kind === 'constant') return dim.value; + if (dim.kind === 'range') return dim.range; + if (dim.kind === 'enum') return dim.choices; + return dim; + }; + + const createAccessor = (kind?: string) => { + return (...names: string[]): any => names.map((name) => dimFn(name, kind)); + }; + + return { + variant: key, + dim: dimFn, + dims: { + any: createAccessor(), + enum: createAccessor('enum'), + range: createAccessor('range'), + constant: createAccessor('constant'), + }, + }; } catch (e: any) { - errors.push(`Variant ${idx}: ${e.message}`); + errors.push(`Variant '${key}': ${e.message}`); continue; } } diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index 3a8b2d244c..e9e0803263 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { softmax } from '../../math'; @@ -77,25 +77,31 @@ export async function createClassifier( const { modelPath, classifierOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [SymbolicTensor('float32', [1, 'N'], ['N'])] - ); - const inpShape = meta.inputTensorMeta[0]!.shape; - const outShape = meta.outputTensorMeta[0]!.shape; - - const numLabels = outShape[outShape.length - 1]!; - if (classifierOpts.labels.length !== numLabels) { + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32(1, 'N')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32('N')] + ), + }); + + const [N, H, W] = dims.constant('N', 'H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = { batched: [1, N], unbatched: [N] }[variant]; + + if (classifierOpts.labels.length !== N) { throw new Error( - `Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${numLabels}).` + `Classifier labels length (${classifierOpts.labels.length}) must match model output dimension (${N}).` ); } - // prettier-ignore const tensors = [ - tensor('float32', outShape), + tensor('float32', outShape), // prettier-ignore tensor('float32', outShape), ] as const; @@ -119,9 +125,8 @@ export async function createClassifier( const tInput = preprocessor.process(input); model.execute('forward', [tInput], [tLogits]); - // prettier-ignore const probas = tLogits - .through(softmax, tProbas) + .through(softmax, tProbas) // prettier-ignore .getData(new Float32Array(tProbas.numel)); return Array.from(probas) diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts index afd4c63383..f4357a2c0c 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; @@ -56,14 +56,22 @@ export async function createImageEmbedder( const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [SymbolicTensor('float32', [1, 'D'], ['D'])] - ); - const inpShape = meta.inputTensorMeta[0]!.shape; - const outShape = meta.outputTensorMeta[0]!.shape; + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32(1, 'D')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32('D')] + ), + }); + + const [D, H, W] = dims.constant('D', 'H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = { batched: [1, D], unbatched: [D] }[variant]; const tensors = [tensor('float32', outShape)] as const; const [tEmbedding] = tensors; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts index d26407ea39..148a1c05ce 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; @@ -115,35 +115,29 @@ export async function createInstanceSegmenter( }> { const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [ - SymbolicTensor('float32', ['N', 4]), - SymbolicTensor('float32', ['N']), - SymbolicTensor('float32', ['N']), - SymbolicTensor('float32', ['N', 'MH', 'MW']), - ] - ); - - const inpShape = meta.inputTensorMeta[0]!.shape; - - const outBoxesShape = meta.outputTensorMeta[0]!.shape; - const outScoresShape = meta.outputTensorMeta[1]!.shape; - const outClassesShape = meta.outputTensorMeta[2]!.shape; - const outMasksShape = meta.outputTensorMeta[3]!.shape; - - const maskH = outMasksShape[1]!; - const maskW = outMasksShape[2]!; - const targetH = inpShape.at(-2)!; - const targetW = inpShape.at(-1)!; + + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', + [f32(1, 3, 'H', 'W')], + [f32('N', 4), f32('N'), f32('N'), f32('N', 'MH', 'MW')] + ), + unbatched: method( + 'forward', + [f32(3, 'H', 'W')], + [f32('N', 4), f32('N'), f32('N'), f32('N', 'MH', 'MW')] + ), + }); + + const [N, H, W, maskH, maskW] = dims.constant('N', 'H', 'W', 'MH', 'MW'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShapes = { boxes: [N, 4], scores: [N], classes: [N], masks: [N, maskH, maskW] }; const tensors = [ - tensor('float32', outBoxesShape), - tensor('float32', outScoresShape), - tensor('float32', outClassesShape), - tensor('float32', outMasksShape), + tensor('float32', outShapes.boxes), + tensor('float32', outShapes.scores), + tensor('float32', outShapes.classes), + tensor('float32', outShapes.masks), tensor('float32', [maskH, maskW, 1]), ] as const; @@ -214,7 +208,7 @@ export async function createInstanceSegmenter( const d = boxes[idx * 4 + 3]!; const box = scaleBox(decodeBox([a, b, c, d], opts.boxFormat), { - from: { width: targetW, height: targetH }, + from: { width: W, height: H }, to: { width: input.width, height: input.height }, resizeMode: 'stretch', }); diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index d6ea03a294..e0f1b108d0 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor, type Tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; @@ -181,27 +181,22 @@ export async function createKeypointDetector( const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [ - SymbolicTensor('float32', ['N', 4]), - SymbolicTensor('float32', ['N']), - SymbolicTensor('float32', ['N']), - ] - ); - - const inpShape = meta.inputTensorMeta[0]!.shape; - const outBoxesShape = meta.outputTensorMeta[0]!.shape; - const outScoresShape = meta.outputTensorMeta[1]!.shape; - const outClassesShape = meta.outputTensorMeta[2]!.shape; - - const targetH = inpShape.at(-2)!; - const targetW = inpShape.at(-1)!; + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32('N', 4), f32('N'), f32('N')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32('N', 4), f32('N'), f32('N')] + ), + }); + + const [N, H, W] = dims.constant('N', 'H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShapes = { boxes: [N, 4], scores: [N], classes: [N] }; const tensors = [ - tensor('float32', outBoxesShape), - tensor('float32', outScoresShape), - tensor('float32', outClassesShape), + tensor('float32', outShapes.boxes), + tensor('float32', outShapes.scores), + tensor('float32', outShapes.classes), ] as const; const [tBoxes, tScores, tClasses] = tensors; @@ -176,7 +174,7 @@ export async function createObjectDetector( label, confidence, box: scaleBox(decodeBox([a, b, c, d], boxFormat), { - from: { width: targetW, height: targetH }, + from: { width: W, height: H }, to: { width: input.width, height: input.height }, ...opts, }), diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts index 3569210481..9d0efd20bb 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/preprocessing.ts @@ -1,5 +1,4 @@ import { tensor, type Tensor } from '../../../core/tensor'; -import { matchShape } from '../../../core/modelSchema'; import type { ImageBuffer } from '../image'; import { @@ -62,15 +61,11 @@ export function createImagePreprocessor( dispose: () => void; } { const numRgbChannels = 3; - const expectedShapes = [ - [numRgbChannels, 'H', 'W'], - [1, numRgbChannels, 'H', 'W'], - ] as const; - - if (!matchShape(outputShape, ...expectedShapes)) { + const isRank3 = outputShape.length === 3 && outputShape[0] === numRgbChannels; + const isRank4 = outputShape.length === 4 && outputShape[1] === numRgbChannels; + if (!isRank3 && !isRank4) { throw new Error( - `preprocessor: got shape [${outputShape}], required one of: ` + - `${expectedShapes.map((s) => `[${s.join(',')}]`).join(' | ')}` + `preprocessor: got shape [${outputShape}], expected [${numRgbChannels}, H, W] or [1, ${numRgbChannels}, H, W]` ); } diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 7747a4517c..46cebfbf11 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, i64, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { randomNormal } from '../../math'; import { loadTokenizer } from '../../nlp/tokenizer'; @@ -79,28 +79,29 @@ export async function createSdxsTextToImage( const model = await wrapAsync(loadModel, runtime)(modelPath); const tokenizer = await wrapAsync(loadTokenizer, runtime)(tokenizerPath); - validateModelSchema( - model, - 'encode', - [SymbolicTensor('int64', [1, CLIP_MAX_TOKENS])], - [SymbolicTensor('float32', [1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE])] - ); - validateModelSchema( - model, - 'denoise', - [ - SymbolicTensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE]), - SymbolicTensor('int64', [1]), - SymbolicTensor('float32', [1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE]), - ], - [SymbolicTensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE])] - ); - validateModelSchema( - model, - 'decode', - [SymbolicTensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE])], - [SymbolicTensor('float32', [1, 3, IMAGE_SIZE, IMAGE_SIZE])] - ); + validateSpec(model.schema, { + default: { + ...method( + 'encode', // prettier-ignore + [i64(1, CLIP_MAX_TOKENS)], + [f32(1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE)] + ), + ...method( + 'denoise', + [ + f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE), + i64(1), + f32(1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE), + ], + [f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE)] + ), + ...method( + 'decode', + [f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE)], + [f32(1, 3, IMAGE_SIZE, IMAGE_SIZE)] + ), + }, + }); const tensors = [ tensor('int64', [1, CLIP_MAX_TOKENS]), diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index 148cf9b267..f6c90cd842 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; @@ -121,18 +121,22 @@ export async function createSemanticSegmenter( const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [SymbolicTensor('float32', [1, 'K', 'H', 'W'], ['K', 'H', 'W'])] - ); - const inpShape = meta.inputTensorMeta[0]!.shape; - const outShape = meta.outputTensorMeta[0]!.shape; + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32(1, 'K', 'H', 'W')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32('K', 'H', 'W')] + ), + }); - const nClasses = outShape.at(-3)!; - const targetH = outShape.at(-2)!; - const targetW = outShape.at(-1)!; + const [nClasses, H, W] = dims.constant('K', 'H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = { batched: [1, nClasses, H, W], unbatched: [nClasses, H, W] }[variant]; // Generate highly distinct, high-contrast colors, see: // https://martin.ankerl.com/2009/12/09/how-to-create-random-colors-programmatically/ @@ -149,11 +153,11 @@ export async function createSemanticSegmenter( const tensors = [ tensor('float32', outShape), - tensor('float32', [nClasses, targetH, targetW]), - tensor('float32', [nClasses, targetH, targetW]), - tensor('float32', [targetH, targetW, nClasses]), - tensor(nClasses > 1 ? 'int32' : 'uint8', [targetH, targetW, 1]), - tensor('uint8', [targetH, targetW, 4]), + tensor('float32', [nClasses, H, W]), + tensor('float32', [nClasses, H, W]), + tensor('float32', [H, W, nClasses]), + tensor(nClasses > 1 ? 'int32' : 'uint8', [H, W, 1]), + tensor('uint8', [H, W, 4]), ] as const; const [tOutput, tReshape, tSigmoid, tChanLast, tMask, tRgba] = tensors; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index 1c1a89cbe5..dfb3397b37 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import type { ImageBuffer } from '../image'; @@ -69,24 +69,29 @@ export async function createStyleTransfer( const { modelPath, opts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])] - ); - const inpShape = meta.inputTensorMeta[0]!.shape; - const outShape = meta.outputTensorMeta[0]!.shape; + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [f32(1, 3, 'H', 'W')], + [f32(1, 3, 'H', 'W')] + ), + unbatched: method( + 'forward', // prettier-ignore + [f32(3, 'H', 'W')], + [f32(3, 'H', 'W')] + ), + }); - const targetH = outShape.at(-2)!; - const targetW = outShape.at(-1)!; + const [H, W] = dims.constant('H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = inpShape; const tensors = [ tensor('float32', outShape), - tensor('float32', [3, targetH, targetW]), - tensor('uint8', [3, targetH, targetW]), - tensor('uint8', [targetH, targetW, 3]), - tensor('uint8', [targetH, targetW, 4]), + tensor('float32', [3, H, W]), + tensor('uint8', [3, H, W]), + tensor('uint8', [H, W, 3]), + tensor('uint8', [H, W, 4]), ] as const; const [tOutput, tReshape, tUint8, tChanLast, tRgba] = tensors; diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index 857457d89f..144e018e63 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, DynamicDim as Dyn, method, i64, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { loadTokenizer } from '../tokenizer'; @@ -67,16 +67,22 @@ export async function createTextEmbedder( // Text embedding models take two int64 inputs: the token ids and the // attention mask, both of shape [1, sequence_length]. - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('int64', [1, 'L']), SymbolicTensor('int64', [1, 'L'])], - [SymbolicTensor('float32', [1, 'D'], ['D'])] - ); - // The models are exported with a dynamic sequence dimension; the declared size - // is the upper bound, used only to truncate over-long inputs. - const maxSeqLen = meta.inputTensorMeta[0]!.shape[1]!; - const outShape = meta.outputTensorMeta[0]!.shape; + const { variant, dims } = validateSpec(model.schema, { + batched: method( + 'forward', // prettier-ignore + [i64(1, Dyn('L')), i64(1, Dyn('L'))], + [f32(1, 'D')] + ), + unbatched: method( + 'forward', // prettier-ignore + [i64(1, Dyn('L')), i64(1, Dyn('L'))], + [f32('D')] + ), + }); + + const [seqLen] = dims.range('L'); + const [D] = dims.constant('D'); + const outShape = { batched: [1, D], unbatched: [D] }[variant]; const tensors = [tensor('float32', outShape)] as const; const [tEmbedding] = tensors; @@ -94,7 +100,7 @@ export async function createTextEmbedder( if (ids.length === 0) { throw new Error('createTextEmbedder: input tokenized to zero tokens'); } - const len = Math.min(ids.length, maxSeqLen); + const len = Math.min(ids.length, seqLen.max); const idsData = new BigInt64Array(len); const maskData = new BigInt64Array(len); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 9225c743ff..8811432118 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, DynamicDim as Dyn, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { extractFrames } from '../utils/vadUtils'; @@ -222,15 +222,15 @@ export async function createFsmnVoiceActivityDetector( // [1, frames, classes] where class 0 is the non-speech class. The output frame // count matches the input at runtime, but ExecuTorch metadata only reports the // static upper bound, so the per-call output tensor is sized explicitly below. - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', ['frames', 'fftLength'])], - [SymbolicTensor('float32', [1, 'frames', 'classes'])] - ); - const maxFrames = meta.inputTensorMeta[0]!.shape[0]!; - const fftLength = meta.inputTensorMeta[0]!.shape[1]!; - const numClass = meta.outputTensorMeta[0]!.shape[2]!; + const { dims } = validateSpec(model.schema, { + default: method( + 'forward', // prettier-ignore + [f32(Dyn('frames'), 'fftLen')], + [f32(1, Dyn('frames'), 'classes')] + ), + }); + const [numClasses, fftLength] = dims.constant('classes', 'fftLen'); + const maxFrames = dims.range('frames')[0].max; // The Hann window is uploaded once and reused by the native framing op across // every call. @@ -264,7 +264,7 @@ export async function createFsmnVoiceActivityDetector( waveform.subarray(startSample, startSample + sampleCount) ); const tInput = tensor('float32', [chunkFrames, fftLength]); - const tOutput = tensor('float32', [1, chunkFrames, numClass]); + const tOutput = tensor('float32', [1, chunkFrames, numClasses]); const outBuffer = new Float32Array(tOutput.numel); try { extractFrames(tWaveform, tHann, tInput, { @@ -281,7 +281,7 @@ export async function createFsmnVoiceActivityDetector( } for (let i = 0; i < realFrames; i++) { - scores[offset + i] = outBuffer[i * numClass]!; + scores[offset + i] = outBuffer[i * numClasses]!; } offset += realFrames; } diff --git a/packages/react-native-executorch/src/index.ts b/packages/react-native-executorch/src/index.ts index 3aad5c8d77..63879c363d 100644 --- a/packages/react-native-executorch/src/index.ts +++ b/packages/react-native-executorch/src/index.ts @@ -33,23 +33,10 @@ export * from './extensions/speech/tasks/fsmnVoiceActivityDetection'; export * from './extensions/speech/tasks/whisperSpeechToText'; // Core primitives — for library builders and power users -export { tensor } from './core/tensor'; -export type { DType, Tensor } from './core/tensor'; - -export { loadModel } from './core/model'; -export type { - Model, - ModelInput, - ModelOutput, - TensorMeta, - ModelMethodMeta, - ExecuTorchTag, -} from './core/model'; - -export { validateModelSchema, SymbolicTensor, matchShape } from './core/modelSchema'; -export type { ValueConstraint, TensorConstraint, SymbolicShape } from './core/modelSchema'; - -export { defaultWorkletRuntime, wrapAsync } from './core/runtime'; +export * from './core/model'; +export * from './core/schema'; +export * from './core/tensor'; +export * from './core/runtime'; export * as math from './extensions/math'; export * as cv from './extensions/cv'; diff --git a/packages/react-native-executorch/src/utils.ts b/packages/react-native-executorch/src/utils.ts index 1c457b3062..154f7c2aa4 100644 --- a/packages/react-native-executorch/src/utils.ts +++ b/packages/react-native-executorch/src/utils.ts @@ -1,5 +1,6 @@ import { rnexecutorchJsi } from './native/bridge'; -import { loadModel, type ModelMethodMeta } from './core/model'; +import { loadModel } from './core/model'; +import type { ModelSpec, ConcreteDim } from './core/schema'; import RNFS from 'react-native-fs'; /** @@ -24,12 +25,13 @@ export function getRegisteredBackends(): string[] { * @category Utils * @experimental Subject to change once the temporary react-native-fs dependency is replaced. See [Issue #1253](https://github.com/software-mansion/react-native-executorch/issues/1253). * @param source The remote HTTP URL or local path to the `.pte` model file. - * @returns A promise resolving to an object containing the model source and - * method signature metadata. + * @returns A promise resolving to an object containing the model source, + * method signature metadata, and per-method backend usage. */ export async function inspectModel(source: string): Promise<{ source: string; - methods: { name: string; meta: ModelMethodMeta }[]; + schema: ModelSpec; + backends: Record; }> { let localPath = source; let downloaded = false; @@ -44,15 +46,7 @@ export async function inspectModel(source: string): Promise<{ try { model = loadModel(localPath); - const methodNames = model.getMethodNames(); - - const methods: { name: string; meta: ModelMethodMeta }[] = []; - for (const method of methodNames) { - const meta = model.getMethodMeta(method); - methods.push({ name: method, meta }); - } - - return { source, methods }; + return { source, schema: model.schema, backends: model.backends }; } finally { if (model) { model.dispose(); From 0416cf51e04903794ad6c6c64ed7ca500582edbf Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 05:20:32 +0200 Subject: [PATCH 10/31] rebase --- .../speech/tasks/whisperSpeechToText.ts | 40 +++++++++---------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index dcd71f9bc4..be935a67db 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -3,8 +3,8 @@ import { scheduleOnRN, createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; import { wrapAsync } from '../../../core/runtime'; +import { f32, i64, method, validateSpec, DynamicDim as Dyn } from '../../../core/schema'; import { argmax } from '../../../extensions/math'; import { loadTokenizer } from '../../nlp/tokenizer'; @@ -166,32 +166,28 @@ export async function createWhisperSpeechToText')!; const isEnglishOnly = supportedLanguages.length === 1 && supportedLanguages[0] === 'en'; - const encMeta = validateModelSchema( - model, - 'encode', - [SymbolicTensor('float32', ['T_audio'])], - [SymbolicTensor('float32', [1, 'Seq', 'State'])] - ); - - const encSeqLen = encMeta.outputTensorMeta[0]!.shape[1]!; - const encStateDim = encMeta.outputTensorMeta[0]!.shape[2]!; - - validateModelSchema( - model, - 'decode', - [ - SymbolicTensor('int64', [1, 'Tokens']), - SymbolicTensor('int64', ['Tokens']), - SymbolicTensor('float32', [1, encSeqLen, encStateDim]), - ], - [SymbolicTensor('float32', [1, 'Tokens', 'Vocab'])] - ); + const { dims } = validateSpec(model.schema, { + default: { + ...method( + 'encode', // prettier-ignore + [f32(Dyn('T_audio'))], + [f32(1, 'SeqLen', 'StateDim')] + ), + ...method( + 'decode', + [i64(1, 'Tokens'), i64('Tokens'), f32(1, 'SeqLen', 'StateDim')], + [f32(1, 'Tokens', 'VocabSize')] + ), + }, + }); + + const [seqLen, stateDim] = dims.constant('SeqLen', 'StateDim'); const tensors = [ tensor('int64', [1]), // tPosition tensor('int64', [1, 1]), // tToken tensor('int32', [1, 1, 1]), // tArgmax - tensor('float32', [1, encSeqLen, encStateDim]), //tEncodings + tensor('float32', [1, seqLen, stateDim]), //tEncodings tensor('float32', [1, 1, tokenizer.getVocabSize()]), // tLogits ] as const; From c1391dacfa88002b7e45631451d4c639144c7655 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 06:02:24 +0200 Subject: [PATCH 11/31] small fix --- .../react-native-executorch/src/core/model.ts | 2 +- .../src/core/schema.ts | 10 +++----- .../src/extensions/cv/tasks/classification.ts | 2 ++ .../src/extensions/cv/tasks/imageEmbedding.ts | 2 ++ .../cv/tasks/instanceSegmentation.ts | 10 ++++---- .../extensions/cv/tasks/keypointDetection.ts | 9 ++++--- .../extensions/cv/tasks/objectDetection.ts | 10 +++++--- .../extensions/cv/tasks/sdxsTextToImage.ts | 25 ++++++++----------- .../cv/tasks/semanticSegmentation.ts | 2 ++ .../src/extensions/cv/tasks/styleTransfer.ts | 2 ++ .../src/extensions/nlp/tasks/textEmbedding.ts | 2 ++ .../tasks/fsmnVoiceActivityDetection.ts | 4 +++ 12 files changed, 46 insertions(+), 34 deletions(-) diff --git a/packages/react-native-executorch/src/core/model.ts b/packages/react-native-executorch/src/core/model.ts index eacd936907..5fd59bb2d4 100644 --- a/packages/react-native-executorch/src/core/model.ts +++ b/packages/react-native-executorch/src/core/model.ts @@ -14,7 +14,7 @@ export type ModelInput = Tensor | number | boolean | null; * A value returned from a model's `execute` method. * @category Types */ -export type ModelOutput = Tensor | number | boolean | null; +export type ModelOutput = Tensor | number | boolean | string | null; /** * A compiled, ready-to-run ExecuTorch model loaded into native memory. diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index eaabef4603..dcaf5af7ae 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -133,7 +133,7 @@ export type ParamSpec = * @category Types */ export type DimRef = { - readonly parameterSide: 'input' | 'output'; + readonly paramSide: 'input' | 'output'; readonly tensorIdx: number; readonly dimIdx: number; }; @@ -447,11 +447,7 @@ function matchModelSpecsSymbols( // ======================================================== function refsEqual(r1: DimRef, r2: DimRef): boolean { - return ( - r1.parameterSide === r2.parameterSide && - r1.tensorIdx === r2.tensorIdx && - r1.dimIdx === r2.dimIdx - ); + return r1.paramSide === r2.paramSide && r1.tensorIdx === r2.tensorIdx && r1.dimIdx === r2.dimIdx; } function constraintsEqual(c1: RuntimeConstraint, c2: RuntimeConstraint): boolean { @@ -483,7 +479,7 @@ function constraintsEqual(c1: RuntimeConstraint, c2: RuntimeConstraint): boolean function resolveDim(methodSpec: MethodSpec, ref: DimRef): D { let tensorSpecs: TensorSpec[]; - switch (ref.parameterSide) { + switch (ref.paramSide) { case 'input': tensorSpecs = methodSpec.inputs.filter((v): v is TensorSpec => v.kind === 'Tensor'); break; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts index e9e0803263..1c0dd34129 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/classification.ts @@ -58,6 +58,7 @@ export async function createClassifier( * Releases all allocated native resources. */ dispose: () => void; + /** * Performs asynchronous image classification on the given input image. * @param input The input image buffer. @@ -68,6 +69,7 @@ export async function createClassifier( * confidence. */ classify: (input: ImageBuffer, options?: { topk?: number }) => Promise[]>; + /** * Synchronous version of {@link classify} to be executed directly on the * caller or worklet thread. diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts index f4357a2c0c..22192f5eda 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/imageEmbedding.ts @@ -41,12 +41,14 @@ export async function createImageEmbedder( * Releases all allocated native resources. */ dispose: () => void; + /** * Asynchronously computes the embedding vector for the given input image. * @param input The input image buffer. * @returns A promise resolving to the embedding vector. */ embed: (input: ImageBuffer) => Promise; + /** * Synchronous version of {@link embed} to be executed directly on the * caller or worklet thread. diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts index 148a1c05ce..59cdbc56d6 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/instanceSegmentation.ts @@ -131,13 +131,13 @@ export async function createInstanceSegmenter( const [N, H, W, maskH, maskW] = dims.constant('N', 'H', 'W', 'MH', 'MW'); const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; - const outShapes = { boxes: [N, 4], scores: [N], classes: [N], masks: [N, maskH, maskW] }; + const outShape = { boxes: [N, 4], scores: [N], classes: [N], masks: [N, maskH, maskW] }; const tensors = [ - tensor('float32', outShapes.boxes), - tensor('float32', outShapes.scores), - tensor('float32', outShapes.classes), - tensor('float32', outShapes.masks), + tensor('float32', outShape.boxes), + tensor('float32', outShape.scores), + tensor('float32', outShape.classes), + tensor('float32', outShape.masks), tensor('float32', [maskH, maskW, 1]), ] as const; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts index e0f1b108d0..e9c2e54085 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/keypointDetection.ts @@ -154,6 +154,7 @@ export async function createKeypointDetector void; + /** * Performs asynchronous keypoint and bounding box detection on the given * input image. @@ -169,6 +170,7 @@ export async function createKeypointDetector Promise[]>; + /** * Synchronous version of {@link detectKeypoints} to be executed directly on * the caller or worklet thread. @@ -192,11 +194,12 @@ export async function createKeypointDetector( * Releases all allocated native resources. */ dispose: () => void; + /** * Performs asynchronous object detection on the given input image. * @param input The input image buffer. @@ -84,6 +85,7 @@ export async function createObjectDetector( input: ImageBuffer, options?: { confidenceThreshold?: number; iouThreshold?: number } ) => Promise[]>; + /** * Synchronous version of {@link detectObjects} to be executed directly on the * caller or worklet thread. @@ -111,12 +113,12 @@ export async function createObjectDetector( const [N, H, W] = dims.constant('N', 'H', 'W'); const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; - const outShapes = { boxes: [N, 4], scores: [N], classes: [N] }; + const outShape = { boxes: [N, 4], scores: [N], classes: [N] }; const tensors = [ - tensor('float32', outShapes.boxes), - tensor('float32', outShapes.scores), - tensor('float32', outShapes.classes), + tensor('float32', outShape.boxes), + tensor('float32', outShape.scores), + tensor('float32', outShape.classes), ] as const; const [tBoxes, tScores, tClasses] = tensors; diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts index 46cebfbf11..0130e6cd07 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/sdxsTextToImage.ts @@ -21,10 +21,9 @@ const CLIP_PAD_TOKEN_ID = 49407; const IMAGE_SIZE = 512; const LATENT_CHANNELS = 4; -// The UNet operates at 1/8 of the image resolution. -const LATENT_SIZE = IMAGE_SIZE / 8; -// Scheduler `init_noise_sigma`: the initial latents are standard normal. -const INIT_NOISE_SIGMA = 1.0; +const LATENT_SIZE = IMAGE_SIZE / 8; // The UNet operates at 1/8 of the image resolution. +const LATENT_SHAPE = [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE]; +const INIT_NOISE_SIGMA = 1.0; // Scheduler's initial latents are standard normal. // At the distilled single step the DEIS update collapses to an exactly linear // combination of the latents and the UNet output: @@ -61,6 +60,7 @@ export async function createSdxsTextToImage( ): Promise<{ /** Releases all allocated native resources. */ dispose: () => void; + /** * Generates an image from a text prompt. * @param prompt The text prompt describing the desired image. @@ -69,6 +69,7 @@ export async function createSdxsTextToImage( * @returns A promise resolving to the generated RGBA image buffer. */ generate: (prompt: string, seed?: number) => Promise; + /** * Synchronous version of {@link generate} to be executed directly on the * caller or worklet thread. @@ -88,16 +89,12 @@ export async function createSdxsTextToImage( ), ...method( 'denoise', - [ - f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE), - i64(1), - f32(1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE), - ], - [f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE)] + [f32(...LATENT_SHAPE), i64(1), f32(1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE)], + [f32(...LATENT_SHAPE)] ), ...method( - 'decode', - [f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE)], + 'decode', // prettier-ignore + [f32(...LATENT_SHAPE)], [f32(1, 3, IMAGE_SIZE, IMAGE_SIZE)] ), }, @@ -107,8 +104,8 @@ export async function createSdxsTextToImage( tensor('int64', [1, CLIP_MAX_TOKENS]), tensor('float32', [1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE]), tensor('int64', [1]), - tensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE]), - tensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE]), + tensor('float32', LATENT_SHAPE), + tensor('float32', LATENT_SHAPE), tensor('float32', [1, 3, IMAGE_SIZE, IMAGE_SIZE]), tensor('float32', [3, IMAGE_SIZE, IMAGE_SIZE]), tensor('uint8', [3, IMAGE_SIZE, IMAGE_SIZE]), diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts index f6c90cd842..b1b19b0385 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/semanticSegmentation.ts @@ -84,6 +84,7 @@ export async function createSemanticSegmenter( * Releases all allocated native resources. */ dispose: () => void; + /** * Runs semantic segmentation asynchronously. * @@ -109,6 +110,7 @@ export async function createSemanticSegmenter( input: ImageBuffer, colormap?: Partial> ) => Promise>; + /** * Runs semantic segmentation synchronously. * @see {@link segment} for details. diff --git a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts index dfb3397b37..ede6b88f0d 100644 --- a/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts +++ b/packages/react-native-executorch/src/extensions/cv/tasks/styleTransfer.ts @@ -54,12 +54,14 @@ export async function createStyleTransfer( * Releases all allocated native resources. */ dispose: () => void; + /** * Performs asynchronous image style transfer on the given input image. * @param input The input image buffer. * @returns A promise resolving to the styled image buffer. */ transferStyle: (input: ImageBuffer) => Promise; + /** * Synchronous version of {@link transferStyle} to be executed directly on the * caller or worklet thread. diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index 144e018e63..ee3e30d80b 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -44,6 +44,7 @@ export async function createTextEmbedder( * Releases all allocated native resources. */ dispose: () => void; + /** * Asynchronously computes the embedding vector for the given input text. * Inputs longer than the model's maximum sequence length are truncated. @@ -53,6 +54,7 @@ export async function createTextEmbedder( * @returns A promise resolving to the embedding vector. */ embed: (input: string, prompt?: string) => Promise; + /** * Synchronous version of {@link embed} to be executed directly on the * caller or worklet thread. diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 8811432118..3eb8faf2ec 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -187,6 +187,7 @@ export async function createFsmnVoiceActivityDetector( * Releases all allocated native resources. */ dispose: () => void; + /** * Asynchronously detects speech segments within a mono waveform sampled at * {@link FSMN_VAD_SAMPLE_RATE_HZ}. @@ -195,11 +196,13 @@ export async function createFsmnVoiceActivityDetector( * @returns A promise resolving to the detected speech segments, in seconds. */ detectVoice: (waveform: Float32Array, options?: VadOptions) => Promise; + /** * Synchronous version of {@link detectVoice} to be executed directly on the caller * or worklet thread. */ detectVoiceWorklet: (waveform: Float32Array, options?: VadOptions) => Segment[]; + /** * Appends a live audio chunk to a bounded rolling window, runs detection over * that window and reports a {@link VadEvent} when speech starts or stops, @@ -209,6 +212,7 @@ export async function createFsmnVoiceActivityDetector( * @param options Optional overrides of the detection thresholds and margin. */ detectVoiceOnStream: (chunk: Float32Array, options?: VadStreamOptions) => VadEvent | undefined; + /** * Clears the rolling window and speaking state used by {@link detectVoiceOnStream}. */ From 8f13d2accbbffe294a4390c1d0f1546040714a0c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 14:27:36 +0200 Subject: [PATCH 12/31] small fix --- .../cpp/core/model.cpp | 44 +++++++++---------- .../react-native-executorch/cpp/core/schema.h | 7 ++- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index ce33d9fa14..5ca219deca 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -46,8 +46,6 @@ namespace conversions = rnexecutorch::core::conversions; using rnexecutorch::core::tensor::TensorHostObject; -constexpr auto kSchemaMethod = "get_model_schema"; - ModelHostObject::ModelHostObject(const std::string &modelPath) : modelPath_(modelPath), etModule_(std::make_unique(modelPath)) { @@ -59,11 +57,12 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) } const auto methodNames = unwrap("loadModel", etModule_->method_names()); + const auto *const getSchemaMethod = "get_model_schema"; schema::ModelSpec overrideSpec; - if (methodNames.contains(kSchemaMethod)) { - auto ctx = std::format("loadModel: '{}'", kSchemaMethod); - auto result = unwrap(ctx, etModule_->execute(kSchemaMethod)); + if (methodNames.contains(getSchemaMethod)) { + auto ctx = std::format("loadModel: '{}'", getSchemaMethod); + auto result = unwrap(ctx, etModule_->execute(getSchemaMethod)); if (result.empty() || result[0].tag != executorch::runtime::Tag::String) { throw std::runtime_error(std::format("{} must return a single string value", ctx)); @@ -188,23 +187,24 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { logFn.callWithThis(rt, consoleObj, {jsi::String::createFromUtf8(rt, info)}); #endif - const auto *error = "execute: Method '{}' failed.\n" - "\n" - "Common causes:\n" - " 1. Backend not registered\n" - " Ensure backends from `model.backends` are registered\n" - " in the ExecuTorch runtime.\n" - "\n" - " 2. Shape/constraint mismatch\n" - " If the model uses dynamic shapes or runtime constraints\n" - " (e.g. equality between dimensions), export a companion\n" - " '{}' method returning a JSON model spec\n" - " (see `src/core/schema.ts` for the JSON structure).\n" - "\n" - " Without it, validation falls back to static metadata\n" - " from ExecuTorch which only contains upper bounds and\n" - " does not capture runtime constraints."; - auto result = unwrap(rt, std::vformat(error, std::make_format_args(methodName, kSchemaMethod)), std::move(executeResult)); + auto result = unwrap(rt, std::format("execute: Method '{}' failed.\n" + "\n" + "Common causes:\n" + " 1. Backend not registered\n" + " Ensure backends from `model.backends` are registered\n" + " in the ExecuTorch runtime.\n" + "\n" + " 2. Shape/constraint mismatch\n" + " If the model uses dynamic shapes or runtime constraints\n" + " (e.g. equality between dimensions), export a companion\n" + " method returning a JSON model spec\n" + " (see `src/core/schema.ts` for the JSON structure).\n" + "\n" + " Without it, validation falls back to static metadata\n" + " from ExecuTorch which only contains upper bounds and\n" + " does not capture runtime constraints.", + methodName), + std::move(executeResult)); auto jsOutputArray = jsi::Array(rt, result.size()); size_t outputIdx = 0; diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h index c1f7047b5a..c24d82daf1 100644 --- a/packages/react-native-executorch/cpp/core/schema.h +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -30,10 +30,9 @@ * ExecuTorch MethodMeta (dtype, rank, static shape). Also validates runtime * constraint values before execution. Runs in native C++. * - * The exported spec originates as JSON from the companion `get_model_schema` - * method (or is derived from MethodMeta when absent). It is parsed by - * `parseModelSpecJson` (C++) and also passed to `validateSpec` (TS) for - * allowed-spec matching. + * The exported spec originates as JSON from the companion method (or is derived + * from MethodMeta when absent). It is parsed by `parseModelSpecJson` (C++) and + * also passed to `validateSpec` (TS) for allowed-spec matching. * * @see src/core/schema.ts for the TypeScript validation layer. */ From eacc19a2f1de892363d24dbff6145dce3aedc5ef Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 17:13:04 +0200 Subject: [PATCH 13/31] feat(schema): add helper methods for runtime constraints and update task specifications --- .agents/skills/README.md | 2 +- .agents/skills/add-task-pipeline/SKILL.md | 38 +-- .agents/skills/core-guidelines/SKILL.md | 2 +- .../skills/model-schema-validation/SKILL.md | 235 +++++++++++------- .../cpp/core/tensor_helpers.cpp | 57 ++--- .../src/core/schema.ts | 61 +++++ .../src/extensions/nlp/tasks/textEmbedding.ts | 8 +- .../tasks/fsmnVoiceActivityDetection.ts | 6 +- .../speech/tasks/whisperSpeechToText.ts | 14 +- 9 files changed, 271 insertions(+), 152 deletions(-) diff --git a/.agents/skills/README.md b/.agents/skills/README.md index fdf0bc9f94..4f9d99a0de 100644 --- a/.agents/skills/README.md +++ b/.agents/skills/README.md @@ -7,6 +7,6 @@ This directory contains specialized skills (recipes) to guide contributors and A - [Core Guidelines](./core-guidelines/SKILL.md) — Symmetrical architecture, conventions, and model rules. - [Add Native Extension](./add-native-extension/SKILL.md) — C++ operations and JSI bindings. - [Add Task Pipeline](./add-task-pipeline/SKILL.md) — TypeScript task pipelines and React hooks. -- [Model Schema Validation](./model-schema-validation/SKILL.md) — SymbolicTensor schemas and validation. +- [Model Schema Validation](./model-schema-validation/SKILL.md) — Model specs, dynamic shapes, and schema validation. - [Verify and Build](./verify-and-build/SKILL.md) — TypeScript typechecking, native rebuilding, and troubleshooting. - [Skills Maintenance](./skills-maintenance/SKILL.md) — Keeping skills synchronized with core primitives. diff --git a/.agents/skills/add-task-pipeline/SKILL.md b/.agents/skills/add-task-pipeline/SKILL.md index 45ec78d485..f8116dfa40 100644 --- a/.agents/skills/add-task-pipeline/SKILL.md +++ b/.agents/skills/add-task-pipeline/SKILL.md @@ -59,17 +59,17 @@ When implementing task constructors like `create` (e.g. `createClassifier` - Handle model-specific configuration parameters (such as unique normalization factors, thresholds, or label arrays) dynamically through the TypeScript task options argument rather than baking them rigidly into JSI C++ code or the model structure. This rule contrasts TypeScript options against values baked into C++ or the model; to choose between a TypeScript **option** and a TypeScript **constant**, see Principle 6. 6. **Options vs. Constants (bucket by who varies the value)**: - - Every parameter belongs in exactly one of three places. Decide by asking *who varies this, and when*: + - Every parameter belongs in exactly one of three places. Decide by asking _who varies this, and when_: - | The value... | Lives as | Example | - | ------------------------------------------------------------------------------------ | -------------------------------------------------------------- | -------------------------------------------------------------- | - | Varies across the shipped models/variants of the pipeline | A task **option**, set per-model in `models.ts` | Whisper `tiny`/`base`/`small` sizes; normalization factors; labels | - | Is fixed by the model architecture or the `.pte` export contract, identical for every shipped variant | A **`const`** in the task file, beside the code that reads it | Static tensor shapes; export-pinned scheduler/decoder scalars | - | Is a per-call choice made by the app developer | An **argument** to the worklet executor | `threshold`, `seed`, `prompt` | + | The value... | Lives as | Example | + | ----------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | ------------------------------------------------------------------ | + | Varies across the shipped models/variants of the pipeline | A task **option**, set per-model in `models.ts` | Whisper `tiny`/`base`/`small` sizes; normalization factors; labels | + | Is fixed by the model architecture or the `.pte` export contract, identical for every shipped variant | A **`const`** in the task file, beside the code that reads it | Static tensor shapes; export-pinned scheduler/decoder scalars | + | Is a per-call choice made by the app developer | An **argument** to the worklet executor | `threshold`, `seed`, `prompt` | - - **A parameter with exactly one valid value is not an option — it is a constant.** For single-model pipelines (e.g. `sdxsTextToImage`), exposing export-pinned scalars or static shapes as options advertises knobs that either fail schema validation or silently corrupt output when touched. Prefer a `const` with a comment naming *why* the value is fixed. - - **Corollary (a quick smell test):** if every variant in `models.ts` passes an *identical* options object, those fields are not configuration — move them into the task file as constants and shrink the model type to the paths that actually differ. - - Do not keep a loop, parameter, or code path solely because it *looks* more general. If the surrounding math is only valid for one value (e.g. coefficients pinned to a single distilled timestep), the generality is fake and the parameter is a correctness trap. + - **A parameter with exactly one valid value is not an option — it is a constant.** For single-model pipelines (e.g. `sdxsTextToImage`), exposing export-pinned scalars or static shapes as options advertises knobs that either fail schema validation or silently corrupt output when touched. Prefer a `const` with a comment naming _why_ the value is fixed. + - **Corollary (a quick smell test):** if every variant in `models.ts` passes an _identical_ options object, those fields are not configuration — move them into the task file as constants and shrink the model type to the paths that actually differ. + - Do not keep a loop, parameter, or code path solely because it _looks_ more general. If the surrounding math is only valid for one value (e.g. coefficients pinned to a single distilled timestep), the generality is fake and the parameter is a correctness trap. ## 🚫 Avoid / Anti-Patterns @@ -95,7 +95,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; +import { validateSpec, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { type ImageBuffer } from '../image'; import { createImagePreprocessor, type ImagePreprocessorOptions } from './preprocessing'; @@ -137,15 +137,15 @@ export async function createMyTask( const { modelPath, taskOpts } = config; const model = await wrapAsync(loadModel, runtime)(modelPath); - // Validate model schema - const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [SymbolicTensor('float32', [1, 10], [10])] - ); - const inpShape = meta.inputTensorMeta[0]!.shape; - const outShape = meta.outputTensorMeta[0]!.shape; + // Validate model spec + const { variant, dims } = validateSpec(model.schema, { + batched: method('forward', [f32(1, 3, 'H', 'W')], [f32(1, 10)]), + unbatched: method('forward', [f32(3, 'H', 'W')], [f32(10)]), + }); + + const [H, W] = dims.constant('H', 'W'); + const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; + const outShape = { batched: [1, 10], unbatched: [10] }[variant]; // 2. Pre-allocate static tensors const tensors = [tensor('float32', outShape)] as const; diff --git a/.agents/skills/core-guidelines/SKILL.md b/.agents/skills/core-guidelines/SKILL.md index 16028711ff..2510c5b475 100644 --- a/.agents/skills/core-guidelines/SKILL.md +++ b/.agents/skills/core-guidelines/SKILL.md @@ -72,7 +72,7 @@ Use the following index to locate the specific procedural guides for your task: | **Add a new native operator or C++ binding** | [SKILL.md](../add-native-extension/SKILL.md) | Procedural guide to implementing C++ functions, exposing them via JSI, and writing TypeScript bridge wrappers. | | **Create a task pipeline or hook** | [SKILL.md](../add-task-pipeline/SKILL.md) | Guide to building end-to-end TS pipelines (e.g. object detection) and exposing them via React hooks. | | **Verify, rebuild, or troubleshoot changes** | [SKILL.md](../verify-and-build/SKILL.md) | Workflows for rebuilding TS/C++ and resolving common JSI runtime errors. | -| **Validate model constraints & schemas** | [SKILL.md](../model-schema-validation/SKILL.md) | Guide on specifying SymbolicTensor constraints and shapes for model signature validation. | +| **Validate model constraints & schemas** | [SKILL.md](../model-schema-validation/SKILL.md) | Guide on specifying model specs, dynamic shapes, and runtime constraints for model validation. | | **Maintain or refactor codebase patterns** | [SKILL.md](../skills-maintenance/SKILL.md) | Guide to keeping workspace skills in sync with codebase state to prevent documentation decay. | --- diff --git a/.agents/skills/model-schema-validation/SKILL.md b/.agents/skills/model-schema-validation/SKILL.md index f9d5e56ea0..7137e2f42f 100644 --- a/.agents/skills/model-schema-validation/SKILL.md +++ b/.agents/skills/model-schema-validation/SKILL.md @@ -1,162 +1,209 @@ --- name: model-schema-validation -description: Use when defining SymbolicTensor constraints, validating ExecuTorch model shapes, checking method signatures, or verifying dynamic tensor dimensions. +description: Use when defining model spec constraints, validating ExecuTorch model shapes, checking method signatures, or resolving dimension symbols. metadata: id: model_schema_validation - scope: src/core/modelSchema.ts, src/extensions/*/tasks/* + scope: src/core/schema.ts, src/extensions/*/tasks/* --- # Skill: Model Schema Validation Guide -Use this guide to define and enforce structural constraints (shapes and data types) on loaded ExecuTorch `.pte` models using `validateModelSchema`. +Use this guide to define and enforce structural constraints (shapes, data types, runtime constraints) on loaded ExecuTorch `.pte` models using `validateSpec`. --- -## 🔍 Why Validate Model Schemas? +## 🔍 Why Validate Model Specs? -Every ExecuTorch model exposes metadata about its execution methods (typically `'forward'`), including: -* Input and output argument counts. -* The expected types (primitives like `number`/`boolean` or `Tensor`). -* The data type (`float32`, `int32`, etc.) and the shape arrays of tensors. +Every ExecuTorch model exposes a `schema` (`ModelSpec`), derived either from standard ExecuTorch `MethodMeta` at load time or from an exported `get_model_schema` companion method returning a JSON model spec. -To prevent runtime crashes and memory allocation mismatches in C++, the TypeScript task pipeline must validate that the provided model matches its expected execution signature *before* allocating static tensors or executing inference. +To prevent runtime crashes, type mismatches, and memory allocation errors in C++, TypeScript task pipelines validate the loaded model's exported schema against allowed spec variants using `validateSpec` _before_ allocating static tensors or executing inference. --- ## 🛠️ Validation API Reference ```typescript -import { validateModelSchema, SymbolicTensor } from '../../../core/modelSchema'; - -const meta = validateModelSchema( - model, - methodName, - expectedInputs, - expectedOutputs -); +import { + validateSpec, + method, + f32, + i64, + i32, + DynamicDim as Dyn, + inp, + out, + eq, + linear, +} from '../../../core/schema'; + +const { variant, dims } = validateSpec(model.schema, { + batched: method('forward', [f32(1, 3, 'H', 'W')], [f32(1, 'N')]), + unbatched: method('forward', [f32(3, 'H', 'W')], [f32('N')]), +}); + +const [N, H, W] = dims.constant('N', 'H', 'W'); ``` -### Constraints Types: -* **Primitives**: `'number' | 'boolean' | 'null'` -* **Tensors**: Defined via the `SymbolicTensor(dtype?, ...shapes)` helper. +### Key Schema Utilities from `src/core/schema.ts`: + +- **`method(name, inputs, outputs, runtimeConstraints?)`**: Constructs a method specification. +- **`f32(...)` / `i64(...)` / `i32(...)` / `ui8(...)`**: Shorthand helpers for tensor parameter specs. +- **`StaticDim('symbol')` / String Literals**: Strings passed to shape helpers (e.g. `'H'`, `'W'`) automatically map to `StaticDim`, acting as **static dimension wildcards**. They bind strictly to `constant` positive integer dimensions in the exported spec. +- **`DynamicDim('symbol')` (or `Dyn('symbol')`)**: Creates a dynamic dimension symbol. Must be used when a dimension genuinely varies at runtime and binds to a `range` or `enum` domain in the exported spec. +- **Constraint Helpers**: + - **`inp(tensorIdx, dimIdx)`**: Creates a reference to an input tensor's dimension. + - **`out(tensorIdx, dimIdx)`**: Creates a reference to an output tensor's dimension. + - **`eq(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. + - **`linear(lhs, rhs, a, b)`**: Creates a linear constraint `lhs = a * rhs + b`. +- **`validateSpec(exportedSchema, allowedVariants)`**: Compares the model's exported schema against named variants and returns `{ variant, dim, dims }`. +- **Symbol Accessors (`dims`)**: + - `dims.constant('N', 'H')`: Extracts constant values for symbols as numbers. + - `dims.range('S')`: Extracts range domains `{ min, max, step }`. + - `dims.enum('E')`: Extracts enum choices `readonly number[]`. + - `dims.any('D')`: Extracts raw `ConcreteDim`. --- ## 📏 Symbolic Dimensions & Dynamic Shapes -Tensors often support dynamic dimensions (such as varying image sizes `'H'`, `'W'` or dynamic batch/prediction counts `'N'`). -The `SymbolicTensor` helper supports specifying **Symbolic Shapes** where integers are static matching constraints, and string names act as runtime variables. +Tensors can have static dimensions (integers), static symbol wildcards (strings), or dynamic symbols (`DynamicDim`). +The `validateSpec` utility matches allowed variants in order: -### How Symbolic Matching Works: 1. **Numbers (Static Match)**: - If a dimension is defined as a number, the loaded model's tensor dimension must match that exact integer. - * *Example*: `[1, 3, 'H', 'W']` requires the batch dimension to be exactly `1`, and channels to be exactly `3`. + Must match the exact static integer in the exported spec. -2. **Strings (Symbolic Match)**: - If a dimension is defined as a string (e.g., `'H'`), the validator binds the actual dimension size to that symbol name. - * *Symbolic Constraint Rules*: Within a single tensor, if a symbol (e.g., `'H'`) appears multiple times, the corresponding dimensions must be equal. +2. **Strings / `StaticDim` (Static Wildcard Match)**: + Binds string symbol names (e.g. `'H'`) to static constant integers in the exported spec. Repeated occurrences of the same static symbol must bind to the same constant value. -3. **Multiple Alternative Shapes**: - You can provide multiple shape variations to `SymbolicTensor` to support models compiled in different layouts (e.g., channels-first vs. channels-last). - * *Example*: `SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])` allows either 4D or 3D float32 layouts. +3. **`DynamicDim` (Dynamic Domain Match)**: + Binds dynamic symbol names to `range` or `enum` domains in the exported spec. + +4. **Variant Selection**: + Specify multiple variant keys in `validateSpec` (e.g. `batched` vs `unbatched`). The validator tests variants sequentially and returns the first matching key and bound symbols. --- -## ⚙️ Runtime Dynamic Dimensions: the `get_dynamic_dims_` companion +## 🔀 Dimension Domains vs. Runtime Constraints + +Understanding the distinction between a dimension's **domain** and its **runtime value** is central to schema design and validation: + +### 1. Dimension Domains & Domain Matching + +- **Dimension Domain**: The set of values a dimension may take: + - `constant`: Exactly `value` (a singleton integer). + - `range`: Any value between `min` and `max` stepping by `step`. + - `enum`: Any value listed in `choices`. +- **Runtime Value**: The concrete integer size of a tensor's dimension in a specific execution call. +- **Domain Matching (`validateSpec`)**: + - `validateSpec` performs static, load-time domain matching. + - Dynamic symbols (`DynamicDim('S')`) bind to exported dimension domains. Reusing a symbol (`'S'`) across tensor inputs or outputs requires every occurrence to bind to the **exact same domain**. + - ⚠️ **Key Rule**: Binding to the same domain does **NOT** mean runtime values coincide! Two dimensions bound to the same domain (e.g., both having range `1..512`) may take _different_ runtime values in a single execution (e.g. length 10 and length 25). + +### 2. Runtime Constraints (`eq` & `linear`) + +- **Runtime Constraints**: Declarations about the **runtime values** of tensor dimensions during execution: + - **Equality Constraint (`eq(...)`)**: Requires all referenced dimensions to take the exact same runtime value in any execution call. + ```typescript + method( + 'forward', + [f32('B', Dyn('S1')), f32('B', Dyn('S2'))], + [f32('B', Dyn('S1'))], + [ + eq(inp(0, 0), inp(1, 0)), // Input 0 batch dim == Input 1 batch dim + ] + ); + ``` + - **Linear Constraint (`linear(...)`)**: Requires two dimensions to satisfy `dimLhs = a * dimRhs + b` at runtime. +- **Validation & Enforcement**: + - `validateSpec` verifies that the exported model spec declares the exact same runtime constraints (1-to-1 declaration match). + - Native C++ validates input runtime constraints before invoking `model.execute()`. -`SymbolicTensor` string symbols let the **validator** accept a range of shapes, but an ExecuTorch `.pte`'s metadata only serializes the **static upper bound** of a dynamic dimension — not its active `[min, max, step]` range. So for an input dimension that genuinely varies at runtime (e.g. a text model's sequence length), the model must be **exported with a companion method** that re-exposes the range to the runtime: +--- -* **Name**: `get_dynamic_dims_` — e.g. `get_dynamic_dims_forward` for `forward`. -* **Signature**: takes no arguments. -* **Returns**: a list of outputs, one per `Tensor` input of the target method (scalar inputs are skipped), each a **2D `int32` tensor of shape `[rank, 3]`** whose rows are `[min, max, step]` constraints for that input's dimensions — e.g. `[1, 1, 1]` for a fixed dimension and `[1, maxTokens, 1]` for the dynamic one. +## ⚙️ Companion JSON Spec (`get_model_schema`) -At load time the C++ core (`Model::parseDynamicInputShapes`) reads this companion and validates inputs against the range; `model.execute` then accepts tensors at their exact runtime length. **Without the companion, a method falls back to exact per-dimension validation** — it only accepts the single shape it was exported with. So a `.pte` whose metadata reports `[1, 512]` but is meant to accept `[1, 1..512]` MUST ship `get_dynamic_dims_forward`, or variable-length inputs are rejected at runtime. +For models with runtime dynamic dimensions or runtime constraints (equality/linear relations between dimensions), the `.pte` model exports a companion method named `get_model_schema` returning a JSON string representation of `ModelSpec`. -This is an **export-side contract** (the export-scripts repo provides a `build_dynamic_dims_program` helper that emits the companion). The TypeScript task only declares the symbol via `SymbolicTensor` and reads the resulting upper bound from `meta.inputTensorMeta`. +When `get_model_schema` is present, the C++ loader parses it to populate `model.schema` with exact `RangeDim` / `EnumDim` domains and runtime constraints. Without it, `model.schema` is populated strictly from static ExecuTorch `MethodMeta`. --- ## 📋 Common Validation Recipes -### 1. Classification -Accepts an image tensor and outputs a 2D class probabilities array: +### 1. Classification (Batched vs Unbatched) + +Accepts an image tensor and outputs class probabilities: + ```typescript -const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], // Input - [SymbolicTensor('float32', [1, 'N'], ['N'])] // Output: logits / probs -); - -const inpShape = meta.inputTensorMeta[0]!.shape; -const outShape = meta.outputTensorMeta[0]!.shape; +const { variant, dims } = validateSpec(model.schema, { + batched: method('forward', [f32(1, 3, 'H', 'W')], [f32(1, 'N')]), + unbatched: method('forward', [f32(3, 'H', 'W')], [f32('N')]), +}); + +const [N, H, W] = dims.constant('N', 'H', 'W'); +const inpShape = { batched: [1, 3, H, W], unbatched: [3, H, W] }[variant]; +const outShape = { batched: [1, N], unbatched: [N] }[variant]; ``` ### 2. Image-to-Image / Style Transfer + Accepts an image tensor and returns a modified image tensor with identical dimensions: + ```typescript -const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], // Input - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])] // Output -); +const { dims } = validateSpec(model.schema, { + default: method('forward', [f32(1, 3, 'H', 'W')], [f32(1, 3, 'H', 'W')]), +}); + +const [H, W] = dims.constant('H', 'W'); ``` -### 3. Object Detection (Dynamic Boxes Count) -Accepts an image tensor, and outputs boxes, scores, and class labels for `N` dynamic detections: +### 3. Object Detection (Dynamic Box Count) + +Accepts an image tensor, and outputs boxes, scores, and class labels for `N` detections: + ```typescript -const meta = validateModelSchema( - model, - 'forward', - [SymbolicTensor('float32', [1, 3, 'H', 'W'], [3, 'H', 'W'])], - [ - SymbolicTensor('float32', ['N', 4]), // Bounding boxes (xyxy / xywh) - SymbolicTensor('float32', ['N']), // Prediction confidence scores - SymbolicTensor('float32', ['N']), // Predicted class labels - ] -); +const { dims } = validateSpec(model.schema, { + default: method('forward', [f32(1, 3, 'H', 'W')], [f32('N', 4), f32('N'), f32('N')]), +}); + +const [N, H, W] = dims.constant('N', 'H', 'W'); ``` -### 4. Single-Model Pipeline (fully static export) +### 4. Single-Model Pipeline (Fully Static Export) -When a pipeline targets exactly one model whose `.pte` is exported with fully static shapes (e.g. `sdxsTextToImage`), the shapes are constants of the export contract. Validate each method against those constants — do **not** invent symbols for dimensions that cannot vary, and do **not** thread the shapes through task options: +Assert exact shape constants from the task file: ```typescript -const IMAGE_SIZE = 512; -const LATENT_CHANNELS = 4; -const LATENT_SIZE = IMAGE_SIZE / 8; - -validateModelSchema( - model, - 'denoise', - [ - SymbolicTensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE]), - SymbolicTensor('int64', [1]), - SymbolicTensor('float32', [1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE]), - ], - [SymbolicTensor('float32', [1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE])] -); +const { dims } = validateSpec(model.schema, { + default: method( + 'denoise', + [ + f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE), + i64(1), + f32(1, CLIP_MAX_TOKENS, CLIP_HIDDEN_SIZE), + ], + [f32(1, LATENT_CHANNELS, LATENT_SIZE, LATENT_SIZE)] + ), +}); ``` -A rigid `.pte` contract is an argument **for** validating, not against it: the stricter the contract, the cheaper and more precise the assertion. Validation is load-time only (it costs nothing per inference) and converts a stale, corrupt, or wrongly re-exported `.pte` into a readable TypeScript error instead of a native crash or silently garbage output. This matters most for pipelines shipping several separately-exported backend variants (XNNPACK / CoreML / MLX), where one variant can break while the others stay healthy. - --- ## 🚫 Avoid / Anti-Patterns -* **Do NOT write imperative size/type checks manually:** Avoid writing custom shape/type assertion blocks (e.g., `if (tensor.shape[0] !== 1)`). Always use the declarative `validateModelSchema` utility, which reports unified, readable mismatch errors. -* **Do NOT use hardcoded integers for dynamic dimensions:** If a shape can vary (e.g., dynamic height, width, or batch sizes), use a string symbol (like `'H'`, `'W'`, `'N'`) to allow dynamic matching. Conversely, a dimension that genuinely cannot vary *should* be a hardcoded integer — reserve symbols for real variability. -* **Do NOT skip validation at startup:** Always validate the model schema *before* creating pre-allocated static tensors, preventing native memory crashes from mismatched layouts. -* **Do NOT skip validation just because the pipeline supports a single model:** "The `.pte` must be very specific anyway" is a reason to assert the exact contract, not to trust it. See Recipe 4. +- **Do NOT write manual imperative shape/type checks:** Always use declarative `validateSpec` variants which report unified, readable mismatch errors. +- **Do NOT use hardcoded integers for dynamic dimensions:** Use string symbols (like `'H'`, `'W'`, `'N'`) for dynamic dimensions. Conversely, fixed export dimensions should be integers. +- **Do NOT skip validation at startup:** Validate `model.schema` before allocating static tensors. +- **Do NOT skip validation for single-model pipelines:** Single-model pipelines should still validate against task shape constants. --- ## 📋 Verification Checklist When specifying model schema validations, verify that: -- [ ] Schema validation is performed immediately after model loading and before tensor initialization. -- [ ] All dynamic dimensions (e.g., dynamic box counts, channels-last heights/widths) are defined as string symbols, and dimensions fixed by the export are plain integers. -- [ ] Single-model pipelines with static exports still validate, asserting against the task file's shape constants rather than skipping validation or routing shapes through options. -- [ ] Multiple shape variations are provided to `SymbolicTensor` if channels-first and channels-last layouts are both supported. -- [ ] Input and output constraints map accurately to standard model specifications (e.g. dense logits, standard bounding boxes layouts). + +- [ ] Schema validation is performed immediately after model loading and before tensor initialization using `validateSpec(model.schema, { ... })`. +- [ ] Dynamic dimensions are defined as string symbols, while fixed dimensions are plain integers. +- [ ] Symbol values are extracted using `dims.constant(...)`, `dims.range(...)`, or `dims.enum(...)`. +- [ ] Multiple shape variants (e.g. `batched` vs `unbatched`) are provided when supported. +- [ ] Input and output constraints map accurately to model specifications. diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp index 0826340ba1..1cdfd8ae1a 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp @@ -51,23 +51,21 @@ namespace { std::string shapeToString(const SymbolicShape &shape) { std::string s; for (const auto &dim : shape) { - if (std::holds_alternative(dim)) { - s += std::get(dim); + if (const auto *str = std::get_if(&dim)) { + s += *str; } - if (std::holds_alternative(dim)) { - s += std::to_string(std::get(dim)); + if (const auto *val = std::get_if(&dim)) { + s += std::to_string(*val); } - if (std::holds_alternative(dim)) { - auto range = std::get(dim); - s += std::format("[{}..{}:{}]", range.min, range.max, range.step); + if (const auto *range = std::get_if(&dim)) { + s += std::format("[{}..{}:{}]", range->min, range->max, range->step); } - if (std::holds_alternative(dim)) { - auto enumeration = std::get(dim); + if (const auto *enumeration = std::get_if(&dim)) { s += "{"; - for (const auto choice : enumeration.choices) { + for (const auto choice : enumeration->choices) { s += std::to_string(choice) + ","; } - if (!enumeration.choices.empty()) { + if (!enumeration->choices.empty()) { s.pop_back(); } s += "}"; @@ -112,36 +110,35 @@ fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, for (size_t i = 0; i < expectedShape->size(); ++i) { const auto &dim = expectedShape->at(i); - if (std::holds_alternative(dim)) { - const auto &symbol = std::get(dim); - if (symbolBinding.contains(symbol) && symbolBinding[symbol] != shape[i]) { - throw jsi::JSError(rt, ""); + if (const auto *symbol = std::get_if(&dim)) { + if (symbolBinding.contains(*symbol) && symbolBinding[*symbol] != shape[i]) { + throw jsi::JSError(rt, std::format("{} must have shape {} (symbol {} mismatch: expected {}, got {})", + ctx, shapeToString(*expectedShape), *symbol, symbolBinding[*symbol], shape[i])); } - symbolBinding[symbol] = shape[i]; + symbolBinding[*symbol] = shape[i]; } - if (std::holds_alternative(dim)) { - if (shape[i] != std::get(dim)) { - throw jsi::JSError(rt, ""); + if (const auto *val = std::get_if(&dim)) { + if (shape[i] != *val) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} mismatch: expected {}, got {})", + ctx, shapeToString(*expectedShape), i, *val, shape[i])); } } - if (std::holds_alternative(dim)) { - auto range = std::get(dim); - if (shape[i] < range.min) { + if (const auto *range = std::get_if(&dim)) { + if (shape[i] < range->min) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} < min {})", - ctx, shapeToString(*expectedShape), i, shape[i], range.min)); + ctx, shapeToString(*expectedShape), i, shape[i], range->min)); } - if (shape[i] > range.max) { + if (shape[i] > range->max) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} > max {})", - ctx, shapeToString(*expectedShape), i, shape[i], range.max)); + ctx, shapeToString(*expectedShape), i, shape[i], range->max)); } - if ((shape[i] - range.min) % range.step != 0) { + if ((shape[i] - range->min) % range->step != 0) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} must be min({}) + k*step({}), got {})", - ctx, shapeToString(*expectedShape), i, range.min, range.step, shape[i])); + ctx, shapeToString(*expectedShape), i, range->min, range->step, shape[i])); } } - if (std::holds_alternative(dim)) { - auto enumeration = std::get(dim); - if (std::ranges::find(enumeration.choices, shape[i]) == enumeration.choices.end()) { + if (const auto *enumeration = std::get_if(&dim)) { + if (std::ranges::find(enumeration->choices, shape[i]) == enumeration->choices.end()) { throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} not allowed: got {})", ctx, shapeToString(*expectedShape), i, shape[i])); } diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index dcaf5af7ae..2d62fff675 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -302,6 +302,67 @@ export const i64 = (...shape: SymbolicShape) => SymbolicTensor('int64', shape); export const i32 = (...shape: SymbolicShape) => SymbolicTensor('int32', shape); export const ui8 = (...shape: SymbolicShape) => SymbolicTensor('uint8', shape); +/** + * Creates an input dimension reference for runtime constraints. + * @param tensorIdx Index of the input tensor parameter. + * @param dimIdx Index of the dimension within the tensor shape. + * @returns The dimension reference. + */ +export const inp = (tensorIdx: number, dimIdx: number): DimRef => ({ + paramSide: 'input', + tensorIdx, + dimIdx, +}); + +/** + * Creates an output dimension reference for runtime constraints. + * @param tensorIdx Index of the output tensor parameter. + * @param dimIdx Index of the dimension within the tensor shape. + * @returns The dimension reference. + */ +export const out = (tensorIdx: number, dimIdx: number): DimRef => ({ + paramSide: 'output', + tensorIdx, + dimIdx, +}); + +/** + * Creates an equality constraint requiring the referenced dimensions to coincide. + * @param dims The dimension references to constrain. + * @returns The equality constraint. + */ +export const eq = (...dims: DimRef[]): EqualityConstraint => ({ kind: 'equality', dims }); + +/** + * Creates a linear constraint: `lhs = a * rhs + b`. + * @param dimLhs Left-hand side dimension reference. + * @param dimRhs Right-hand side dimension reference. + * @param a Multiplicative coefficient. + * @param b Additive constant offset. Defaults to 0. + * @returns The linear constraint. + */ +export const linear = ( + dimLhs: DimRef, + dimRhs: DimRef, + a: number, + b: number = 0 +): LinearConstraint => ({ + kind: 'linear', + dimLhs, + dimRhs, + coefficients: [a, b], +}); + +/** + * Constructs a method specification mapping a method name to its parameter + * specs and runtime constraints. + * @param name The execution method name (e.g., `'forward'`). + * @param inputs Ordered list of parameter specifications for method inputs. + * @param outputs Ordered list of parameter specifications for method outputs. + * @param constraints Optional array of {@link RuntimeConstraint} declarations + * (equality or linear relations across dimensions). + * @returns A record mapping `name` to its corresponding {@link MethodSpec}. + */ export function method( name: string, inputs: ParamSpec[], diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index ee3e30d80b..9bc95652cb 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, i64, f32 } from '../../../core/schema'; +import { validateSpec, DynamicDim as Dyn, method, i64, f32, eq, inp } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { loadTokenizer } from '../tokenizer'; @@ -73,12 +73,14 @@ export async function createTextEmbedder( batched: method( 'forward', // prettier-ignore [i64(1, Dyn('L')), i64(1, Dyn('L'))], - [f32(1, 'D')] + [f32(1, 'D')], + [eq(inp(0, 1), inp(1, 1))] ), unbatched: method( 'forward', // prettier-ignore [i64(1, Dyn('L')), i64(1, Dyn('L'))], - [f32('D')] + [f32('D')], + [eq(inp(0, 1), inp(1, 1))] ), }); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 3eb8faf2ec..a733296d77 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -2,7 +2,8 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, f32 } from '../../../core/schema'; +import { validateSpec, DynamicDim as Dyn, method, f32, eq, inp, out } from '../../../core/schema'; + import { wrapAsync } from '../../../core/runtime'; import { extractFrames } from '../utils/vadUtils'; @@ -230,7 +231,8 @@ export async function createFsmnVoiceActivityDetector( default: method( 'forward', // prettier-ignore [f32(Dyn('frames'), 'fftLen')], - [f32(1, Dyn('frames'), 'classes')] + [f32(1, Dyn('frames'), 'classes')], + [eq(inp(0, 0), out(0, 1))] ), }); const [numClasses, fftLength] = dims.constant('classes', 'fftLen'); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index be935a67db..13e30016fe 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -4,7 +4,16 @@ import { scheduleOnRN, createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; import { wrapAsync } from '../../../core/runtime'; -import { f32, i64, method, validateSpec, DynamicDim as Dyn } from '../../../core/schema'; +import { + f32, + i64, + method, + validateSpec, + DynamicDim as Dyn, + eq, + inp, + out, +} from '../../../core/schema'; import { argmax } from '../../../extensions/math'; import { loadTokenizer } from '../../nlp/tokenizer'; @@ -176,7 +185,8 @@ export async function createWhisperSpeechToText Date: Tue, 28 Jul 2026 17:38:19 +0200 Subject: [PATCH 14/31] refactor(schema): namespace schema exports and cleanup constraint helpers - Re-export schema utilities under a namespaced 'schema' export in index.ts - Re-export types flatly using 'export type *' from ./core/schema - Group constraint helpers under 'constr' object and inline DimRef object literals in task pipelines - Remove redundant static dimension constraint from Whisper task pipeline --- .../src/core/schema.ts | 59 +++---------------- .../src/extensions/nlp/tasks/textEmbedding.ts | 16 ++++- .../tasks/fsmnVoiceActivityDetection.ts | 9 ++- .../speech/tasks/whisperSpeechToText.ts | 14 +---- packages/react-native-executorch/src/index.ts | 4 +- 5 files changed, 34 insertions(+), 68 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 2d62fff675..41b249acb9 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -302,56 +302,15 @@ export const i64 = (...shape: SymbolicShape) => SymbolicTensor('int64', shape); export const i32 = (...shape: SymbolicShape) => SymbolicTensor('int32', shape); export const ui8 = (...shape: SymbolicShape) => SymbolicTensor('uint8', shape); -/** - * Creates an input dimension reference for runtime constraints. - * @param tensorIdx Index of the input tensor parameter. - * @param dimIdx Index of the dimension within the tensor shape. - * @returns The dimension reference. - */ -export const inp = (tensorIdx: number, dimIdx: number): DimRef => ({ - paramSide: 'input', - tensorIdx, - dimIdx, -}); - -/** - * Creates an output dimension reference for runtime constraints. - * @param tensorIdx Index of the output tensor parameter. - * @param dimIdx Index of the dimension within the tensor shape. - * @returns The dimension reference. - */ -export const out = (tensorIdx: number, dimIdx: number): DimRef => ({ - paramSide: 'output', - tensorIdx, - dimIdx, -}); - -/** - * Creates an equality constraint requiring the referenced dimensions to coincide. - * @param dims The dimension references to constrain. - * @returns The equality constraint. - */ -export const eq = (...dims: DimRef[]): EqualityConstraint => ({ kind: 'equality', dims }); - -/** - * Creates a linear constraint: `lhs = a * rhs + b`. - * @param dimLhs Left-hand side dimension reference. - * @param dimRhs Right-hand side dimension reference. - * @param a Multiplicative coefficient. - * @param b Additive constant offset. Defaults to 0. - * @returns The linear constraint. - */ -export const linear = ( - dimLhs: DimRef, - dimRhs: DimRef, - a: number, - b: number = 0 -): LinearConstraint => ({ - kind: 'linear', - dimLhs, - dimRhs, - coefficients: [a, b], -}); +/** Helper namespace for declaring runtime constraints. */ +export const constr = { + eq: (...dims: DimRef[]): EqualityConstraint => { + return { kind: 'equality', dims }; + }, + linear: (dimLhs: DimRef, dimRhs: DimRef, a: number, b: number = 0): LinearConstraint => { + return { kind: 'linear', dimLhs, dimRhs, coefficients: [a, b] }; + }, +}; /** * Constructs a method specification mapping a method name to its parameter diff --git a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts index 9bc95652cb..3aa88f0b05 100644 --- a/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts +++ b/packages/react-native-executorch/src/extensions/nlp/tasks/textEmbedding.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, i64, f32, eq, inp } from '../../../core/schema'; +import { validateSpec, DynamicDim as Dyn, method, i64, f32, constr } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { loadTokenizer } from '../tokenizer'; @@ -74,13 +74,23 @@ export async function createTextEmbedder( 'forward', // prettier-ignore [i64(1, Dyn('L')), i64(1, Dyn('L'))], [f32(1, 'D')], - [eq(inp(0, 1), inp(1, 1))] + [ + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 1 } + ), + ] ), unbatched: method( 'forward', // prettier-ignore [i64(1, Dyn('L')), i64(1, Dyn('L'))], [f32('D')], - [eq(inp(0, 1), inp(1, 1))] + [ + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 1 } + ), + ] ), }); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index a733296d77..2f8197e878 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, f32, eq, inp, out } from '../../../core/schema'; +import { validateSpec, DynamicDim as Dyn, method, f32, constr } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { extractFrames } from '../utils/vadUtils'; @@ -232,7 +232,12 @@ export async function createFsmnVoiceActivityDetector( 'forward', // prettier-ignore [f32(Dyn('frames'), 'fftLen')], [f32(1, Dyn('frames'), 'classes')], - [eq(inp(0, 0), out(0, 1))] + [ + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, + { paramSide: 'output', tensorIdx: 0, dimIdx: 1 } + ), + ] ), }); const [numClasses, fftLength] = dims.constant('classes', 'fftLen'); diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index 13e30016fe..be935a67db 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -4,16 +4,7 @@ import { scheduleOnRN, createSynchronizable } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; import { wrapAsync } from '../../../core/runtime'; -import { - f32, - i64, - method, - validateSpec, - DynamicDim as Dyn, - eq, - inp, - out, -} from '../../../core/schema'; +import { f32, i64, method, validateSpec, DynamicDim as Dyn } from '../../../core/schema'; import { argmax } from '../../../extensions/math'; import { loadTokenizer } from '../../nlp/tokenizer'; @@ -185,8 +176,7 @@ export async function createWhisperSpeechToText Date: Tue, 28 Jul 2026 17:42:03 +0200 Subject: [PATCH 15/31] docs(skills): update model-schema-validation skill to reflect constr helper and inline DimRef objects --- .../skills/model-schema-validation/SKILL.md | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.agents/skills/model-schema-validation/SKILL.md b/.agents/skills/model-schema-validation/SKILL.md index 7137e2f42f..ba680cc141 100644 --- a/.agents/skills/model-schema-validation/SKILL.md +++ b/.agents/skills/model-schema-validation/SKILL.md @@ -30,10 +30,7 @@ import { i64, i32, DynamicDim as Dyn, - inp, - out, - eq, - linear, + constr, } from '../../../core/schema'; const { variant, dims } = validateSpec(model.schema, { @@ -50,11 +47,10 @@ const [N, H, W] = dims.constant('N', 'H', 'W'); - **`f32(...)` / `i64(...)` / `i32(...)` / `ui8(...)`**: Shorthand helpers for tensor parameter specs. - **`StaticDim('symbol')` / String Literals**: Strings passed to shape helpers (e.g. `'H'`, `'W'`) automatically map to `StaticDim`, acting as **static dimension wildcards**. They bind strictly to `constant` positive integer dimensions in the exported spec. - **`DynamicDim('symbol')` (or `Dyn('symbol')`)**: Creates a dynamic dimension symbol. Must be used when a dimension genuinely varies at runtime and binds to a `range` or `enum` domain in the exported spec. -- **Constraint Helpers**: - - **`inp(tensorIdx, dimIdx)`**: Creates a reference to an input tensor's dimension. - - **`out(tensorIdx, dimIdx)`**: Creates a reference to an output tensor's dimension. - - **`eq(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. - - **`linear(lhs, rhs, a, b)`**: Creates a linear constraint `lhs = a * rhs + b`. +- **Constraint Helpers (`constr`)**: + - **`DimRef` Object Literal (`{ paramSide: 'input' | 'output', tensorIdx, dimIdx }`)**: Explicit reference to a tensor's dimension. + - **`constr.eq(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. + - **`constr.linear(lhs, rhs, a, b)`**: Creates a linear constraint `lhs = a * rhs + b`. - **`validateSpec(exportedSchema, allowedVariants)`**: Compares the model's exported schema against named variants and returns `{ variant, dim, dims }`. - **Symbol Accessors (`dims`)**: - `dims.constant('N', 'H')`: Extracts constant values for symbols as numbers. @@ -99,21 +95,24 @@ Understanding the distinction between a dimension's **domain** and its **runtime - Dynamic symbols (`DynamicDim('S')`) bind to exported dimension domains. Reusing a symbol (`'S'`) across tensor inputs or outputs requires every occurrence to bind to the **exact same domain**. - ⚠️ **Key Rule**: Binding to the same domain does **NOT** mean runtime values coincide! Two dimensions bound to the same domain (e.g., both having range `1..512`) may take _different_ runtime values in a single execution (e.g. length 10 and length 25). -### 2. Runtime Constraints (`eq` & `linear`) +### 2. Runtime Constraints (`constr.eq` & `constr.linear`) - **Runtime Constraints**: Declarations about the **runtime values** of tensor dimensions during execution: - - **Equality Constraint (`eq(...)`)**: Requires all referenced dimensions to take the exact same runtime value in any execution call. + - **Equality Constraint (`constr.eq(...)`)**: Requires all referenced dimensions to take the exact same runtime value in any execution call. ```typescript method( 'forward', [f32('B', Dyn('S1')), f32('B', Dyn('S2'))], [f32('B', Dyn('S1'))], [ - eq(inp(0, 0), inp(1, 0)), // Input 0 batch dim == Input 1 batch dim + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 0 } + ), ] ); ``` - - **Linear Constraint (`linear(...)`)**: Requires two dimensions to satisfy `dimLhs = a * dimRhs + b` at runtime. + - **Linear Constraint (`constr.linear(...)`)**: Requires two dimensions to satisfy `dimLhs = a * dimRhs + b` at runtime. - **Validation & Enforcement**: - `validateSpec` verifies that the exported model spec declares the exact same runtime constraints (1-to-1 declaration match). - Native C++ validates input runtime constraints before invoking `model.execute()`. From e622b5f448ca14c4a9f21fd260208737f69806d5 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 18:05:04 +0200 Subject: [PATCH 16/31] docs(schema): document SpecMatch.dims and add dynamic dimension accessor --- .../src/core/schema.ts | 57 ++++++++++++++++++- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 41b249acb9..894ac4e78c 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -639,11 +639,12 @@ function validateDimDomains(modelSpec: ModelSpec): void { /** * Result of validating an exported model spec against allowed variants. - * @typeParam V The variant key type. + * @typeParam K The variant key type. */ export type SpecMatch = { /** Key of the matched variant. */ readonly variant: K; + /** * Returns the concrete value for a symbol. * @param name The symbol name. @@ -654,12 +655,53 @@ export type SpecMatch = { dim(name: string, kind: 'constant'): number; dim(name: string, kind: 'range'): Range; dim(name: string, kind: 'enum'): readonly number[]; + dim(name: string, kind: 'dynamic'): Exclude; dim(name: string): ConcreteDim; + /** + * Batch accessors for retrieving multiple symbol dimensions at once as typed tuples. + */ dims: { + /** + * Retrieves concrete dimension objects for multiple symbols. + * @param names Symbol names to retrieve. + * @returns A tuple of {@link ConcreteDim} objects corresponding to `names`. + * @throws {Error} If any symbol is not found. + */ any(...names: S): { [I in keyof S]: ConcreteDim }; + + /** + * Retrieves choice arrays for multiple enumerated dynamic symbols. + * @param names Symbol names expected to be enum dimensions. + * @returns A tuple of choice arrays corresponding to `names`. + * @throws {Error} If any symbol is not found or is not an enum dimension. + */ enum(...names: S): { [I in keyof S]: readonly number[] }; + + /** + * Retrieves range objects for multiple dynamic symbols. + * @param names Symbol names expected to be range dimensions. + * @returns A tuple of {@link Range} objects corresponding to `names`. + * @throws {Error} If any symbol is not found or is not a range dimension. + */ range(...names: S): { [I in keyof S]: Range }; + + /** + * Retrieves dynamic dimension objects (range or enum) for multiple symbols. + * @param names Symbol names expected to be dynamic dimensions (range or enum). + * @returns A tuple of dynamic {@link ConcreteDim} objects corresponding to `names`. + * @throws {Error} If any symbol is not found or is a constant dimension. + */ + dynamic( + ...names: S + ): { [I in keyof S]: Exclude }; + + /** + * Retrieves constant numeric values for multiple static symbols. + * @param names Symbol names expected to be constant dimensions. + * @returns A tuple of numbers corresponding to `names`. + * @throws {Error} If any symbol is not found or is not a constant dimension. + */ constant(...names: S): { [I in keyof S]: number }; }; }; @@ -710,8 +752,16 @@ export function validateSpec Date: Tue, 28 Jul 2026 20:26:16 +0200 Subject: [PATCH 17/31] docs(schema): expand JSDoc comments and clarify validation phases - Expand schema.h file-level docblock to detail the two-phase validation model (load-time validateSpec + runtime validateRuntimeConstraints), explain the two spec source paths (companion method vs MethodMeta fallback), and document output-dimension skipping behaviour pre-execution - Improve individual function docstrings for parseModelSpecJson, validateSpec, and validateRuntimeConstraints to reflect actual behaviour more precisely - Fix whisperSpeechToText decode method spec: replace named symbol strings for static constant dimensions with literal integers (1) to match schema semantics --- .../cpp/core/model.cpp | 7 +- .../react-native-executorch/cpp/core/model.h | 31 +++++ .../react-native-executorch/cpp/core/schema.h | 108 +++++++++++++----- .../speech/tasks/whisperSpeechToText.ts | 4 +- 4 files changed, 114 insertions(+), 36 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 5ca219deca..0f429f699c 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -57,12 +57,11 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) } const auto methodNames = unwrap("loadModel", etModule_->method_names()); - const auto *const getSchemaMethod = "get_model_schema"; schema::ModelSpec overrideSpec; - if (methodNames.contains(getSchemaMethod)) { - auto ctx = std::format("loadModel: '{}'", getSchemaMethod); - auto result = unwrap(ctx, etModule_->execute(getSchemaMethod)); + if (methodNames.contains(kGetModelSchemaMethod)) { + auto ctx = std::format("loadModel: '{}'", kGetModelSchemaMethod); + auto result = unwrap(ctx, etModule_->execute(kGetModelSchemaMethod)); if (result.empty() || result[0].tag != executorch::runtime::Tag::String) { throw std::runtime_error(std::format("{} must return a single string value", ctx)); diff --git a/packages/react-native-executorch/cpp/core/model.h b/packages/react-native-executorch/cpp/core/model.h index 34ace946f9..594e28679b 100644 --- a/packages/react-native-executorch/cpp/core/model.h +++ b/packages/react-native-executorch/cpp/core/model.h @@ -15,6 +15,13 @@ namespace rnexecutorch::core::model { namespace jsi = facebook::jsi; + +/** + * Optional companion method name inside a `.pte` ExecuTorch module that exports + * a JSON model spec (containing dynamic dimension domains or runtime constraints). + */ +inline constexpr auto kGetModelSchemaMethod = "get_model_schema"; + /** * JSI HostObject wrapping an ExecuTorch Model instance * (`executorch::extension::Module`). @@ -26,19 +33,43 @@ namespace jsi = facebook::jsi; class ModelHostObject : public jsi::HostObject, public std::enable_shared_from_this { public: + /** + * Loads the ExecuTorch model from the specified file path, initializes its + * method metadata, parses schemas, and populates backend delegates. + * + * @param modelPath Absolute file system path to the `.pte` model binary. + */ explicit ModelHostObject(const std::string &modelPath); jsi::Value get(jsi::Runtime &rt, const jsi::PropNameID &name) override; std::vector getPropertyNames(jsi::Runtime &rt) override; private: + /// File system path to the loaded ExecuTorch `.pte` model binary. std::string modelPath_; + + /** + * Parsed model schema mapping method names to their input/output parameter + * specs and runtime constraints. + */ schema::ModelSpec spec_; + + /** + * Map of method names to the list of backend delegate identifiers used by + * each method. + */ std::unordered_map> backends_; + /** + * Mutex serializing access to `etModule_` to prevent concurrent inference + * execution on the same model. + */ std::mutex mutex_; + + /// Unique pointer to the underlying ExecuTorch module instance. std::unique_ptr etModule_; }; void install_loadModel(jsi::Runtime &rt, jsi::Object &module); + } // namespace rnexecutorch::core::model diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h index c24d82daf1..df651c4c18 100644 --- a/packages/react-native-executorch/cpp/core/schema.h +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -16,25 +16,63 @@ #include /** - * Model spec types and validation. + * Model spec types, JSON parsing/serialization, and validation. * * This module is the C++ counterpart of `src/core/schema.ts`. Both files - * define the same data model (MethodSpec, ParamSpec, ConcreteDim, etc.) - * and validation logic, but serve different roles: + * define the exact same structural data model (`ModelSpec`, `MethodSpec`, + * `ParamSpec`, `ConcreteDim`, `RuntimeConstraint`, etc.) and validation + * semantics, but operate at different phases of the lifecycle: * - * - **schema.ts** — validates *allowed* (symbolic) specs written by pipeline - * authors against *exported* (concrete) specs. Handles symbol binding, - * dim-domain matching, and 1-to-1 constraint matching. Runs in JS. + * - **schema.ts (TypeScript)** — Validates *allowed* (symbolic) specs written by + * pipeline authors against *exported* (concrete) specs. Handles symbol binding, + * dimension domain matching, and 1-to-1 constraint matching. Runs in JavaScript/TypeScript. * - * - **schema.cpp / schema.h** — validates exported specs at load time against - * ExecuTorch MethodMeta (dtype, rank, static shape). Also validates runtime - * constraint values before execution. Runs in native C++. + * - **schema.h / schema.cpp (Native C++)** — Parses exported JSON specs, validates + * them at model load time against ExecuTorch `MethodMeta`, and performs dynamic + * shape and constraint validation prior to model execution. Runs in native C++. + + * Model specifications originate from one of two sources at load time: + * 1. **Optional JSON Companion Method**: When a `.pte` model binary exports the + * companion method `rnexecutorch::core::model::kGetModelSchemaMethod`, + * `ModelHostObject` calls it to retrieve a JSON string. This string is + * parsed by `parseModelSpecJson` into a `ModelSpec` containing rich dynamic + * dimension domains (`RangeDim`, `EnumDim`) and `RuntimeConstraint`s. + * 2. **Fallback MethodMeta**: If companion is absent, `methodSpecFromMetadata` + * derives a basic `ModelSpec` directly from static ExecuTorch `MethodMeta` + * (where all dimensions are static `int32_t` constants and no constraints + * are present). + * + * The types defined in `schema.h` mirror their TypeScript counterparts in + * `src/core/schema.ts`: + + * Validation in C++ occurs in two distinct phases: * - * The exported spec originates as JSON from the companion method (or is derived - * from MethodMeta when absent). It is parsed by `parseModelSpecJson` (C++) and - * also passed to `validateSpec` (TS) for allowed-spec matching. + * 1. **Load-Time Spec Validation (`validateSpec`)**: + * Executed inside `ModelHostObject` constructor when loading a model. It checks + * the parsed exported spec (`MethodSpec`) against ExecuTorch's `MethodMeta` metadata. + * Verifies that parameter counts, primitive tags, tensor `DType`s, and static shape + * dimensions match ExecuTorch's compiled program requirements. + * + * 2. **Dynamic Runtime Validation (`validateRuntimeConstraints`)**: + * Executed natively in C++ inside `ModelHostObject::execute` immediately prior to + * running model inference. It inspects the actual concrete dimensions of user-provided + * input tensors and asserts that: + * - Dynamic dimensions fall within their declared domains (`RangeDim` + * min/max/step or `EnumDim` choices). + * - `EqualityConstraint`: Referenced input tensor dimensions evaluate to + * identical integer values. + * - `LinearConstraint`: Referenced input dimensions satisfy `dimLhs == a * + * dimRhs + b`. + * + * *Note on Output Dimension References*: Pre-execution validation only checks + * constraints over input dimensions (`ParamSide::input`). References to output + * dimensions (`ParamSide::output`) are skipped prior to execution (for `EqualityConstraint`, + * output dimension references are filtered out; for `LinearConstraint`, constraints referencing + * an output dimension are skipped). After execution, concrete output shapes produced by + * ExecuTorch are verified when copying data into user-provided output buffers. * * @see src/core/schema.ts for the TypeScript validation layer. + * @see rnexecutorch::core::model::kGetModelSchemaMethod for the companion method constant. */ namespace rnexecutorch::core::schema { namespace jsi = facebook::jsi; @@ -118,7 +156,7 @@ struct LinearConstraint { /** * A requirement on the runtime values of a method's tensor dimensions: the * concrete tensors passed to and produced by the method must satisfy it in - * any given execution. Matched as a declaration during spec validation. + * any given execution. */ using RuntimeConstraint = std::variant; @@ -147,10 +185,10 @@ using ModelSpec = std::unordered_map; // ======================================================== /** - * Parses a JSON-encoded ModelSpec using nlohmann/json, validating - * every value (positive constants, ordered range bounds, known kinds and tags, - * in-range indices). Throws std::runtime_error with `ctx` context on invalid - * JSON or a malformed spec. + * Parses a JSON-encoded ModelSpec into a C++ `ModelSpec` structure using + * nlohmann::json. Throws `std::runtime_error` with `ctx` context on invalid + * JSON syntax or unrecognized kinds and tags. Semantic validation against model + * metadata is performed separately by `validateSpec`. * * @param ctx Context description used for error messages. * @param json The JSON string to parse. @@ -206,30 +244,40 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & // ======================================================== /** - * Validates a MethodSpec against ExecuTorch MethodMeta at load time. - * Checks input/output counts, tags, tensor dtypes, and static shape dimensions. - * Dynamic dimensions are skipped. + * Validates a MethodSpec against ExecuTorch MethodMeta at load time. Performs + * the following checks: + * 1. Dimension domain validity: positive constant values, ordered range bounds + * (`min > 0`, `max >= min`, `step > 0`), and non-empty positive enum + * choices. + * 2. Parameter metadata matching: input/output counts, parameter primitive + * tags, tensor `DType`s, tensor ranks, and exact values of static constant + * dimensions against ExecuTorch `MethodMeta` (dynamic dimensions are + * skipped). + * 3. Constraint structure: verifies `EqualityConstraint` has at least 2 + * dimensions and all `DimRef` tensor and dimension indices are within valid + * input/output rank bounds. * * @param spec The method spec to validate. * @param meta The method metadata from the .pte program. - * @param methodName Method name for error messages. - * @throws std::runtime_error on any mismatch. + * @param ctx Context string for error messages. + * @throws std::runtime_error on any mismatch or invalid spec. */ void validateSpec(const MethodSpec &spec, const executorch::runtime::MethodMeta &meta, const std::string &ctx); /** - * Validates runtime constraints on input tensors before execution. - * Only input-only constraints are checked. Constraints referencing outputs - * are skipped because ExecuTorch constructs the output tensors internally, - * guaranteeing they satisfy the declared constraints by construction. - * After execution, the output shapes are validated separately against the - * shapes returned by ExecuTorch when copying data to the user-provided buffers. + * Validates runtime constraints against actual concrete input tensor shapes before execution. + * + * References to output dimensions (`ParamSide::output`) are skipped prior to + * execution (for `EqualityConstraint`, output dimension references are filtered + * out; for `LinearConstraint`, constraints referencing an output dimension are + * skipped). After execution, concrete output shapes produced by ExecuTorch are + * verified when copying data into user-provided output buffers. * * @param rt The JSI runtime instance (for error reporting). - * @param constraints The constraints to validate. - * @param inputShapes Per-tensor input shapes (indexed by DimRef.tensorIdx). + * @param constraints The list of runtime constraints to validate. + * @param inputShapes Per-tensor input shapes (indexed by `DimRef.tensorIdx`). * @param ctx Context string for error messages (e.g. method name). * @throws jsi::JSError on constraint violation. */ diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index be935a67db..919d10caf3 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -175,8 +175,8 @@ export async function createWhisperSpeechToText Date: Tue, 28 Jul 2026 22:09:00 +0200 Subject: [PATCH 18/31] small fix --- packages/react-native-executorch/cpp/core/model.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 0f429f699c..ae649e0601 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -56,11 +56,11 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) throw std::runtime_error(std::format("Failed to load model: {}", errorMsg)); } - const auto methodNames = unwrap("loadModel", etModule_->method_names()); + const auto methodNames = unwrap("method names", etModule_->method_names()); schema::ModelSpec overrideSpec; if (methodNames.contains(kGetModelSchemaMethod)) { - auto ctx = std::format("loadModel: '{}'", kGetModelSchemaMethod); + auto ctx = std::format("Execute '{}'", kGetModelSchemaMethod); auto result = unwrap(ctx, etModule_->execute(kGetModelSchemaMethod)); if (result.empty() || result[0].tag != executorch::runtime::Tag::String) { @@ -72,7 +72,9 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) } for (const auto &methodName : methodNames) { - auto methodMeta = unwrap("loadModel", etModule_->method_meta(methodName)); + auto ctx = std::format("Method '{}'", methodName); + auto methodMeta = unwrap(ctx, etModule_->method_meta(methodName)); + spec_[methodName] = schema::methodSpecFromMetadata(methodMeta); backends_[methodName] = schema::getUsedBackends(methodMeta); @@ -80,7 +82,6 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) spec_[methodName] = std::move(overrideSpec[methodName]); } - auto ctx = std::format("loadModel: method '{}'", methodName); schema::validateSpec(spec_[methodName], methodMeta, ctx); } } From 7bdb7a5e0e93d534756eead522422351ff1cc049 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Tue, 28 Jul 2026 23:35:56 +0200 Subject: [PATCH 19/31] fix attribute name --- packages/react-native-executorch/cpp/core/schema.cpp | 10 +++++----- packages/react-native-executorch/cpp/core/schema.h | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp index c8943feaed..74fb46d847 100644 --- a/packages/react-native-executorch/cpp/core/schema.cpp +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -79,7 +79,7 @@ T unwrap(const std::string &ctx, executorch::runtime::Result result) { NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RangeDim, min, max, step) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EnumDim, choices) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(DimRef, side, tensorIdx, dimIdx) +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(DimRef, paramSide, tensorIdx, dimIdx) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EqualityConstraint, dims) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(LinearConstraint, dimLhs, dimRhs, coefficients) NLOHMANN_JSON_SERIALIZE_ENUM(ParamSide, {{ParamSide::input, "input"}, {ParamSide::output, "output"}}) @@ -358,7 +358,7 @@ void validateDimRef(const DimRef &ref, const std::vector &inputRanks, const std::vector &outputRanks, const std::string &ctx) { - bool isInput = (ref.side == ParamSide::input); + bool isInput = (ref.paramSide == ParamSide::input); const auto &ranks = isInput ? inputRanks : outputRanks; if (std::cmp_greater_equal(ref.tensorIdx, ranks.size())) { throw std::runtime_error(std::format("{}: tensorIdx {} out of range", ctx, ref.tensorIdx)); @@ -461,7 +461,7 @@ void validateRuntimeConstraints(jsi::Runtime &rt, if (const auto *eq = std::get_if(&constraints[i])) { std::vector inputVals; for (const auto &d : eq->dims) { - if (d.side == ParamSide::input) { + if (d.paramSide == ParamSide::input) { inputVals.push_back(getInputDimValue(d, inputShapes)); } } @@ -476,8 +476,8 @@ void validateRuntimeConstraints(jsi::Runtime &rt, } if (const auto *lin = std::get_if(&constraints[i])) { - if (lin->dimLhs.side == ParamSide::output || - lin->dimRhs.side == ParamSide::output) { + if (lin->dimLhs.paramSide == ParamSide::output || + lin->dimRhs.paramSide == ParamSide::output) { continue; } int32_t lhs = getInputDimValue(lin->dimLhs, inputShapes); diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h index df651c4c18..cf5cf39d0e 100644 --- a/packages/react-native-executorch/cpp/core/schema.h +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -129,7 +129,7 @@ enum class ParamSide { input, * with ExecuTorch's `inputTensorMeta` / `outputTensorMeta` ordering. */ struct DimRef { - ParamSide side = ParamSide::input; + ParamSide paramSide = ParamSide::input; int32_t tensorIdx = 0; int32_t dimIdx = 0; }; From 05fbc5e669e862c0ca918b9fe8df4d405fb70a9c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 01:31:39 +0200 Subject: [PATCH 20/31] fix --- packages/react-native-executorch/cpp/core/model.cpp | 2 +- packages/react-native-executorch/cpp/core/schema.cpp | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index ae649e0601..2b5421da56 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -82,7 +82,7 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) spec_[methodName] = std::move(overrideSpec[methodName]); } - schema::validateSpec(spec_[methodName], methodMeta, ctx); + schema::validateSpec(spec_[methodName], methodMeta, ctx + " metadata validation"); } } diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp index 74fb46d847..2921d1a094 100644 --- a/packages/react-native-executorch/cpp/core/schema.cpp +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -78,7 +78,6 @@ T unwrap(const std::string &ctx, executorch::runtime::Result result) { // ======================================================== NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(RangeDim, min, max, step) -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EnumDim, choices) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(DimRef, paramSide, tensorIdx, dimIdx) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(EqualityConstraint, dims) NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(LinearConstraint, dimLhs, dimRhs, coefficients) @@ -92,7 +91,7 @@ void from_json(const json &j, ConcreteDim &d) { } else if (kind == "range") { d = j.at("range").get(); } else if (kind == "enum") { - d = j.at("choices").get(); + d = EnumDim{.choices = j.at("choices").get>()}; } else { throw std::runtime_error(std::format("unsupported dim kind '{}'", kind)); } @@ -106,7 +105,7 @@ void to_json(json &j, const ConcreteDim &d) { j = json::object({{"kind", "range"}, {"range", *r}}); } if (const auto *e = std::get_if(&d)) { - j = json::object({{"kind", "enum"}, {"choices", *e}}); + j = json::object({{"kind", "enum"}, {"choices", e->choices}}); } } From 99b4d5f9897b8f847967d21a2126401e3d678e77 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 01:38:42 +0200 Subject: [PATCH 21/31] fix --- .../speech/tasks/fsmnVoiceActivityDetection.ts | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts index 2f8197e878..0fd6f4885a 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/fsmnVoiceActivityDetection.ts @@ -2,7 +2,7 @@ import type { WorkletRuntime } from 'react-native-worklets'; import { tensor } from '../../../core/tensor'; import { loadModel } from '../../../core/model'; -import { validateSpec, DynamicDim as Dyn, method, f32, constr } from '../../../core/schema'; +import { validateSpec, DynamicDim as Dyn, method, f32 } from '../../../core/schema'; import { wrapAsync } from '../../../core/runtime'; import { extractFrames } from '../utils/vadUtils'; @@ -231,13 +231,7 @@ export async function createFsmnVoiceActivityDetector( default: method( 'forward', // prettier-ignore [f32(Dyn('frames'), 'fftLen')], - [f32(1, Dyn('frames'), 'classes')], - [ - constr.eq( - { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, - { paramSide: 'output', tensorIdx: 0, dimIdx: 1 } - ), - ] + [f32(1, Dyn('frames'), 'classes')] ), }); const [numClasses, fftLength] = dims.constant('classes', 'fftLen'); From 7dee00b87c026957884d5d1a70283e8804a51dfc Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 02:40:17 +0200 Subject: [PATCH 22/31] docs(schema): document get_model_schema companion method and spec source merge behavior --- .../src/core/schema.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 894ac4e78c..3a482c3647 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -45,6 +45,25 @@ * decided statically — equal constants are equal at runtime. Linear * constraints, in contrast, are never evaluated against domains here, even * between constants. + * + * **Exported spec source.** An exported model's `ModelSpec` + * is populated at load time from one of two sources: + * + * 1. **ExecuTorch `MethodMeta`** (default) — when the `.pte` only carries + * static metadata, every dimension domain is `constant`. This is + * sufficient for models whose input/output shapes are fully fixed at + * export time. + * + * 2. **Companion `get_model_schema` method** — for models whose tensors + * have dynamic or enumerated dimensions (e.g. variable-length sequences) + * or that declare runtime constraints, the `.pte` exports a method named + * `get_model_schema` that returns a JSON-encoded `ModelSpec` + * string. The native loader calls this method after loading the model and + * merges the result into `model.schema`, overlaying precise `range`, + * `enum`, and `RuntimeConstraint` entries onto the base `MethodMeta`. + * Only methods that actually need overrides need to appear in the JSON; + * methods absent from the companion spec are kept as-is from + * `MethodMeta`. * @packageDocumentation */ import type { DType } from './tensor'; From f8d94a89f836f92372e2a59491af660bcc6c88b6 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 16:38:19 +0200 Subject: [PATCH 23/31] refactor: use std::visit pattern for variant matching in tensor_helpers and schema.cpp --- .../cpp/core/model.cpp | 3 +- .../cpp/core/schema.cpp | 219 ++++++++++-------- .../react-native-executorch/cpp/core/schema.h | 6 +- .../cpp/core/tensor_helpers.cpp | 114 ++++----- 4 files changed, 194 insertions(+), 148 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 2b5421da56..b21356c380 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -127,7 +127,8 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto outputTensorsArray = conversions::asType(rt, "execute: outputTensors", args[2]); if (inputsArray.size(rt) != methodSpec.inputs.size()) { - throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs")); + throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs: got {}, expected {}", + inputsArray.size(rt), methodSpec.inputs.size())); } std::vector inputs(methodSpec.inputs.size()); diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp index 2921d1a094..87549a3066 100644 --- a/packages/react-native-executorch/cpp/core/schema.cpp +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -63,6 +63,11 @@ using nlohmann::json; namespace { +template +struct overloaded : Ts... { + using Ts::operator()...; +}; + template T unwrap(const std::string &ctx, executorch::runtime::Result result) { if (!result.ok()) { @@ -98,15 +103,13 @@ void from_json(const json &j, ConcreteDim &d) { } // NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. void to_json(json &j, const ConcreteDim &d) { - if (const auto *c = std::get_if(&d)) { - j = json::object({{"kind", "constant"}, {"value", *c}}); - } - if (const auto *r = std::get_if(&d)) { - j = json::object({{"kind", "range"}, {"range", *r}}); - } - if (const auto *e = std::get_if(&d)) { - j = json::object({{"kind", "enum"}, {"choices", e->choices}}); - } + // clang-format off + std::visit(overloaded{ + [&](int32_t c) { j = json::object({{"kind", "constant"}, {"value", c}}); }, + [&](const RangeDim &r) { j = json::object({{"kind", "range"}, {"range", r}}); }, + [&](const EnumDim &e) { j = json::object({{"kind", "enum"}, {"choices", e.choices}}); }, + }, d); + // clang-format on } // NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. @@ -141,15 +144,17 @@ void from_json(const json &j, RuntimeConstraint &c) { } // NOLINTNEXTLINE(misc-use-internal-linkage): ADL requires external linkage. void to_json(json &j, const RuntimeConstraint &c) { - if (const auto *eq = std::get_if(&c)) { - j = json::object({{"kind", "equality"}, {"dims", eq->dims}}); - } - if (const auto *lin = std::get_if(&c)) { - j = json::object({{"kind", "linear"}, - {"dimLhs", lin->dimLhs}, - {"dimRhs", lin->dimRhs}, - {"coefficients", lin->coefficients}}); - } + // clang-format off + std::visit(overloaded{ + [&](const EqualityConstraint &eq) { j = json::object({{"kind", "equality"}, {"dims", eq.dims}}); }, + [&](const LinearConstraint &lin) { + j = json::object({{"kind", "linear"}, + {"dimLhs", lin.dimLhs}, + {"dimRhs", lin.dimRhs}, + {"coefficients", lin.coefficients}}); + }, + }, c); + // clang-format on } NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(MethodSpec, inputs, outputs, runtimeConstraints) @@ -228,13 +233,12 @@ jsi::Object backendsToJs(jsi::Runtime &rt, namespace { ParamSpec tensorMetaToParamSpec(const executorch::runtime::TensorInfo &tensorMeta) { - ParamSpec p{.tag = Tag::Tensor, .dtype = types::fromScalarType(tensorMeta.scalar_type())}; const auto sizes = tensorMeta.sizes(); - p.shape.reserve(sizes.size()); - for (const auto size : sizes) { - p.shape.emplace_back(size); - } - return p; + return ParamSpec{ + .tag = Tag::Tensor, + .dtype = types::fromScalarType(tensorMeta.scalar_type()), + .shape = std::vector(sizes.begin(), sizes.end()), + }; } } // namespace @@ -288,32 +292,36 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & namespace { void validateConcreteDim(const ConcreteDim &dim, const std::string &ctx) { - if (const auto *c = std::get_if(&dim)) { - if (*c <= 0) { - throw std::runtime_error(std::format("{}: constant dim must be positive", ctx)); - } - } - if (const auto *r = std::get_if(&dim)) { - if (r->min < 0) { - throw std::runtime_error(std::format("{}: range min must be non-negative", ctx)); - } - if (r->max < r->min) { - throw std::runtime_error(std::format("{}: range max must be >= min", ctx)); - } - if (r->step <= 0) { - throw std::runtime_error(std::format("{}: range step must be positive", ctx)); - } - } - if (const auto *e = std::get_if(&dim)) { - if (e->choices.empty()) { - throw std::runtime_error(std::format("{}: enum must have at least one choice", ctx)); - } - for (const auto &choice : e->choices) { - if (choice <= 0) { - throw std::runtime_error(std::format("{}: enum choices must be positive", ctx)); + // clang-format off + std::visit(overloaded{ + [&](int32_t c) { + if (c <= 0) { + throw std::runtime_error(std::format("{}: constant dim must be positive", ctx)); } - } - } + }, + [&](const RangeDim &r) { + if (r.min < 0) { + throw std::runtime_error(std::format("{}: range min must be non-negative", ctx)); + } + if (r.max < r.min) { + throw std::runtime_error(std::format("{}: range max must be >= min", ctx)); + } + if (r.step <= 0) { + throw std::runtime_error(std::format("{}: range step must be positive", ctx)); + } + }, + [&](const EnumDim &e) { + if (e.choices.empty()) { + throw std::runtime_error(std::format("{}: enum must have at least one choice", ctx)); + } + for (const auto &choice : e.choices) { + if (choice <= 0) { + throw std::runtime_error(std::format("{}: enum choices must be positive", ctx)); + } + } + }, + }, dim); + // clang-format on } void validateSpecDimDomains(const MethodSpec &spec, const std::string &ctx) { @@ -346,10 +354,31 @@ void validateTensorParam(const ParamSpec ¶m, } for (size_t d = 0; d < param.shape.size(); ++d) { - if (std::holds_alternative(param.shape[d]) && - std::get(param.shape[d]) != metaShape[d]) { - throw std::runtime_error(std::format("{}: shape[{}] mismatch", ctx, d)); - } + auto bound = static_cast(metaShape[d]); + // clang-format off + std::visit(overloaded{ + [&](int32_t c) { + if (c != bound) { + throw std::runtime_error(std::format("{}: shape[{}] mismatch", ctx, d)); + } + }, + [&](const RangeDim &r) { + if (r.max > bound) { + throw std::runtime_error(std::format("{}: shape[{}] range max {} exceeds compiled bound {}", + ctx, d, r.max, bound)); + } + }, + [&](const EnumDim &e) { + for (const auto choice : e.choices) { + if (choice > bound) { + throw std::runtime_error(std::format("{}: shape[{}] enum choice {} exceeds compiled bound {}", + ctx, d, choice, bound)); + } + } + }, + }, + param.shape[d]); + // clang-format on } } @@ -385,19 +414,22 @@ void validateConstraintSpecs(const MethodSpec &spec, const std::string &ctx) { auto cctx = std::format("{} constraint[{}]", ctx, i); const auto &constraint = spec.runtimeConstraints[i]; - if (const auto *eq = std::get_if(&constraint)) { - if (eq->dims.size() < 2) { - throw std::runtime_error(std::format("{}: equality needs at least two dims", cctx)); - } - for (const auto &dim : eq->dims) { - validateDimRef(dim, inputRanks, outputRanks, cctx); - } - } - - if (const auto *lin = std::get_if(&constraint)) { - validateDimRef(lin->dimLhs, inputRanks, outputRanks, cctx); - validateDimRef(lin->dimRhs, inputRanks, outputRanks, cctx); - } + // clang-format off + std::visit(overloaded{ + [&](const EqualityConstraint &eq) { + if (eq.dims.size() < 2) { + throw std::runtime_error(std::format("{}: equality needs at least two dims", cctx)); + } + for (const auto &dim : eq.dims) { + validateDimRef(dim, inputRanks, outputRanks, cctx); + } + }, + [&](const LinearConstraint &lin) { + validateDimRef(lin.dimLhs, inputRanks, outputRanks, cctx); + validateDimRef(lin.dimRhs, inputRanks, outputRanks, cctx); + }, + }, constraint); + // clang-format on } } @@ -457,34 +489,37 @@ void validateRuntimeConstraints(jsi::Runtime &rt, for (size_t i = 0; i < constraints.size(); ++i) { auto cctx = std::format("{} constraint[{}]", ctx, i); - if (const auto *eq = std::get_if(&constraints[i])) { - std::vector inputVals; - for (const auto &d : eq->dims) { - if (d.paramSide == ParamSide::input) { - inputVals.push_back(getInputDimValue(d, inputShapes)); + // clang-format off + std::visit(overloaded{ + [&](const EqualityConstraint &eq) { + std::vector inputVals; + for (const auto &d : eq.dims) { + if (d.paramSide == ParamSide::input) { + inputVals.push_back(getInputDimValue(d, inputShapes)); + } } - } - if (inputVals.size() < 2) { - continue; - } - for (size_t j = 1; j < inputVals.size(); ++j) { - if (inputVals[j] != inputVals[0]) { - throw jsi::JSError(rt, std::format("{}: equality constraint violated", cctx)); + if (inputVals.size() < 2) { + return; } - } - } - - if (const auto *lin = std::get_if(&constraints[i])) { - if (lin->dimLhs.paramSide == ParamSide::output || - lin->dimRhs.paramSide == ParamSide::output) { - continue; - } - int32_t lhs = getInputDimValue(lin->dimLhs, inputShapes); - int32_t rhs = getInputDimValue(lin->dimRhs, inputShapes); - if (lhs != lin->coefficients[0] * rhs + lin->coefficients[1]) { - throw jsi::JSError(rt, std::format("{}: linear constraint violated", cctx)); - } - } + for (size_t j = 1; j < inputVals.size(); ++j) { + if (inputVals[j] != inputVals[0]) { + throw jsi::JSError(rt, std::format("{}: equality constraint violated", cctx)); + } + } + }, + [&](const LinearConstraint &lin) { + if (lin.dimLhs.paramSide == ParamSide::output || + lin.dimRhs.paramSide == ParamSide::output) { + return; + } + int32_t lhs = getInputDimValue(lin.dimLhs, inputShapes); + int32_t rhs = getInputDimValue(lin.dimRhs, inputShapes); + if (lhs != lin.coefficients[0] * rhs + lin.coefficients[1]) { + throw jsi::JSError(rt, std::format("{}: linear constraint violated", cctx)); + } + }, + }, constraints[i]); + // clang-format on } } diff --git a/packages/react-native-executorch/cpp/core/schema.h b/packages/react-native-executorch/cpp/core/schema.h index cf5cf39d0e..484108eb65 100644 --- a/packages/react-native-executorch/cpp/core/schema.h +++ b/packages/react-native-executorch/cpp/core/schema.h @@ -250,9 +250,9 @@ std::vector getUsedBackends(const executorch::runtime::MethodMeta & * (`min > 0`, `max >= min`, `step > 0`), and non-empty positive enum * choices. * 2. Parameter metadata matching: input/output counts, parameter primitive - * tags, tensor `DType`s, tensor ranks, and exact values of static constant - * dimensions against ExecuTorch `MethodMeta` (dynamic dimensions are - * skipped). + * tags, tensor `DType`s, tensor ranks, exact values of static constant + * dimensions, and dynamic dimension upper bounds (`RangeDim` max and + * `EnumDim` choices <= compiled `MethodMeta` allocation bound). * 3. Constraint structure: verifies `EqualityConstraint` has at least 2 * dimensions and all `DimRef` tensor and dimension indices are within valid * input/output rank bounds. diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp index 1cdfd8ae1a..c7fa13c582 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp @@ -48,28 +48,34 @@ void checkNotSameTensor(jsi::Runtime &rt, } namespace { + +template +struct overloaded : Ts... { + using Ts::operator()...; +}; + std::string shapeToString(const SymbolicShape &shape) { std::string s; for (const auto &dim : shape) { - if (const auto *str = std::get_if(&dim)) { - s += *str; - } - if (const auto *val = std::get_if(&dim)) { - s += std::to_string(*val); - } - if (const auto *range = std::get_if(&dim)) { - s += std::format("[{}..{}:{}]", range->min, range->max, range->step); - } - if (const auto *enumeration = std::get_if(&dim)) { - s += "{"; - for (const auto choice : enumeration->choices) { - s += std::to_string(choice) + ","; - } - if (!enumeration->choices.empty()) { - s.pop_back(); - } - s += "}"; - } + // clang-format off + std::visit(overloaded{ + [&](const std::string &str) { s += str; }, + [&](int32_t val) { s += std::to_string(val); }, + [&](const schema::RangeDim &range) { + s += std::format("[{}..{}:{}]", range.min, range.max, range.step); + }, + [&](const schema::EnumDim &enumeration) { + s += "{"; + for (const auto choice : enumeration.choices) { + s += std::to_string(choice) + ","; + } + if (!enumeration.choices.empty()) { + s.pop_back(); + } + s += "}"; + }, + }, dim); + // clang-format on s += ","; } if (!shape.empty()) { @@ -110,39 +116,43 @@ fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, for (size_t i = 0; i < expectedShape->size(); ++i) { const auto &dim = expectedShape->at(i); - if (const auto *symbol = std::get_if(&dim)) { - if (symbolBinding.contains(*symbol) && symbolBinding[*symbol] != shape[i]) { - throw jsi::JSError(rt, std::format("{} must have shape {} (symbol {} mismatch: expected {}, got {})", - ctx, shapeToString(*expectedShape), *symbol, symbolBinding[*symbol], shape[i])); - } - symbolBinding[*symbol] = shape[i]; - } - if (const auto *val = std::get_if(&dim)) { - if (shape[i] != *val) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} mismatch: expected {}, got {})", - ctx, shapeToString(*expectedShape), i, *val, shape[i])); - } - } - if (const auto *range = std::get_if(&dim)) { - if (shape[i] < range->min) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} < min {})", - ctx, shapeToString(*expectedShape), i, shape[i], range->min)); - } - if (shape[i] > range->max) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} > max {})", - ctx, shapeToString(*expectedShape), i, shape[i], range->max)); - } - if ((shape[i] - range->min) % range->step != 0) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} must be min({}) + k*step({}), got {})", - ctx, shapeToString(*expectedShape), i, range->min, range->step, shape[i])); - } - } - if (const auto *enumeration = std::get_if(&dim)) { - if (std::ranges::find(enumeration->choices, shape[i]) == enumeration->choices.end()) { - throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} not allowed: got {})", - ctx, shapeToString(*expectedShape), i, shape[i])); - } - } + // clang-format off + std::visit(overloaded{ + [&](const std::string &symbol) { + if (symbolBinding.contains(symbol) && symbolBinding[symbol] != shape[i]) { + throw jsi::JSError(rt, std::format("{} must have shape {} (symbol {} mismatch: expected {}, got {})", + ctx, shapeToString(*expectedShape), symbol, symbolBinding[symbol], shape[i])); + } + symbolBinding[symbol] = shape[i]; + }, + [&](int32_t val) { + if (shape[i] != val) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} mismatch: expected {}, got {})", + ctx, shapeToString(*expectedShape), i, val, shape[i])); + } + }, + [&](const schema::RangeDim &range) { + if (shape[i] < range.min) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} < min {})", + ctx, shapeToString(*expectedShape), i, shape[i], range.min)); + } + if (shape[i] > range.max) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} out of range: {} > max {})", + ctx, shapeToString(*expectedShape), i, shape[i], range.max)); + } + if ((shape[i] - range.min) % range.step != 0) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} must be min({}) + k*step({}), got {})", + ctx, shapeToString(*expectedShape), i, range.min, range.step, shape[i])); + } + }, + [&](const schema::EnumDim &enumeration) { + if (std::ranges::find(enumeration.choices, shape[i]) == enumeration.choices.end()) { + throw jsi::JSError(rt, std::format("{} must have shape {} (dim {} not allowed: got {})", + ctx, shapeToString(*expectedShape), i, shape[i])); + } + }, + }, dim); + // clang-format on } return tensor; From 5183a3ac52748d6e643792c34fa796ecf7398ec1 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 16:55:39 +0200 Subject: [PATCH 24/31] docs: clarify audio padding comment in whisperSpeechToText --- .../src/extensions/speech/tasks/whisperSpeechToText.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts index 919d10caf3..d5a202eaca 100644 --- a/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts +++ b/packages/react-native-executorch/src/extensions/speech/tasks/whisperSpeechToText.ts @@ -170,7 +170,7 @@ export async function createWhisperSpeechToText Date: Wed, 29 Jul 2026 17:00:00 +0200 Subject: [PATCH 25/31] fix(schema): throw on step === 0 in RangeDim builder --- packages/react-native-executorch/src/core/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 3a482c3647..6a145bf485 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -293,7 +293,7 @@ export const RangeDim = (min: number, max: number, step?: number): ConcreteDim = if (!Number.isInteger(max)) { throw new Error(`Invalid range max (${max}): must be a non-negative integer.`); } - if (step && (step <= 0 || !Number.isInteger(step))) { + if (step !== undefined && (step <= 0 || !Number.isInteger(step))) { throw new Error(`Invalid range step (${step}): must be a positive integer.`); } return { kind: 'range', range: { min, max, step: step ?? 1 } }; From 917047799149de37edfb41cdd6f866b8ab04fb67 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Wed, 29 Jul 2026 17:07:56 +0200 Subject: [PATCH 26/31] docs: update model-schema-validation skill example and accessors --- .../skills/model-schema-validation/SKILL.md | 33 ++++++++++++++++--- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/.agents/skills/model-schema-validation/SKILL.md b/.agents/skills/model-schema-validation/SKILL.md index ba680cc141..c102fd72ef 100644 --- a/.agents/skills/model-schema-validation/SKILL.md +++ b/.agents/skills/model-schema-validation/SKILL.md @@ -34,11 +34,32 @@ import { } from '../../../core/schema'; const { variant, dims } = validateSpec(model.schema, { - batched: method('forward', [f32(1, 3, 'H', 'W')], [f32(1, 'N')]), - unbatched: method('forward', [f32(3, 'H', 'W')], [f32('N')]), + batched: method( + 'forward', + [i64(1, Dyn('L')), i64(1, Dyn('L'))], + [f32(1, 'D')], + [ + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 1 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 1 } + ), + ] + ), + unbatched: method( + 'forward', + [i64(Dyn('L')), i64(Dyn('L'))], + [f32('D')], + [ + constr.eq( + { paramSide: 'input', tensorIdx: 0, dimIdx: 0 }, + { paramSide: 'input', tensorIdx: 1, dimIdx: 0 } + ), + ] + ), }); -const [N, H, W] = dims.constant('N', 'H', 'W'); +const [D] = dims.constant('D'); +const L = dims.range('L'); ``` ### Key Schema Utilities from `src/core/schema.ts`: @@ -52,11 +73,13 @@ const [N, H, W] = dims.constant('N', 'H', 'W'); - **`constr.eq(...dims)`**: Creates an equality constraint requiring the referenced dimensions to take the exact same value at runtime. - **`constr.linear(lhs, rhs, a, b)`**: Creates a linear constraint `lhs = a * rhs + b`. - **`validateSpec(exportedSchema, allowedVariants)`**: Compares the model's exported schema against named variants and returns `{ variant, dim, dims }`. -- **Symbol Accessors (`dims`)**: +- **Symbol Accessors (`dims` & `dim`)**: - `dims.constant('N', 'H')`: Extracts constant values for symbols as numbers. - `dims.range('S')`: Extracts range domains `{ min, max, step }`. - `dims.enum('E')`: Extracts enum choices `readonly number[]`. - - `dims.any('D')`: Extracts raw `ConcreteDim`. + - `dims.dynamic('L')`: Extracts dynamic `range` or `enum` as raw `ConcreteDim`. + - `dims.any('D')`: Extracts raw dimension value (`number`, `Range`, `readonly number[]`, or `ConcreteDim`). + - `dim('N')`: Extracts value for a single symbol. --- From 212851b49732e1dd6a9d78917b71b412422603ed Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 30 Jul 2026 16:57:19 +0200 Subject: [PATCH 27/31] fix(schema): restore bind terminology in schema matching and document get_model_schema --- .../src/core/schema.ts | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/packages/react-native-executorch/src/core/schema.ts b/packages/react-native-executorch/src/core/schema.ts index 6a145bf485..4ef0325041 100644 --- a/packages/react-native-executorch/src/core/schema.ts +++ b/packages/react-native-executorch/src/core/schema.ts @@ -64,6 +64,12 @@ * Only methods that actually need overrides need to appear in the JSON; * methods absent from the companion spec are kept as-is from * `MethodMeta`. + * + * **Tip for model export:** + * Embed the companion schema method during ExecuTorch compilation in Python + * by passing `constant_methods={"get_model_schema": schema_json}` when + * lowering with `to_edge_transform_and_lower(...)` where `schema_json` is + * the JSON string encoding the model's `ModelSpec`. * @packageDocumentation */ import type { DType } from './tensor'; @@ -366,7 +372,12 @@ function choicesEqual(c1: readonly number[], c2: readonly number[]): boolean { return s1.size === s2.size && [...s1].every((elem) => s2.has(elem)); } -function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, dims: SymbolBindings, ctx: string): void { +function matchDim( + sDim: SymbolicDim, + cDim: ConcreteDim, + bindings: SymbolBindings, + ctx: string +): void { if (sDim.kind === 'constant' && cDim.kind === 'constant') { if (sDim.value !== cDim.value) { throw new Error(`${ctx}: Constant dimension mismatch.`); @@ -389,30 +400,30 @@ function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, dims: SymbolBindings, ct } if (sDim.kind === 'static' && cDim.kind === 'constant') { - const bind = dims.get(sDim.symbol); + const bind = bindings.get(sDim.symbol); if (bind) { if (bind.kind !== 'constant' || bind.value !== cDim.value) { - throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent dims.`); + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); } return; } - dims.set(sDim.symbol, cDim); + bindings.set(sDim.symbol, cDim); return; } if (sDim.kind === 'dynamic' && (cDim.kind === 'range' || cDim.kind === 'enum')) { - const bind = dims.get(sDim.symbol); + const bind = bindings.get(sDim.symbol); if (bind) { const consistentRange = bind.kind === 'range' && cDim.kind === 'range' && rangesEqual(bind.range, cDim.range); const consistentEnum = bind.kind === 'enum' && cDim.kind === 'enum' && choicesEqual(bind.choices, cDim.choices); if (!consistentRange && !consistentEnum) { - throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent dims.`); + throw new Error(`${ctx}: Symbol '${sDim.symbol}' has inconsistent bindings.`); } return; } - dims.set(sDim.symbol, cDim); + bindings.set(sDim.symbol, cDim); return; } @@ -422,7 +433,7 @@ function matchDim(sDim: SymbolicDim, cDim: ConcreteDim, dims: SymbolBindings, ct function matchMethodSpecs( allowedMethodSpec: MethodSpec, exportedMethodSpec: MethodSpec, - dims: SymbolBindings, + bindings: SymbolBindings, ctx: string ): void { if (allowedMethodSpec.inputs.length !== exportedMethodSpec.inputs.length) { @@ -462,7 +473,7 @@ function matchMethodSpecs( for (let d = 0; d < allowedTensorSpec.shape.length; ++d) { const dimCtx = `${paramSpecCtx} Tensor dim #${d}`; - matchDim(allowedTensorSpec.shape[d]!, exportedTensorSpec.shape[d]!, dims, dimCtx); + matchDim(allowedTensorSpec.shape[d]!, exportedTensorSpec.shape[d]!, bindings, dimCtx); } } } @@ -470,14 +481,14 @@ function matchMethodSpecs( function matchModelSpecsSymbols( allowedModelSpec: ModelSpec, exportedModelSpec: ModelSpec, - dims: SymbolBindings + bindings: SymbolBindings ): void { for (const [methodName, allowedMethodSpec] of Object.entries(allowedModelSpec)) { const exportedMethodSpec = exportedModelSpec[methodName]; if (!exportedMethodSpec) { throw new Error(`Method '${methodName}' not found in exported model spec.`); } - matchMethodSpecs(allowedMethodSpec, exportedMethodSpec, dims, `Method '${methodName}'`); + matchMethodSpecs(allowedMethodSpec, exportedMethodSpec, bindings, `Method '${methodName}'`); } } @@ -730,7 +741,7 @@ export type SpecMatch = { * the allowed (symbolic) model specs — variants are tried in order and the * first match wins. For a variant to match: * - Every method exists in the exported spec and its signature matches, - * dim each symbol to a constant value (static) or a range/enum + * binding each symbol to a constant value (static) or a range/enum * (dynamic). Repeated symbols must bind consistently across the whole spec. * - The exported spec declares exactly the same runtime constraints per * method (1-to-1, no missing, no extras). Constraints are matched as @@ -762,14 +773,14 @@ export function validateSpec { - const dim = dims.get(name); + const dim = bindings.get(name); if (!dim) { - throw new Error(`Symbol '${name}' not found in dims.`); + throw new Error(`Symbol '${name}' not found in bindings.`); } if (kind) { if (kind === 'dynamic') { From cbb69c5c7c846991642cd77981648e4f8b594509 Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 30 Jul 2026 17:20:09 +0200 Subject: [PATCH 28/31] fix(model): fix error string --- packages/react-native-executorch/cpp/core/model.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index b21356c380..634280db26 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -200,11 +200,10 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { " (e.g. equality between dimensions), export a companion\n" " method returning a JSON model spec\n" " (see `src/core/schema.ts` for the JSON structure).\n" - "\n" " Without it, validation falls back to static metadata\n" " from ExecuTorch which only contains upper bounds and\n" " does not capture runtime constraints.", - methodName), + "\n", "Error:", methodName), std::move(executeResult)); auto jsOutputArray = jsi::Array(rt, result.size()); From 3f0a77285b7683e3c4b3c0576f9cf031e7f1696c Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 30 Jul 2026 20:48:38 +0200 Subject: [PATCH 29/31] fix(core): improve error messages for model, schema, and tensor_helpers --- .../cpp/core/model.cpp | 17 ++++++++----- .../cpp/core/schema.cpp | 25 +++++++++++++------ .../cpp/core/tensor_helpers.cpp | 2 +- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 634280db26..493327f6b8 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -53,7 +53,8 @@ ModelHostObject::ModelHostObject(const std::string &modelPath) auto error = etModule_->load(); if (!etModule_->is_loaded()) { const std::string errorMsg = executorch::runtime::to_string(error); - throw std::runtime_error(std::format("Failed to load model: {}", errorMsg)); + throw std::runtime_error(std::format("Failed to load model from '{}': {}", + modelPath_, errorMsg)); } const auto methodNames = unwrap("method names", etModule_->method_names()); @@ -127,8 +128,8 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { auto outputTensorsArray = conversions::asType(rt, "execute: outputTensors", args[2]); if (inputsArray.size(rt) != methodSpec.inputs.size()) { - throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs: got {}, expected {}", - inputsArray.size(rt), methodSpec.inputs.size())); + throw jsi::JSError(rt, std::format("execute: Incorrect size for inputs of method '{}': got {}, expected {}", + methodName, inputsArray.size(rt), methodSpec.inputs.size())); } std::vector inputs(methodSpec.inputs.size()); @@ -202,8 +203,10 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { " (see `src/core/schema.ts` for the JSON structure).\n" " Without it, validation falls back to static metadata\n" " from ExecuTorch which only contains upper bounds and\n" - " does not capture runtime constraints.", - "\n", "Error:", methodName), + " does not capture runtime constraints.\n" + "\n" + "Error", + methodName), std::move(executeResult)); auto jsOutputArray = jsi::Array(rt, result.size()); @@ -214,7 +217,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { switch (output.tag) { case executorch::runtime::Tag::Tensor: { if (tensorOutputIdx >= outputTensorsArray.size(rt)) { - throw jsi::JSError(rt, "execute: Not enough tensor output placeholders in outputTensors"); + throw jsi::JSError(rt, std::format("execute: Not enough tensor output placeholders in outputTensors" + " (provided {}, expected at least {})", + outputTensorsArray.size(rt), tensorOutputIdx + 1)); } auto ctx = std::format("execute: outputTensors[{}]", tensorOutputIdx); diff --git a/packages/react-native-executorch/cpp/core/schema.cpp b/packages/react-native-executorch/cpp/core/schema.cpp index 87549a3066..e26e6dfd5e 100644 --- a/packages/react-native-executorch/cpp/core/schema.cpp +++ b/packages/react-native-executorch/cpp/core/schema.cpp @@ -345,12 +345,14 @@ void validateTensorParam(const ParamSpec ¶m, const std::string &ctx) { auto metaDtype = types::fromScalarType(tensorMeta.scalar_type()); if (param.dtype != metaDtype) { - throw std::runtime_error(std::format("{}: dtype mismatch", ctx)); + throw std::runtime_error(std::format("{}: dtype mismatch (spec type '{}' != compiled metadata type '{}')", + ctx, types::toString(param.dtype), types::toString(metaDtype))); } auto metaShape = tensorMeta.sizes(); if (param.shape.size() != metaShape.size()) { - throw std::runtime_error(std::format("{}: rank mismatch", ctx)); + throw std::runtime_error(std::format("{}: rank mismatch (spec rank {} != compiled metadata rank {})", + ctx, param.shape.size(), metaShape.size())); } for (size_t d = 0; d < param.shape.size(); ++d) { @@ -359,7 +361,8 @@ void validateTensorParam(const ParamSpec ¶m, std::visit(overloaded{ [&](int32_t c) { if (c != bound) { - throw std::runtime_error(std::format("{}: shape[{}] mismatch", ctx, d)); + throw std::runtime_error(std::format("{}: shape[{}] mismatch (spec constant {} != compiled bound {})", + ctx, d, c, bound)); } }, [&](const RangeDim &r) { @@ -443,7 +446,9 @@ void validateParamsAgainstMeta(const std::vector ¶ms, : unwrap(pctx, meta.output_tag(i)); if (params[i].tag != tagResult) { - throw std::runtime_error(std::format("{}: tag mismatch", pctx)); + throw std::runtime_error(std::format("{}: tag mismatch (spec tag {} != compiled metadata tag {})", + pctx, executorch::runtime::tag_to_string(params[i].tag), + executorch::runtime::tag_to_string(tagResult))); } if (tagResult == Tag::Tensor) { @@ -461,10 +466,12 @@ void validateSpec(const MethodSpec &spec, const std::string &ctx) { if (spec.inputs.size() != meta.num_inputs()) { - throw std::runtime_error(std::format("{}: input count mismatch", ctx)); + throw std::runtime_error(std::format("{}: input count mismatch (spec has {}, model metadata has {})", + ctx, spec.inputs.size(), meta.num_inputs())); } if (spec.outputs.size() != meta.num_outputs()) { - throw std::runtime_error(std::format("{}: output count mismatch", ctx)); + throw std::runtime_error(std::format("{}: output count mismatch (spec has {}, model metadata has {})", + ctx, spec.outputs.size(), meta.num_outputs())); } validateSpecDimDomains(spec, ctx); @@ -503,7 +510,8 @@ void validateRuntimeConstraints(jsi::Runtime &rt, } for (size_t j = 1; j < inputVals.size(); ++j) { if (inputVals[j] != inputVals[0]) { - throw jsi::JSError(rt, std::format("{}: equality constraint violated", cctx)); + throw jsi::JSError(rt, std::format("{}: equality constraint violated (dimension value {} != {})", + cctx, inputVals[0], inputVals[j])); } } }, @@ -515,7 +523,8 @@ void validateRuntimeConstraints(jsi::Runtime &rt, int32_t lhs = getInputDimValue(lin.dimLhs, inputShapes); int32_t rhs = getInputDimValue(lin.dimRhs, inputShapes); if (lhs != lin.coefficients[0] * rhs + lin.coefficients[1]) { - throw jsi::JSError(rt, std::format("{}: linear constraint violated", cctx)); + throw jsi::JSError(rt, std::format("{}: linear constraint violated (LHS {} != {} * RHS {} + {})", + cctx, lhs, lin.coefficients[0], rhs, lin.coefficients[1])); } }, }, constraints[i]); diff --git a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp index c7fa13c582..de77106ae2 100644 --- a/packages/react-native-executorch/cpp/core/tensor_helpers.cpp +++ b/packages/react-native-executorch/cpp/core/tensor_helpers.cpp @@ -99,7 +99,7 @@ fromJs(jsi::Runtime &rt, const std::string &ctx, const jsi::Value &value, const auto &shape = tensor->shape_; if (expectedDtype && dtype != *expectedDtype) { - throw jsi::JSError(rt, std::format("{} must be of type {}", ctx, types::toString(*expectedDtype))); + throw jsi::JSError(rt, std::format("{} must be of type {} (got {})", ctx, types::toString(*expectedDtype), types::toString(dtype))); } if (!expectedShape) { From c8c6c62f35694e3ff2d613267949a55ecad8546f Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 30 Jul 2026 23:15:35 +0200 Subject: [PATCH 30/31] fix: error message --- packages/react-native-executorch/cpp/core/model.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 493327f6b8..639f1054c3 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -194,7 +194,8 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { "Common causes:\n" " 1. Backend not registered\n" " Ensure backends from `model.backends` are registered\n" - " in the ExecuTorch runtime.\n" + " in the ExecuTorch runtime\n" + " (use `getRegisteredBackends()` to check registered backends).\n" "\n" " 2. Shape/constraint mismatch\n" " If the model uses dynamic shapes or runtime constraints\n" From 41467071474a99393199a4c95da5bd3f5686d67e Mon Sep 17 00:00:00 2001 From: Bartosz Hanc Date: Thu, 30 Jul 2026 23:16:44 +0200 Subject: [PATCH 31/31] fix(model): add bad model export cause to execution error message --- packages/react-native-executorch/cpp/core/model.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/react-native-executorch/cpp/core/model.cpp b/packages/react-native-executorch/cpp/core/model.cpp index 639f1054c3..2ff28cd2c3 100644 --- a/packages/react-native-executorch/cpp/core/model.cpp +++ b/packages/react-native-executorch/cpp/core/model.cpp @@ -206,6 +206,9 @@ jsi::Value ModelHostObject::get(jsi::Runtime &rt, const jsi::PropNameID &name) { " from ExecuTorch which only contains upper bounds and\n" " does not capture runtime constraints.\n" "\n" + " 3. Bad model export\n" + " The model export itself might be broken or invalid.\n" + "\n" "Error", methodName), std::move(executeResult));