feat(realtime): lucy-2.5 + optional sampleFrameData on set/setImage

packages/sdk/index.html

@@ -325,6 +325,7 @@ <h3>Configuration</h3>
                 </optgroup>
                 <optgroup label="Canonical">
                     <option value="lucy-2.1" selected>Lucy 2.1</option>
+                    <option value="lucy-2.5">Lucy 2.5</option>
                     <option value="lucy-2.1-vton">Lucy 2.1 VTON</option>
                     <option value="lucy-vton-2">Lucy VTON 2</option>
                     <option value="lucy-restyle-2">Lucy Restyle 2</option>
@@ -442,6 +443,12 @@ <h3>Image Reference (Style Transfer)</h3>
         <div class="inline-controls">
             <button id="send-image" disabled>Send Reference Image</button>
         </div>
+        <div class="control-group" style="display: flex; align-items: center; gap: 8px; margin-top: 8px;">
+            <input type="checkbox" id="include-reference-frame">
+            <label for="include-reference-frame" style="margin: 0; font-weight: normal;">
+                Also capture &amp; attach camera reference frame
+            </label>
+        </div>
         <div id="image-status" style="margin-top: 10px; padding: 10px; background: #f8f9fa; border-radius: 6px; display: none;">
             <strong>Status:</strong> <span id="image-status-text">Ready</span>
         </div>
@@ -549,6 +556,39 @@ <h3>Console Logs</h3>
         let decartClient = null;
         let decartRealtime = null;
         let localStream = null;
+
+        // Captures a low-res JPEG snapshot from the local camera stream.
+        // Mirrors the pattern used in tryonv1 so the bouncer's prompt enhancer can see what the person is currently wearing.
+        async function captureReferenceFrame() {
+            if (!localStream) return null;
+            const video = document.createElement('video');
+            video.srcObject = localStream;
+            video.muted = true;
+            video.playsInline = true;
+            await video.play();
+            await new Promise((resolve) => {
+                if ('requestVideoFrameCallback' in video) {
+                    video.requestVideoFrameCallback(() => resolve());
+                } else {
+                    requestAnimationFrame(() => requestAnimationFrame(() => resolve()));
+                }
+            });
+            if (!video.videoWidth) {
+                video.pause();
+                video.srcObject = null;
+                return null;
+            }
+            const scale = 320 / video.videoWidth;
+            const canvas = document.createElement('canvas');
+            canvas.width = 320;
+            canvas.height = Math.round(video.videoHeight * scale);
+            canvas.getContext('2d').drawImage(video, 0, 0, canvas.width, canvas.height);
+            video.pause();
+            video.srcObject = null;
+            return new Promise((resolve) => {
+                canvas.toBlob((blob) => resolve(blob), 'image/jpeg', 0.5);
+            });
+        }
         let isConnected = false;
         let activeConnectionMode = null;
         let processedVideoUrl = null;
@@ -591,6 +631,7 @@ <h3>Console Logs</h3>
             logsContainer: document.getElementById('logs-container'),
             // Image reference elements
             referenceImage: document.getElementById('reference-image'),
+            includeReferenceFrame: document.getElementById('include-reference-frame'),
             setImage: document.getElementById('send-image'),
             imageStatus: document.getElementById('image-status'),
             imageStatusText: document.getElementById('image-status-text'),
@@ -1217,7 +1258,17 @@ <h3>Console Logs</h3>
 
                 addLog('Sending reference image...', 'info');
 
-                await decartRealtime.setImage(file);
+                let sampleFrameData = null;
+                if (elements.includeReferenceFrame.checked) {
+                    sampleFrameData = await captureReferenceFrame();
+                    if (sampleFrameData) {
+                        addLog(`Captured reference frame from camera (${(sampleFrameData.size / 1024).toFixed(1)} KB)`, 'info');
+                    } else {
+                        addLog('Camera stream not available — skipping reference frame', 'warning');
+                    }
+                }
+
+                await decartRealtime.setImage(file, sampleFrameData ? { sampleFrameData } : undefined);
 
                 elements.imageStatusText.textContent = '✅ Image sent successfully!';
                 elements.imageStatusText.style.color = '#4CAF50';

packages/sdk/src/realtime/client.ts

@@ -85,7 +85,12 @@ export type RealTimeClient = {
    * - `"file_..."` id (from `client.files.upload(...).id`): sent as a server-side reference.
    * - `null`: clear the current image.
    */
-  setImage: (image: Blob | File | string | null, options?: ImageSetOptions) => Promise<void>;
+  setImage: (
+    image: Blob | File | string | null,
+    options?: Omit<ImageSetOptions, "sampleFrameData"> & {
+      sampleFrameData?: Blob | File | string | null;
+    },
+  ) => Promise<void>;
 };
 
 export const createRealTimeClient = (opts: RealTimeClientOptions) => {
@@ -219,13 +224,23 @@ export const createRealTimeClient = (opts: RealTimeClientOptions) => {
           return subscribeToken;
         },
         getSubscribeToken: () => subscribeToken,
-        setImage: async (image: Blob | File | string | null, imgOptions?: ImageSetOptions) => {
+        setImage: async (
+          image: Blob | File | string | null,
+          imgOptions?: Omit<ImageSetOptions, "sampleFrameData"> & {
+            sampleFrameData?: Blob | File | string | null;
+          },
+        ) => {
+          const { sampleFrameData, ...rest } = imgOptions ?? {};
+          const sessionOpts: ImageSetOptions = { ...rest };
+          if (sampleFrameData !== undefined) {
+            sessionOpts.sampleFrameData = sampleFrameData === null ? null : await imageToBase64(sampleFrameData);
+          }
           if (isFileRefId(image)) {
-            return activeSession.setImage({ kind: "ref", ref: image }, imgOptions);
+            return activeSession.setImage({ kind: "ref", ref: image }, sessionOpts);
           }
-          if (image === null) return activeSession.setImage({ kind: "data", data: null }, imgOptions);
+          if (image === null) return activeSession.setImage({ kind: "data", data: null }, sessionOpts);
           const base64 = await imageToBase64(image);
-          return activeSession.setImage({ kind: "data", data: base64 }, imgOptions);
+          return activeSession.setImage({ kind: "data", data: base64 }, sessionOpts);
         },
       };
 

packages/sdk/src/realtime/methods.ts

@@ -13,6 +13,7 @@ const setInputSchema = z
      * - `"file_..."` id (from `client.files.upload(...).id`): sent as a server-side reference.
      */
     image: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(),
+    sampleFrameData: z.union([z.instanceof(Blob), z.instanceof(File), z.string(), z.null()]).optional(),
   })
   .refine((data) => data.prompt !== undefined || data.image !== undefined, {
     message: "At least one of 'prompt' or 'image' must be provided",
@@ -33,8 +34,15 @@ export const realtimeMethods = (
     const parsed = setInputSchema.safeParse(input);
     if (!parsed.success) throw parsed.error;
 
-    const { prompt, enhance, image } = parsed.data;
-    const options = { prompt, enhance, timeout: REALTIME_CONFIG.methods.updateTimeoutMs };
+    const { prompt, enhance, image, sampleFrameData } = parsed.data;
+    const options: { prompt?: string; enhance: boolean; timeout: number; sampleFrameData?: string | null } = {
+      prompt,
+      enhance,
+      timeout: REALTIME_CONFIG.methods.updateTimeoutMs,
+    };
+    if (sampleFrameData !== undefined) {
+      options.sampleFrameData = sampleFrameData === null ? null : await imageToBase64(sampleFrameData);
+    }
 
     if (isFileRefId(image)) {
       await session.setImage({ kind: "ref", ref: image }, options);

packages/sdk/src/realtime/signaling-channel.ts

@@ -172,6 +172,7 @@ export class SignalingChannel {
         : { type: "set_image", image_data: payload.data };
     if (opts.prompt !== undefined) message.prompt = opts.prompt;
     if (opts.enhance !== undefined) message.enhance_prompt = opts.enhance;
+    if (opts.sampleFrameData !== undefined) message.sample_frame_data = opts.sampleFrameData;
 
     const ack = await this.request<SetImageAckMessage>({
       message,

packages/sdk/src/realtime/types.ts

@@ -23,6 +23,7 @@ export type SetImageMessage = {
   image_ref?: string;
   prompt?: string | null;
   enhance_prompt?: boolean;
+  sample_frame_data?: string | null;
 };
 
 export type SetImagePayload = { kind: "data"; data: string | null } | { kind: "ref"; ref: string };
@@ -117,6 +118,8 @@ export type ImageSetOptions = {
   prompt?: string | null;
   enhance?: boolean;
   timeout?: number;
+  /** Optional base64-encoded sample frame (e.g. current camera frame) for prompt-enhancement context. */
+  sampleFrameData?: string | null;
 };
 
 // Incoming message types (from server)

packages/sdk/src/shared/model.ts

@@ -3,14 +3,21 @@ import { createModelNotFoundError } from "../utils/errors";
 
 const CANONICAL_MODEL_NAMES = [
   "lucy-2.1",
+  "lucy-2.5",
   "lucy-2.1-vton",
   "lucy-vton-2",
   "lucy-restyle-2",
   "lucy-clip",
   "lucy-image-2",
 ] as const;
 
-const CANONICAL_REALTIME_MODEL_NAMES = ["lucy-2.1", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2"] as const;
+const CANONICAL_REALTIME_MODEL_NAMES = [
+  "lucy-2.1",
+  "lucy-2.5",
+  "lucy-2.1-vton",
+  "lucy-vton-2",
+  "lucy-restyle-2",
+] as const;
 const CANONICAL_VIDEO_MODEL_NAMES = [
   "lucy-clip",
   "lucy-2.1",
@@ -59,6 +66,7 @@ function warnDeprecated(model: string): void {
 export const realtimeModels = z.union([
   // Canonical names
   z.literal("lucy-2.1"),
+  z.literal("lucy-2.5"),
   z.literal("lucy-2.1-vton"),
   z.literal("lucy-vton-2"),
   z.literal("lucy-restyle-2"),
@@ -255,6 +263,7 @@ export const modelInputSchemas = {
   "lucy-image-2": imageEditSchema,
   "lucy-restyle-2": restyleSchema,
   "lucy-2.1": videoEdit2Schema,
+  "lucy-2.5": videoEdit2Schema,
   "lucy-2.1-vton": videoEdit2Schema,
   "lucy-vton-2": videoEdit2Schema,
   // Latest aliases (server-side resolution)
@@ -342,6 +351,14 @@ const _models = {
       height: 624,
       inputSchema: z.object({}),
     },
+    "lucy-2.5": {
+      urlPath: "/v1/stream",
+      name: "lucy-2.5" as const,
+      fps: 20,
+      width: 1088,
+      height: 624,
+      inputSchema: z.object({}),
+    },
     "lucy-2.1-vton": {
       urlPath: "/v1/stream",
       name: "lucy-2.1-vton" as const,
@@ -609,6 +626,7 @@ export const models = {
    *
    * Available options:
    *   - `"lucy-2.1"` - Lucy 2.1 realtime video editing
+   *   - `"lucy-2.5"` - Lucy 2.5 realtime video editing
    *   - `"lucy-2.1-vton"` - Lucy 2.1 virtual try-on
    *   - `"lucy-vton-2"` - Lucy virtual try-on 2
    *   - `"lucy-restyle-2"` - Realtime video restyling

packages/sdk/tests/realtime.unit.test.ts

@@ -298,6 +298,36 @@ describe("set()", () => {
       expect.objectContaining({ timeout: REALTIME_CONFIG.methods.updateTimeoutMs }),
     );
   });
+
+  it("converts sampleFrameData Blob to base64 and forwards it on options", async () => {
+    const refBlob = new Blob(["ref-frame"], { type: "image/jpeg" });
+    mockImageToBase64.mockImplementation(async (input) => (input === refBlob ? "refbase64" : "imgbase64"));
+
+    await methods.set({ image: "raw", sampleFrameData: refBlob });
+
+    expect(mockImageToBase64).toHaveBeenCalledWith(refBlob);
+    expect(mockSession.setImage).toHaveBeenCalledWith(
+      { kind: "data", data: "imgbase64" },
+      expect.objectContaining({ sampleFrameData: "refbase64" }),
+    );
+  });
+
+  it("forwards null sampleFrameData without calling imageToBase64 on it", async () => {
+    mockImageToBase64.mockResolvedValue("imgbase64");
+    await methods.set({ image: "raw", sampleFrameData: null });
+
+    expect(mockImageToBase64).toHaveBeenCalledTimes(1);
+    expect(mockSession.setImage).toHaveBeenCalledWith(
+      { kind: "data", data: "imgbase64" },
+      expect.objectContaining({ sampleFrameData: null }),
+    );
+  });
+
+  it("omits sampleFrameData from options when not provided", async () => {
+    await methods.set({ prompt: "a cat" });
+    const call = mockSession.setImage.mock.calls[0]?.[1] as Record<string, unknown>;
+    expect("sampleFrameData" in call).toBe(false);
+  });
 });
 
 describe("Subscribe Token", () => {

packages/sdk/tests/unit.test.ts

@@ -2367,7 +2367,13 @@ describe("Canonical Model Names", () => {
         expect(canonicalModelSchema.safeParse(alias).success).toBe(false);
       }
 
-      expect(canonicalRealtimeModels.options).toEqual(["lucy-2.1", "lucy-2.1-vton", "lucy-vton-2", "lucy-restyle-2"]);
+      expect(canonicalRealtimeModels.options).toEqual([
+        "lucy-2.1",
+        "lucy-2.5",
+        "lucy-2.1-vton",
+        "lucy-vton-2",
+        "lucy-restyle-2",
+      ]);
       expect(canonicalVideoModels.options).toEqual([
         "lucy-clip",
         "lucy-2.1",
@@ -2433,7 +2439,7 @@ describe("Canonical Model Names", () => {
     it("lists all models when called without options", () => {
       const listedModels = listModels();
 
-      expect(listedModels).toHaveLength(26);
+      expect(listedModels).toHaveLength(27);
       expect(listedModels.some((model) => model.kind === "realtime" && model.name === "lucy-2.1")).toBe(true);
       expect(listedModels.some((model) => model.kind === "video" && model.name === "lucy-clip")).toBe(true);
       expect(listedModels.some((model) => model.kind === "image" && model.name === "lucy-image-2")).toBe(true);
@@ -2499,6 +2505,15 @@ describe("Canonical Model Names", () => {
       expect(model.height).toBe(624);
     });
 
+    it("lucy-2.5 canonical name works", () => {
+      const model = models.realtime("lucy-2.5");
+      expect(model.name).toBe("lucy-2.5");
+      expect(model.urlPath).toBe("/v1/stream");
+      expect(model.fps).toBe(20);
+      expect(model.width).toBe(1088);
+      expect(model.height).toBe(624);
+    });
+
     it("lucy-2.1-vton canonical name works", () => {
       const model = models.realtime("lucy-2.1-vton");
       expect(model.name).toBe("lucy-2.1-vton");