Skip to content

Fix exact presentation timing and harden openIMIS replay #257

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abrichr merged 2 commits into from
Jul 26, 2026
Merged

Fix exact presentation timing and harden openIMIS replay #257

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7a883e0
Fix exact presentation media timing
abrichr Jul 26, 2026
5cded0b
Harden current openIMIS Standard replay
abrichr Jul 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 24 additions & 17 deletions openadapt_flow/backends/playwright_backend.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,20 @@ class _FramePoint:
if (cursor === row) break;
cursor = parent;
}
// Retain the complete live DOM state for the bounded record boundary,
// excluding only OpenAdapt's own random lease attribute. This remains in
// the page-local guard store; it is never returned in a report. Comparing
// the final state, rather than treating every MutationObserver callback as
// irreversible, admits framework-owned hover/actionability churn only
// when the exact same Element and DOM state have been restored.
const boundary = (row || node).cloneNode(true);
for (const candidate of [boundary, ...boundary.querySelectorAll('*')]) {
for (const attr of Array.from(candidate.attributes || [])) {
if (attr.name.startsWith('data-openadapt-actuation-')) {
candidate.removeAttribute(attr.name);
}
}
}
return {
descriptor: JSON.stringify([
1,
Expand All @@ -194,6 +208,7 @@ class _FramePoint:
],
ancestry,
rowIdentity,
boundary.outerHTML,
]),
rowIdentity: rowIdentity,
row: row,
Expand Down Expand Up @@ -253,17 +268,6 @@ class _FramePoint:
tokenMap.delete(args.token);
};
entry.invalidate = invalidate;
// Any mutation in the target's record boundary after arming invalidates
// the lease. Descriptor equality is deliberately irrelevant: hidden
// attributes and pixel-identical node replacement can still change the
// action that a click performs.
entry.observer = new MutationObserver(invalidate);
entry.observer.observe(observed.row || el, {
attributes: true,
childList: true,
characterData: true,
subtree: true,
});
if (args.requireFocused) {
if (document.activeElement !== el) {
invalidate();
Expand Down Expand Up @@ -1361,12 +1365,15 @@ def act_structural(
"""Atomically verify and click the exact DOM target resolved earlier.

A strict locator must still resolve to the token-bound Element and its
unchanged target/row descriptor. A short-lived MutationObserver removes
the token on intervening identity mutation. The final action is a
Playwright click/dblclick on that unique random-token locator, preserving
Playwright's native pointer sequence and actionability checks rather
than synthesizing DOM events. A replacement element or changed record
row is a refusal, never a coordinate fallback.
unchanged complete bounded DOM descriptor. The same checks are repeated
after Playwright's actionability trial and immediately before the real
input, so transient framework presentation churn is admitted only when
the exact same Element and DOM state have been restored. The final
action is a Playwright click/dblclick on that unique random-token
locator, preserving Playwright's native pointer sequence rather than
synthesizing DOM events. A replacement element, lasting hidden
attribute change, or changed record row is a refusal, never a coordinate
fallback.
"""

fingerprint = handle.target_fingerprint
Expand Down
102 changes: 88 additions & 14 deletions scripts/export_public_demo_evidence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -355,6 +355,55 @@ def _probe_frame_presentation_times_us(
return presentation_times_us


def _probe_video_sample_end_us(
ffprobe: str,
media_path: Path,
) -> int:
"""Return the exact end of the final encoded video sample.

A frame timestamp identifies the start of a sample, not the media duration.
The public viewer must retain the complete terminal hold, so bind the
timeline to ``max(PTS + duration)`` across the encoded video packets.
"""

result = subprocess.run(
[
ffprobe,
"-v",
"error",
"-select_streams",
"v:0",
"-show_entries",
"packet=pts_time,duration_time",
"-of",
"json",
str(media_path),
],
check=True,
capture_output=True,
text=True,
)
payload = json.loads(result.stdout)
raw_packets = payload.get("packets") if isinstance(payload, dict) else None
if not isinstance(raw_packets, list) or not raw_packets:
raise EvidencePackError(f"ffprobe found no video packets in {media_path}")
sample_end_us = 0
for item in raw_packets:
pts = item.get("pts_time") if isinstance(item, dict) else None
duration = item.get("duration_time") if isinstance(item, dict) else None
if not isinstance(pts, str) or not isinstance(duration, str):
raise EvidencePackError(f"ffprobe omitted packet timing in {media_path}")
end_us = (Decimal(pts) + Decimal(duration)) * Decimal(1_000_000)
if end_us != end_us.to_integral_value():
raise EvidencePackError(
f"ffprobe returned a sub-microsecond sample end in {media_path}"
)
sample_end_us = max(sample_end_us, int(end_us))
if sample_end_us <= 0:
raise EvidencePackError(f"ffprobe returned an empty duration for {media_path}")
return sample_end_us


def _write_presentation_clip(
*,
capture: _PresentationCapture,
Expand Down Expand Up @@ -397,7 +446,7 @@ def _write_presentation_clip(

ffmpeg, ffprobe = _presentation_tools()
presentation_dir.mkdir(parents=True, exist_ok=True)
media_path = presentation_dir / f"{clip_id}.webm"
media_path = presentation_dir / f"{clip_id}.mp4"
timeline_path = presentation_dir / f"{clip_id}.control-overlay.v2.json"
pts_path = presentation_dir / f"{clip_id}.frame-pts-us.json"
with tempfile.TemporaryDirectory(
Expand Down Expand Up @@ -453,17 +502,23 @@ def _write_presentation_clip(
str(concat_path),
"-an",
"-fps_mode",
"vfr",
"passthrough",
"-enc_time_base",
"demux",
"-c:v",
"libvpx-vp9",
"-lossless",
"1",
"-deadline",
"good",
"-cpu-used",
"2",
"libx264",
"-preset",
"slow",
"-crf",
"18",
"-bf",
"0",
"-pix_fmt",
"yuv420p",
"-video_track_timescale",
"1000000",
"-movflags",
"+faststart",
str(media_path),
],
check=True,
Expand Down Expand Up @@ -493,7 +548,16 @@ def _write_presentation_clip(
)

media_sha256 = _sha256(media_path)
duration_ms = presentation_times_us[-1] // 1_000
sample_end_us = _probe_video_sample_end_us(ffprobe, media_path)
if sample_end_us <= presentation_times_us[-1]:
raise EvidencePackError(
f"{clip_id} final decoded sample has no retained duration"
)
if sample_end_us % 1_000:
raise EvidencePackError(
f"{clip_id} final decoded sample does not end on a media millisecond"
)
duration_ms = sample_end_us // 1_000
timeline = build_runtime_control_overlay_timeline_v2(
capture.frames,
data_classification=ControlOverlayDataClassification.SYNTHETIC,
Expand Down Expand Up @@ -1247,6 +1311,7 @@ def _media_type(path: Path) -> str:
".html": "text/html",
".py": "text/x-python",
".png": "image/png",
".mp4": "video/mp4",
".webm": "video/webm",
}.get(path.suffix.lower(), "application/octet-stream")

Expand All @@ -1258,7 +1323,7 @@ def _role(path: str) -> str:
return "compiled_bundle"
if "/qualification/" in path:
return "qualification"
if path.endswith(".webm"):
if path.endswith((".mp4", ".webm")):
return "media"
if "/cases/" in path:
return "case_evidence"
Expand Down Expand Up @@ -1345,7 +1410,7 @@ def _assemble_manifest(

def presentation_media(clip_id: str) -> dict[str, Any]:
return {
"media": _ref_for(root, presentation / f"{clip_id}.webm"),
"media": _ref_for(root, presentation / f"{clip_id}.mp4"),
"timeline": _ref_for(
root, presentation / f"{clip_id}.control-overlay.v2.json"
),
Expand Down Expand Up @@ -1754,7 +1819,7 @@ def _validate_presentation_artifacts(
pts_relative = str(clip["frame_pts"]["path"])
expected_prefix = f"artifacts/presentation/{clip_id}"
if (
media_relative != f"{expected_prefix}.webm"
media_relative not in {f"{expected_prefix}.mp4", f"{expected_prefix}.webm"}
or timeline_relative != f"{expected_prefix}.control-overlay.v2.json"
or pts_relative != f"{expected_prefix}.frame-pts-us.json"
):
Expand Down Expand Up @@ -1806,13 +1871,22 @@ def _validate_presentation_artifacts(
raise EvidencePackError(
f"exact-PTS sidecar does not bind decoded media: {pts_relative}"
)
expected_duration_ms = presentation_times_us[-1] // 1_000
if media_path.suffix.lower() == ".mp4":
_ffmpeg, ffprobe = _presentation_tools()
sample_end_us = _probe_video_sample_end_us(ffprobe, media_path)
if sample_end_us % 1_000:
raise EvidencePackError(
f"presentation media does not end on a millisecond: {media_relative}"
)
expected_duration_ms = sample_end_us // 1_000
if (
timeline.get("schema_version") != "openadapt.control-overlay-timeline/v2"
or timeline.get("data_classification") != "synthetic"
or timeline.get("evidence_pack_id") != pack_id
or timeline.get("media_sha256") != media_sha256
or timeline.get("media_frame_count") != len(presentation_times_us)
or timeline.get("duration_ms") != presentation_times_us[-1] // 1_000
or timeline.get("duration_ms") != expected_duration_ms
):
raise EvidencePackError(
f"control-overlay timeline does not bind media: {timeline_relative}"
Expand Down
84 changes: 7 additions & 77 deletions scripts/openimis_eligibility_demo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,9 +72,8 @@
OpenIMISFixture,
)

from openadapt_flow.backend import StructuralResolutionRefused # noqa: E402
from openadapt_flow.backends.playwright_backend import PlaywrightBackend # noqa: E402
from openadapt_flow.ir import StructuralHandle, StructuralLocator # noqa: E402
from openadapt_flow.ir import StructuralLocator # noqa: E402

HERE = Path(__file__).resolve().parent
DEPLOYMENT_YAML = (
Expand All @@ -99,7 +98,10 @@ class names/ids, and the service options in a portal outside the dialog.
the checked policyholder. This narrow adapter records stable semantic
locators and a structured identity string that includes the run's insuree
number for dialog-scoped actions. The compiler parameterizes that value,
so replay verifies the current run's policyholder before acting.
so replay verifies the current run's policyholder before acting. Structural
delivery remains inherited from :class:`PlaywrightBackend`, including its
unique-candidate refusal and one-shot mutation fingerprint between resolve
and actuation.
"""

_STRUCTURAL_JS = r"""([px, py]) => {
Expand All @@ -109,14 +111,14 @@ class names/ids, and the service options in a portal outside the dialog.
const placeholder = input ? (input.getAttribute('placeholder') || '') : '';
if (placeholder.startsWith('Insuree enquiry')) {
return {
selector: 'input[placeholder^="Insuree enquiry"]',
selector: 'input[placeholder^="Insuree enquiry"]:visible',
role: 'textbox',
name: 'Insuree enquiry'
};
}
if (placeholder.startsWith('Search Service')) {
return {
selector: 'input[placeholder^="Search Service"]',
selector: 'input[placeholder^="Search Service"]:visible',
role: 'textbox',
name: 'Search Service'
};
Expand Down Expand Up @@ -210,78 +212,6 @@ def structured_text_at(self, x: int, y: int) -> str | None:
context["insurance_no"] = dialogs[0][0]
return json.dumps(context, separators=(",", ":"))

def locate_structural(self, locator: StructuralLocator) -> StructuralHandle | None:
"""Refuse duplicate semantic candidates instead of falling to pixels.

The generic browser backend treats an ambiguous structural locator as
a miss so less specialized workflows may continue down the evidence
ladder. This reference adapter authors exact semantic locators for
every clicked control, so more than one live candidate is a record/
dialog ambiguity and must halt rather than let a visual rung pick one.
"""
if locator.selector:
candidates = self.page.locator(locator.selector)
elif locator.role and locator.name:
candidates = self.page.get_by_role(
locator.role, # type: ignore[arg-type]
name=locator.name,
exact=True,
)
else:
return None
try:
visible_indices = candidates.evaluate_all(
"""elements => elements.flatMap((el, index) => {
const style = window.getComputedStyle(el);
const rect = el.getBoundingClientRect();
const visible = style.display !== 'none'
&& style.visibility !== 'hidden'
&& style.visibility !== 'collapse'
&& rect.width > 0 && rect.height > 0
&& rect.right > 0 && rect.bottom > 0
&& rect.left < window.innerWidth
&& rect.top < window.innerHeight;
return visible ? [index] : [];
})"""
)
if not isinstance(visible_indices, list) or not all(
isinstance(index, int) for index in visible_indices
):
return None
if len(visible_indices) > 1:
raise StructuralResolutionRefused(
"openIMIS eligibility target is structurally ambiguous"
)
if not visible_indices:
return None
candidate = candidates.nth(visible_indices[0])
box = candidate.bounding_box()
if not box or box["width"] <= 0 or box["height"] <= 0:
return None
cx = int(round(box["x"] + box["width"] / 2))
cy = int(round(box["y"] + box["height"] / 2))
vw, vh = self.viewport
if not (0 <= cx < vw and 0 <= cy < vh):
return None
topmost = candidate.evaluate(
"""(el, pt) => {
const n = document.elementFromPoint(pt[0], pt[1]);
return !!n && (n === el || el.contains(n));
}""",
[cx, cy],
)
if not topmost:
return None
return StructuralHandle(point=(cx, cy), candidate_count=1)
except StructuralResolutionRefused:
raise
except Exception:
# Observation failures are a miss; the resolver may use other
# evidence, but a proven multi-candidate ambiguity above is a
# terminal refusal and never reaches a weaker rung.
return None


def eligibility_effects() -> list[Any]:
"""The eligibility outcome bound to the run's ``insurance_no``.

Expand Down
Loading