From e78604103d1e3b17c7343a21478d9ab7b3b898a7 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 13 Jul 2026 15:15:10 -0700 Subject: [PATCH 1/6] Fix sub-workflow checkpoint restore to preserve sub-workflow state Add Runner.capture_checkpoint_object/restore_from_checkpoint_object (quiescent-only nested checkpoint) and embed a sub_workflow_checkpoint in WorkflowExecutor.on_checkpoint_save/on_checkpoint_restore so a resumed parent restores each sub-workflow's mid-progress state instead of only replaying pending request-info events. Keeps a backward-compat fallback when sub_workflow_checkpoint is absent. --- .../agent_framework/_workflows/_runner.py | 67 ++++++++++++++++++ .../_workflows/_workflow_executor.py | 25 +++++-- .../core/tests/workflow/test_runner.py | 70 +++++++++++++++++++ .../core/tests/workflow/test_sub_workflow.py | 54 ++++++++++++++ 4 files changed, 209 insertions(+), 7 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_runner.py b/python/packages/core/agent_framework/_workflows/_runner.py index 543fe0d93d8..ff7c1a67b3f 100644 --- a/python/packages/core/agent_framework/_workflows/_runner.py +++ b/python/packages/core/agent_framework/_workflows/_runner.py @@ -340,6 +340,73 @@ async def restore_from_checkpoint( logger.error(f"Failed to restore from checkpoint {checkpoint_id}: {e}") raise WorkflowCheckpointException(f"Failed to restore from checkpoint {checkpoint_id}") from e + async def capture_checkpoint_object(self) -> WorkflowCheckpoint: + """Capture the current runner state as an in-memory ``WorkflowCheckpoint``. + + Builds a checkpoint from committed shared state, executor snapshots, and any + pending request_info events, without writing to a storage backend. The caller + owns the returned object - for example, a parent ``WorkflowExecutor`` embedding + a child workflow's checkpoint in its own checkpoint payload. + + This is only valid when the runner is quiescent (no in-flight executor + messages). That holds at a checkpoint boundary because a sub-workflow runs to + idle within a single parent superstep before the parent checkpoints. + + Returns: + A ``WorkflowCheckpoint`` snapshot of the current runner state. + + Raises: + WorkflowCheckpointException: If in-flight executor messages are present. + """ + if await self._ctx.has_messages(): + raise WorkflowCheckpointException( + "Cannot capture a checkpoint object while in-flight executor messages are present." + ) + + # Persist executor snapshots into committed shared state before exporting it. + await self._prepare_checkpoint_state() + pending_request_info_events = await self._ctx.get_pending_request_info_events() + return WorkflowCheckpoint( + workflow_name=self._workflow_name, + graph_signature_hash=self._graph_signature_hash, + previous_checkpoint_id=None, + messages={}, + state=self._state.export_state(), + pending_request_info_events=pending_request_info_events, + iteration_count=self._iteration, + ) + + async def restore_from_checkpoint_object(self, checkpoint: WorkflowCheckpoint) -> None: + """Restore runner state from an in-memory ``WorkflowCheckpoint`` object. + + Unlike :meth:`restore_from_checkpoint`, this does not load from a storage + backend; it applies a checkpoint the caller already holds - for example, a + child workflow checkpoint embedded in a parent ``WorkflowExecutor``'s state. + + Restores shared state, executor snapshots, in-flight messages, and pending + request_info events, then marks the runner as resumed. + + Args: + checkpoint: The checkpoint whose state should be restored. + + Raises: + WorkflowCheckpointException: If the checkpoint's graph signature does not + match this runner's workflow. + """ + if self._graph_signature_hash != checkpoint.graph_signature_hash: + raise WorkflowCheckpointException( + "Workflow graph has changed since the checkpoint was created. " + "Please rebuild the original workflow before resuming." + ) + + # Clear first so import_state (which merges) does not leak stale keys from a + # prior run on this Workflow instance. + self._state.clear() + self._state.import_state(checkpoint.state) + await self._restore_executor_states() + await self._ctx.apply_checkpoint(checkpoint) + self._mark_resumed(checkpoint) + async def _save_executor_states(self) -> None: """Populate executor state by calling checkpoint hooks on executors.""" for exec_id, executor in self._executors.items(): diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index a77887ba053..55ac7739840 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -478,6 +478,12 @@ async def on_checkpoint_save(self) -> dict[str, Any]: execution_id: execution_context for execution_id, execution_context in self._execution_contexts.items() }, "request_to_execution": dict(self._request_to_execution), + # Embed the sub-workflow's own checkpoint so its full state (shared state, + # executor snapshots, and pending request_info events) is restored faithfully + # on resume, instead of rehydrating only the pending requests. The sub-workflow + # is quiescent here: it ran to idle within this parent superstep before the + # parent checkpoints. + "sub_workflow_checkpoint": await self.workflow._runner.capture_checkpoint_object(), # pyright: ignore[reportPrivateUsage] } @override @@ -517,13 +523,18 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: self._execution_contexts = execution_contexts self._request_to_execution = request_to_execution - # Add the `request_info_event`s back to the sub workflow. - # This is only a temporary solution to rehydrate the sub workflow with the requests. - # The proper way would be to rehydrate the workflow from a checkpoint on a Workflow - # API instead of the '_runner_context' object that should be hidden. And the sub workflow - # should be rehydrated from a checkpoint object instead of from a subset of the state. - # TODO(@taochen): Issue #1614 - how to handle the case when the parent workflow has checkpointing - # set up but not the sub workflow? + # Rehydrate the sub-workflow from its embedded checkpoint so its full state is + # restored (shared state, executor snapshots, in-flight messages, and pending + # request_info events) - not just the pending requests. + sub_workflow_checkpoint = state.get("sub_workflow_checkpoint") + if sub_workflow_checkpoint is not None: + sub_workflow_checkpoint = decode_checkpoint_value(sub_workflow_checkpoint) + await self.workflow._runner.restore_from_checkpoint_object(sub_workflow_checkpoint) # pyright: ignore[reportPrivateUsage] + return + + # Backward-compatibility fallback for checkpoints saved before the sub-workflow + # checkpoint was embedded: re-add only the pending request_info events. Such + # checkpoints cannot restore the sub-workflow's deeper executor state. request_info_events = [ request_info_event for execution_context in self._execution_contexts.values() diff --git a/python/packages/core/tests/workflow/test_runner.py b/python/packages/core/tests/workflow/test_runner.py index 9c88febf8c4..f2dce2502ad 100644 --- a/python/packages/core/tests/workflow/test_runner.py +++ b/python/packages/core/tests/workflow/test_runner.py @@ -512,6 +512,76 @@ async def test_runner_reset_iteration_count(): assert runner._iteration == 0 # pyright: ignore[reportPrivateUsage] +async def test_runner_capture_and_restore_checkpoint_object_roundtrip(): + """capture_checkpoint_object() then restore_from_checkpoint_object() must roundtrip. + + Shared state and executor snapshots are captured into an in-memory ``WorkflowCheckpoint`` + and restored from it without any storage backend (the path the parent WorkflowExecutor + uses to checkpoint a nested sub-workflow). + """ + + class CounterExecutor(Executor): + def __init__(self, id: str) -> None: + super().__init__(id=id) + self.count = 0 + + @handler + async def handle(self, message: MockMessage, ctx: WorkflowContext[Any, int]) -> None: + self.count += message.data + + async def on_checkpoint_save(self) -> dict[str, Any]: + return {"count": self.count} + + async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: + self.count = int(state.get("count", 0)) + + executor = CounterExecutor(id="counter") + state = State() + ctx = InProcRunnerContext() + runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash") + + # Establish some state to capture. + executor.count = 7 + state.set("shared_key", "shared_value") + state.commit() + + checkpoint = await runner.capture_checkpoint_object() + assert checkpoint.graph_signature_hash == "test_hash" + + # Mutate after capture; restoring must roll back to the captured snapshot. + executor.count = 999 + state.set("shared_key", "mutated") + state.commit() + + await runner.restore_from_checkpoint_object(checkpoint) + + assert executor.count == 7 + assert state.get("shared_key") == "shared_value" + assert runner._resumed_from_checkpoint is True # pyright: ignore[reportPrivateUsage] + + +async def test_runner_capture_checkpoint_object_rejects_in_flight_messages(): + """capture_checkpoint_object() must reject capture while in-flight messages are present.""" + executor = MockExecutor(id="executor_a") + state = State() + ctx = InProcRunnerContext() + runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash") + + await ctx.send_message(WorkflowMessage(data=MockMessage(data=1), source_id="START")) + + with pytest.raises(WorkflowCheckpointException, match="in-flight executor messages"): + await runner.capture_checkpoint_object() + + +async def test_runner_restore_from_checkpoint_object_rejects_graph_mismatch(): + """restore_from_checkpoint_object() must reject a checkpoint from a different graph.""" + runner = Runner([], {}, State(), InProcRunnerContext(), "test_name", graph_signature_hash="hash-a") + + foreign = WorkflowCheckpoint(workflow_name="test_name", graph_signature_hash="hash-b") + with pytest.raises(WorkflowCheckpointException, match="Workflow graph has changed"): + await runner.restore_from_checkpoint_object(foreign) + + class CheckpointingContext(InProcRunnerContext): """A context that supports checkpointing for testing.""" diff --git a/python/packages/core/tests/workflow/test_sub_workflow.py b/python/packages/core/tests/workflow/test_sub_workflow.py index f5e2e8fa26f..8b865638c38 100644 --- a/python/packages/core/tests/workflow/test_sub_workflow.py +++ b/python/packages/core/tests/workflow/test_sub_workflow.py @@ -15,6 +15,7 @@ WorkflowContext, WorkflowEvent, WorkflowExecutor, + WorkflowRunState, handler, response_handler, ) @@ -619,6 +620,59 @@ async def test_sub_workflow_checkpoint_restore_no_duplicate_requests() -> None: assert request_events[0].data.prompt == "Second request" +async def test_sub_workflow_checkpoint_restore_preserves_sub_workflow_state() -> None: + """Resuming a sub-workflow mid-progress must restore its internal executor state. + + Regression guard for the issue where only the WorkflowExecutor's bookkeeping (pending + requests) was checkpointed, so a sub-workflow executor that accumulates state across + multiple request/response cycles (here ``TwoStepSubWorkflowExecutor._responses``) lost + that state on resume. With the sub-workflow's own checkpoint embedded in the parent + checkpoint, the second response now completes the two-step flow instead of triggering a + spurious third request. + """ + storage = InMemoryCheckpointStorage() + + # Step 1: run until the first request. + workflow1 = _build_checkpoint_test_workflow(storage) + first_request_id: str | None = None + async for event in workflow1.run("test_value", stream=True): + if event.type == "request_info": + first_request_id = event.request_id + assert first_request_id is not None + + # Step 2: answer the first request so the sub-workflow accumulates internal state + # (``_responses == ["first_answer"]``) and emits the second request. This mid-progress + # point is what we checkpoint and resume from - the case the no-duplicate test (which + # checkpoints at the first request, before any state accrues) does not cover. + second_request_id: str | None = None + async for event in workflow1.run(stream=True, responses={first_request_id: "first_answer"}): + if event.type == "request_info": + second_request_id = event.request_id + assert second_request_id is not None + + # Resume from the latest checkpoint (captured after the second request was made). + checkpoints = await storage.list_checkpoints(workflow_name=workflow1.name) + checkpoint_id = max(checkpoints, key=lambda cp: cp.iteration_count).checkpoint_id + + workflow2 = _build_checkpoint_test_workflow(storage) + resumed_second_request_id: str | None = None + async for event in workflow2.run(checkpoint_id=checkpoint_id, stream=True): + if event.type == "request_info": + resumed_second_request_id = event.request_id + assert resumed_second_request_id is not None + assert resumed_second_request_id == second_request_id + + # Step 3: answer the second request. With the sub-workflow's state restored, the two-step + # executor completes instead of emitting a spurious third request. If the internal state + # were lost, the second answer would be treated as a first answer and a third request + # ("Second request") would be emitted. + result = await workflow2.run(responses={resumed_second_request_id: "second_answer"}) + assert result.get_request_info_events() == [], ( + "Sub-workflow internal state was lost on resume: a spurious extra request was emitted" + ) + assert result.get_final_state() == WorkflowRunState.IDLE + + async def test_sub_workflow_intermediate_outputs_propagate_to_parent() -> None: """A child workflow's intermediate emissions must bubble up through the parent. From a486374fd8a9d6fd4ffdeaf8a77dc83419630e37 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Mon, 13 Jul 2026 15:57:37 -0700 Subject: [PATCH 2/6] Move checkpoint-object construction into the runner context Add RunnerContext.create_checkpoint_object alongside create_checkpoint (create_checkpoint now delegates to it and persists), so Runner.capture_checkpoint_object builds the snapshot via the context instead of a one-off get_messages peek primitive. In-flight messages are captured non-destructively (per-source lists copied). The checkpoint-less capturing contexts (azurefunctions, durabletask) raise NotImplementedError to match create_checkpoint. --- .../_context.py | 12 ++++ .../agent_framework/_workflows/_runner.py | 38 ++++------- .../_workflows/_runner_context.py | 65 ++++++++++++++++--- .../_workflows/runner_context.py | 11 ++++ 4 files changed, 93 insertions(+), 33 deletions(-) diff --git a/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py b/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py index 4912fe4cc92..6d76e179964 100644 --- a/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py +++ b/python/packages/azurefunctions/agent_framework_azurefunctions/_context.py @@ -115,6 +115,18 @@ async def create_checkpoint( """Checkpointing not supported in activity context.""" raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context") + async def create_checkpoint_object( + self, + workflow_name: str, + graph_signature_hash: str, + state: State, + previous_checkpoint_id: str | None, + iteration_count: int, + metadata: dict[str, Any] | None = None, + ) -> WorkflowCheckpoint: + """Checkpointing not supported in activity context.""" + raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context") + async def load_checkpoint(self, checkpoint_id: str) -> WorkflowCheckpoint | None: """Checkpointing not supported in activity context.""" raise NotImplementedError("Checkpointing is not supported in Azure Functions activity context") diff --git a/python/packages/core/agent_framework/_workflows/_runner.py b/python/packages/core/agent_framework/_workflows/_runner.py index ff7c1a67b3f..360e85099a0 100644 --- a/python/packages/core/agent_framework/_workflows/_runner.py +++ b/python/packages/core/agent_framework/_workflows/_runner.py @@ -343,37 +343,27 @@ async def restore_from_checkpoint( async def capture_checkpoint_object(self) -> WorkflowCheckpoint: """Capture the current runner state as an in-memory ``WorkflowCheckpoint``. - Builds a checkpoint from committed shared state, executor snapshots, and any - pending request_info events, without writing to a storage backend. The caller - owns the returned object - for example, a parent ``WorkflowExecutor`` embedding - a child workflow's checkpoint in its own checkpoint payload. + Builds a checkpoint from committed shared state, executor snapshots, any in-flight + messages, and any pending request_info events, without writing to a storage backend. + The caller owns the returned object - for example, a parent ``WorkflowExecutor`` + embedding a child workflow's checkpoint in its own checkpoint payload. - This is only valid when the runner is quiescent (no in-flight executor - messages). That holds at a checkpoint boundary because a sub-workflow runs to - idle within a single parent superstep before the parent checkpoints. + Like any checkpoint, the snapshot is only internally consistent when captured at a + stable point (e.g. a superstep boundary) while no iteration is concurrently mutating + the runner. This mirrors the normal per-superstep checkpoint path and makes no + assumption about which caller is taking the checkpoint. Returns: A ``WorkflowCheckpoint`` snapshot of the current runner state. - - Raises: - WorkflowCheckpointException: If in-flight executor messages are present. """ - if await self._ctx.has_messages(): - raise WorkflowCheckpointException( - "Cannot capture a checkpoint object while in-flight executor messages are present." - ) - # Persist executor snapshots into committed shared state before exporting it. await self._prepare_checkpoint_state() - pending_request_info_events = await self._ctx.get_pending_request_info_events() - return WorkflowCheckpoint( - workflow_name=self._workflow_name, - graph_signature_hash=self._graph_signature_hash, - previous_checkpoint_id=None, - messages={}, - state=self._state.export_state(), - pending_request_info_events=pending_request_info_events, - iteration_count=self._iteration, + return await self._ctx.create_checkpoint_object( + self._workflow_name, + self._graph_signature_hash, + self._state, + None, + self._iteration, ) async def restore_from_checkpoint_object(self, checkpoint: WorkflowCheckpoint) -> None: diff --git a/python/packages/core/agent_framework/_workflows/_runner_context.py b/python/packages/core/agent_framework/_workflows/_runner_context.py index 6cbee0b039c..5a11afd8b60 100644 --- a/python/packages/core/agent_framework/_workflows/_runner_context.py +++ b/python/packages/core/agent_framework/_workflows/_runner_context.py @@ -195,6 +195,34 @@ def is_streaming(self) -> bool: """ ... + async def create_checkpoint_object( + self, + workflow_name: str, + graph_signature_hash: str, + state: State, + previous_checkpoint_id: CheckpointID | None, + iteration_count: int, + metadata: dict[str, Any] | None = None, + ) -> WorkflowCheckpoint: + """Build an in-memory checkpoint of the current context state without persisting it. + + Unlike :meth:`create_checkpoint`, this does not require checkpoint storage and does + not save anything; it returns the checkpoint object for the caller to own. + + Args: + workflow_name: The name of the workflow for which the checkpoint is being created. + graph_signature_hash: Hash of the workflow graph topology to + validate checkpoint compatibility during restore. + state: The state to include in the checkpoint. + previous_checkpoint_id: The ID of the previous checkpoint, if any, to form a checkpoint chain. + iteration_count: The current iteration count of the workflow. + metadata: Optional metadata to associate with the checkpoint. + + Returns: + A ``WorkflowCheckpoint`` snapshot of the current context state. + """ + ... + async def create_checkpoint( self, workflow_name: str, @@ -204,7 +232,7 @@ async def create_checkpoint( iteration_count: int, metadata: dict[str, Any] | None = None, ) -> CheckpointID: - """Create a checkpoint of the current workflow state. + """Create and persist a checkpoint of the current workflow state. Args: workflow_name: The name of the workflow for which the checkpoint is being created. @@ -381,7 +409,7 @@ def clear_runtime_checkpoint_storage(self) -> None: def has_checkpointing(self) -> bool: return self._get_effective_checkpoint_storage() is not None - async def create_checkpoint( + async def create_checkpoint_object( self, workflow_name: str, graph_signature_hash: str, @@ -389,21 +417,40 @@ async def create_checkpoint( previous_checkpoint_id: CheckpointID | None, iteration_count: int, metadata: dict[str, Any] | None = None, - ) -> CheckpointID: - storage = self._get_effective_checkpoint_storage() - if not storage: - raise ValueError("Checkpoint storage not configured") - - checkpoint = WorkflowCheckpoint( + ) -> WorkflowCheckpoint: + return WorkflowCheckpoint( workflow_name=workflow_name, graph_signature_hash=graph_signature_hash, previous_checkpoint_id=previous_checkpoint_id, - messages=dict(self._messages), + # Copy the per-source lists so the snapshot is isolated from later context mutations. + messages={source_id: list(messages) for source_id, messages in self._messages.items()}, state=state.export_state(), pending_request_info_events=dict(self._pending_request_info_events), iteration_count=iteration_count, metadata=metadata or {}, ) + + async def create_checkpoint( + self, + workflow_name: str, + graph_signature_hash: str, + state: State, + previous_checkpoint_id: CheckpointID | None, + iteration_count: int, + metadata: dict[str, Any] | None = None, + ) -> CheckpointID: + storage = self._get_effective_checkpoint_storage() + if not storage: + raise ValueError("Checkpoint storage not configured") + + checkpoint = await self.create_checkpoint_object( + workflow_name, + graph_signature_hash, + state, + previous_checkpoint_id, + iteration_count, + metadata, + ) checkpoint_id = await storage.save(checkpoint) logger.debug(f"Created checkpoint {checkpoint_id}") return checkpoint_id diff --git a/python/packages/durabletask/agent_framework_durabletask/_workflows/runner_context.py b/python/packages/durabletask/agent_framework_durabletask/_workflows/runner_context.py index 1851339bb37..d59d10bcce3 100644 --- a/python/packages/durabletask/agent_framework_durabletask/_workflows/runner_context.py +++ b/python/packages/durabletask/agent_framework_durabletask/_workflows/runner_context.py @@ -98,6 +98,17 @@ async def create_checkpoint( ) -> str: raise NotImplementedError("Checkpointing is not supported in activity context") + async def create_checkpoint_object( + self, + workflow_name: str, + graph_signature_hash: str, + state: State, + previous_checkpoint_id: str | None, + iteration_count: int, + metadata: dict[str, Any] | None = None, + ) -> WorkflowCheckpoint: + raise NotImplementedError("Checkpointing is not supported in activity context") + async def load_checkpoint(self, checkpoint_id: str) -> WorkflowCheckpoint | None: raise NotImplementedError("Checkpointing is not supported in activity context") From 93719f4a34fcb9b1d84af7bc1b3a4226bc5947cf Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 14 Jul 2026 10:46:14 -0700 Subject: [PATCH 3/6] Remove per-execution bookkeeping from WorkflowExecutor The sub-workflow is a single shared instance, so per-execution ExecutionContext/request routing never provided real isolation. Delegate request/response tracking to the sub-workflow itself: can_handle accepts targeted propagated responses, _handle_response validates against the sub-workflow's pending requests and forwards responses immediately, and on_checkpoint_save embeds only the sub-workflow checkpoint (on_checkpoint_restore keeps a legacy reader for older checkpoints). Also emit the fresh-message/checkpoint-while-pending warning from FunctionalWorkflow.run to match Workflow.run. --- .../agent_framework/_workflows/_functional.py | 18 + .../_workflows/_workflow_executor.py | 354 ++++++------------ .../core/tests/workflow/test_sub_workflow.py | 58 +++ 3 files changed, 195 insertions(+), 235 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_functional.py b/python/packages/core/agent_framework/_workflows/_functional.py index 098700bea55..2ffe99807c0 100644 --- a/python/packages/core/agent_framework/_workflows/_functional.py +++ b/python/packages/core/agent_framework/_workflows/_functional.py @@ -811,6 +811,24 @@ def run( execution is not allowed). """ self._validate_run_params(message, responses, checkpoint_id) + # Warn (but don't block) when a fresh message or a checkpoint restore begins while a prior + # run left request_info events pending. Mirrors Workflow.run. Delivering responses is the + # normal way to complete the pending cycle and is intentionally not warned. + if (message is not None or checkpoint_id is not None) and self._last_pending_request_ids: + logger.warning( + "Workflow %s received %s while %d request_info event(s) are still pending from an " + "unfinished request/response cycle; %s. Deliver responses (responses=...) to complete " + "the pending cycle before starting new input.", + self.name, + "a fresh message" if message is not None else "a checkpoint restore", + len(self._last_pending_request_ids), + ( + "those requests remain answerable, but this run advances workflow state, so a " + "response that arrives later may apply to a workflow that has moved on" + if message is not None + else "those pending requests will be overwritten by the checkpoint's state" + ), + ) if responses and checkpoint_id is None: # Require at least one response key to match a currently-pending # request; prevents silent replay against stale state while still diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index 55ac7739840..849e7b12e0f 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -4,7 +4,6 @@ import logging import sys import types -import uuid from dataclasses import dataclass from typing import TYPE_CHECKING, Any @@ -36,7 +35,12 @@ @dataclass class ExecutionContext: - """Context for tracking a single sub-workflow execution.""" + """Legacy per-execution bookkeeping. + + Retained only to decode checkpoints written before the sub-workflow's own checkpoint was + embedded (see ``WorkflowExecutor.on_checkpoint_restore``). It is no longer used at runtime - + the wrapped sub-workflow is the single source of truth for its pending requests. + """ # The ID of the execution context execution_id: str @@ -161,11 +165,9 @@ class WorkflowExecutor(Executor): # The response handler expects a SubWorkflowResponseMessage wrapping the response data. ### State Management - WorkflowExecutor maintains execution state across request/response cycles: - - Tracks pending requests by request_id - - Accumulates responses until all expected responses are received - - Resumes sub-workflow execution with complete response batch - - Handles concurrent executions and multiple pending requests + WorkflowExecutor keeps no request/response bookkeeping of its own. The wrapped sub-workflow + is the single source of truth for its pending requests; responses are forwarded to it and + validated against its own pending request_info events. ## Type System Integration WorkflowExecutor inherits its type signature from the wrapped workflow: @@ -194,46 +196,21 @@ class WorkflowExecutor(Executor): - Converts to error event in parent context - Provides detailed error information including sub-workflow ID - ## Concurrent Execution Support - WorkflowExecutor fully supports multiple concurrent sub-workflow executions: - - ### Per-Execution State Isolation - Each sub-workflow invocation creates an isolated ExecutionContext: - - .. code-block:: python - - # Multiple concurrent invocations are supported - workflow_executor = WorkflowExecutor(my_workflow, id="concurrent_executor") - - # Each invocation gets its own execution context - # Execution 1: processes input_1 independently - # Execution 2: processes input_2 independently - # No state interference between executions - - ### Request/Response Coordination - Responses are correctly routed to the originating execution: - - Each execution tracks its own pending requests and expected responses - - Request-to-execution mapping ensures responses reach the correct sub-workflow - - Response accumulation is isolated per execution - - Automatic cleanup when execution completes - - ### Memory Management - - Unlimited concurrent executions supported - - Each execution has unique UUID-based identification - - Cleanup of completed execution contexts - - Thread-safe state management for concurrent access - - ### Important Considerations - **Shared Workflow Instance**: All concurrent executions use the same underlying workflow instance. - For proper isolation, ensure that the wrapped workflow and its executors are stateless. + ## Overlapping Executions + A ``WorkflowExecutor`` wraps a single shared sub-workflow instance and keeps no per-execution + state. If a new input arrives while the sub-workflow still has pending request_info events from + an unfinished request/response cycle, the new input advances the shared sub-workflow state and + can interfere with that cycle - a response arriving later may apply to a sub-workflow that has + moved on. This is allowed but logs a warning, and is only safe when the wrapped workflow (and + its executors) are stateless. .. code-block:: python - # Avoid: Stateful executor with instance variables + # Avoid: stateful executor whose instance variables are shared across overlapping runs class StatefulExecutor(Executor): def __init__(self): super().__init__(id="stateful") - self.data = [] # This will be shared across concurrent executions! + self.data = [] # Shared across overlapping sub-workflow executions! ## Integration with Parent Workflows Parent workflows can intercept sub-workflow requests: @@ -256,11 +233,13 @@ async def handle_subworkflow_request( await ctx.request_info(request.source_event, response_type=request.source_event.response_type) ## Implementation Notes - - Sub-workflows run to completion before processing their results - - Event processing is atomic - all outputs are forwarded before requests - - Response accumulation ensures sub-workflows receive complete response batches - - Execution state is maintained for proper resumption after external requests - - Concurrent executions are fully isolated and do not interfere with each other + - Sub-workflows run to completion (or to idle-with-pending-requests) before their results are processed + - Event processing is ordered - outputs are forwarded before requests + - Responses are forwarded to the sub-workflow as they arrive; the sub-workflow tracks its own + pending requests and resumes when they are answered + - The WorkflowExecutor keeps no per-execution bookkeeping; the sub-workflow is the single source + of truth for its pending requests. Starting a new execution while the sub-workflow still has + pending requests logs a warning and is only safe when the wrapped workflow is stateless """ def __init__( @@ -274,19 +253,18 @@ def __init__( """Initialize the WorkflowExecutor. Args: - workflow: The workflow to execute as a sub-workflow. + workflow: The workflow to execute as a sub-workflow. This workflow instance (including + the executor instances within it) must be unique. If the same instances are shared + across multiple WorkflowExecutor instances, it may lead to incorrect behavior. id: Unique identifier for this executor. - allow_direct_output: Whether to allow direct output from the sub-workflow. - By default, outputs from the sub-workflow are sent to - other executors in the parent workflow as messages. - When this is set to true, the outputs are yielded - directly from the WorkflowExecutor to the parent - workflow's event stream. - propagate_request: Whether to propagate requests from the sub-workflow to the - parent workflow. If set to true, requests from the sub-workflow - will be propagated as the original WorkflowEvent to the parent - workflow. Otherwise, they will be wrapped in a SubWorkflowRequestMessage, - which should be handled by an executor in the parent workflow. + allow_direct_output: Whether to allow direct output from the sub-workflow. By default, + outputs from the sub-workflow are sent to other executors in the parent workflow as + messages. When this is set to true, the outputs are yielded directly from the + WorkflowExecutor to the parent workflow's event stream. + propagate_request: Whether to propagate requests from the sub-workflow to the parent + workflow. If set to true, requests from the sub-workflow will be propagated as the + original WorkflowEvent to the parent workflow. Otherwise, they will be wrapped in a + SubWorkflowRequestMessage, which should be handled by an executor in the parent workflow. Keyword Args: **kwargs: Additional keyword arguments passed to the parent constructor. @@ -294,11 +272,6 @@ def __init__( super().__init__(id, **kwargs) self.workflow = workflow self.allow_direct_output = allow_direct_output - - # Track execution contexts for concurrent sub-workflow executions - self._execution_contexts: dict[str, ExecutionContext] = {} # execution_id -> ExecutionContext - # Map request_id to execution_id for response routing - self._request_to_execution: dict[str, str] = {} # request_id -> execution_id self._propagate_request = propagate_request @property @@ -351,11 +324,10 @@ def can_handle(self, message: WorkflowMessage) -> bool: # Always handle SubWorkflowResponseMessage return True - if ( - message.original_request_info_event is not None - and message.original_request_info_event.request_id in self._request_to_execution - ): - # Handle propagated responses for known requests + if message.original_request_info_event is not None: + # A propagated response is target-routed back to the executor that issued the request, + # so if one reaches this WorkflowExecutor it belongs to our sub-workflow. _handle_response + # validates it against the sub-workflow's pending requests and ignores anything unknown. return True # For other messages, only handle if the wrapped workflow can accept them as input @@ -372,58 +344,52 @@ async def process_workflow(self, input_data: object, ctx: WorkflowContext[Any, A input_data: The input data to send to the sub-workflow. ctx: The workflow context from the parent. """ - # Create execution context for this sub-workflow run - execution_id = str(uuid.uuid4()) - execution_context = ExecutionContext( - execution_id=execution_id, - collected_responses={}, - expected_response_count=0, - pending_requests={}, - ) - self._execution_contexts[execution_id] = execution_context - - logger.debug(f"WorkflowExecutor {self.id} starting sub-workflow {self.workflow.id} execution {execution_id}") - - try: - # Get kwargs from parent workflow's State to propagate to subworkflow - parent_kwargs: dict[str, Any] = ctx.get_state(WORKFLOW_RUN_KWARGS_KEY, {}) - - # Extract invocation kwargs recognised by Workflow.run() - # The state stores resolved format (with __global__ wrapper for global kwargs). - # Unwrap __global__ before passing to the subworkflow so it gets re-resolved - # against the subworkflow's own executor IDs. - fi_kwargs: dict[str, Any] | None = None - ci_kwargs: dict[str, Any] | None = None - for key in ("function_invocation_kwargs", "client_kwargs"): - resolved = parent_kwargs.get(key) - if isinstance(resolved, dict): - # Unwrap global sentinel; pass per-executor dicts as-is - unwrapped: dict[str, Any] = resolved.get(GLOBAL_KWARGS_KEY, resolved) # type: ignore - if key == "function_invocation_kwargs": - fi_kwargs = unwrapped # type: ignore - else: - ci_kwargs = unwrapped # type: ignore - - # Run the sub-workflow and collect all events, passing parent kwargs - result = await self.workflow.run( - input_data, - function_invocation_kwargs=fi_kwargs, # type: ignore - client_kwargs=ci_kwargs, # type: ignore + # The sub-workflow is a single shared instance. If it still has pending request_info events + # from an unfinished request/response cycle, a new input advances its shared state and can + # interfere with that cycle - a response arriving later may apply to a sub-workflow that has + # moved on. We allow it (the sub-workflow may be stateless) but warn so the risk is visible. + pending_requests = await self.workflow._runner_context.get_pending_request_info_events() # pyright: ignore[reportPrivateUsage] + if pending_requests: + logger.warning( + f"WorkflowExecutor {self.id} received a new input message while its sub-workflow " + f"({self.workflow.id}) still has {len(pending_requests)} pending request(s) from an " + f"unfinished request/response cycle. The sub-workflow is a single shared instance, so the " + f"new input advances shared state and can interfere with the in-flight cycle. Ensure the " + f"sub-workflow is stateless, or complete the pending cycle before sending new input." ) - logger.debug( - f"WorkflowExecutor {self.id} sub-workflow {self.workflow.id} " - f"execution {execution_id} completed with {len(result)} events" - ) + logger.debug(f"WorkflowExecutor {self.id} starting sub-workflow {self.workflow.id}") + + # Get kwargs from parent workflow's State to propagate to subworkflow + parent_kwargs: dict[str, Any] = ctx.get_state(WORKFLOW_RUN_KWARGS_KEY, {}) + + # Extract invocation kwargs recognised by Workflow.run() + # The state stores resolved format (with __global__ wrapper for global kwargs). + # Unwrap __global__ before passing to the subworkflow so it gets re-resolved + # against the subworkflow's own executor IDs. + fi_kwargs: dict[str, Any] | None = None + ci_kwargs: dict[str, Any] | None = None + for key in ("function_invocation_kwargs", "client_kwargs"): + resolved = parent_kwargs.get(key) + if isinstance(resolved, dict): + # Unwrap global sentinel; pass per-executor dicts as-is + unwrapped: dict[str, Any] = resolved.get(GLOBAL_KWARGS_KEY, resolved) # type: ignore + if key == "function_invocation_kwargs": + fi_kwargs = unwrapped # type: ignore + else: + ci_kwargs = unwrapped # type: ignore - # Process the workflow result using shared logic - await self._process_workflow_result(result, execution_context, ctx) - finally: - # Clean up execution context if it's completed (no pending requests) - if execution_id in self._execution_contexts: - exec_ctx = self._execution_contexts[execution_id] - if not exec_ctx.pending_requests: - del self._execution_contexts[execution_id] + # Run the sub-workflow and collect all events, passing parent kwargs + result = await self.workflow.run( + input_data, + function_invocation_kwargs=fi_kwargs, # type: ignore + client_kwargs=ci_kwargs, # type: ignore + ) + + logger.debug(f"WorkflowExecutor {self.id} sub-workflow {self.workflow.id} completed with {len(result)} events") + + # Process the workflow result using shared logic + await self._process_workflow_result(result, ctx) @handler async def handle_message_wrapped_request_response( @@ -433,8 +399,8 @@ async def handle_message_wrapped_request_response( ) -> None: """Handle response from parent for a forwarded request. - This handler accumulates responses and only resumes the sub-workflow - when all expected responses have been received for that execution. + Forwards the response to the sub-workflow, which resumes and validates it against its + own pending requests. Args: response: The response to a previous request. @@ -474,72 +440,38 @@ async def handle_propagated_request_response( async def on_checkpoint_save(self) -> dict[str, Any]: """Get the current state of the WorkflowExecutor for checkpointing purposes.""" return { - "execution_contexts": { - execution_id: execution_context for execution_id, execution_context in self._execution_contexts.items() - }, - "request_to_execution": dict(self._request_to_execution), - # Embed the sub-workflow's own checkpoint so its full state (shared state, - # executor snapshots, and pending request_info events) is restored faithfully - # on resume, instead of rehydrating only the pending requests. The sub-workflow - # is quiescent here: it ran to idle within this parent superstep before the - # parent checkpoints. + # The sub-workflow's own checkpoint carries everything needed to resume: shared state, + # executor snapshots, in-flight messages, and pending request_info events. The + # WorkflowExecutor keeps no separate request/response bookkeeping of its own. The + # sub-workflow is quiescent here: it ran to idle within this parent superstep before + # the parent checkpoints. "sub_workflow_checkpoint": await self.workflow._runner.capture_checkpoint_object(), # pyright: ignore[reportPrivateUsage] } @override async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: """Restore the WorkflowExecutor state from a checkpoint snapshot.""" - # Validate the state contains the right keys - if "execution_contexts" not in state: - raise KeyError("Missing 'execution_contexts' in WorkflowExecutor state.") - if "request_to_execution" not in state: - raise KeyError("Missing 'request_to_execution' in WorkflowExecutor state.") - - # Validate the execution contexts stored in the state have the right keys and values - execution_contexts: dict[str, ExecutionContext] | None = None - try: - execution_contexts = { - key: decode_checkpoint_value(value) for key, value in state["execution_contexts"].items() - } - except Exception as ex: - raise RuntimeError("Failed to deserialize execution context.") from ex - - if not all( - isinstance(key, str) and isinstance(value, ExecutionContext) for key, value in execution_contexts.items() - ): - raise ValueError("Execution contexts must have 'str' as key and 'ExecutionContext' as value.") - if not all(key == value.execution_id for key, value in execution_contexts.items()): - raise ValueError("Execution contexts must have matching keys and IDs.") - - # Validate the request_to_execution map contain the right data - request_to_execution = state["request_to_execution"] - if not all(isinstance(key, str) and isinstance(value, str) for key, value in request_to_execution.items()): - raise ValueError("Request to execution map must have 'str' as key and 'str' as value.") - if not all(value in execution_contexts for value in request_to_execution.values()): - raise ValueError( - "'request_to_execution` contains unknown execution ID that is not part of the execution contexts." - ) - - self._execution_contexts = execution_contexts - self._request_to_execution = request_to_execution - - # Rehydrate the sub-workflow from its embedded checkpoint so its full state is - # restored (shared state, executor snapshots, in-flight messages, and pending - # request_info events) - not just the pending requests. + # Preferred path: rehydrate the sub-workflow from its embedded checkpoint so its full + # state is restored (shared state, executor snapshots, in-flight messages, and pending + # request_info events). sub_workflow_checkpoint = state.get("sub_workflow_checkpoint") if sub_workflow_checkpoint is not None: sub_workflow_checkpoint = decode_checkpoint_value(sub_workflow_checkpoint) await self.workflow._runner.restore_from_checkpoint_object(sub_workflow_checkpoint) # pyright: ignore[reportPrivateUsage] return - # Backward-compatibility fallback for checkpoints saved before the sub-workflow - # checkpoint was embedded: re-add only the pending request_info events. Such - # checkpoints cannot restore the sub-workflow's deeper executor state. - request_info_events = [ - request_info_event - for execution_context in self._execution_contexts.values() - for request_info_event in execution_context.pending_requests.values() - ] + # Backward-compatibility fallback for checkpoints written before the sub-workflow checkpoint + # was embedded. Those stored per-execution bookkeeping; recover only the pending + # request_info events so the sub-workflow re-emits its pending requests. The sub-workflow's + # deeper executor/shared state cannot be restored from these older checkpoints. + legacy_execution_contexts = state.get("execution_contexts") + if not legacy_execution_contexts: + return + request_info_events: list[WorkflowEvent[Any]] = [] + for encoded_context in legacy_execution_contexts.values(): + execution_context = decode_checkpoint_value(encoded_context) + if isinstance(execution_context, ExecutionContext): + request_info_events.extend(execution_context.pending_requests.values()) await asyncio.gather(*[ self.workflow._runner_context.add_request_info_event(event) # pyright: ignore[reportPrivateUsage] for event in request_info_events @@ -548,7 +480,6 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: async def _process_workflow_result( self, result: WorkflowRunResult, - execution_context: ExecutionContext, ctx: WorkflowContext[Any], ) -> None: """Process the result from a workflow execution. @@ -558,7 +489,6 @@ async def _process_workflow_result( Args: result: The workflow execution result. - execution_context: The execution context for this sub-workflow run. ctx: The workflow context. """ # Collect all events from the workflow @@ -597,10 +527,6 @@ async def _forward_intermediate_output(output: Any) -> None: for event in request_info_events: request_id = event.request_id response_type = event.response_type - # Track the pending request in execution context - execution_context.pending_requests[request_id] = event - # Map request to execution for response routing - self._request_to_execution[request_id] = execution_context.execution_id if self._propagate_request: # In a workflow where the parent workflow does not handle the request, the request # should be propagated via the `request_info` mechanism to an external source. And @@ -611,9 +537,6 @@ async def _forward_intermediate_output(output: Any) -> None: # request and handle it directly, a message should be sent. await ctx.send_message(SubWorkflowRequestMessage(source_event=event, executor_id=self.id)) - # Update expected response count for this execution - execution_context.expected_response_count = len(request_info_events) - # Handle final state if workflow_run_state == WorkflowRunState.FAILED: # Find the failed event (type='failed'). @@ -632,26 +555,18 @@ async def _forward_intermediate_output(output: Any) -> None: await ctx.add_event(error_event) elif workflow_run_state == WorkflowRunState.IDLE: # Sub-workflow is idle - nothing more to do now - logger.debug( - f"Sub-workflow {self.workflow.id} is idle with {len(self._execution_contexts)} active executions" - ) + logger.debug(f"Sub-workflow {self.workflow.id} is idle") elif workflow_run_state == WorkflowRunState.CANCELLED: # Sub-workflow was cancelled - treat as completion - logger.debug( - f"Sub-workflow {self.workflow.id} was cancelled with {len(self._execution_contexts)} active executions" - ) + logger.debug(f"Sub-workflow {self.workflow.id} was cancelled") elif workflow_run_state == WorkflowRunState.IN_PROGRESS_PENDING_REQUESTS: # Sub-workflow is still running with pending requests logger.debug( - f"Sub-workflow {self.workflow.id} is still in progress with {len(request_info_events)} " - f"pending requests with {len(self._execution_contexts)} active executions" + f"Sub-workflow {self.workflow.id} is still in progress with {len(request_info_events)} pending requests" ) elif workflow_run_state == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS: # Sub-workflow is idle but has pending requests - logger.debug( - f"Sub-workflow {self.workflow.id} is idle with pending requests: " - f"{len(request_info_events)} with {len(self._execution_contexts)} active executions" - ) + logger.debug(f"Sub-workflow {self.workflow.id} is idle with pending requests: {len(request_info_events)}") else: raise RuntimeError(f"Unexpected workflow run state: {workflow_run_state}") @@ -661,48 +576,17 @@ async def _handle_response( response: Any, ctx: WorkflowContext[Any], ) -> None: - execution_id = self._request_to_execution.get(request_id) - if not execution_id or execution_id not in self._execution_contexts: + # The sub-workflow is the source of truth for what it is awaiting. Validate the response + # against its pending requests and ignore anything unknown or already handled. + pending_requests = await self.workflow._runner_context.get_pending_request_info_events() # pyright: ignore[reportPrivateUsage] + if request_id not in pending_requests: logger.warning( - f"WorkflowExecutor {self.id} received response for unknown request_id: {request_id}. " - "This response will be ignored." + f"WorkflowExecutor {self.id} received a response for an unknown or already-handled " + f"request_id: {request_id}. This response will be ignored." ) return - execution_context = self._execution_contexts[execution_id] - - # Check if we have this pending request in the execution context - if request_id not in execution_context.pending_requests: - logger.warning( - f"WorkflowExecutor {self.id} received response for unknown request_id: " - f"{request_id} in execution {execution_id}, ignoring" - ) - return - - # Remove the request from pending list and request mapping - execution_context.pending_requests.pop(request_id, None) - self._request_to_execution.pop(request_id, None) - - # Accumulate the response in this execution's context - execution_context.collected_responses[request_id] = response - # Check if we have all expected responses for this execution - if len(execution_context.collected_responses) < execution_context.expected_response_count: - logger.debug( - f"WorkflowExecutor {self.id} execution {execution_id} waiting for more responses: " - f"{len(execution_context.collected_responses)}/{execution_context.expected_response_count} received" - ) - return # Wait for more responses - - # Send all collected responses to the sub-workflow - responses_to_send = dict(execution_context.collected_responses) - execution_context.collected_responses.clear() # Clear for next batch - - try: - # Resume the sub-workflow with all collected responses - result = await self.workflow.run(responses=responses_to_send) - # Process the workflow result using shared logic - await self._process_workflow_result(result, execution_context, ctx) - finally: - # Clean up execution context if it's completed (no pending requests) - if not execution_context.pending_requests: - del self._execution_contexts[execution_id] + # Forward the response to the sub-workflow, which resumes and validates it against its own + # pending requests, then process whatever the sub-workflow produces. + result = await self.workflow.run(responses={request_id: response}) + await self._process_workflow_result(result, ctx) diff --git a/python/packages/core/tests/workflow/test_sub_workflow.py b/python/packages/core/tests/workflow/test_sub_workflow.py index 8b865638c38..f320146e3f5 100644 --- a/python/packages/core/tests/workflow/test_sub_workflow.py +++ b/python/packages/core/tests/workflow/test_sub_workflow.py @@ -1,9 +1,11 @@ # Copyright (c) Microsoft. All rights reserved. +import logging from dataclasses import dataclass, field from typing import Any from uuid import uuid4 +import pytest from typing_extensions import Never from agent_framework import ( @@ -466,6 +468,62 @@ async def collect_result(self, result: ValidationResult, ctx: WorkflowContext) - # (This is implicitly tested by the fact that we got correct results for all emails) +async def test_sub_workflow_warns_on_overlapping_execution(caplog: pytest.LogCaptureFixture) -> None: + """A new input while a prior sub-workflow execution is awaiting responses logs a warning. + + Overlapping executions share one sub-workflow instance and its state, so WorkflowExecutor + allows the new execution but warns that it is only safe when the wrapped workflow is stateless. + """ + + class TwoInputParent(Executor): + def __init__(self) -> None: + super().__init__(id="two_input_parent") + self._pending: dict[str, SubWorkflowRequestMessage] = {} + + @handler + async def start(self, emails: list[str], ctx: WorkflowContext[EmailValidationRequest]) -> None: + for email in emails: + await ctx.send_message(EmailValidationRequest(email=email)) + + @handler + async def handle_domain_request( + self, + sub_workflow_request: SubWorkflowRequestMessage, + ctx: WorkflowContext[SubWorkflowResponseMessage], + ) -> None: + domain_request = sub_workflow_request.source_event.data + assert isinstance(domain_request, DomainCheckRequest) + self._pending[domain_request.id] = sub_workflow_request + await ctx.request_info(domain_request, bool) + + @handler + async def collect(self, result: ValidationResult, ctx: WorkflowContext) -> None: ... + + parent = TwoInputParent() + workflow_executor = WorkflowExecutor(create_email_validation_workflow(), "email_workflow") + main_workflow = ( + WorkflowBuilder(start_executor=parent) + .add_edge(parent, workflow_executor) + .add_edge(workflow_executor, parent) + .build() + ) + + with caplog.at_level(logging.WARNING, logger="agent_framework._workflows._workflow_executor"): + result = await main_workflow.run(["a@domain1.com", "b@domain2.com"]) + + # Two inputs are delivered to the same WorkflowExecutor in one superstep: the second execution + # starts while the sub-workflow still has a pending request, producing exactly one overlap + # warning from the WorkflowExecutor. (The substring is unique to the WorkflowExecutor warning so + # it is not confused with the core Workflow.run pending-request warning.) + assert len(result.get_request_info_events()) == 2 + overlap_warnings = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "new input message while its sub-workflow" in record.getMessage() + ] + assert len(overlap_warnings) == 1 + + # region Checkpoint-related message types and executors for sub-workflow tests From 1389f304f235745b8591bb8a60c7c4ae25308bb3 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 14 Jul 2026 11:11:49 -0700 Subject: [PATCH 4/6] Drop redundant decode in WorkflowExecutor.on_checkpoint_restore The storage backend already materializes the full checkpoint on load (FileCheckpointStorage decodes recursively; InMemoryCheckpointStorage deep-copies), so the embedded sub_workflow_checkpoint (and legacy execution_contexts) arrive already decoded - like every other executor's on_checkpoint_restore state. Remove the no-op decode_checkpoint_value calls and the now-unused import. --- .../agent_framework/_workflows/_workflow_executor.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index 849e7b12e0f..8b48eece2a4 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -10,7 +10,6 @@ if TYPE_CHECKING: from ._workflow import Workflow -from ._checkpoint_encoding import decode_checkpoint_value from ._const import GLOBAL_KWARGS_KEY, WORKFLOW_RUN_KWARGS_KEY from ._events import ( WorkflowEvent, @@ -451,12 +450,12 @@ async def on_checkpoint_save(self) -> dict[str, Any]: @override async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: """Restore the WorkflowExecutor state from a checkpoint snapshot.""" - # Preferred path: rehydrate the sub-workflow from its embedded checkpoint so its full - # state is restored (shared state, executor snapshots, in-flight messages, and pending - # request_info events). + # The storage backend fully materializes the checkpoint on load (FileCheckpointStorage + # decodes recursively; InMemoryCheckpointStorage deep-copies without encoding), so nested + # executor state - including this embedded checkpoint - already arrives as a live object, + # exactly as every other executor consumes its already-decoded on_checkpoint_restore state. sub_workflow_checkpoint = state.get("sub_workflow_checkpoint") if sub_workflow_checkpoint is not None: - sub_workflow_checkpoint = decode_checkpoint_value(sub_workflow_checkpoint) await self.workflow._runner.restore_from_checkpoint_object(sub_workflow_checkpoint) # pyright: ignore[reportPrivateUsage] return @@ -468,8 +467,7 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: if not legacy_execution_contexts: return request_info_events: list[WorkflowEvent[Any]] = [] - for encoded_context in legacy_execution_contexts.values(): - execution_context = decode_checkpoint_value(encoded_context) + for execution_context in legacy_execution_contexts.values(): if isinstance(execution_context, ExecutionContext): request_info_events.extend(execution_context.pending_requests.values()) await asyncio.gather(*[ From a4f6c2699003b63c46a10fc65b764acc1c0b9c20 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 14 Jul 2026 11:28:48 -0700 Subject: [PATCH 5/6] Clean up --- .../agent_framework/_workflows/_runner.py | 6 ++- .../agent_framework/_workflows/_workflow.py | 34 +++++++++++++++-- .../_workflows/_workflow_executor.py | 5 +-- .../workflow/test_functional_workflow.py | 37 ++++++++++++++++++ .../core/tests/workflow/test_runner.py | 38 +++++++++++++++++-- .../core/tests/workflow/test_workflow.py | 33 ++++++++++++++++ 6 files changed, 139 insertions(+), 14 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_runner.py b/python/packages/core/agent_framework/_workflows/_runner.py index 360e85099a0..a444594c073 100644 --- a/python/packages/core/agent_framework/_workflows/_runner.py +++ b/python/packages/core/agent_framework/_workflows/_runner.py @@ -358,13 +358,15 @@ async def capture_checkpoint_object(self) -> WorkflowCheckpoint: """ # Persist executor snapshots into committed shared state before exporting it. await self._prepare_checkpoint_state() - return await self._ctx.create_checkpoint_object( + checkpoint = await self._ctx.create_checkpoint_object( self._workflow_name, self._graph_signature_hash, self._state, - None, + self._previous_checkpoint_id, self._iteration, ) + self._previous_checkpoint_id = checkpoint.checkpoint_id + return checkpoint async def restore_from_checkpoint_object(self, checkpoint: WorkflowCheckpoint) -> None: """Restore runner state from an in-memory ``WorkflowCheckpoint`` object. diff --git a/python/packages/core/agent_framework/_workflows/_workflow.py b/python/packages/core/agent_framework/_workflows/_workflow.py index 099e03df2ff..cef0f8b12c9 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow.py +++ b/python/packages/core/agent_framework/_workflows/_workflow.py @@ -815,10 +815,9 @@ async def _run_core( # runner context has fully drained from any prior run. If it still # has in-flight executor messages, the prior run didn't complete - # the caller must either resume from a checkpoint or wait for the - # prior run to drain. (Pending request_info events are intentionally - # NOT blocked here: a follow-up run with message=... is the normal - # way to deliver a response to those pending requests, e.g. via - # WorkflowAgent._process_pending_requests.) + # prior run to drain. Pending request_info events are intentionally + # NOT blocked here (they are answered via a follow-up ``responses=...`` + # run); the warning below surfaces the abandon/overwrite cases instead. # NOTE: _validate_run_params already enforces that ``message`` is # mutually exclusive with both ``checkpoint_id`` and ``responses``, # so we don't need to re-check those here. @@ -831,6 +830,33 @@ async def _run_core( "checkpointing; there is no in-process recovery path." ) + # Warn (but don't block) when a fresh message or a checkpoint restore begins while the + # workflow still has pending request_info events from an unfinished request/response + # cycle. A fresh ``message`` does NOT drop those pending requests - they remain pending and + # can still be answered later - but the new run advances executor and shared state, so when + # a response for an earlier request eventually arrives the workflow may have moved on, + # yielding inconsistent results. A ``checkpoint_id`` restore instead replaces the context's + # pending requests with the checkpoint's state. Delivering ``responses`` is the normal way to + # answer pending requests and is intentionally not warned. Mirrors the WorkflowExecutor + # warning for overlapping sub-workflow executions. + if message is not None or checkpoint_id is not None: + pending_request_info_events = await self._runner.context.get_pending_request_info_events() + if pending_request_info_events: + logger.warning( + "Workflow %s received %s while %d request_info event(s) are still pending from an " + "unfinished request/response cycle; %s. Deliver responses (responses=...) to complete " + "the pending cycle before starting new input.", + self.id, + "a fresh message" if message is not None else "a checkpoint restore", + len(pending_request_info_events), + ( + "those requests remain pending, but this run advances executor and shared state, " + "so a response that arrives later may apply to a workflow that has moved on" + if message is not None + else "those pending requests will be overwritten by the checkpoint's state" + ), + ) + initial_executor_fn = self._resolve_execution_mode(message, responses, checkpoint_id, checkpoint_storage) async for event in self._run_workflow_with_tracing( diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index 8b48eece2a4..f54e74f1939 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -450,10 +450,7 @@ async def on_checkpoint_save(self) -> dict[str, Any]: @override async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: """Restore the WorkflowExecutor state from a checkpoint snapshot.""" - # The storage backend fully materializes the checkpoint on load (FileCheckpointStorage - # decodes recursively; InMemoryCheckpointStorage deep-copies without encoding), so nested - # executor state - including this embedded checkpoint - already arrives as a live object, - # exactly as every other executor consumes its already-decoded on_checkpoint_restore state. + # The storage backend fully materializes the checkpoint on load, checkpointed data arrives as live objects. sub_workflow_checkpoint = state.get("sub_workflow_checkpoint") if sub_workflow_checkpoint is not None: await self.workflow._runner.restore_from_checkpoint_object(sub_workflow_checkpoint) # pyright: ignore[reportPrivateUsage] diff --git a/python/packages/core/tests/workflow/test_functional_workflow.py b/python/packages/core/tests/workflow/test_functional_workflow.py index 4ae4069061c..c4313e4f4e9 100644 --- a/python/packages/core/tests/workflow/test_functional_workflow.py +++ b/python/packages/core/tests/workflow/test_functional_workflow.py @@ -257,6 +257,43 @@ async def review_wf(doc: str, ctx: RunContext) -> str: assert outputs == ["Final: Looks great!"] assert result2.get_final_state() == WorkflowRunState.IDLE + async def test_fresh_message_while_pending_requests_warns(self, caplog: pytest.LogCaptureFixture) -> None: + """A fresh message while request_info events are pending is allowed but logs a warning.""" + + @workflow + async def review_wf(doc: str, ctx: RunContext) -> str: + feedback = await ctx.request_info({"draft": doc}, response_type=str, request_id="req1") + return f"Final: {feedback}" + + result1 = await review_wf.run("my doc") + assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS + + # Starting fresh input while a request is pending does not abandon it, but advances + # workflow state so a later response may apply to a moved-on workflow -> warn (but proceed). + with caplog.at_level(logging.WARNING): + await review_wf.run("another doc") + + assert "request_info event(s) are still pending" in caplog.text + assert "a fresh message" in caplog.text + + async def test_responses_while_pending_requests_does_not_warn(self, caplog: pytest.LogCaptureFixture) -> None: + """Delivering responses is the normal completion path and must not warn.""" + + @workflow + async def review_wf(doc: str, ctx: RunContext) -> str: + feedback = await ctx.request_info({"draft": doc}, response_type=str, request_id="req1") + return f"Final: {feedback}" + + result1 = await review_wf.run("my doc") + assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS + + caplog.clear() + with caplog.at_level(logging.WARNING): + result2 = await review_wf.run(responses={"req1": "Looks great!"}) + + assert result2.get_final_state() == WorkflowRunState.IDLE + assert "still pending" not in caplog.text + async def test_untyped_ctx_parameter(self): """ctx is injected by parameter name even without a RunContext annotation.""" diff --git a/python/packages/core/tests/workflow/test_runner.py b/python/packages/core/tests/workflow/test_runner.py index f2dce2502ad..3b706b318e9 100644 --- a/python/packages/core/tests/workflow/test_runner.py +++ b/python/packages/core/tests/workflow/test_runner.py @@ -560,8 +560,8 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: assert runner._resumed_from_checkpoint is True # pyright: ignore[reportPrivateUsage] -async def test_runner_capture_checkpoint_object_rejects_in_flight_messages(): - """capture_checkpoint_object() must reject capture while in-flight messages are present.""" +async def test_runner_capture_checkpoint_object_includes_in_flight_messages(): + """capture_checkpoint_object() must snapshot in-flight messages non-destructively.""" executor = MockExecutor(id="executor_a") state = State() ctx = InProcRunnerContext() @@ -569,8 +569,38 @@ async def test_runner_capture_checkpoint_object_rejects_in_flight_messages(): await ctx.send_message(WorkflowMessage(data=MockMessage(data=1), source_id="START")) - with pytest.raises(WorkflowCheckpointException, match="in-flight executor messages"): - await runner.capture_checkpoint_object() + checkpoint = await runner.capture_checkpoint_object() + + # The in-flight message is captured in the snapshot ... + assert list(checkpoint.messages.keys()) == ["START"] + assert len(checkpoint.messages["START"]) == 1 + # ... without draining it from the runner (capture is non-destructive). + assert await ctx.has_messages() is True + + +async def test_runner_capture_checkpoint_object_advances_previous_checkpoint_id(): + """capture_checkpoint_object() must advance _previous_checkpoint_id so a later capture chains to it.""" + executor = MockExecutor(id="executor_a") + state = State() + ctx = InProcRunnerContext() + runner = Runner([], {executor.id: executor}, state, ctx, "test_name", graph_signature_hash="test_hash") + + # Pre-condition: nothing captured yet, so there is no parent to chain back to. + assert runner._previous_checkpoint_id is None # pyright: ignore[reportPrivateUsage] + + first = await runner.capture_checkpoint_object() + + # Capturing advances the tracked checkpoint id to the newly-created checkpoint ... + assert runner._previous_checkpoint_id == first.checkpoint_id # pyright: ignore[reportPrivateUsage] + # ... and a fresh capture begins a new lineage with no parent. + assert first.previous_checkpoint_id is None + + second = await runner.capture_checkpoint_object() + + # The tracked id advances again ... + assert runner._previous_checkpoint_id == second.checkpoint_id # pyright: ignore[reportPrivateUsage] + # ... and the second checkpoint chains back to the first. + assert second.previous_checkpoint_id == first.checkpoint_id async def test_runner_restore_from_checkpoint_object_rejects_graph_mismatch(): diff --git a/python/packages/core/tests/workflow/test_workflow.py b/python/packages/core/tests/workflow/test_workflow.py index c77688c03c2..1d7087b6fed 100644 --- a/python/packages/core/tests/workflow/test_workflow.py +++ b/python/packages/core/tests/workflow/test_workflow.py @@ -2,6 +2,7 @@ import asyncio import gc +import logging import tempfile from collections.abc import AsyncIterable, Awaitable, Sequence from dataclasses import dataclass, field @@ -109,6 +110,38 @@ async def mock_handler_b( await ctx.send_message(NumberMessage(data=data)) +async def test_fresh_message_while_pending_advances_state_without_abandoning_requests( + caplog: pytest.LogCaptureFixture, +) -> None: + """A fresh message while a request is pending is allowed but hazardous. + + A fresh ``message`` does NOT abandon the pending request - it can still be answered + later - but the new run advances executor state, so a response for the earlier request + applies to a workflow that has moved on. The run is allowed and a warning is emitted. + """ + executor = MockExecutorRequestApproval(id="approver") + workflow = WorkflowBuilder(start_executor=executor).build() + + # Turn 1: request approval for data=1 -> workflow idles with a pending request. + result1 = await workflow.run(NumberMessage(data=1)) + assert result1.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS + original_request_id = result1.get_request_info_events()[0].request_id + + # Turn 2: a fresh message for data=2 while the first request is still pending. This is + # allowed but warns, and advances the executor's stored state from 1 to 2. + with caplog.at_level(logging.WARNING): + result2 = await workflow.run(NumberMessage(data=2)) + assert "request_info event(s) are still pending" in caplog.text + assert "a fresh message" in caplog.text + assert result2.get_final_state() == WorkflowRunState.IDLE_WITH_PENDING_REQUESTS + + # Turn 3: the ORIGINAL request is still answerable, proving the fresh message did not + # abandon it. But because the executor state moved on to 2, the response applies to the + # moved-on state and yields 2, not the original 1. + result3 = await workflow.run(responses={original_request_id: ApprovalMessage(approved=True)}) + assert result3.get_outputs() == [2] + + async def test_workflow_run_streaming() -> None: """Test the workflow run stream.""" executor_a = IncrementExecutor(id="executor_a") From e0b0b79d9eea40ab981ba940f30181b1b2e9dbd5 Mon Sep 17 00:00:00 2001 From: Tao Chen Date: Tue, 14 Jul 2026 14:30:46 -0700 Subject: [PATCH 6/6] Do not allow checkpoint storage in sub workflow --- .../agent_framework/_workflows/_runner.py | 46 ++++++++++--------- .../_workflows/_workflow_executor.py | 14 +++++- .../core/tests/workflow/test_runner.py | 18 ++++---- .../core/tests/workflow/test_sub_workflow.py | 24 ++++++++++ 4 files changed, 71 insertions(+), 31 deletions(-) diff --git a/python/packages/core/agent_framework/_workflows/_runner.py b/python/packages/core/agent_framework/_workflows/_runner.py index a444594c073..bf56bedfa06 100644 --- a/python/packages/core/agent_framework/_workflows/_runner.py +++ b/python/packages/core/agent_framework/_workflows/_runner.py @@ -245,7 +245,13 @@ async def _prepare_checkpoint_state(self) -> None: self._state.commit() async def create_checkpoint_if_enabled(self) -> None: - """Create a checkpoint if checkpointing is enabled and attach a label and metadata.""" + """Create a checkpoint and save the checkpoint to the configured storage if one is configured. + + Note: + 1. This method has no effect if checkpointing is not enabled in the context. + 2. Do not use this method with ``create_checkpoint_object`` as both methods advance the + ``previous_checkpoint_id`` and may result in unexpected checkpoint lineage. + """ if not self._ctx.has_checkpointing(): return @@ -340,21 +346,15 @@ async def restore_from_checkpoint( logger.error(f"Failed to restore from checkpoint {checkpoint_id}: {e}") raise WorkflowCheckpointException(f"Failed to restore from checkpoint {checkpoint_id}") from e - async def capture_checkpoint_object(self) -> WorkflowCheckpoint: - """Capture the current runner state as an in-memory ``WorkflowCheckpoint``. + async def create_checkpoint_object(self) -> WorkflowCheckpoint: + """Create a checkpoint object. - Builds a checkpoint from committed shared state, executor snapshots, any in-flight - messages, and any pending request_info events, without writing to a storage backend. - The caller owns the returned object - for example, a parent ``WorkflowExecutor`` - embedding a child workflow's checkpoint in its own checkpoint payload. - - Like any checkpoint, the snapshot is only internally consistent when captured at a - stable point (e.g. a superstep boundary) while no iteration is concurrently mutating - the runner. This mirrors the normal per-superstep checkpoint path and makes no - assumption about which caller is taking the checkpoint. + Note: + 1. Do not use this method with ``create_checkpoint_if_enabled`` as both methods advance the + ``previous_checkpoint_id`` and may result in unexpected checkpoint lineage. Returns: - A ``WorkflowCheckpoint`` snapshot of the current runner state. + A ``WorkflowCheckpoint``. """ # Persist executor snapshots into committed shared state before exporting it. await self._prepare_checkpoint_state() @@ -383,7 +383,7 @@ async def restore_from_checkpoint_object(self, checkpoint: WorkflowCheckpoint) - Raises: WorkflowCheckpointException: If the checkpoint's graph signature does not - match this runner's workflow. + match this runner's workflow, or if restoration otherwise fails. """ if self._graph_signature_hash != checkpoint.graph_signature_hash: raise WorkflowCheckpointException( @@ -391,13 +391,17 @@ async def restore_from_checkpoint_object(self, checkpoint: WorkflowCheckpoint) - "Please rebuild the original workflow before resuming." ) - # Clear first so import_state (which merges) does not leak stale keys from a - # prior run on this Workflow instance. - self._state.clear() - self._state.import_state(checkpoint.state) - await self._restore_executor_states() - await self._ctx.apply_checkpoint(checkpoint) - self._mark_resumed(checkpoint) + try: + # Clear first so import_state (which merges) does not leak stale keys from a + # prior run on this Workflow instance. + self._state.clear() + self._state.import_state(checkpoint.state) + await self._restore_executor_states() + await self._ctx.apply_checkpoint(checkpoint) + self._mark_resumed(checkpoint) + except Exception as e: + logger.error(f"Failed to restore from checkpoint {checkpoint.checkpoint_id}: {e}") + raise WorkflowCheckpointException(f"Failed to restore from checkpoint {checkpoint.checkpoint_id}") from e async def _save_executor_states(self) -> None: """Populate executor state by calling checkpoint hooks on executors.""" diff --git a/python/packages/core/agent_framework/_workflows/_workflow_executor.py b/python/packages/core/agent_framework/_workflows/_workflow_executor.py index f54e74f1939..972387c643f 100644 --- a/python/packages/core/agent_framework/_workflows/_workflow_executor.py +++ b/python/packages/core/agent_framework/_workflows/_workflow_executor.py @@ -7,6 +7,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from ..exceptions import WorkflowCheckpointException + if TYPE_CHECKING: from ._workflow import Workflow @@ -231,6 +233,10 @@ async def handle_subworkflow_request( # Forward to external handler await ctx.request_info(request.source_event, response_type=request.source_event.response_type) + ## Checkpointing + The provided sub workflow may not have its own checkpoint storage. The sub workflow checkpointed states will + be managed by the parent workflow. + ## Implementation Notes - Sub-workflows run to completion (or to idle-with-pending-requests) before their results are processed - Event processing is ordered - outputs are forwarded before requests @@ -273,6 +279,12 @@ def __init__( self.allow_direct_output = allow_direct_output self._propagate_request = propagate_request + if self.workflow._runner_context.has_checkpointing(): # type: ignore + raise WorkflowCheckpointException( + "The wrapped sub workflow must not have checkpointing enabled. " + "Sub workflow checkpointing is managed by the parent workflow." + ) + @property def input_types(self) -> list[type[Any] | types.UnionType]: """Get the input types based on the underlying workflow's input types plus WorkflowExecutor-specific types. @@ -444,7 +456,7 @@ async def on_checkpoint_save(self) -> dict[str, Any]: # WorkflowExecutor keeps no separate request/response bookkeeping of its own. The # sub-workflow is quiescent here: it ran to idle within this parent superstep before # the parent checkpoints. - "sub_workflow_checkpoint": await self.workflow._runner.capture_checkpoint_object(), # pyright: ignore[reportPrivateUsage] + "sub_workflow_checkpoint": await self.workflow._runner.create_checkpoint_object(), # pyright: ignore[reportPrivateUsage] } @override diff --git a/python/packages/core/tests/workflow/test_runner.py b/python/packages/core/tests/workflow/test_runner.py index 3b706b318e9..8133c3080f5 100644 --- a/python/packages/core/tests/workflow/test_runner.py +++ b/python/packages/core/tests/workflow/test_runner.py @@ -513,7 +513,7 @@ async def test_runner_reset_iteration_count(): async def test_runner_capture_and_restore_checkpoint_object_roundtrip(): - """capture_checkpoint_object() then restore_from_checkpoint_object() must roundtrip. + """create_checkpoint_object() then restore_from_checkpoint_object() must roundtrip. Shared state and executor snapshots are captured into an in-memory ``WorkflowCheckpoint`` and restored from it without any storage backend (the path the parent WorkflowExecutor @@ -545,7 +545,7 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: state.set("shared_key", "shared_value") state.commit() - checkpoint = await runner.capture_checkpoint_object() + checkpoint = await runner.create_checkpoint_object() assert checkpoint.graph_signature_hash == "test_hash" # Mutate after capture; restoring must roll back to the captured snapshot. @@ -560,8 +560,8 @@ async def on_checkpoint_restore(self, state: dict[str, Any]) -> None: assert runner._resumed_from_checkpoint is True # pyright: ignore[reportPrivateUsage] -async def test_runner_capture_checkpoint_object_includes_in_flight_messages(): - """capture_checkpoint_object() must snapshot in-flight messages non-destructively.""" +async def test_runner_create_checkpoint_object_includes_in_flight_messages(): + """create_checkpoint_object() must snapshot in-flight messages non-destructively.""" executor = MockExecutor(id="executor_a") state = State() ctx = InProcRunnerContext() @@ -569,7 +569,7 @@ async def test_runner_capture_checkpoint_object_includes_in_flight_messages(): await ctx.send_message(WorkflowMessage(data=MockMessage(data=1), source_id="START")) - checkpoint = await runner.capture_checkpoint_object() + checkpoint = await runner.create_checkpoint_object() # The in-flight message is captured in the snapshot ... assert list(checkpoint.messages.keys()) == ["START"] @@ -578,8 +578,8 @@ async def test_runner_capture_checkpoint_object_includes_in_flight_messages(): assert await ctx.has_messages() is True -async def test_runner_capture_checkpoint_object_advances_previous_checkpoint_id(): - """capture_checkpoint_object() must advance _previous_checkpoint_id so a later capture chains to it.""" +async def test_runner_create_checkpoint_object_advances_previous_checkpoint_id(): + """create_checkpoint_object() must advance _previous_checkpoint_id so a later capture chains to it.""" executor = MockExecutor(id="executor_a") state = State() ctx = InProcRunnerContext() @@ -588,14 +588,14 @@ async def test_runner_capture_checkpoint_object_advances_previous_checkpoint_id( # Pre-condition: nothing captured yet, so there is no parent to chain back to. assert runner._previous_checkpoint_id is None # pyright: ignore[reportPrivateUsage] - first = await runner.capture_checkpoint_object() + first = await runner.create_checkpoint_object() # Capturing advances the tracked checkpoint id to the newly-created checkpoint ... assert runner._previous_checkpoint_id == first.checkpoint_id # pyright: ignore[reportPrivateUsage] # ... and a fresh capture begins a new lineage with no parent. assert first.previous_checkpoint_id is None - second = await runner.capture_checkpoint_object() + second = await runner.create_checkpoint_object() # The tracked id advances again ... assert runner._previous_checkpoint_id == second.checkpoint_id # pyright: ignore[reportPrivateUsage] diff --git a/python/packages/core/tests/workflow/test_sub_workflow.py b/python/packages/core/tests/workflow/test_sub_workflow.py index f320146e3f5..974a1a4e91d 100644 --- a/python/packages/core/tests/workflow/test_sub_workflow.py +++ b/python/packages/core/tests/workflow/test_sub_workflow.py @@ -14,6 +14,7 @@ SubWorkflowResponseMessage, Workflow, WorkflowBuilder, + WorkflowCheckpointException, WorkflowContext, WorkflowEvent, WorkflowExecutor, @@ -630,6 +631,29 @@ def _build_checkpoint_test_workflow(storage: InMemoryCheckpointStorage) -> Workf ) +def test_workflow_executor_rejects_sub_workflow_with_checkpointing() -> None: + """WorkflowExecutor must reject a sub-workflow that already has checkpointing enabled. + + Sub-workflow checkpointing is managed by the parent workflow, so wrapping a sub-workflow + that carries its own checkpoint storage is a configuration error and must fail fast at + construction time. + """ + + class _Passthrough(Executor): + @handler + async def run(self, value: str, ctx: WorkflowContext[Any, str]) -> None: + await ctx.yield_output(value) + + storage = InMemoryCheckpointStorage() + sub_workflow = WorkflowBuilder( + start_executor=_Passthrough(id="passthrough"), + checkpoint_storage=storage, + ).build() + + with pytest.raises(WorkflowCheckpointException, match="must not have checkpointing enabled"): + WorkflowExecutor(sub_workflow, id="sub_workflow_executor") + + async def test_sub_workflow_checkpoint_restore_no_duplicate_requests() -> None: """Test that resuming a sub-workflow from checkpoint does not emit duplicate requests.