-
Notifications
You must be signed in to change notification settings - Fork 153
[Feature][api][runtime] Support auto resolve memory reference for passing data across actions. #950
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
9bde274
1c9da03
600b901
e4ffa8e
e2aa09e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,7 +21,13 @@ | |
| import com.fasterxml.jackson.annotation.JsonCreator; | ||
| import com.fasterxml.jackson.annotation.JsonIgnore; | ||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import com.fasterxml.jackson.core.JsonParser; | ||
| import com.fasterxml.jackson.databind.DeserializationContext; | ||
| import com.fasterxml.jackson.databind.JsonDeserializer; | ||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import com.fasterxml.jackson.databind.ObjectMapper; | ||
| import com.fasterxml.jackson.databind.annotation.JsonDeserialize; | ||
| import org.apache.flink.agents.api.context.MemoryRef; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
|
|
@@ -38,6 +44,11 @@ public class Event { | |
| private final String type; | ||
| private final Map<String, Object> attributes; | ||
|
|
||
| // Keep the annotation on the field as well as the creator parameter so it also applies when | ||
| // Jackson constructs Event subclasses whose creators do not declare attachments. | ||
| @JsonDeserialize(contentUsing = AttachmentValueDeserializer.class) | ||
| private final Map<String, Object> attachments; | ||
|
|
||
| /** | ||
| * Runtime-internal timestamp from the source record. Not part of the cross-language event | ||
| * contract; used by the Flink runtime for timestamp propagation. | ||
|
|
@@ -46,25 +57,36 @@ public class Event { | |
|
|
||
| /** Unified event with user-defined type and attributes. */ | ||
| public Event(String type, Map<String, Object> attributes) { | ||
| this(UUID.randomUUID(), type, attributes); | ||
| this(UUID.randomUUID(), type, attributes, new HashMap<>()); | ||
| } | ||
|
|
||
| /** Unified event with user-defined type and empty attributes. */ | ||
| public Event(String type) { | ||
| this(type, new HashMap<>()); | ||
| } | ||
|
|
||
| @JsonCreator | ||
| public Event( | ||
| @JsonProperty("id") UUID id, | ||
| @JsonProperty("type") String type, | ||
| @JsonProperty("attributes") Map<String, Object> attributes) { | ||
| this(id, type, attributes, new HashMap<>()); | ||
| } | ||
|
|
||
| @JsonCreator | ||
| public Event( | ||
| @JsonProperty("id") UUID id, | ||
| @JsonProperty("type") String type, | ||
| @JsonProperty("attributes") Map<String, Object> attributes, | ||
| @JsonProperty("attachments") | ||
| @JsonDeserialize(contentUsing = AttachmentValueDeserializer.class) | ||
| Map<String, Object> attachments) { | ||
| if (type == null || type.isEmpty()) { | ||
| throw new IllegalArgumentException("Event 'type' must not be null or empty."); | ||
| } | ||
| this.id = id; | ||
| this.type = type; | ||
| this.attributes = attributes != null ? attributes : new HashMap<>(); | ||
| this.attributes = attributes != null ? new HashMap<>(attributes) : new HashMap<>(); | ||
| this.attachments = attachments != null ? new HashMap<>(attachments) : new HashMap<>(); | ||
| } | ||
|
|
||
| public UUID getId() { | ||
|
|
@@ -81,6 +103,10 @@ public Map<String, Object> getAttributes() { | |
| return attributes; | ||
| } | ||
|
|
||
| public Map<String, Object> getAttachments() { | ||
| return attachments; | ||
| } | ||
|
|
||
| public Object getAttr(String name) { | ||
| return attributes.get(name); | ||
| } | ||
|
|
@@ -89,6 +115,14 @@ public void setAttr(String name, Object value) { | |
| attributes.put(name, value); | ||
| } | ||
|
|
||
| public Object getAttachment(String name) { | ||
| return attachments.get(name); | ||
| } | ||
|
|
||
| public void setAttachment(String name, Object value) { | ||
| attachments.put(name, value); | ||
| } | ||
|
|
||
| @JsonIgnore | ||
| public boolean hasSourceTimestamp() { | ||
| return sourceTimestamp != null; | ||
|
|
@@ -105,12 +139,17 @@ public void setSourceTimestamp(long timestamp) { | |
| } | ||
|
|
||
| /** | ||
| * Creates a base Event from another Event, copying id, type, and attributes. Subclasses | ||
| * override this to reconstruct typed event objects with proper field deserialization. | ||
| * Creates a base Event from another Event, copying id, type, attributes, and attachments. | ||
| * Subclasses override this to reconstruct typed event objects with proper field | ||
| * deserialization. | ||
| */ | ||
| public static Event fromEvent(Event event) { | ||
| Event copy = | ||
| new Event(event.getId(), event.getType(), new HashMap<>(event.getAttributes())); | ||
| new Event( | ||
| event.getId(), | ||
| event.getType(), | ||
| new HashMap<>(event.getAttributes()), | ||
| new HashMap<>(event.attachments)); | ||
| if (event.hasSourceTimestamp()) { | ||
| copy.setSourceTimestamp(event.getSourceTimestamp()); | ||
| } | ||
|
|
@@ -128,18 +167,34 @@ public static Event fromJson(String json) throws IOException { | |
| return MAPPER.readValue(json, Event.class); | ||
| } | ||
|
|
||
| /** Deserializes one attachment value, preserving explicitly tagged memory references. */ | ||
| static final class AttachmentValueDeserializer extends JsonDeserializer<Object> { | ||
|
|
||
| @Override | ||
| public Object deserialize(JsonParser parser, DeserializationContext context) | ||
| throws IOException { | ||
| JsonNode node = parser.getCodec().readTree(parser); | ||
| if (node.isObject() | ||
| && MemoryRef.TYPE_VALUE.equals(node.path(MemoryRef.TYPE_FIELD).asText())) { | ||
| return parser.getCodec().treeToValue(node, MemoryRef.class); | ||
| } | ||
| return parser.getCodec().treeToValue(node, Object.class); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public boolean equals(Object o) { | ||
| if (this == o) return true; | ||
| if (o == null || getClass() != o.getClass()) return false; | ||
| Event other = (Event) o; | ||
| return Objects.equals(this.id, other.id) | ||
| && Objects.equals(this.getType(), other.getType()) | ||
| && Objects.equals(this.attributes, other.attributes); | ||
| && Objects.equals(this.attributes, other.attributes) | ||
| && Objects.equals(this.attachments, other.attachments); | ||
| } | ||
|
|
||
| @Override | ||
| public int hashCode() { | ||
| return Objects.hash(id, getType(), attributes); | ||
| return Objects.hash(id, getType(), attributes, attachments); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Including A fan-out with durable execution on is where that shows: Adding |
||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -54,6 +54,7 @@ public OutputEvent( | |
| */ | ||
| public static OutputEvent fromEvent(Event event) { | ||
| OutputEvent result = new OutputEvent(event.getId(), new HashMap<>(event.getAttributes())); | ||
| result.getAttachments().putAll(event.getAttachments()); | ||
|
Collaborator
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Both runtimes reject an What is the intended contract for attachments on |
||
| if (event.hasSourceTimestamp()) { | ||
| result.setSourceTimestamp(event.getSourceTimestamp()); | ||
| } | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.