From 6eac1a7feb6572e6626336642c595a2b961877a5 Mon Sep 17 00:00:00 2001 From: Vladimir Aseev Date: Wed, 22 Jul 2026 18:08:11 +0200 Subject: [PATCH 1/2] support prompt caching and add tests --- .../ai/sdk/orchestration/CacheControl.java | 14 ++++ .../ai/sdk/orchestration/CacheablePrompt.java | 9 ++ .../ConfigToRequestTransformer.java | 82 +++++++++++++++++- .../com/sap/ai/sdk/orchestration/Message.java | 32 ++++++- .../orchestration/OrchestrationPrompt.java | 15 +++- .../orchestration/PromptCachingConfig.java | 76 +++++++++++++++++ .../ai/sdk/orchestration/SystemMessage.java | 54 +++++++++++- .../sap/ai/sdk/orchestration/TextItem.java | 49 ++++++++++- .../sap/ai/sdk/orchestration/UserMessage.java | 53 +++++++++++- .../app/controllers/OrchestrationTest.java | 84 +++++++++++++++++++ .../test/resources/fixture-for-caching.txt | 51 +++++++++++ 11 files changed, 505 insertions(+), 14 deletions(-) create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java create mode 100644 orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java create mode 100644 sample-code/spring-app/src/test/resources/fixture-for-caching.txt diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java new file mode 100644 index 000000000..e57b62eb4 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java @@ -0,0 +1,14 @@ +package com.sap.ai.sdk.orchestration; + +import lombok.Getter; + +import javax.annotation.Nonnull; + +@Getter +public final class CacheControl{ + private final String ttl; + + public CacheControl(@Nonnull String ttl) { + this.ttl = ttl; + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java new file mode 100644 index 000000000..67e7e5f60 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java @@ -0,0 +1,9 @@ +package com.sap.ai.sdk.orchestration; + +import javax.annotation.Nullable; + +public sealed interface CacheablePrompt permits TextItem { + + @Nullable + CacheControl getCacheControl(); +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java index a2dc2f326..01e3eac99 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java @@ -16,10 +16,15 @@ import com.sap.ai.sdk.orchestration.model.Template; import com.sap.ai.sdk.orchestration.model.TemplateRef; import com.sap.ai.sdk.orchestration.model.TranslationModuleConfig; +import com.sap.ai.sdk.orchestration.model.UserChatMessage; +import io.vavr.collection.HashMap; import io.vavr.control.Option; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.function.UnaryOperator; +import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -37,8 +42,9 @@ static CompletionRequestConfiguration toCompletionPostRequest( @Nonnull final OrchestrationModuleConfig config, @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { + final var cachingConfig = resolveCachingConfig(config, fallbackConfigs); final UnaryOperator copyWithImmutableTemplateConfig = - c -> c.withTemplateConfig(toTemplateModuleConfig(prompt, c.getTemplateConfig())); + c -> c.withTemplateConfig(toTemplateModuleConfig(prompt, cachingConfig, c.getTemplateConfig())); val configList = Lists.asList(config, fallbackConfigs); val configsCopy = Lists.transform(configList, copyWithImmutableTemplateConfig::apply); @@ -59,9 +65,18 @@ static CompletionRequestConfiguration toCompletionPostRequest( .messagesHistory(messageHistory); } + @Nonnull + static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( + @Nonnull final OrchestrationPrompt prompt, + @Nullable final PromptTemplatingModuleConfigPrompt config + ) { + return toTemplateModuleConfig(prompt, PromptCachingConfig.noCaching(), config); + } + @Nonnull static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( @Nonnull final OrchestrationPrompt prompt, + @Nonnull final PromptCachingConfig cachingConfig, @Nullable final PromptTemplatingModuleConfigPrompt config) { /* * Currently, we have to merge the prompt into the template configuration. @@ -78,8 +93,9 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( val messages = template.getTemplate(); val messagesWithPrompt = new ArrayList<>(messages); - messagesWithPrompt.addAll( - prompt.getMessages().stream().map(Message::createChatMessage).toList()); + var promptMessages = withCachingConstraintsApplied(prompt.getMessages(), cachingConfig); + + messagesWithPrompt.addAll(promptMessages.stream().map(Message::createChatMessage).toList()); if (messagesWithPrompt.isEmpty()) { throw new IllegalStateException( "A prompt is required. Pass at least one message or configure a template with messages or a template reference."); @@ -98,6 +114,66 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( return result; } + static PromptCachingConfig resolveCachingConfig(@Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { + + var modelName = "unknown"; + if (config.getLlmConfig() != null) { + modelName = config.getLlmConfig().getName(); + } else { + for (var fallbackConfig : fallbackConfigs) { + if (fallbackConfig.getLlmConfig() != null) { + modelName = fallbackConfig.getLlmConfig().getName(); + break; + } + } + } + return PromptCachingConfig.forModel(modelName); + } + + static List withCachingConstraintsApplied(@Nonnull final List promptMessages, + @Nonnull final PromptCachingConfig cachingConfig) { + var outputMessages = new ArrayList(promptMessages.size()); + var remainingCacheableCheckpoints = cachingConfig.getMaxCheckpointsPerRequest(); + for (int i = 0; i < promptMessages.size(); i++) { + var message = promptMessages.get(i); + var contentItems = new ArrayList(message.content().items().size()); + var messageSupportsCache = false; + if (message instanceof UserMessage || message instanceof SystemMessage) { + messageSupportsCache = true; + } + for (int j = 0; j < message.content().items().size(); j++) { + var contentItem = message.content().items().get(j); + if (contentItem instanceof TextItem textItem) { + var existingCacheControl = textItem.getCacheControl(); + if (existingCacheControl != null) { + if (remainingCacheableCheckpoints <= 0 || !messageSupportsCache) { + // skipping text control if there is already too many cache checkpoints + contentItem = new TextItem(textItem.text()); + } else if (cachingConfig.getMinTokensPerCheckpoint() > textItem.getText().chars().filter(c -> c == ' ').count() + 1) { + // to few tokens to cache, skipping to respect model limitation + contentItem = new TextItem(textItem.text()); + } else { + remainingCacheableCheckpoints--; + if (!cachingConfig.getSupportedTTLValues().matcher(existingCacheControl.getTtl()).matches()) { + contentItem = new TextItem(textItem.text(), new CacheControl(cachingConfig.getDefaultTTLValue())); + } + } + } + } + contentItems.add(contentItem); + } + if (message instanceof UserMessage) { + message = new UserMessage(new MessageContent(contentItems)); + } else if (message instanceof SystemMessage) { + message = new SystemMessage(new MessageContent(contentItems)); + } + + outputMessages.add(message); + } + return outputMessages; + } + @Nonnull static InnerModuleConfigs toModuleConfigs(@Nonnull final OrchestrationModuleConfig config) { return OrchestrationConfigModules.createInnerModuleConfigs(setupModuleConfigs(config)); diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java index ec616e281..f13a1552d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java @@ -4,6 +4,7 @@ import java.nio.file.Path; import java.util.List; import javax.annotation.Nonnull; +import javax.annotation.Nullable; /** Interface representing convenience wrappers of chat message to the orchestration service. */ public sealed interface Message permits AssistantMessage, SystemMessage, ToolMessage, UserMessage { @@ -16,7 +17,20 @@ public sealed interface Message permits AssistantMessage, SystemMessage, ToolMes */ @Nonnull static UserMessage user(@Nonnull final String message) { - return new UserMessage(message); + return user(message, null); + } + + /** + * A convenience method to create a user message from a string. + * + * @since 1.23.0 + * @param message the message content. + * @param cacheControl cache checkpoint configuration + * @return the user message. + */ + @Nonnull + static UserMessage user(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + return new UserMessage(message, cacheControl); } /** @@ -62,7 +76,21 @@ static AssistantMessage assistant(@Nonnull final String message) { */ @Nonnull static SystemMessage system(@Nonnull final String message) { - return new SystemMessage(message); + return system(message, null); + } + + /** + * A convenience method to create a system message from a string allowing to configure + * cache checkpoint + * + * @since 1.23.0 + * @param message the message content + * @param cacheControl optional cache checkpoint configuration + * @return the system message + */ + @Nonnull + static SystemMessage system(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + return new SystemMessage(message, cacheControl); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java index 01dc4d670..8dd490fbe 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java @@ -7,6 +7,8 @@ import java.util.Map; import java.util.TreeMap; import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import lombok.AccessLevel; import lombok.Getter; import lombok.Value; @@ -31,7 +33,18 @@ public class OrchestrationPrompt { * @param message A user message. */ public OrchestrationPrompt(@Nonnull final String message) { - messages.add(new UserMessage(message)); + this(message, null); + } + + /** + * Initialize a prompt with the given user message with optional cache checkpoint configuration + * + * @since 1.23.0 + * @param message A user message. + * @param cacheControl optional cache checkpoint configuration + */ + public OrchestrationPrompt(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + messages.add(new UserMessage(message, cacheControl)); } /** diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java new file mode 100644 index 000000000..2c9d325a4 --- /dev/null +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java @@ -0,0 +1,76 @@ +package com.sap.ai.sdk.orchestration; + +import lombok.Getter; + +import javax.annotation.Nonnull; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Describes supported caching properties of a model + */ +@Getter +public final class PromptCachingConfig { + + private static final PromptCachingConfig NOT_SUPPORTED = + new PromptCachingConfig(0, 0, "0m", "5m"); + + private static final Map PROMPT_CACHING_SUPPORT = Collections.unmodifiableMap(new HashMap<>(){{ + put(OrchestrationAiModel.CLAUDE_4_5_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_6_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_5_SONNET.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_6_SONNET.getName(), new PromptCachingConfig(1024, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_5_HAIKU.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m", "5m")); + put(OrchestrationAiModel.CLAUDE_4_7_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put(OrchestrationAiModel.CLAUDE_4_8_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + }}); + + + /** + * Caching checkpoint can only be made for so few prompt input tokens and not fewer + */ + private final int minTokensPerCheckpoint; + + /** + * Only up to this number of caching points can be created per request + */ + private final int maxCheckpointsPerRequest; + + /** + * Pattern of supported TTL values, which can be passed + */ + private final Pattern supportedTTLValues; + + /** + * Caching TTL value to use if TTL has not been explicitly specified + */ + private final String defaultTTLValue; + + private PromptCachingConfig( + int minTokensPerCheckpoint, + int maxCheckpointsPerRequest, + String ttlPattern, + String defaultTTLValue + ){ + this.minTokensPerCheckpoint = minTokensPerCheckpoint; + this.maxCheckpointsPerRequest = maxCheckpointsPerRequest; + this.supportedTTLValues = Pattern.compile(ttlPattern); + this.defaultTTLValue = defaultTTLValue; + } + + @Nonnull + public static PromptCachingConfig forModel(String modelName) { + if (modelName == null || modelName.isEmpty()) { + return NOT_SUPPORTED; + } + return PROMPT_CACHING_SUPPORT.getOrDefault(modelName, NOT_SUPPORTED); + } + + public static PromptCachingConfig noCaching() { + return NOT_SUPPORTED; + } + +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java index c69fffde1..c8255c5b0 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java @@ -1,14 +1,21 @@ package com.sap.ai.sdk.orchestration; import static com.sap.ai.sdk.orchestration.model.SystemChatMessage.RoleEnum.SYSTEM; +import static com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem.TypeEnum.TEXT; +import com.sap.ai.sdk.orchestration.model.CacheControl; import com.sap.ai.sdk.orchestration.model.ChatMessage; import com.sap.ai.sdk.orchestration.model.ChatMessageContent; import com.sap.ai.sdk.orchestration.model.SystemChatMessage; import com.sap.ai.sdk.orchestration.model.TextContent; import java.util.LinkedList; import java.util.List; +import java.util.function.Function; +import java.util.function.UnaryOperator; import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +import com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -36,7 +43,18 @@ public class SystemMessage implements Message { */ @Tolerate public SystemMessage(@Nonnull final String message) { - content = new MessageContent(List.of(new TextItem(message))); + this(message, null); + } + + /** + * Creates a new system message from a string, allows for cache checkpoint configuration + * + * @since 1.23.0 + * @param message the first message. + */ + public SystemMessage(@Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + content = new MessageContent(List.of(new TextItem(message, cacheControl))); } /** @@ -48,15 +66,45 @@ public SystemMessage(@Nonnull final String message) { */ @Nonnull public SystemMessage withText(@Nonnull final String message) { + return withText(message, null); + } + + /** + * Add text to the message + * + * @since 1.23.0 + * @param message the text to add + * @param cacheControl optional cache checkpoint configuration + * @return the new message + */ + @Nonnull + public SystemMessage withText(@Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { final var contentItems = new LinkedList<>(content.items()); - contentItems.add(new TextItem(message)); + contentItems.add(new TextItem(message, cacheControl)); return new SystemMessage(new MessageContent(contentItems)); } @Nonnull @Override public ChatMessage createChatMessage() { + final Function toTextContent = (item) -> { + var convertedItem = TextContent.create().type(TextContent.TypeEnum.TEXT).text(item.text()); + var cacheControl = item.getCacheControl(); + if (cacheControl != null) { + var cacheControlConverted = com.sap.ai.sdk.orchestration.model.CacheControl.create() + .type(com.sap.ai.sdk.orchestration.model.CacheControl.TypeEnum.EPHEMERAL) + .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); + convertedItem.setCacheControl(cacheControlConverted); + } + return convertedItem; + }; if (content.items().size() == 1 && content.items().get(0) instanceof TextItem textItem) { + if (textItem.getCacheControl() != null) { + return SystemChatMessage.create() + .role(SYSTEM) + .content(ChatMessageContent.createListOfTextContents(List.of(toTextContent.apply(textItem)))); + } return SystemChatMessage.create() .role(SYSTEM) .content(ChatMessageContent.create(textItem.text())); @@ -65,7 +113,7 @@ public ChatMessage createChatMessage() { content.items().stream() .filter(item -> item instanceof TextItem) .map(item -> (TextItem) item) - .map(item -> TextContent.create().type(TextContent.TypeEnum.TEXT).text(item.text())) + .map(toTextContent) .toList(); return SystemChatMessage.create() .role(SYSTEM) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java index d8119683a..8e136c0c3 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java @@ -1,11 +1,56 @@ package com.sap.ai.sdk.orchestration; +import com.sap.ai.sdk.orchestration.model.ChatCompletionTool; +import lombok.Getter; +import lombok.ToString; + import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Objects; /** * Represents a text item in a {@link MessageContent} object. * - * @param text the text of the item * @since 1.3.0 */ -public record TextItem(@Nonnull String text) implements ContentItem {} +@Getter +@ToString +public final class TextItem implements ContentItem, CacheablePrompt { + + private final String text; + private final CacheControl cacheControl; + + public TextItem(@Nonnull final String text, @Nullable final CacheControl cacheControl) { + this.text = text; + this.cacheControl = cacheControl; + } + + /** + * Compatibility constructor conforming with the previous API to avoid breaking changes + * @param text value of the item + */ + public TextItem(@Nonnull final String text) { + this(text, null); + } + + /** + * Compatibility method to support conversion from record to class without breaking changes, + * the same as {@link #getText()} + * @return text + */ + public String text() { + return text; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + TextItem textItem = (TextItem) o; + return Objects.equals(text, textItem.text) && Objects.equals(cacheControl, textItem.cacheControl); + } + + @Override + public int hashCode() { + return Objects.hash(text, cacheControl); + } +} diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java index bcc387ae1..d7eee6deb 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java @@ -5,6 +5,7 @@ import static com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem.TypeEnum.IMAGE_URL; import static com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem.TypeEnum.TEXT; +import com.sap.ai.sdk.orchestration.model.CacheControl; import com.sap.ai.sdk.orchestration.model.ChatMessage; import com.sap.ai.sdk.orchestration.model.FileContent; import com.sap.ai.sdk.orchestration.model.ImageContentUrl; @@ -18,6 +19,8 @@ import java.util.LinkedList; import java.util.List; import java.util.Locale; +import java.util.function.Function; +import java.util.function.UnaryOperator; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -49,7 +52,18 @@ public class UserMessage implements Message { */ @Tolerate public UserMessage(@Nonnull final String message) { - this(new MessageContent(List.of(new TextItem(message)))); + this(message, null); + } + + + /** + * Creates a new user message from a string with a cache checkpoint + * @param message the first message + * @param cacheControl caching checkpoint configuration + */ + public UserMessage(@Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + this(new MessageContent(List.of(new TextItem(message, cacheControl)))); } /** @@ -61,8 +75,22 @@ public UserMessage(@Nonnull final String message) { */ @Nonnull public UserMessage withText(@Nonnull final String message) { + return withText(message, null); + } + + /** + * Add text to the message with optional cache checkpoint configuration + * + * @param message the text to add. + * @param cacheControl optional cache checkpoint configuration. + * @return the new message. + * @since 1.23.0 + */ + @Nonnull + public UserMessage withText(@Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { final var contentItems = new LinkedList<>(content.items()); - contentItems.add(new TextItem(message)); + contentItems.add(new TextItem(message, cacheControl)); return new UserMessage(new MessageContent(contentItems)); } @@ -166,14 +194,33 @@ private static void warnNotPdfFile(@Nullable final String filename) { public ChatMessage createChatMessage() { final var contentList = new LinkedList(); + final Function toContentItem = (textItem) -> { + var contentItem = UserChatMessageContentItem.create().type(TEXT).text(textItem.text()); + var cacheControl = textItem.getCacheControl(); + if (cacheControl != null) { + var cacheControlConverted = CacheControl.create() + .type(CacheControl.TypeEnum.EPHEMERAL) + .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); + contentItem.setCacheControl(cacheControlConverted); + } + return contentItem; + }; + if (content.items().size() == 1 && content.items().get(0) instanceof TextItem textItem) { + if (textItem.getCacheControl() != null) { + return UserChatMessage + .create() + .content(UserChatMessageContent + .createListOfUserChatMessageContentItems(List.of(toContentItem.apply(textItem)))) + .role(USER); + } return UserChatMessage.create() .content(UserChatMessageContent.create(textItem.text())) .role(USER); } for (final ContentItem item : content.items()) { if (item instanceof TextItem textItem) { - contentList.add(UserChatMessageContentItem.create().type(TEXT).text(textItem.text())); + contentList.add(toContentItem.apply(textItem)); } else if (item instanceof ImageItem imageItem) { final var detail = imageItem.detailLevel().toString(); final var img = ImageContentUrl.create().url(imageItem.imageUrl()).detail(detail); diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java index a3c4b1f43..cf4190b8b 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java @@ -1,5 +1,6 @@ package com.sap.ai.sdk.app.controllers; +import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.CLAUDE_4_6_SONNET; import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GEMINI_2_5_FLASH; import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.GPT_5; import static com.sap.ai.sdk.orchestration.OrchestrationAiModel.Parameter.TEMPERATURE; @@ -8,12 +9,14 @@ import static java.util.Locale.ROOT; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.withinPercentage; import com.sap.ai.sdk.app.services.OrchestrationService; import com.sap.ai.sdk.app.services.OrchestrationService.Translation; import com.sap.ai.sdk.orchestration.AssistantMessage; import com.sap.ai.sdk.orchestration.AzureContentFilter; import com.sap.ai.sdk.orchestration.AzureFilterThreshold; +import com.sap.ai.sdk.orchestration.CacheControl; import com.sap.ai.sdk.orchestration.DpiMasking; import com.sap.ai.sdk.orchestration.Message; import com.sap.ai.sdk.orchestration.OrchestrationClient; @@ -21,10 +24,14 @@ import com.sap.ai.sdk.orchestration.OrchestrationFilterException; import com.sap.ai.sdk.orchestration.OrchestrationModuleConfig; import com.sap.ai.sdk.orchestration.OrchestrationPrompt; +import com.sap.ai.sdk.orchestration.SystemMessage; import com.sap.ai.sdk.orchestration.TemplateConfig; import com.sap.ai.sdk.orchestration.TextItem; +import com.sap.ai.sdk.orchestration.UserMessage; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.InputTranslationModuleResult; + +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.HttpURLConnection; @@ -44,6 +51,19 @@ @Slf4j class OrchestrationTest { + + private static final String PROMPT_FIXTURE; + private static final int PROMPT_FIXTURE_LENGTH_TOKENS = 1895; + + static { + try { + ClassLoader classLoader = OrchestrationTest.class.getClassLoader(); + File file = new File(classLoader.getResource("fixture-for-caching.txt").getFile()); + PROMPT_FIXTURE = Files.readString(file.toPath(), StandardCharsets.UTF_8); + } catch (IOException | NullPointerException e) { + throw new RuntimeException("failed to initialize test", e); + } + } private final OrchestrationClient client = new OrchestrationClient(); private final OrchestrationModuleConfig config = new OrchestrationModuleConfig().withLlmConfig(GEMINI_2_5_FLASH.withParam(TEMPERATURE, 0.0)); @@ -84,6 +104,70 @@ void testStreamChatCompletion() { assertThat(filledDeltaCount.get()).isGreaterThan(0); } + @Test + void testPromptCaching() { + val prompt = new OrchestrationPrompt( + new UserMessage(PROMPT_FIXTURE, new CacheControl("5m")), + new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") + ); + val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); + val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); + + val readFromCacheOrJustCached = Math.max( + res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), + res.getTokenUsage().getPromptTokensDetails().getCachedTokens() + ); + assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + } + + @Test + void testPromptCachingSystemMessageCanBeCached() { + val prompt = new OrchestrationPrompt( + new SystemMessage(PROMPT_FIXTURE, new CacheControl("5m")), + new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") + ); + val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); + val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); + + val readFromCacheOrJustCached = Math.max( + res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), + res.getTokenUsage().getPromptTokensDetails().getCachedTokens() + ); + assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + } + + @Test + void testPromptCachingIgnoredForShortInput() { + val prompt = new OrchestrationPrompt( + new UserMessage("Recent developments in production automation and robotics are spectacular", new CacheControl("5m")), + new UserMessage("Will we see advanced home robots like cooking humanoids in foreseeable future?") + ); + val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); + val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); + + val readFromCacheOrJustCached = Math.max( + res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), + res.getTokenUsage().getPromptTokensDetails().getCachedTokens() + ); + assertThat(readFromCacheOrJustCached).isEqualTo(0); + } + + @Test + void testPromptCachingInvalidTTLGetsResetToDefault() { + val prompt = new OrchestrationPrompt( + new UserMessage(PROMPT_FIXTURE, new CacheControl("0s")), + new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") + ); + val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); + val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); + + val readFromCacheOrJustCached = Math.max( + res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), + res.getTokenUsage().getPromptTokensDetails().getCachedTokens() + ); + assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + } + @Test void testTemplate() { assertThat(service.getConfig().getLlmConfig()).isNotNull(); diff --git a/sample-code/spring-app/src/test/resources/fixture-for-caching.txt b/sample-code/spring-app/src/test/resources/fixture-for-caching.txt new file mode 100644 index 000000000..59711bc92 --- /dev/null +++ b/sample-code/spring-app/src/test/resources/fixture-for-caching.txt @@ -0,0 +1,51 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and +You must cause any modified files to carry prominent notices stating that You changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS \ No newline at end of file From 3a4e81a6123d2d90c05d68628b7ae508a146e128 Mon Sep 17 00:00:00 2001 From: SAP Cloud SDK Bot Date: Wed, 22 Jul 2026 16:10:05 +0000 Subject: [PATCH 2/2] Formatting --- .../ai/sdk/orchestration/CacheControl.java | 13 +- .../ai/sdk/orchestration/CacheablePrompt.java | 4 +- .../ConfigToRequestTransformer.java | 36 +++--- .../com/sap/ai/sdk/orchestration/Message.java | 10 +- .../orchestration/OrchestrationPrompt.java | 4 +- .../orchestration/PromptCachingConfig.java | 118 +++++++++--------- .../ai/sdk/orchestration/SystemMessage.java | 45 +++---- .../sap/ai/sdk/orchestration/TextItem.java | 83 ++++++------ .../sap/ai/sdk/orchestration/UserMessage.java | 47 +++---- .../app/controllers/OrchestrationTest.java | 67 +++++----- 10 files changed, 227 insertions(+), 200 deletions(-) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java index e57b62eb4..9ea6ad3cc 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheControl.java @@ -1,14 +1,13 @@ package com.sap.ai.sdk.orchestration; -import lombok.Getter; - import javax.annotation.Nonnull; +import lombok.Getter; @Getter -public final class CacheControl{ - private final String ttl; +public final class CacheControl { + private final String ttl; - public CacheControl(@Nonnull String ttl) { - this.ttl = ttl; - } + public CacheControl(@Nonnull String ttl) { + this.ttl = ttl; + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java index 67e7e5f60..f575c2369 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/CacheablePrompt.java @@ -4,6 +4,6 @@ public sealed interface CacheablePrompt permits TextItem { - @Nullable - CacheControl getCacheControl(); + @Nullable + CacheControl getCacheControl(); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java index 01e3eac99..f01b54b93 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/ConfigToRequestTransformer.java @@ -16,15 +16,10 @@ import com.sap.ai.sdk.orchestration.model.Template; import com.sap.ai.sdk.orchestration.model.TemplateRef; import com.sap.ai.sdk.orchestration.model.TranslationModuleConfig; -import com.sap.ai.sdk.orchestration.model.UserChatMessage; -import io.vavr.collection.HashMap; import io.vavr.control.Option; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.function.UnaryOperator; -import java.util.regex.Pattern; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -44,7 +39,9 @@ static CompletionRequestConfiguration toCompletionPostRequest( final var cachingConfig = resolveCachingConfig(config, fallbackConfigs); final UnaryOperator copyWithImmutableTemplateConfig = - c -> c.withTemplateConfig(toTemplateModuleConfig(prompt, cachingConfig, c.getTemplateConfig())); + c -> + c.withTemplateConfig( + toTemplateModuleConfig(prompt, cachingConfig, c.getTemplateConfig())); val configList = Lists.asList(config, fallbackConfigs); val configsCopy = Lists.transform(configList, copyWithImmutableTemplateConfig::apply); @@ -67,9 +64,8 @@ static CompletionRequestConfiguration toCompletionPostRequest( @Nonnull static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( - @Nonnull final OrchestrationPrompt prompt, - @Nullable final PromptTemplatingModuleConfigPrompt config - ) { + @Nonnull final OrchestrationPrompt prompt, + @Nullable final PromptTemplatingModuleConfigPrompt config) { return toTemplateModuleConfig(prompt, PromptCachingConfig.noCaching(), config); } @@ -114,8 +110,9 @@ static PromptTemplatingModuleConfigPrompt toTemplateModuleConfig( return result; } - static PromptCachingConfig resolveCachingConfig(@Nonnull final OrchestrationModuleConfig config, - @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { + static PromptCachingConfig resolveCachingConfig( + @Nonnull final OrchestrationModuleConfig config, + @Nonnull final OrchestrationModuleConfig... fallbackConfigs) { var modelName = "unknown"; if (config.getLlmConfig() != null) { @@ -131,8 +128,9 @@ static PromptCachingConfig resolveCachingConfig(@Nonnull final OrchestrationModu return PromptCachingConfig.forModel(modelName); } - static List withCachingConstraintsApplied(@Nonnull final List promptMessages, - @Nonnull final PromptCachingConfig cachingConfig) { + static List withCachingConstraintsApplied( + @Nonnull final List promptMessages, + @Nonnull final PromptCachingConfig cachingConfig) { var outputMessages = new ArrayList(promptMessages.size()); var remainingCacheableCheckpoints = cachingConfig.getMaxCheckpointsPerRequest(); for (int i = 0; i < promptMessages.size(); i++) { @@ -150,13 +148,19 @@ static List withCachingConstraintsApplied(@Nonnull final List if (remainingCacheableCheckpoints <= 0 || !messageSupportsCache) { // skipping text control if there is already too many cache checkpoints contentItem = new TextItem(textItem.text()); - } else if (cachingConfig.getMinTokensPerCheckpoint() > textItem.getText().chars().filter(c -> c == ' ').count() + 1) { + } else if (cachingConfig.getMinTokensPerCheckpoint() + > textItem.getText().chars().filter(c -> c == ' ').count() + 1) { // to few tokens to cache, skipping to respect model limitation contentItem = new TextItem(textItem.text()); } else { remainingCacheableCheckpoints--; - if (!cachingConfig.getSupportedTTLValues().matcher(existingCacheControl.getTtl()).matches()) { - contentItem = new TextItem(textItem.text(), new CacheControl(cachingConfig.getDefaultTTLValue())); + if (!cachingConfig + .getSupportedTTLValues() + .matcher(existingCacheControl.getTtl()) + .matches()) { + contentItem = + new TextItem( + textItem.text(), new CacheControl(cachingConfig.getDefaultTTLValue())); } } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java index f13a1552d..89e667c6c 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/Message.java @@ -29,7 +29,8 @@ static UserMessage user(@Nonnull final String message) { * @return the user message. */ @Nonnull - static UserMessage user(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + static UserMessage user( + @Nonnull final String message, @Nullable final CacheControl cacheControl) { return new UserMessage(message, cacheControl); } @@ -80,8 +81,8 @@ static SystemMessage system(@Nonnull final String message) { } /** - * A convenience method to create a system message from a string allowing to configure - * cache checkpoint + * A convenience method to create a system message from a string allowing to configure cache + * checkpoint * * @since 1.23.0 * @param message the message content @@ -89,7 +90,8 @@ static SystemMessage system(@Nonnull final String message) { * @return the system message */ @Nonnull - static SystemMessage system(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + static SystemMessage system( + @Nonnull final String message, @Nullable final CacheControl cacheControl) { return new SystemMessage(message, cacheControl); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java index 8dd490fbe..4221c9ad8 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/OrchestrationPrompt.java @@ -8,7 +8,6 @@ import java.util.TreeMap; import javax.annotation.Nonnull; import javax.annotation.Nullable; - import lombok.AccessLevel; import lombok.Getter; import lombok.Value; @@ -43,7 +42,8 @@ public OrchestrationPrompt(@Nonnull final String message) { * @param message A user message. * @param cacheControl optional cache checkpoint configuration */ - public OrchestrationPrompt(@Nonnull final String message, @Nullable final CacheControl cacheControl) { + public OrchestrationPrompt( + @Nonnull final String message, @Nullable final CacheControl cacheControl) { messages.add(new UserMessage(message, cacheControl)); } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java index 2c9d325a4..d3948d44d 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/PromptCachingConfig.java @@ -1,76 +1,82 @@ package com.sap.ai.sdk.orchestration; -import lombok.Getter; - -import javax.annotation.Nonnull; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +import javax.annotation.Nonnull; +import lombok.Getter; -/** - * Describes supported caching properties of a model - */ +/** Describes supported caching properties of a model */ @Getter public final class PromptCachingConfig { - private static final PromptCachingConfig NOT_SUPPORTED = - new PromptCachingConfig(0, 0, "0m", "5m"); + private static final PromptCachingConfig NOT_SUPPORTED = + new PromptCachingConfig(0, 0, "0m", "5m"); - private static final Map PROMPT_CACHING_SUPPORT = Collections.unmodifiableMap(new HashMap<>(){{ - put(OrchestrationAiModel.CLAUDE_4_5_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_6_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_5_SONNET.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_6_SONNET.getName(), new PromptCachingConfig(1024, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_5_HAIKU.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m", "5m")); - put(OrchestrationAiModel.CLAUDE_4_7_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - put(OrchestrationAiModel.CLAUDE_4_8_OPUS.getName(), new PromptCachingConfig(4096, 4, "5m|1h", "5m")); - }}); + private static final Map PROMPT_CACHING_SUPPORT = + Collections.unmodifiableMap( + new HashMap<>() { + { + put( + OrchestrationAiModel.CLAUDE_4_5_OPUS.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_6_OPUS.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_5_SONNET.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_6_SONNET.getName(), + new PromptCachingConfig(1024, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_5_HAIKU.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_OPUS.getName(), + new PromptCachingConfig(4096, 4, "5m", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_7_OPUS.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + put( + OrchestrationAiModel.CLAUDE_4_8_OPUS.getName(), + new PromptCachingConfig(4096, 4, "5m|1h", "5m")); + } + }); + /** Caching checkpoint can only be made for so few prompt input tokens and not fewer */ + private final int minTokensPerCheckpoint; - /** - * Caching checkpoint can only be made for so few prompt input tokens and not fewer - */ - private final int minTokensPerCheckpoint; + /** Only up to this number of caching points can be created per request */ + private final int maxCheckpointsPerRequest; - /** - * Only up to this number of caching points can be created per request - */ - private final int maxCheckpointsPerRequest; + /** Pattern of supported TTL values, which can be passed */ + private final Pattern supportedTTLValues; - /** - * Pattern of supported TTL values, which can be passed - */ - private final Pattern supportedTTLValues; + /** Caching TTL value to use if TTL has not been explicitly specified */ + private final String defaultTTLValue; - /** - * Caching TTL value to use if TTL has not been explicitly specified - */ - private final String defaultTTLValue; - - private PromptCachingConfig( - int minTokensPerCheckpoint, - int maxCheckpointsPerRequest, - String ttlPattern, - String defaultTTLValue - ){ - this.minTokensPerCheckpoint = minTokensPerCheckpoint; - this.maxCheckpointsPerRequest = maxCheckpointsPerRequest; - this.supportedTTLValues = Pattern.compile(ttlPattern); - this.defaultTTLValue = defaultTTLValue; - } - - @Nonnull - public static PromptCachingConfig forModel(String modelName) { - if (modelName == null || modelName.isEmpty()) { - return NOT_SUPPORTED; - } - return PROMPT_CACHING_SUPPORT.getOrDefault(modelName, NOT_SUPPORTED); - } + private PromptCachingConfig( + int minTokensPerCheckpoint, + int maxCheckpointsPerRequest, + String ttlPattern, + String defaultTTLValue) { + this.minTokensPerCheckpoint = minTokensPerCheckpoint; + this.maxCheckpointsPerRequest = maxCheckpointsPerRequest; + this.supportedTTLValues = Pattern.compile(ttlPattern); + this.defaultTTLValue = defaultTTLValue; + } - public static PromptCachingConfig noCaching() { - return NOT_SUPPORTED; + @Nonnull + public static PromptCachingConfig forModel(String modelName) { + if (modelName == null || modelName.isEmpty()) { + return NOT_SUPPORTED; } + return PROMPT_CACHING_SUPPORT.getOrDefault(modelName, NOT_SUPPORTED); + } + public static PromptCachingConfig noCaching() { + return NOT_SUPPORTED; + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java index c8255c5b0..1265522a0 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/SystemMessage.java @@ -1,7 +1,6 @@ package com.sap.ai.sdk.orchestration; import static com.sap.ai.sdk.orchestration.model.SystemChatMessage.RoleEnum.SYSTEM; -import static com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem.TypeEnum.TEXT; import com.sap.ai.sdk.orchestration.model.CacheControl; import com.sap.ai.sdk.orchestration.model.ChatMessage; @@ -11,11 +10,8 @@ import java.util.LinkedList; import java.util.List; import java.util.function.Function; -import java.util.function.UnaryOperator; import javax.annotation.Nonnull; import javax.annotation.Nullable; - -import com.sap.ai.sdk.orchestration.model.UserChatMessageContentItem; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -52,8 +48,9 @@ public SystemMessage(@Nonnull final String message) { * @since 1.23.0 * @param message the first message. */ - public SystemMessage(@Nonnull final String message, - @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + public SystemMessage( + @Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { content = new MessageContent(List.of(new TextItem(message, cacheControl))); } @@ -78,8 +75,9 @@ public SystemMessage withText(@Nonnull final String message) { * @return the new message */ @Nonnull - public SystemMessage withText(@Nonnull final String message, - @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + public SystemMessage withText( + @Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { final var contentItems = new LinkedList<>(content.items()); contentItems.add(new TextItem(message, cacheControl)); return new SystemMessage(new MessageContent(contentItems)); @@ -88,22 +86,27 @@ public SystemMessage withText(@Nonnull final String message, @Nonnull @Override public ChatMessage createChatMessage() { - final Function toTextContent = (item) -> { - var convertedItem = TextContent.create().type(TextContent.TypeEnum.TEXT).text(item.text()); - var cacheControl = item.getCacheControl(); - if (cacheControl != null) { - var cacheControlConverted = com.sap.ai.sdk.orchestration.model.CacheControl.create() - .type(com.sap.ai.sdk.orchestration.model.CacheControl.TypeEnum.EPHEMERAL) - .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); - convertedItem.setCacheControl(cacheControlConverted); - } - return convertedItem; - }; + final Function toTextContent = + (item) -> { + var convertedItem = + TextContent.create().type(TextContent.TypeEnum.TEXT).text(item.text()); + var cacheControl = item.getCacheControl(); + if (cacheControl != null) { + var cacheControlConverted = + com.sap.ai.sdk.orchestration.model.CacheControl.create() + .type(com.sap.ai.sdk.orchestration.model.CacheControl.TypeEnum.EPHEMERAL) + .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); + convertedItem.setCacheControl(cacheControlConverted); + } + return convertedItem; + }; if (content.items().size() == 1 && content.items().get(0) instanceof TextItem textItem) { if (textItem.getCacheControl() != null) { return SystemChatMessage.create() - .role(SYSTEM) - .content(ChatMessageContent.createListOfTextContents(List.of(toTextContent.apply(textItem)))); + .role(SYSTEM) + .content( + ChatMessageContent.createListOfTextContents( + List.of(toTextContent.apply(textItem)))); } return SystemChatMessage.create() .role(SYSTEM) diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java index 8e136c0c3..62903ee16 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/TextItem.java @@ -1,12 +1,10 @@ package com.sap.ai.sdk.orchestration; -import com.sap.ai.sdk.orchestration.model.ChatCompletionTool; -import lombok.Getter; -import lombok.ToString; - +import java.util.Objects; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.Objects; +import lombok.Getter; +import lombok.ToString; /** * Represents a text item in a {@link MessageContent} object. @@ -17,40 +15,43 @@ @ToString public final class TextItem implements ContentItem, CacheablePrompt { - private final String text; - private final CacheControl cacheControl; - - public TextItem(@Nonnull final String text, @Nullable final CacheControl cacheControl) { - this.text = text; - this.cacheControl = cacheControl; - } - - /** - * Compatibility constructor conforming with the previous API to avoid breaking changes - * @param text value of the item - */ - public TextItem(@Nonnull final String text) { - this(text, null); - } - - /** - * Compatibility method to support conversion from record to class without breaking changes, - * the same as {@link #getText()} - * @return text - */ - public String text() { - return text; - } - - @Override - public boolean equals(Object o) { - if (o == null || getClass() != o.getClass()) return false; - TextItem textItem = (TextItem) o; - return Objects.equals(text, textItem.text) && Objects.equals(cacheControl, textItem.cacheControl); - } - - @Override - public int hashCode() { - return Objects.hash(text, cacheControl); - } + private final String text; + private final CacheControl cacheControl; + + public TextItem(@Nonnull final String text, @Nullable final CacheControl cacheControl) { + this.text = text; + this.cacheControl = cacheControl; + } + + /** + * Compatibility constructor conforming with the previous API to avoid breaking changes + * + * @param text value of the item + */ + public TextItem(@Nonnull final String text) { + this(text, null); + } + + /** + * Compatibility method to support conversion from record to class without breaking changes, the + * same as {@link #getText()} + * + * @return text + */ + public String text() { + return text; + } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) return false; + TextItem textItem = (TextItem) o; + return Objects.equals(text, textItem.text) + && Objects.equals(cacheControl, textItem.cacheControl); + } + + @Override + public int hashCode() { + return Objects.hash(text, cacheControl); + } } diff --git a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java index d7eee6deb..6e5e1ca03 100644 --- a/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java +++ b/orchestration/src/main/java/com/sap/ai/sdk/orchestration/UserMessage.java @@ -20,7 +20,6 @@ import java.util.List; import java.util.Locale; import java.util.function.Function; -import java.util.function.UnaryOperator; import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.AccessLevel; @@ -55,14 +54,15 @@ public UserMessage(@Nonnull final String message) { this(message, null); } - /** * Creates a new user message from a string with a cache checkpoint + * * @param message the first message * @param cacheControl caching checkpoint configuration */ - public UserMessage(@Nonnull final String message, - @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + public UserMessage( + @Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { this(new MessageContent(List.of(new TextItem(message, cacheControl)))); } @@ -87,8 +87,9 @@ public UserMessage withText(@Nonnull final String message) { * @since 1.23.0 */ @Nonnull - public UserMessage withText(@Nonnull final String message, - @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { + public UserMessage withText( + @Nonnull final String message, + @Nullable final com.sap.ai.sdk.orchestration.CacheControl cacheControl) { final var contentItems = new LinkedList<>(content.items()); contentItems.add(new TextItem(message, cacheControl)); return new UserMessage(new MessageContent(contentItems)); @@ -194,25 +195,27 @@ private static void warnNotPdfFile(@Nullable final String filename) { public ChatMessage createChatMessage() { final var contentList = new LinkedList(); - final Function toContentItem = (textItem) -> { - var contentItem = UserChatMessageContentItem.create().type(TEXT).text(textItem.text()); - var cacheControl = textItem.getCacheControl(); - if (cacheControl != null) { - var cacheControlConverted = CacheControl.create() - .type(CacheControl.TypeEnum.EPHEMERAL) - .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); - contentItem.setCacheControl(cacheControlConverted); - } - return contentItem; - }; + final Function toContentItem = + (textItem) -> { + var contentItem = UserChatMessageContentItem.create().type(TEXT).text(textItem.text()); + var cacheControl = textItem.getCacheControl(); + if (cacheControl != null) { + var cacheControlConverted = + CacheControl.create() + .type(CacheControl.TypeEnum.EPHEMERAL) + .ttl(CacheControl.TtlEnum.fromValue(cacheControl.getTtl())); + contentItem.setCacheControl(cacheControlConverted); + } + return contentItem; + }; if (content.items().size() == 1 && content.items().get(0) instanceof TextItem textItem) { if (textItem.getCacheControl() != null) { - return UserChatMessage - .create() - .content(UserChatMessageContent - .createListOfUserChatMessageContentItems(List.of(toContentItem.apply(textItem)))) - .role(USER); + return UserChatMessage.create() + .content( + UserChatMessageContent.createListOfUserChatMessageContentItems( + List.of(toContentItem.apply(textItem)))) + .role(USER); } return UserChatMessage.create() .content(UserChatMessageContent.create(textItem.text())) diff --git a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java index cf4190b8b..54fc26099 100644 --- a/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java +++ b/sample-code/spring-app/src/test/java/com/sap/ai/sdk/app/controllers/OrchestrationTest.java @@ -30,7 +30,6 @@ import com.sap.ai.sdk.orchestration.UserMessage; import com.sap.ai.sdk.orchestration.model.DPIEntities; import com.sap.ai.sdk.orchestration.model.InputTranslationModuleResult; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -64,6 +63,7 @@ class OrchestrationTest { throw new RuntimeException("failed to initialize test", e); } } + private final OrchestrationClient client = new OrchestrationClient(); private final OrchestrationModuleConfig config = new OrchestrationModuleConfig().withLlmConfig(GEMINI_2_5_FLASH.withParam(TEMPERATURE, 0.0)); @@ -106,66 +106,75 @@ void testStreamChatCompletion() { @Test void testPromptCaching() { - val prompt = new OrchestrationPrompt( + val prompt = + new OrchestrationPrompt( new UserMessage(PROMPT_FIXTURE, new CacheControl("5m")), - new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") - ); + new UserMessage( + "Does this license permit free of charge commercial use of the library licensed under it?")); val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); - val readFromCacheOrJustCached = Math.max( + val readFromCacheOrJustCached = + Math.max( res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), - res.getTokenUsage().getPromptTokensDetails().getCachedTokens() - ); - assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + res.getTokenUsage().getPromptTokensDetails().getCachedTokens()); + assertThat(readFromCacheOrJustCached) + .isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); } @Test void testPromptCachingSystemMessageCanBeCached() { - val prompt = new OrchestrationPrompt( + val prompt = + new OrchestrationPrompt( new SystemMessage(PROMPT_FIXTURE, new CacheControl("5m")), - new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") - ); + new UserMessage( + "Does this license permit free of charge commercial use of the library licensed under it?")); val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); - val readFromCacheOrJustCached = Math.max( + val readFromCacheOrJustCached = + Math.max( res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), - res.getTokenUsage().getPromptTokensDetails().getCachedTokens() - ); - assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + res.getTokenUsage().getPromptTokensDetails().getCachedTokens()); + assertThat(readFromCacheOrJustCached) + .isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); } @Test void testPromptCachingIgnoredForShortInput() { - val prompt = new OrchestrationPrompt( - new UserMessage("Recent developments in production automation and robotics are spectacular", new CacheControl("5m")), - new UserMessage("Will we see advanced home robots like cooking humanoids in foreseeable future?") - ); + val prompt = + new OrchestrationPrompt( + new UserMessage( + "Recent developments in production automation and robotics are spectacular", + new CacheControl("5m")), + new UserMessage( + "Will we see advanced home robots like cooking humanoids in foreseeable future?")); val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); - val readFromCacheOrJustCached = Math.max( + val readFromCacheOrJustCached = + Math.max( res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), - res.getTokenUsage().getPromptTokensDetails().getCachedTokens() - ); + res.getTokenUsage().getPromptTokensDetails().getCachedTokens()); assertThat(readFromCacheOrJustCached).isEqualTo(0); } @Test void testPromptCachingInvalidTTLGetsResetToDefault() { - val prompt = new OrchestrationPrompt( + val prompt = + new OrchestrationPrompt( new UserMessage(PROMPT_FIXTURE, new CacheControl("0s")), - new UserMessage("Does this license permit free of charge commercial use of the library licensed under it?") - ); + new UserMessage( + "Does this license permit free of charge commercial use of the library licensed under it?")); val cfgCacheSupportingModel = new OrchestrationModuleConfig().withLlmConfig(CLAUDE_4_6_SONNET); val res = new OrchestrationClient().chatCompletion(prompt, cfgCacheSupportingModel); - val readFromCacheOrJustCached = Math.max( + val readFromCacheOrJustCached = + Math.max( res.getTokenUsage().getPromptTokensDetails().getCacheCreationTokens(), - res.getTokenUsage().getPromptTokensDetails().getCachedTokens() - ); - assertThat(readFromCacheOrJustCached).isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); + res.getTokenUsage().getPromptTokensDetails().getCachedTokens()); + assertThat(readFromCacheOrJustCached) + .isCloseTo(PROMPT_FIXTURE_LENGTH_TOKENS, withinPercentage(5)); } @Test