diff --git a/src/main/java/br/com/arquivolivre/myjavagenie/service/PromptBuilder.java b/src/main/java/br/com/arquivolivre/myjavagenie/service/PromptBuilder.java index 2327715..62c5cfc 100644 --- a/src/main/java/br/com/arquivolivre/myjavagenie/service/PromptBuilder.java +++ b/src/main/java/br/com/arquivolivre/myjavagenie/service/PromptBuilder.java @@ -12,14 +12,30 @@ @Component public class PromptBuilder { + /** + * Fixed reply the model is instructed to return whenever the retrieved context does not answer + * the question (e.g. an off-topic question). Exposed so callers can detect the refusal and avoid + * attaching irrelevant source citations to it. + */ + public static final String OUT_OF_SCOPE_ANSWER = + "I can only answer questions about the Java 25 documentation, and I couldn't find " + + "anything relevant to your question in my sources."; + private static final String SYSTEM_PROMPT = - "You are an expert on Java 25 documentation. " - + "Answer questions accurately based on the provided context. " - + "If the context doesn't contain relevant information, say so. " - + "Keep answers concise and cite sources when possible."; + "You are an assistant that answers strictly from the provided Java 25 documentation. " + + "Use ONLY the information in the Context section below to answer the Question. " + + "Do not use any outside or prior knowledge, and never make anything up. " + + "If the Context does not contain information that answers the Question, or the " + + "Question is not about Java 25, reply with exactly the following sentence and nothing " + + "else: \"" + + OUT_OF_SCOPE_ANSWER + + "\" When the Context does answer the Question, be accurate and cite the source files " + + "you used."; /** - * Builds a complete prompt for the language model. + * Builds a complete prompt for the language model. The strict grounding instructions are + * prepended so the model only answers from the retrieved context (the provider sends this string + * verbatim, so the instructions must live inside it). * * @param question the user's question * @param retrievedChunks the relevant document chunks retrieved from the vector database @@ -31,7 +47,7 @@ public String buildPrompt(String question, List retrievedChunks) } String context = formatContext(retrievedChunks); - return "Context:\n" + context + "\n\nQuestion: " + question + "\n\nAnswer:"; + return SYSTEM_PROMPT + "\n\nContext:\n" + context + "\n\nQuestion: " + question + "\n\nAnswer:"; } /** diff --git a/src/main/java/br/com/arquivolivre/myjavagenie/service/QueryService.java b/src/main/java/br/com/arquivolivre/myjavagenie/service/QueryService.java index bcc25ff..5f7b9e3 100644 --- a/src/main/java/br/com/arquivolivre/myjavagenie/service/QueryService.java +++ b/src/main/java/br/com/arquivolivre/myjavagenie/service/QueryService.java @@ -128,9 +128,14 @@ public QueryResponse processQuery(String question) { span.setAttribute("llm.provider", languageModel.getProviderName()); } - // Step 4: Extract source references + // Step 4: Extract source references. When the model returned the fixed out-of-scope reply + // (the retrieved context did not answer the question), do not attach irrelevant sources. logger.debug("Step 4: Extracting source references"); - List sources = extractSourceReferences(scoredChunks); + boolean outOfScope = + answer != null + && answer.contains("I can only answer questions about the Java 25 documentation"); + List sources = + outOfScope ? List.of() : extractSourceReferences(scoredChunks); // Step 5: Track token usage logger.debug("Step 5: Recording token usage");