From e43c16db503fd9da8713c54449da59bf628914a9 Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 4 Jul 2026 09:51:01 +0100 Subject: [PATCH 1/2] Store answer: reasoning-heavy map (full tree-walk per document) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the store answer's map stage: instead of fanning the section-selection strategy across documents (retrieve-then-generate, chunk-shaped), run the FULL agentic tree-walk on EACH document — read its structure, navigate hop by hop to a grounded per-document answer + page citations — then synthesise the per-document answers into one. Every document is now reasoned over exactly like /v1/answer/treewalk; no chunking, no embeddings anywhere in the collection path. - max_depth controls the per-document tree-walk hop budget (0 = the engine's full/default depth), as requested. - max_docs bounds how many documents are tree-walked (0 = all) — a cost valve for large collections; the per-doc walks run with bounded concurrency. - Citations are one-per-contributing-document, carrying the document's cited page span + overlapping section ids (from the tree-walk's CitedPages) and a short quote from its answer. Refusals/empty per-doc answers are dropped before synthesis. Returns 501 unless the tree-walk strategy + an LLM are configured. --- internal/handler/answer_store.go | 383 ++++++++++++++------------ internal/handler/answer_store_test.go | 12 +- internal/handler/router.go | 2 +- 3 files changed, 211 insertions(+), 186 deletions(-) diff --git a/internal/handler/answer_store.go b/internal/handler/answer_store.go index 6e0c9aa..b1212e7 100644 --- a/internal/handler/answer_store.go +++ b/internal/handler/answer_store.go @@ -4,83 +4,96 @@ import ( "context" "encoding/json" "fmt" - "io" "log/slog" "net/http" "sort" "strings" + "sync" "time" "github.com/hallelx2/llmgate" + "golang.org/x/sync/errgroup" + enginecfg "github.com/hallelx2/vectorless-engine/pkg/config" "github.com/hallelx2/vectorless-engine/pkg/retrieval" - "github.com/hallelx2/vectorless-engine/pkg/storage" "github.com/hallelx2/vectorless-engine/pkg/tree" ) +// storeTreeLoader is the narrow slice of *db.Pool the store handler needs +// to materialise each document's section tree. *db.Pool satisfies it. +type storeTreeLoader interface { + LoadTree(ctx context.Context, docID tree.DocumentID, orgID, storeID string) (*tree.Tree, error) +} + // AnswerStoreHandler answers a question across a SET of documents (a // "store" / collection) and synthesises a SINGLE grounded answer with -// per-document citations. It is the map-reduce companion to the -// single-document tree-walk: +// per-document citations. It is reasoning-first, not chunk-retrieval: // -// map — retrieval.MultiDoc fans the strategy across every document -// and returns the relevant sections per document. -// reduce — one LLM call reads the gathered, source-labelled evidence -// and writes one answer, citing sources with [n] markers. +// map — the FULL agentic tree-walk runs on EACH document: it reads +// the document's structure and navigates it hop by hop to a +// grounded per-document answer + page citations. No chunking, +// no embeddings — every document is reasoned over exactly like +// the single-document /v1/answer/treewalk. +// reduce — one LLM call reads the per-document answers (each labelled +// with its title) and writes one coherent answer, citing the +// documents it drew on with [n] markers. // -// The caller passes the document_ids that make up the store (the -// dashboard resolves store → its ready documents); the org + store scope -// come from the injected headers, so a key can only ever read its own -// documents. +// max_depth controls the per-document tree-walk hop budget (0 = the +// engine's full/default depth). max_docs bounds how many documents are +// tree-walked (0 = all supplied) — a cost valve for large collections. type AnswerStoreHandler struct { - logger *slog.Logger - storage storage.Storage - multiDoc *retrieval.MultiDoc - llm llmgate.Client - llmModel string + logger *slog.Logger + db storeTreeLoader + treewalk *retrieval.TreeWalkStrategy + treeWalkEnabled bool + llm llmgate.Client + llmModel string } -// NewAnswerStoreHandler creates an AnswerStoreHandler. llm/multiDoc may be -// nil in configurations without an LLM; the handler then returns 501. +// NewAnswerStoreHandler creates an AnswerStoreHandler. It returns 501 at +// request time when the tree-walk strategy or LLM is not configured. func NewAnswerStoreHandler( logger *slog.Logger, - store storage.Storage, - multiDoc *retrieval.MultiDoc, + db storeTreeLoader, + treewalk *retrieval.TreeWalkStrategy, + treeWalkCfg enginecfg.TreeWalkBlock, llm llmgate.Client, llmModel string, ) *AnswerStoreHandler { return &AnswerStoreHandler{ - logger: logger, - storage: store, - multiDoc: multiDoc, - llm: llm, - llmModel: llmModel, + logger: logger, + db: db, + treewalk: treewalk, + treeWalkEnabled: treeWalkCfg.Enabled, + llm: llm, + llmModel: llmModel, } } type answerStoreRequest struct { - DocumentIDs []tree.DocumentID `json:"document_ids"` - Query string `json:"query"` - Model string `json:"model"` - MaxSectionsPerDoc int `json:"max_sections_per_doc"` - MaxTokens int `json:"max_tokens"` + DocumentIDs []tree.DocumentID `json:"document_ids"` + Query string `json:"query"` + Model string `json:"model"` + MaxDepth int `json:"max_depth"` // per-doc tree-walk hop budget; 0 = full + MaxDocs int `json:"max_docs"` // cap documents tree-walked; 0 = all + MaxTokens int `json:"max_tokens"` } -// evidenceBlock is one source-labelled snippet handed to the reducer. -type evidenceBlock struct { - index int // 1-based global citation index +// docAnswer is one document's tree-walk result in the map stage. +type docAnswer struct { + index int // 1-based citation index (assigned to contributing docs) docID tree.DocumentID docTitle string + answer string + confidence float64 startPage int endPage int sectionIDs []tree.SectionID - content string } const ( - storeAnswerDefaultSectionsPerDoc = 3 - storeAnswerEvidenceBudget = 14000 // chars of evidence sent to the reducer - storeAnswerQuoteLen = 240 + storeAnswerMapConcurrency = 6 + storeAnswerQuoteLen = 240 ) // HandleAnswerStore is POST /v1/answer/store. @@ -99,13 +112,14 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re writeErr(w, http.StatusBadRequest, "document_ids (non-empty) and query are required") return } - if h.multiDoc == nil || h.llm == nil { - writeErr(w, http.StatusNotImplemented, "store answers require an LLM-backed configuration") + if h.llm == nil || h.treewalk == nil || !h.treeWalkEnabled { + writeErr(w, http.StatusNotImplemented, "store answers require the tree-walk strategy + an LLM (retrieval.treewalk.enabled=true)") return } - perDoc := body.MaxSectionsPerDoc - if perDoc <= 0 { - perDoc = storeAnswerDefaultSectionsPerDoc + + docIDs := body.DocumentIDs + if body.MaxDocs > 0 && len(docIDs) > body.MaxDocs { + docIDs = docIDs[:body.MaxDocs] } model := body.Model if model == "" { @@ -114,25 +128,26 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re started := time.Now() - // ── MAP ─────────────────────────────────────────────────────── - budget := retrieval.ContextBudget{ModelName: body.Model, MaxTokens: 100000, ReservedForPrompt: 4000, MaxParallelCalls: 8} - md, err := h.multiDoc.Query(r.Context(), orgID, storeID(r), body.DocumentIDs, body.Query, budget) - if err != nil { - h.logger.Error("answer/store: map failed", "err", err) - writeErr(w, http.StatusInternalServerError, "store retrieval failed: "+err.Error()) - return - } + // ── MAP: full tree-walk on every document, in parallel ───────── + perDoc, mapUsage := h.mapTreeWalk(r.Context(), orgID, storeID(r), docIDs, body.Query, model, body.MaxDepth) - evidence, matched := h.gatherEvidence(r.Context(), md, perDoc) - totalUsage := md.TotalUsage + // Keep only documents that produced a real, non-refusal answer. + contributing := make([]docAnswer, 0, len(perDoc)) + for _, d := range perDoc { + if isNonAnswer(d.answer) { + continue + } + d.index = len(contributing) + 1 + contributing = append(contributing, d) + } - // No document surfaced anything relevant → honest refusal. - if len(evidence) == 0 { + totalUsage := mapUsage + if len(contributing) == 0 { writeJSON(w, http.StatusOK, map[string]any{ "query": body.Query, - "answer": "The documents in this store do not appear to address this query.", + "answer": "The documents in this collection do not appear to address this query.", "citations": []any{}, - "documents_searched": len(body.DocumentIDs), + "documents_searched": len(docIDs), "documents_with_matches": 0, "confidence": 0.0, "usage": usageMap(totalUsage), @@ -141,8 +156,8 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re return } - // ── REDUCE ──────────────────────────────────────────────────── - answer, citedIdx, synthUsage, err := h.synthesise(r.Context(), model, body.Query, evidence, body.MaxTokens) + // ── REDUCE: synthesise the per-document answers ──────────────── + answer, citedIdx, synthUsage, err := h.synthesise(r.Context(), model, body.Query, contributing, body.MaxTokens) if err != nil { h.logger.Error("answer/store: reduce failed", "err", err) writeErr(w, http.StatusInternalServerError, "answer synthesis failed: "+err.Error()) @@ -150,118 +165,115 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re } totalUsage.Add(synthUsage) - citations := buildStoreCitations(evidence, citedIdx) - writeJSON(w, http.StatusOK, map[string]any{ "query": body.Query, "answer": answer, - "citations": citations, - "documents_searched": len(body.DocumentIDs), - "documents_with_matches": matched, + "citations": buildStoreCitations(contributing, citedIdx), + "documents_searched": len(docIDs), + "documents_with_matches": len(contributing), "usage": usageMap(totalUsage), "elapsed_ms": time.Since(started).Milliseconds(), }) } -// gatherEvidence turns the per-document retrieval result into a flat, -// globally-indexed, source-labelled evidence list, newest-relevance -// first (documents that matched more sections lead), capped by the -// evidence char budget. Returns the evidence and the count of documents -// that contributed at least one block. -func (h *AnswerStoreHandler) gatherEvidence(ctx context.Context, md *retrieval.MultiDocResult, perDoc int) ([]evidenceBlock, int) { - // Deterministic doc order: more selected sections first, then by id. - type docEntry struct { - id tree.DocumentID - dr *retrieval.DocResult - } - docs := make([]docEntry, 0, len(md.Documents)) - for id, dr := range md.Documents { - docs = append(docs, docEntry{id, dr}) - } - sort.Slice(docs, func(i, j int) bool { - if len(docs[i].dr.SelectedIDs) != len(docs[j].dr.SelectedIDs) { - return len(docs[i].dr.SelectedIDs) > len(docs[j].dr.SelectedIDs) - } - return docs[i].id < docs[j].id - }) - +// mapTreeWalk runs the full tree-walk on each document concurrently and +// returns each document's grounded answer + citations, plus the summed +// usage. Per-document failures are logged and skipped (partial results +// are fine). maxDepth overrides the per-document hop budget (0 = default). +func (h *AnswerStoreHandler) mapTreeWalk(ctx context.Context, orgID, storeID string, docIDs []tree.DocumentID, query, model string, maxDepth int) ([]docAnswer, retrieval.Usage) { var ( - out []evidenceBlock - spent int - matched int - idx = 1 + mu sync.Mutex + out = make([]docAnswer, 0, len(docIDs)) + usage retrieval.Usage ) - for _, d := range docs { - if d.dr.Tree == nil { - continue - } - contributed := false - for i, sid := range d.dr.SelectedIDs { - if i >= perDoc || spent >= storeAnswerEvidenceBudget { - break + budget := retrieval.ContextBudget{ModelName: model, MaxTokens: 100000, ReservedForPrompt: 4000} + + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(storeAnswerMapConcurrency) + for _, docID := range docIDs { + g.Go(func() error { + t, err := h.db.LoadTree(gctx, docID, orgID, storeID) + if err != nil || t == nil { + h.logger.Warn("answer/store: load tree", "doc", docID, "err", err) + return nil } - sec := d.dr.Tree.FindByID(sid) - if sec == nil || sec.ContentRef == "" { - continue + // Per-request copy so a hop override never mutates the shared + // strategy other goroutines are reading. + perReq := *h.treewalk + perReq.OnEvent = nil // non-streaming + if maxDepth > 0 { + perReq.MaxHops = maxDepth } - content := h.loadContent(ctx, sec.ContentRef) - if strings.TrimSpace(content) == "" { - continue + res, err := perReq.SelectWithCost(gctx, t, query, budget) + if err != nil { + h.logger.Warn("answer/store: treewalk", "doc", docID, "err", err) + return nil } - remaining := storeAnswerEvidenceBudget - spent - if len(content) > remaining { - content = content[:remaining] + + // Resolve cited page ranges → section ids + a page span. + var ( + secIDs []tree.SectionID + minStart, maxEnd int + seen = map[tree.SectionID]struct{}{} + ) + for _, pr := range res.CitedPages { + if minStart == 0 || pr[0] < minStart { + minStart = pr[0] + } + if pr[1] > maxEnd { + maxEnd = pr[1] + } + for _, id := range retrieval.SectionIDsOverlapping(t, pr[0], pr[1]) { + if _, dup := seen[id]; !dup { + seen[id] = struct{}{} + secIDs = append(secIDs, id) + } + } } - spent += len(content) - out = append(out, evidenceBlock{ - index: idx, - docID: d.id, - docTitle: d.dr.Tree.Title, - startPage: sec.PageStart, - endPage: sec.PageEnd, - sectionIDs: []tree.SectionID{sid}, - content: content, + + mu.Lock() + out = append(out, docAnswer{ + docID: docID, + docTitle: t.Title, + answer: strings.TrimSpace(res.Reasoning), + confidence: res.Confidence, + startPage: minStart, + endPage: maxEnd, + sectionIDs: secIDs, }) - idx++ - contributed = true - } - if contributed { - matched++ - } + usage.Add(res.Usage) + mu.Unlock() + return nil + }) } - return out, matched -} + _ = g.Wait() -func (h *AnswerStoreHandler) loadContent(ctx context.Context, ref string) string { - rc, _, err := h.storage.Get(ctx, ref) - if err != nil { - return "" - } - defer func() { _ = rc.Close() }() - raw, err := io.ReadAll(rc) - if err != nil { - return "" - } - return string(raw) + // Deterministic order: highest confidence first, then title. + sort.SliceStable(out, func(i, j int) bool { + if out[i].confidence != out[j].confidence { + return out[i].confidence > out[j].confidence + } + return out[i].docTitle < out[j].docTitle + }) + return out, usage } -// synthesise runs the single reducer LLM call. It returns the answer -// text (with [n] citation markers), the set of cited evidence indices, -// and usage. -func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string, evidence []evidenceBlock, maxTokens int) (string, []int, retrieval.Usage, error) { +// synthesise runs the single reducer LLM call over the per-document +// answers. Returns the final answer (with [n] markers), the cited doc +// indices, and usage. +func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string, docs []docAnswer, maxTokens int) (string, []int, retrieval.Usage, error) { var ev strings.Builder - for _, b := range evidence { - pages := formatPageRange(b.startPage, b.endPage) - title := b.docTitle + for _, d := range docs { + title := d.docTitle if title == "" { - title = string(b.docID) + title = string(d.docID) } - fmt.Fprintf(&ev, "[%d] %s", b.index, title) - if pages != "" { - fmt.Fprintf(&ev, " (%s)", pages) + fmt.Fprintf(&ev, "[%d] %s", d.index, title) + if p := formatPageRange(d.startPage, d.endPage); p != "" { + fmt.Fprintf(&ev, " (%s)", p) } ev.WriteString(":\n") - ev.WriteString(strings.TrimSpace(b.content)) + ev.WriteString(d.answer) ev.WriteString("\n\n") } @@ -274,7 +286,7 @@ func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string Temperature: 0, Messages: []llmgate.Message{ {Role: llmgate.RoleSystem, Content: storeAnswerSystemPrompt}, - {Role: llmgate.RoleUser, Content: fmt.Sprintf("QUESTION:\n%s\n\nEVIDENCE:\n%s\nReply with ONLY the JSON object.", query, ev.String())}, + {Role: llmgate.RoleUser, Content: fmt.Sprintf("QUESTION:\n%s\n\nPER-DOCUMENT ANSWERS:\n%s\nReply with ONLY the JSON object.", query, ev.String())}, }, } resp, err := h.llm.Complete(ctx, req) @@ -288,21 +300,23 @@ func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string CostUSD: resp.Usage.CostUSD, LLMCalls: 1, } - - answer, cited := parseStoreAnswer(resp.Content, len(evidence)) + answer, cited := parseStoreAnswer(resp.Content, len(docs)) return answer, cited, usage, nil } -const storeAnswerSystemPrompt = `You answer a question using ONLY the supplied EVIDENCE, which is drawn from several documents in a collection. Each evidence block is prefixed with a number in brackets, e.g. [1], [2]. +const storeAnswerSystemPrompt = `You are given a QUESTION and a set of ANSWERS, each produced by carefully reading ONE document in a collection. Each is prefixed with a number in brackets, e.g. [1], [2], and its source title. + +Your job is to write ONE coherent answer to the question by synthesising across these per-document answers. RULES: -- Answer ONLY from the evidence. Never invent facts. If the evidence does not answer the question, say so plainly. -- Cite every claim inline with the bracketed number(s) of the evidence you used, e.g. "Consent must be freely given [1]." Cite the specific block(s) that support each statement. -- Synthesise across documents when the answer draws on more than one — the whole point is one coherent answer over the collection. -- Be concise and well-structured. Use short paragraphs or a short list. +- Use ONLY the supplied per-document answers. Do not invent facts. +- Cite the document(s) you draw on inline with their bracketed number, e.g. "Consent must be freely given [1]." Cite every claim. +- Synthesise: when several documents contribute, combine them into one answer and note agreements or differences. When only some are relevant, use only those. +- If none of the answers actually address the question, say so plainly. +- Be concise and well-structured. Reply with EXACTLY one JSON object, no markdown fences: -{"answer":"","cited":[]}` +{"answer":"","cited":[]}` // storeAnswerJSON is the reducer's expected output shape. type storeAnswerJSON struct { @@ -324,7 +338,6 @@ func parseStoreAnswer(raw string, maxIdx int) (string, []int) { } } } - // Fallback: use the raw text and scrape [n] markers from it. return s, scrapeMarkers(s, maxIdx) } @@ -365,45 +378,57 @@ func scrapeMarkers(s string, maxIdx int) []int { return dedupeInRange(nums, maxIdx) } -// buildStoreCitations turns the cited evidence indices into the response -// citations array, preserving cited order and carrying document id/title, -// page range, section ids, and a short verbatim quote. -func buildStoreCitations(evidence []evidenceBlock, cited []int) []map[string]any { - byIdx := make(map[int]evidenceBlock, len(evidence)) - for _, b := range evidence { - byIdx[b.index] = b +// buildStoreCitations turns the cited document indices into the response +// citations array — one citation per contributing document, carrying its +// id, title, page span, section ids, and a short quote from its answer. +func buildStoreCitations(docs []docAnswer, cited []int) []map[string]any { + byIdx := make(map[int]docAnswer, len(docs)) + for _, d := range docs { + byIdx[d.index] = d } - // If the model cited nothing parseable, fall back to citing all - // evidence so the user still sees the sources the answer used. - if len(cited) == 0 { - for _, b := range evidence { - cited = append(cited, b.index) + if len(cited) == 0 { // model cited nothing parseable → show all sources + for _, d := range docs { + cited = append(cited, d.index) } } out := make([]map[string]any, 0, len(cited)) for _, n := range cited { - b, ok := byIdx[n] + d, ok := byIdx[n] if !ok { continue } c := map[string]any{ - "index": b.index, - "document_id": b.docID, - "section_ids": b.sectionIDs, - "quote": snippet(b.content, storeAnswerQuoteLen), + "index": d.index, + "document_id": d.docID, + "section_ids": d.sectionIDs, + "quote": snippet(d.answer, storeAnswerQuoteLen), + "confidence": d.confidence, } - if b.docTitle != "" { - c["document_title"] = b.docTitle + if d.docTitle != "" { + c["document_title"] = d.docTitle } - if b.startPage > 0 { - c["start_page"] = b.startPage - c["end_page"] = b.endPage + if d.startPage > 0 { + c["start_page"] = d.startPage + c["end_page"] = d.endPage } out = append(out, c) } return out } +// isNonAnswer reports whether a per-document tree-walk answer is a refusal +// or empty, so it can be dropped from the synthesis. +func isNonAnswer(s string) bool { + s = strings.TrimSpace(strings.ToLower(s)) + if len(s) < 3 { + return true + } + return strings.Contains(s, "does not address") || + strings.Contains(s, "not address this query") || + strings.Contains(s, "no relevant information") || + strings.Contains(s, "does not contain") +} + func snippet(s string, n int) string { s = strings.TrimSpace(strings.Join(strings.Fields(s), " ")) if len(s) <= n { diff --git a/internal/handler/answer_store_test.go b/internal/handler/answer_store_test.go index ba0caec..e0615d2 100644 --- a/internal/handler/answer_store_test.go +++ b/internal/handler/answer_store_test.go @@ -45,13 +45,13 @@ func TestParseStoreAnswer(t *testing.T) { } func TestBuildStoreCitations(t *testing.T) { - evidence := []evidenceBlock{ - {index: 1, docID: "doc_a", docTitle: "GDPR", startPage: 32, endPage: 33, sectionIDs: []tree.SectionID{"sec_1"}, content: "Consent must be freely given, specific, informed and unambiguous."}, - {index: 2, docID: "doc_b", docTitle: "PRISMA", startPage: 4, endPage: 4, sectionIDs: []tree.SectionID{"sec_2"}, content: "Report the review using the checklist."}, + docs := []docAnswer{ + {index: 1, docID: "doc_a", docTitle: "GDPR", startPage: 32, endPage: 33, sectionIDs: []tree.SectionID{"sec_1"}, answer: "Consent must be freely given, specific, informed and unambiguous."}, + {index: 2, docID: "doc_b", docTitle: "PRISMA", startPage: 4, endPage: 4, sectionIDs: []tree.SectionID{"sec_2"}, answer: "Report the review using the checklist."}, } t.Run("cited subset in order with metadata", func(t *testing.T) { - cits := buildStoreCitations(evidence, []int{2}) + cits := buildStoreCitations(docs, []int{2}) if len(cits) != 1 { t.Fatalf("want 1 citation, got %d", len(cits)) } @@ -63,8 +63,8 @@ func TestBuildStoreCitations(t *testing.T) { } }) - t.Run("empty cited falls back to all evidence", func(t *testing.T) { - cits := buildStoreCitations(evidence, nil) + t.Run("empty cited falls back to all documents", func(t *testing.T) { + cits := buildStoreCitations(docs, nil) if len(cits) != 2 { t.Fatalf("want 2 (fallback to all), got %d", len(cits)) } diff --git a/internal/handler/router.go b/internal/handler/router.go index 9e4fa51..f034ed1 100644 --- a/internal/handler/router.go +++ b/internal/handler/router.go @@ -147,7 +147,7 @@ func Router(d Deps) http.Handler { queryStreamMulti := NewQueryStreamMultiHandler(d.Logger, d.Storage, d.MultiDoc) answer := NewAnswerHandler(d.Logger, d.DB, d.Storage, d.Strategy, d.LLM, d.LLMModel, d.AnswerSpan, d.Answer, d.Replay) answerTreeWalk := NewAnswerTreeWalkHandler(d.Logger, d.DB, d.Storage, d.LLM, d.LLMModel, d.AnswerSpan, d.Replay, d.TreeWalkStrategy, d.TreeWalk) - answerStore := NewAnswerStoreHandler(d.Logger, d.Storage, d.MultiDoc, d.LLM, d.LLMModel) + answerStore := NewAnswerStoreHandler(d.Logger, d.DB, d.TreeWalkStrategy, d.TreeWalk, d.LLM, d.LLMModel) webhook := NewWebhookHandler(d.Logger, d.Queue) // ── Connect-RPC Handlers (generated stubs, three-transport) ─── From e59aad538800900b8b71234a3106283399c744cc Mon Sep 17 00:00:00 2001 From: hallelx2 Date: Sat, 4 Jul 2026 10:00:23 +0100 Subject: [PATCH 2/2] Store answer: reason over structure+summaries, then full section content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rework the store map to the Vectorless primitive the collection query should use — reason over each document's llms.txt-style structure + summaries, select the relevant sections, pull their FULL content, then generate: 1. per document (parallel): load the tree, render its section outline (title + one-line summary per section), and ask the LLM which sections' full text are relevant — reasoning over summaries, not raw pages, and not chunks. 2. fetch the FULL content of every selected section (a selected heading pulls its subsection's leaves). 3. one generation call over the full content of ALL relevant sections → a single answer citing sections + documents with [n] markers. Two cheap selection calls over compact summaries + one generation call, instead of N page-based tree-walks. No chunking, no embeddings, no truncated snippets — full section content drives the answer. Controls: max_docs (0 = all), max_sections (0 = all relevant), plus a total content budget so a large selection still fits the model context. Citations are per section, carrying document + section title + page span + a quote. Returns 501 without an LLM. --- internal/handler/answer_store.go | 493 ++++++++++++++++---------- internal/handler/answer_store_test.go | 22 +- internal/handler/router.go | 2 +- 3 files changed, 330 insertions(+), 187 deletions(-) diff --git a/internal/handler/answer_store.go b/internal/handler/answer_store.go index b1212e7..ea1d365 100644 --- a/internal/handler/answer_store.go +++ b/internal/handler/answer_store.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "log/slog" "net/http" "sort" @@ -14,8 +15,8 @@ import ( "github.com/hallelx2/llmgate" "golang.org/x/sync/errgroup" - enginecfg "github.com/hallelx2/vectorless-engine/pkg/config" "github.com/hallelx2/vectorless-engine/pkg/retrieval" + "github.com/hallelx2/vectorless-engine/pkg/storage" "github.com/hallelx2/vectorless-engine/pkg/tree" ) @@ -27,73 +28,67 @@ type storeTreeLoader interface { // AnswerStoreHandler answers a question across a SET of documents (a // "store" / collection) and synthesises a SINGLE grounded answer with -// per-document citations. It is reasoning-first, not chunk-retrieval: +// per-section, per-document citations. It is reasoning-first over +// structure + summaries — the Vectorless primitive, no chunking, no +// embeddings: // -// map — the FULL agentic tree-walk runs on EACH document: it reads -// the document's structure and navigates it hop by hop to a -// grounded per-document answer + page citations. No chunking, -// no embeddings — every document is reasoned over exactly like -// the single-document /v1/answer/treewalk. -// reduce — one LLM call reads the per-document answers (each labelled -// with its title) and writes one coherent answer, citing the -// documents it drew on with [n] markers. +// 1. reason over each document's STRUCTURE + SUMMARIES (its llms.txt-style +// outline) to SELECT the sections whose full text is relevant. +// 2. fetch the FULL content of every selected relevant section. +// 3. GENERATE one answer from that full content, citing the sections and +// documents it drew on. // -// max_depth controls the per-document tree-walk hop budget (0 = the -// engine's full/default depth). max_docs bounds how many documents are -// tree-walked (0 = all supplied) — a cost valve for large collections. +// max_docs bounds how many documents are considered (0 = all). max_sections +// caps the total relevant sections whose full content is pulled (0 = all +// relevant). A content budget caps the total characters handed to the +// generator so a huge selection still fits the model context. type AnswerStoreHandler struct { - logger *slog.Logger - db storeTreeLoader - treewalk *retrieval.TreeWalkStrategy - treeWalkEnabled bool - llm llmgate.Client - llmModel string + logger *slog.Logger + db storeTreeLoader + storage storage.Storage + llm llmgate.Client + llmModel string } // NewAnswerStoreHandler creates an AnswerStoreHandler. It returns 501 at -// request time when the tree-walk strategy or LLM is not configured. +// request time when no LLM is configured. func NewAnswerStoreHandler( logger *slog.Logger, db storeTreeLoader, - treewalk *retrieval.TreeWalkStrategy, - treeWalkCfg enginecfg.TreeWalkBlock, + store storage.Storage, llm llmgate.Client, llmModel string, ) *AnswerStoreHandler { - return &AnswerStoreHandler{ - logger: logger, - db: db, - treewalk: treewalk, - treeWalkEnabled: treeWalkCfg.Enabled, - llm: llm, - llmModel: llmModel, - } + return &AnswerStoreHandler{logger: logger, db: db, storage: store, llm: llm, llmModel: llmModel} } type answerStoreRequest struct { DocumentIDs []tree.DocumentID `json:"document_ids"` Query string `json:"query"` Model string `json:"model"` - MaxDepth int `json:"max_depth"` // per-doc tree-walk hop budget; 0 = full - MaxDocs int `json:"max_docs"` // cap documents tree-walked; 0 = all + MaxDocs int `json:"max_docs"` // cap documents considered; 0 = all + MaxSections int `json:"max_sections"` // cap relevant sections pulled; 0 = all MaxTokens int `json:"max_tokens"` } -// docAnswer is one document's tree-walk result in the map stage. -type docAnswer struct { - index int // 1-based citation index (assigned to contributing docs) - docID tree.DocumentID - docTitle string - answer string - confidence float64 - startPage int - endPage int - sectionIDs []tree.SectionID +// relevantSection is one selected section with its full content, ready to +// hand to the generator. +type relevantSection struct { + index int // 1-based global citation index + docID tree.DocumentID + docTitle string + sectionID tree.SectionID + sectionTitle string + startPage int + endPage int + content string } const ( - storeAnswerMapConcurrency = 6 - storeAnswerQuoteLen = 240 + storeMapConcurrency = 6 + storeContentBudget = 40000 // total chars of full section content sent to the generator + storeSectionCharsCap = 8000 // per-section content cap + storeAnswerQuoteLen = 240 ) // HandleAnswerStore is POST /v1/answer/store. @@ -112,8 +107,8 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re writeErr(w, http.StatusBadRequest, "document_ids (non-empty) and query are required") return } - if h.llm == nil || h.treewalk == nil || !h.treeWalkEnabled { - writeErr(w, http.StatusNotImplemented, "store answers require the tree-walk strategy + an LLM (retrieval.treewalk.enabled=true)") + if h.llm == nil { + writeErr(w, http.StatusNotImplemented, "store answers require an LLM-backed configuration") return } @@ -128,39 +123,30 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re started := time.Now() - // ── MAP: full tree-walk on every document, in parallel ───────── - perDoc, mapUsage := h.mapTreeWalk(r.Context(), orgID, storeID(r), docIDs, body.Query, model, body.MaxDepth) - - // Keep only documents that produced a real, non-refusal answer. - contributing := make([]docAnswer, 0, len(perDoc)) - for _, d := range perDoc { - if isNonAnswer(d.answer) { - continue - } - d.index = len(contributing) + 1 - contributing = append(contributing, d) - } - + // ── 1+2. Per document: reason over structure+summaries → select + // relevant sections → fetch their full content (parallel). ────── + sections, docsMatched, mapUsage := h.selectRelevant(r.Context(), orgID, storeID(r), docIDs, body.Query, model, body.MaxSections) totalUsage := mapUsage - if len(contributing) == 0 { + + if len(sections) == 0 { writeJSON(w, http.StatusOK, map[string]any{ "query": body.Query, - "answer": "The documents in this collection do not appear to address this query.", + "answer": "No sections in this collection appear relevant to the query.", "citations": []any{}, "documents_searched": len(docIDs), "documents_with_matches": 0, - "confidence": 0.0, + "sections_used": 0, "usage": usageMap(totalUsage), "elapsed_ms": time.Since(started).Milliseconds(), }) return } - // ── REDUCE: synthesise the per-document answers ──────────────── - answer, citedIdx, synthUsage, err := h.synthesise(r.Context(), model, body.Query, contributing, body.MaxTokens) + // ── 3. Generate one answer from the full section content. ────── + answer, cited, synthUsage, err := h.generate(r.Context(), model, body.Query, sections, body.MaxTokens) if err != nil { - h.logger.Error("answer/store: reduce failed", "err", err) - writeErr(w, http.StatusInternalServerError, "answer synthesis failed: "+err.Error()) + h.logger.Error("answer/store: generate failed", "err", err) + writeErr(w, http.StatusInternalServerError, "answer generation failed: "+err.Error()) return } totalUsage.Add(synthUsage) @@ -168,112 +154,191 @@ func (h *AnswerStoreHandler) HandleAnswerStore(w http.ResponseWriter, r *http.Re writeJSON(w, http.StatusOK, map[string]any{ "query": body.Query, "answer": answer, - "citations": buildStoreCitations(contributing, citedIdx), + "citations": buildStoreCitations(sections, cited), "documents_searched": len(docIDs), - "documents_with_matches": len(contributing), + "documents_with_matches": docsMatched, + "sections_used": len(sections), "usage": usageMap(totalUsage), "elapsed_ms": time.Since(started).Milliseconds(), }) } -// mapTreeWalk runs the full tree-walk on each document concurrently and -// returns each document's grounded answer + citations, plus the summed -// usage. Per-document failures are logged and skipped (partial results -// are fine). maxDepth overrides the per-document hop budget (0 = default). -func (h *AnswerStoreHandler) mapTreeWalk(ctx context.Context, orgID, storeID string, docIDs []tree.DocumentID, query, model string, maxDepth int) ([]docAnswer, retrieval.Usage) { +// manifestRef maps a manifest index to a section id + whether it carries +// content directly. +type manifestRef struct { + id tree.SectionID + hasContent bool +} + +// selectRelevant runs, per document and in parallel: load tree → render +// the structure+summary outline → LLM-select the relevant sections → +// fetch their full content. It returns the flat list of relevant sections +// (globally indexed), the number of documents that contributed at least +// one section, and the summed selection usage. A total content budget +// (and optional max_sections) bounds the material. +func (h *AnswerStoreHandler) selectRelevant(ctx context.Context, orgID, storeID string, docIDs []tree.DocumentID, query, model string, maxSections int) ([]relevantSection, int, retrieval.Usage) { + type docPick struct { + secs []relevantSection + usage retrieval.Usage + } var ( mu sync.Mutex - out = make([]docAnswer, 0, len(docIDs)) - usage retrieval.Usage + picks = make(map[tree.DocumentID]docPick, len(docIDs)) ) - budget := retrieval.ContextBudget{ModelName: model, MaxTokens: 100000, ReservedForPrompt: 4000} g, gctx := errgroup.WithContext(ctx) - g.SetLimit(storeAnswerMapConcurrency) + g.SetLimit(storeMapConcurrency) for _, docID := range docIDs { g.Go(func() error { t, err := h.db.LoadTree(gctx, docID, orgID, storeID) - if err != nil || t == nil { + if err != nil || t == nil || t.Root == nil { h.logger.Warn("answer/store: load tree", "doc", docID, "err", err) return nil } - // Per-request copy so a hop override never mutates the shared - // strategy other goroutines are reading. - perReq := *h.treewalk - perReq.OnEvent = nil // non-streaming - if maxDepth > 0 { - perReq.MaxHops = maxDepth + manifest, refs := buildSectionManifest(t.Root) + if len(refs) == 0 { + return nil } - res, err := perReq.SelectWithCost(gctx, t, query, budget) + idxs, u, err := h.selectSections(gctx, model, query, t.Title, manifest, len(refs)) if err != nil { - h.logger.Warn("answer/store: treewalk", "doc", docID, "err", err) + h.logger.Warn("answer/store: select", "doc", docID, "err", err) return nil } - // Resolve cited page ranges → section ids + a page span. - var ( - secIDs []tree.SectionID - minStart, maxEnd int - seen = map[tree.SectionID]struct{}{} - ) - for _, pr := range res.CitedPages { - if minStart == 0 || pr[0] < minStart { - minStart = pr[0] + // Resolve selected indices → content-bearing sections + // (a selected internal node pulls its content leaves). + var out []relevantSection + seen := map[tree.SectionID]struct{}{} + for _, n := range idxs { + if n < 1 || n > len(refs) { + continue } - if pr[1] > maxEnd { - maxEnd = pr[1] + sec := t.FindByID(refs[n-1].id) + if sec == nil { + continue } - for _, id := range retrieval.SectionIDsOverlapping(t, pr[0], pr[1]) { - if _, dup := seen[id]; !dup { - seen[id] = struct{}{} - secIDs = append(secIDs, id) + for _, cs := range collectContentSections(sec) { + if _, dup := seen[cs.ID]; dup { + continue + } + seen[cs.ID] = struct{}{} + content := h.loadContent(gctx, cs.ContentRef) + if strings.TrimSpace(content) == "" { + continue } + out = append(out, relevantSection{ + docID: docID, + docTitle: t.Title, + sectionID: cs.ID, + sectionTitle: cs.Title, + startPage: cs.PageStart, + endPage: cs.PageEnd, + content: content, + }) } } mu.Lock() - out = append(out, docAnswer{ - docID: docID, - docTitle: t.Title, - answer: strings.TrimSpace(res.Reasoning), - confidence: res.Confidence, - startPage: minStart, - endPage: maxEnd, - sectionIDs: secIDs, - }) - usage.Add(res.Usage) + picks[docID] = docPick{secs: out, usage: u} mu.Unlock() return nil }) } _ = g.Wait() - // Deterministic order: highest confidence first, then title. - sort.SliceStable(out, func(i, j int) bool { - if out[i].confidence != out[j].confidence { - return out[i].confidence > out[j].confidence + // Assemble in caller-supplied document order for determinism, apply + // the total content budget + optional max_sections, and assign global + // citation indices. + var ( + flat []relevantSection + spent int + matched int + usage retrieval.Usage + idx = 1 + ) + for _, docID := range docIDs { + p, ok := picks[docID] + if !ok { + continue } - return out[i].docTitle < out[j].docTitle - }) - return out, usage + usage.Add(p.usage) + if len(p.secs) == 0 { + continue + } + matched++ + for _, s := range p.secs { + if maxSections > 0 && len(flat) >= maxSections { + break + } + if spent >= storeContentBudget { + break + } + if len(s.content) > storeSectionCharsCap { + s.content = s.content[:storeSectionCharsCap] + } + if remaining := storeContentBudget - spent; len(s.content) > remaining { + s.content = s.content[:remaining] + } + spent += len(s.content) + s.index = idx + idx++ + flat = append(flat, s) + } + } + return flat, matched, usage +} + +// selectSections asks the model, over a document's structure+summary +// outline (NOT its full text), which sections' full content is relevant. +func (h *AnswerStoreHandler) selectSections(ctx context.Context, model, query, docTitle, manifest string, maxIdx int) ([]int, retrieval.Usage, error) { + req := llmgate.Request{ + Model: model, + MaxTokens: 256, + Temperature: 0, + Messages: []llmgate.Message{ + {Role: llmgate.RoleSystem, Content: storeSelectSystemPrompt}, + {Role: llmgate.RoleUser, Content: fmt.Sprintf("QUESTION:\n%s\n\nDOCUMENT: %s\nSECTION OUTLINE (number, title, summary):\n%s\nReply with ONLY the JSON object.", query, docTitle, manifest)}, + }, + } + resp, err := h.llm.Complete(ctx, req) + if err != nil { + return nil, retrieval.Usage{}, err + } + u := retrieval.Usage{ + InputTokens: resp.Usage.InputTokens, + OutputTokens: resp.Usage.OutputTokens, + TotalTokens: resp.Usage.TotalTokens, + CostUSD: resp.Usage.CostUSD, + LLMCalls: 1, + } + return parseRelevant(resp.Content, maxIdx), u, nil } -// synthesise runs the single reducer LLM call over the per-document -// answers. Returns the final answer (with [n] markers), the cited doc -// indices, and usage. -func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string, docs []docAnswer, maxTokens int) (string, []int, retrieval.Usage, error) { +const storeSelectSystemPrompt = `You are given a QUESTION and the OUTLINE of ONE document — its sections, each with a number in brackets [n], a title, and a one-line summary. You are NOT given the section text. + +Identify which sections' FULL text you would need to read to answer the question. Choose by relevance, favouring precision: pick the sections that actually bear on the question, not every loosely related one. If none are relevant, return an empty list. + +Reply with EXACTLY one JSON object, no markdown fences: +{"relevant":[]}` + +// generate produces the final answer from the FULL content of the +// selected sections, citing the sections it uses with [n] markers. +func (h *AnswerStoreHandler) generate(ctx context.Context, model, query string, sections []relevantSection, maxTokens int) (string, []int, retrieval.Usage, error) { var ev strings.Builder - for _, d := range docs { - title := d.docTitle + for _, s := range sections { + title := s.docTitle if title == "" { - title = string(d.docID) + title = string(s.docID) + } + fmt.Fprintf(&ev, "[%d] %s", s.index, title) + if s.sectionTitle != "" { + fmt.Fprintf(&ev, " › %s", s.sectionTitle) } - fmt.Fprintf(&ev, "[%d] %s", d.index, title) - if p := formatPageRange(d.startPage, d.endPage); p != "" { + if p := formatPageRange(s.startPage, s.endPage); p != "" { fmt.Fprintf(&ev, " (%s)", p) } ev.WriteString(":\n") - ev.WriteString(d.answer) + ev.WriteString(strings.TrimSpace(s.content)) ev.WriteString("\n\n") } @@ -285,8 +350,8 @@ func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string MaxTokens: maxTokens, Temperature: 0, Messages: []llmgate.Message{ - {Role: llmgate.RoleSystem, Content: storeAnswerSystemPrompt}, - {Role: llmgate.RoleUser, Content: fmt.Sprintf("QUESTION:\n%s\n\nPER-DOCUMENT ANSWERS:\n%s\nReply with ONLY the JSON object.", query, ev.String())}, + {Role: llmgate.RoleSystem, Content: storeGenerateSystemPrompt}, + {Role: llmgate.RoleUser, Content: fmt.Sprintf("QUESTION:\n%s\n\nSECTIONS:\n%s\nReply with ONLY the JSON object.", query, ev.String())}, }, } resp, err := h.llm.Complete(ctx, req) @@ -300,34 +365,114 @@ func (h *AnswerStoreHandler) synthesise(ctx context.Context, model, query string CostUSD: resp.Usage.CostUSD, LLMCalls: 1, } - answer, cited := parseStoreAnswer(resp.Content, len(docs)) + answer, cited := parseStoreAnswer(resp.Content, len(sections)) return answer, cited, usage, nil } -const storeAnswerSystemPrompt = `You are given a QUESTION and a set of ANSWERS, each produced by carefully reading ONE document in a collection. Each is prefixed with a number in brackets, e.g. [1], [2], and its source title. - -Your job is to write ONE coherent answer to the question by synthesising across these per-document answers. +const storeGenerateSystemPrompt = `You answer a QUESTION using ONLY the supplied SECTIONS, which are the full text of the relevant sections drawn from several documents in a collection. Each section is prefixed with a number in brackets [n] and its document title / page range. RULES: -- Use ONLY the supplied per-document answers. Do not invent facts. -- Cite the document(s) you draw on inline with their bracketed number, e.g. "Consent must be freely given [1]." Cite every claim. -- Synthesise: when several documents contribute, combine them into one answer and note agreements or differences. When only some are relevant, use only those. -- If none of the answers actually address the question, say so plainly. +- Answer ONLY from the sections. Never invent facts. If they do not answer the question, say so plainly. +- Cite every claim inline with the bracketed number(s) of the section(s) it rests on, e.g. "Consent must be freely given [1]." +- Synthesise across documents when the answer draws on more than one — one coherent answer over the collection. - Be concise and well-structured. Reply with EXACTLY one JSON object, no markdown fences: -{"answer":"","cited":[]}` +{"answer":"","cited":[]}` + +// buildSectionManifest renders a document's structure+summary outline as +// a numbered, indented list and returns the render plus the ordered +// index→section refs. Every non-root section gets a number; the LLM +// selects by number and we map back via refs. +func buildSectionManifest(root *tree.Section) (string, []manifestRef) { + var ( + b strings.Builder + refs []manifestRef + ) + var walk func(s *tree.Section, depth int) + walk = func(s *tree.Section, depth int) { + for _, c := range s.Children { + refs = append(refs, manifestRef{id: c.ID, hasContent: strings.TrimSpace(c.ContentRef) != ""}) + n := len(refs) + title := strings.TrimSpace(c.Title) + if title == "" { + title = string(c.ID) + } + fmt.Fprintf(&b, "[%d]%s %s", n, strings.Repeat(" ", depth), title) + if pr := formatPageRange(c.PageStart, c.PageEnd); pr != "" { + fmt.Fprintf(&b, " (%s)", pr) + } + if sum := strings.Join(strings.Fields(c.Summary), " "); sum != "" { + fmt.Fprintf(&b, " — %s", sum) + } + b.WriteByte('\n') + walk(c, depth+1) + } + } + walk(root, 0) + return b.String(), refs +} + +// collectContentSections returns the content-bearing sections rooted at +// sec: sec itself if it has content, plus any descendants that do. A +// selected heading therefore pulls the full text of its subsection. +func collectContentSections(sec *tree.Section) []*tree.Section { + var out []*tree.Section + var walk func(s *tree.Section) + walk = func(s *tree.Section) { + if strings.TrimSpace(s.ContentRef) != "" { + out = append(out, s) + } + for _, c := range s.Children { + walk(c) + } + } + walk(sec) + return out +} + +func (h *AnswerStoreHandler) loadContent(ctx context.Context, ref string) string { + rc, _, err := h.storage.Get(ctx, ref) + if err != nil { + return "" + } + defer func() { _ = rc.Close() }() + raw, err := io.ReadAll(rc) + if err != nil { + return "" + } + return string(raw) +} + +// ── output parsing + citation building ───────────────────────────── + +type relevantJSON struct { + Relevant []int `json:"relevant"` +} + +// parseRelevant extracts the selected section numbers from the selector's +// reply, tolerant of code fences / prose, clamped to [1,maxIdx]. +func parseRelevant(raw string, maxIdx int) []int { + s := strings.TrimSpace(raw) + if i := strings.IndexByte(s, '{'); i >= 0 { + if j := strings.LastIndexByte(s, '}'); j > i { + var parsed relevantJSON + if err := json.Unmarshal([]byte(s[i:j+1]), &parsed); err == nil { + return dedupeInRange(parsed.Relevant, maxIdx) + } + } + } + return dedupeInRange(scrapeMarkers(s, maxIdx), maxIdx) +} -// storeAnswerJSON is the reducer's expected output shape. type storeAnswerJSON struct { Answer string `json:"answer"` Cited []int `json:"cited"` } -// parseStoreAnswer extracts the answer + cited indices from the model's -// reply, tolerant of code fences / surrounding prose. Falls back to the -// raw text (and citation markers scraped from it) when the JSON can't be -// parsed, so a formatting slip never drops the answer. +// parseStoreAnswer extracts the answer + cited indices from the generator +// reply, tolerant of code fences / prose. Falls back to the raw text (and +// scraped [n] markers) so a formatting slip never drops the answer. func parseStoreAnswer(raw string, maxIdx int) (string, []int) { s := strings.TrimSpace(raw) if i := strings.IndexByte(s, '{'); i >= 0 { @@ -358,7 +503,6 @@ func dedupeInRange(idx []int, maxIdx int) []int { return out } -// scrapeMarkers pulls [n] indices out of answer prose (fallback path). func scrapeMarkers(s string, maxIdx int) []int { var nums []int for i := 0; i < len(s); i++ { @@ -378,57 +522,46 @@ func scrapeMarkers(s string, maxIdx int) []int { return dedupeInRange(nums, maxIdx) } -// buildStoreCitations turns the cited document indices into the response -// citations array — one citation per contributing document, carrying its -// id, title, page span, section ids, and a short quote from its answer. -func buildStoreCitations(docs []docAnswer, cited []int) []map[string]any { - byIdx := make(map[int]docAnswer, len(docs)) - for _, d := range docs { - byIdx[d.index] = d - } - if len(cited) == 0 { // model cited nothing parseable → show all sources - for _, d := range docs { - cited = append(cited, d.index) +// buildStoreCitations turns cited section indices into the response +// citations — one per cited section, carrying document + section +// provenance and a short quote from the section's full content. +func buildStoreCitations(sections []relevantSection, cited []int) []map[string]any { + byIdx := make(map[int]relevantSection, len(sections)) + for _, s := range sections { + byIdx[s.index] = s + } + if len(cited) == 0 { // generator cited nothing parseable → show all sources + for _, s := range sections { + cited = append(cited, s.index) } } out := make([]map[string]any, 0, len(cited)) for _, n := range cited { - d, ok := byIdx[n] + s, ok := byIdx[n] if !ok { continue } c := map[string]any{ - "index": d.index, - "document_id": d.docID, - "section_ids": d.sectionIDs, - "quote": snippet(d.answer, storeAnswerQuoteLen), - "confidence": d.confidence, + "index": s.index, + "document_id": s.docID, + "section_ids": []tree.SectionID{s.sectionID}, + "quote": snippet(s.content, storeAnswerQuoteLen), + } + if s.docTitle != "" { + c["document_title"] = s.docTitle } - if d.docTitle != "" { - c["document_title"] = d.docTitle + if s.sectionTitle != "" { + c["section_title"] = s.sectionTitle } - if d.startPage > 0 { - c["start_page"] = d.startPage - c["end_page"] = d.endPage + if s.startPage > 0 { + c["start_page"] = s.startPage + c["end_page"] = s.endPage } out = append(out, c) } return out } -// isNonAnswer reports whether a per-document tree-walk answer is a refusal -// or empty, so it can be dropped from the synthesis. -func isNonAnswer(s string) bool { - s = strings.TrimSpace(strings.ToLower(s)) - if len(s) < 3 { - return true - } - return strings.Contains(s, "does not address") || - strings.Contains(s, "not address this query") || - strings.Contains(s, "no relevant information") || - strings.Contains(s, "does not contain") -} - func snippet(s string, n int) string { s = strings.TrimSpace(strings.Join(strings.Fields(s), " ")) if len(s) <= n { diff --git a/internal/handler/answer_store_test.go b/internal/handler/answer_store_test.go index e0615d2..3a81505 100644 --- a/internal/handler/answer_store_test.go +++ b/internal/handler/answer_store_test.go @@ -45,13 +45,13 @@ func TestParseStoreAnswer(t *testing.T) { } func TestBuildStoreCitations(t *testing.T) { - docs := []docAnswer{ - {index: 1, docID: "doc_a", docTitle: "GDPR", startPage: 32, endPage: 33, sectionIDs: []tree.SectionID{"sec_1"}, answer: "Consent must be freely given, specific, informed and unambiguous."}, - {index: 2, docID: "doc_b", docTitle: "PRISMA", startPage: 4, endPage: 4, sectionIDs: []tree.SectionID{"sec_2"}, answer: "Report the review using the checklist."}, + secs := []relevantSection{ + {index: 1, docID: "doc_a", docTitle: "GDPR", sectionID: "sec_1", sectionTitle: "Consent", startPage: 32, endPage: 33, content: "Consent must be freely given, specific, informed and unambiguous."}, + {index: 2, docID: "doc_b", docTitle: "PRISMA", sectionID: "sec_2", sectionTitle: "Reporting", startPage: 4, endPage: 4, content: "Report the review using the checklist."}, } t.Run("cited subset in order with metadata", func(t *testing.T) { - cits := buildStoreCitations(docs, []int{2}) + cits := buildStoreCitations(secs, []int{2}) if len(cits) != 1 { t.Fatalf("want 1 citation, got %d", len(cits)) } @@ -63,14 +63,24 @@ func TestBuildStoreCitations(t *testing.T) { } }) - t.Run("empty cited falls back to all documents", func(t *testing.T) { - cits := buildStoreCitations(docs, nil) + t.Run("empty cited falls back to all sections", func(t *testing.T) { + cits := buildStoreCitations(secs, nil) if len(cits) != 2 { t.Fatalf("want 2 (fallback to all), got %d", len(cits)) } }) } +func TestParseRelevant(t *testing.T) { + got := parseRelevant(`{"relevant":[2,5,2,99,0]}`, 6) + if len(got) != 2 || got[0] != 2 || got[1] != 5 { + t.Errorf("parseRelevant = %v, want [2 5]", got) + } + if len(parseRelevant("nonsense no json", 6)) != 0 { + t.Errorf("expected empty on non-JSON with no markers") + } +} + func TestFormatPageRange(t *testing.T) { cases := []struct { s, e int diff --git a/internal/handler/router.go b/internal/handler/router.go index f034ed1..106acdf 100644 --- a/internal/handler/router.go +++ b/internal/handler/router.go @@ -147,7 +147,7 @@ func Router(d Deps) http.Handler { queryStreamMulti := NewQueryStreamMultiHandler(d.Logger, d.Storage, d.MultiDoc) answer := NewAnswerHandler(d.Logger, d.DB, d.Storage, d.Strategy, d.LLM, d.LLMModel, d.AnswerSpan, d.Answer, d.Replay) answerTreeWalk := NewAnswerTreeWalkHandler(d.Logger, d.DB, d.Storage, d.LLM, d.LLMModel, d.AnswerSpan, d.Replay, d.TreeWalkStrategy, d.TreeWalk) - answerStore := NewAnswerStoreHandler(d.Logger, d.DB, d.TreeWalkStrategy, d.TreeWalk, d.LLM, d.LLMModel) + answerStore := NewAnswerStoreHandler(d.Logger, d.DB, d.Storage, d.LLM, d.LLMModel) webhook := NewWebhookHandler(d.Logger, d.Queue) // ── Connect-RPC Handlers (generated stubs, three-transport) ───