diff --git a/internal/handler/documents.go b/internal/handler/documents.go index 80dca58..07d48c8 100644 --- a/internal/handler/documents.go +++ b/internal/handler/documents.go @@ -178,10 +178,11 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R } var ( - filename string - contentType string - body io.Reader - size int64 + filename string + contentType string + body io.Reader + size int64 + titleOverride string ) ct := r.Header.Get("Content-Type") @@ -201,12 +202,15 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R contentType = header.Header.Get("Content-Type") body = file size = header.Size + // Optional multipart "title" field overrides the discovered title. + titleOverride = strings.TrimSpace(r.FormValue("title")) case strings.HasPrefix(ct, "application/json"): var payload struct { Filename string `json:"filename"` ContentType string `json:"content_type"` Content string `json:"content"` + Title string `json:"title"` } if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { writeErr(w, http.StatusBadRequest, "invalid json: "+err.Error()) @@ -220,6 +224,7 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R contentType = payload.ContentType body = strings.NewReader(payload.Content) size = int64(len(payload.Content)) + titleOverride = strings.TrimSpace(payload.Title) default: writeErr(w, http.StatusUnsupportedMediaType, @@ -240,7 +245,10 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R return } - title := filename + title := titleOverride + if title == "" { + title = filename + } if title == "" { title = string(docID) } @@ -281,6 +289,7 @@ func (h *DocumentsHandler) HandleIngestDocument(w http.ResponseWriter, r *http.R ContentType: contentType, Filename: filename, SourceRef: key, + Title: titleOverride, Profile: r.Header.Get("X-Vectorless-Profile"), }) if err := h.queue.Enqueue(ctx, queue.Job{ diff --git a/pkg/ingest/ingest.go b/pkg/ingest/ingest.go index 4df956c..f4ab4f1 100644 --- a/pkg/ingest/ingest.go +++ b/pkg/ingest/ingest.go @@ -67,6 +67,13 @@ type Payload struct { ContentType string `json:"content_type"` Filename string `json:"filename"` SourceRef string `json:"source_ref"` // storage key of the original bytes + // Title, when set, is an explicit caller-supplied document title that + // overrides whatever the parser discovers from the content. It is + // "sticky": persistTree will NOT overwrite it with the parsed title. + // Useful when the PDF's own title text is unreliable (rotated arXiv + // margin stamps, ORCID iD markers, letter-spaced cover pages) or the + // caller simply wants a curated display name. Empty = auto-discover. + Title string `json:"title,omitempty"` // Profile selects domain-aware structuring/summarization prompts // ("generic", "research", "medical"). Empty = generic. Sourced from // the document's store (the control plane injects X-Vectorless-Profile). @@ -376,7 +383,7 @@ func (p *Pipeline) Run(ctx context.Context, pl Payload) error { } log.Info("ingest: parsed", "sections", len(parsed.Flatten()), "title", parsed.Title) - if err := p.persistTree(ctx, p.DB, pl.DocumentID, parsed); err != nil { + if err := p.persistTree(ctx, p.DB, pl.DocumentID, parsed, pl.Title); err != nil { p.fail(ctx, p.DB, pl.DocumentID, "persist tree", err) return err } @@ -674,7 +681,7 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa } log.Info("ingest: parsed", "sections", len(parsed.Flatten()), "title", parsed.Title) - if err := p.persistTree(ctx, store, pl.DocumentID, parsed); err != nil { + if err := p.persistTree(ctx, store, pl.DocumentID, parsed, pl.Title); err != nil { p.fail(ctx, store, pl.DocumentID, "persist tree", err) return err } @@ -695,13 +702,18 @@ func (p *Pipeline) runMinimal(ctx context.Context, store docPersister, pl Payloa // The DB operations go through the narrow docPersister interface so the // persist path can be exercised (e.g. by the minimal-mode test) without // a live Postgres; production callers pass p.DB, which satisfies it. -func (p *Pipeline) persistTree(ctx context.Context, store docPersister, docID tree.DocumentID, doc *parser.ParsedDoc) error { - // Only overwrite the row's title (which was seeded with the - // filename at upload time) when the parsed title looks usable. - // Watermarked PDFs whose overlay text shares a Y coordinate with - // the real title produce mojibake like "GGlloobbaall SSttrraatteeggyy" - // — we'd rather keep the original filename than show that to a user. - if doc.Title != "" && !isLikelyMojibakeTitle(doc.Title) { +func (p *Pipeline) persistTree(ctx context.Context, store docPersister, docID tree.DocumentID, doc *parser.ParsedDoc, titleOverride string) error { + // An explicit caller-supplied title is sticky: keep it and never let + // the parsed title clobber it. The row was already seeded with this + // value at upload time, so there is nothing to write here. + if strings.TrimSpace(titleOverride) != "" { + // no-op: the override is already the row's title. + } else if doc.Title != "" && !isLikelyMojibakeTitle(doc.Title) { + // Otherwise only overwrite the row's title (seeded with the + // filename at upload time) when the parsed title looks usable. + // Watermarked PDFs whose overlay text shares a Y coordinate with + // the real title produce mojibake like "GGlloobbaall SSttrraatteeggyy" + // — we'd rather keep the original filename than show that to a user. if err := store.SetDocumentTitle(ctx, docID, doc.Title); err != nil { return err } diff --git a/pkg/ingest/persist_content_ref_test.go b/pkg/ingest/persist_content_ref_test.go index 8627a0a..0496352 100644 --- a/pkg/ingest/persist_content_ref_test.go +++ b/pkg/ingest/persist_content_ref_test.go @@ -9,6 +9,43 @@ import ( "github.com/hallelx2/vectorless-engine/pkg/storage" ) +// TestPersistTree_TitleOverrideIsSticky verifies that an explicit +// caller-supplied title is never clobbered by the parsed title, while a +// blank override still lets a usable parsed title through. +func TestPersistTree_TitleOverrideIsSticky(t *testing.T) { + store, err := storage.NewLocal(t.TempDir()) + if err != nil { + t.Fatalf("NewLocal: %v", err) + } + p := &Pipeline{Storage: store} + doc := &parser.ParsedDoc{ + Title: "Some Parsed Title", + Sections: []parser.Section{ + {Level: 1, Title: "S", Content: "body", PageStart: 1, PageEnd: 1}, + }, + } + + // With an override, persistTree must NOT push the parsed title (the row + // already carries the override from upload time → SetDocumentTitle + // stays uncalled, so the fake's title remains empty). + fake := &fakeDocStore{} + if err := p.persistTree(context.Background(), fake, "doc_x", doc, "Attention Is All You Need"); err != nil { + t.Fatalf("persistTree (override): %v", err) + } + if fake.title != "" { + t.Errorf("override present: parsed title must not overwrite it; SetDocumentTitle called with %q", fake.title) + } + + // With no override, a usable parsed title IS applied. + fake2 := &fakeDocStore{} + if err := p.persistTree(context.Background(), fake2, "doc_y", doc, ""); err != nil { + t.Fatalf("persistTree (no override): %v", err) + } + if fake2.title != "Some Parsed Title" { + t.Errorf("no override: parsed title should apply, got %q", fake2.title) + } +} + // TestPersistTree_ContentRefMatchesStoredObjects is the HAL-316 regression: // a leaf only gets a ContentRef when its content was actually written. An // empty-after-clean leaf must get NO ref (and no stored object), so later @@ -35,7 +72,7 @@ func TestPersistTree_ContentRefMatchesStoredObjects(t *testing.T) { }, } - if err := p.persistTree(context.Background(), fake, "doc_x", doc); err != nil { + if err := p.persistTree(context.Background(), fake, "doc_x", doc, ""); err != nil { t.Fatalf("persistTree: %v", err) }