diff --git a/pkg/parser/pdf.go b/pkg/parser/pdf.go index 168397a..9cc961f 100644 --- a/pkg/parser/pdf.go +++ b/pkg/parser/pdf.go @@ -1152,6 +1152,17 @@ func extractPDFRows(doc pdftable.Document) ([]pdfRow, error) { return b } for _, wd := range words { + // Drop rotated / vertical text runs — arXiv left-margin + // stamps, page-edge watermarks, rotated figure labels. These + // are page furniture, not body prose, and pdftable reads a + // rotated run in an order that surfaces as reversed text + // ("3202 guA" for a rotated "Aug 2023", "]LC.sc[" for + // "[cs.CL]"), which then pollutes the outline with junk + // headings/titles. Upright left-to-right / right-to-left + // prose (including real RTL languages) is kept. + if isRotatedRun(wd.Upright, wd.Direction) { + continue + } b := find(wd.Y1) b.words = append(b.words, wd) if wd.FontSize > b.maxFS { @@ -1531,6 +1542,24 @@ func looksLikeHeading(s string) bool { return true } +// isRotatedRun reports whether a pdftable word run is rotated / vertical +// text — page furniture (margin stamps, watermarks, rotated labels) +// rather than body prose. pdftable marks such a run as not Upright and/or +// with a vertical reading Direction ("ttb"/"btt"); it reads them in an +// order that surfaces as reversed text, so dropping them keeps the outline +// clean. Upright horizontal prose in either LTR or RTL scripts is body +// text and is never dropped. +func isRotatedRun(upright bool, direction string) bool { + if !upright { + return true + } + switch strings.ToLower(direction) { + case "ttb", "btt": + return true + } + return false +} + // isBoldFont reports whether a PDF font name denotes a bold weight. SEC filing // section headings are typically bold at body font size (not larger), so this is // how we recover them — a size-only heuristic misses them entirely. diff --git a/pkg/parser/rotated_text_test.go b/pkg/parser/rotated_text_test.go new file mode 100644 index 0000000..adcc5ed --- /dev/null +++ b/pkg/parser/rotated_text_test.go @@ -0,0 +1,31 @@ +package parser + +import "testing" + +// TestIsRotatedRun locks the orientation filter that keeps rotated / +// vertical page furniture (arXiv margin stamps, watermarks) out of the +// extracted row stream while preserving upright prose in any script. +func TestIsRotatedRun(t *testing.T) { + cases := []struct { + name string + upright bool + direction string + want bool + }{ + {"upright ltr prose is kept", true, "ltr", false}, + {"upright rtl prose is kept", true, "rtl", false}, + {"upright with empty direction is kept", true, "", false}, + {"not upright is dropped", false, "ltr", true}, + {"vertical top-to-bottom is dropped", true, "ttb", true}, + {"vertical bottom-to-top is dropped", true, "btt", true}, + {"vertical, case-insensitive", true, "TTB", true}, + {"not upright AND vertical is dropped", false, "btt", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isRotatedRun(tc.upright, tc.direction); got != tc.want { + t.Errorf("isRotatedRun(%v, %q) = %v, want %v", tc.upright, tc.direction, got, tc.want) + } + }) + } +}