From 6923195da2d10621b230b236283cfda291709843 Mon Sep 17 00:00:00 2001 From: "youchang.kim" Date: Wed, 22 Jul 2026 20:18:21 +0900 Subject: [PATCH 1/2] table: opt-in layout union, grid refinement, merged-cell/header model, to_html Adds an opt-in table model to page.find_tables(): - use_layout= makes the 1.28.0 layout gating explicit (default True, behavior unchanged). - refine= splits rows merged by the ruling-line grid (using cell background shading), under-segmented columns and over-merged body rows, then resolves merged-cell structure into Table.placements -- a row-major grid of SpanCell placements carrying colspan/rowspan and td/th tags -- with header metadata in Table.header_rows and Table.section_rows. - union= (with use_layout=True) fuses the layout analyzer's table grids with the line finder's candidates: grid replacement, split of over-wide grids, append of tables the layout missed. - Table.to_html() serializes the tagged model, as a companion to to_markdown() and to_pandas(). Also one always-on correctness fix: the detector's global CHARS/EDGES working state becomes call-local (ContextVar) and every Table snapshots its characters -- in released 1.28.0, re-extracting a table after a later find_tables() call returns the later call's text. A deterministic regression test is included. The model lives in four private sibling modules (_table_refine, _table_spans, _table_union, _table_headers), re-exported through pymupdf.table. All new keywords default off; default-path output is byte-identical (verified by output hashing) at ~+1% find_tables timing. On ParseBench's table group (503 pages, GTRM) the official stack scores 56.73 and the opted-in stack 72.11 together with the companion pymupdf4llm change. Adds 30 tests in tests/test_tables.py. --- docs/page.rst | 4 +- setup.py | 4 + src/_table_headers.py | 976 ++++++++++++++++++++++++++++++++++++++++++ src/_table_refine.py | 867 +++++++++++++++++++++++++++++++++++++ src/_table_spans.py | 596 ++++++++++++++++++++++++++ src/_table_union.py | 364 ++++++++++++++++ src/table.py | 433 +++++++++++++++---- tests/test_pylint.py | 4 + tests/test_tables.py | 489 +++++++++++++++++++++ 9 files changed, 3663 insertions(+), 74 deletions(-) create mode 100644 src/_table_headers.py create mode 100644 src/_table_refine.py create mode 100644 src/_table_spans.py create mode 100644 src/_table_union.py diff --git a/docs/page.rst b/docs/page.rst index 4c4c9ef4f..0bc029f44 100644 --- a/docs/page.rst +++ b/docs/page.rst @@ -452,7 +452,7 @@ In a nutshell, this is what you can do with PyMuPDF: :arg bool final_filter: If `True` (default), the method will to remove rectangles having width or height smaller than the respective tolerance value. If `False` no such filtering is done. - .. method:: find_tables(clip=None, strategy=None, vertical_strategy=None, horizontal_strategy=None, vertical_lines=None, horizontal_lines=None, snap_tolerance=None, snap_x_tolerance=None, snap_y_tolerance=None, join_tolerance=None, join_x_tolerance=None, join_y_tolerance=None, edge_min_length=3, min_words_vertical=3, min_words_horizontal=1, intersection_tolerance=None, intersection_x_tolerance=None, intersection_y_tolerance=None, text_tolerance=None, text_x_tolerance=None, text_y_tolerance=None, add_lines=None, add_boxes=None, paths=None) + .. method:: find_tables(clip=None, strategy=None, vertical_strategy=None, horizontal_strategy=None, vertical_lines=None, horizontal_lines=None, snap_tolerance=None, snap_x_tolerance=None, snap_y_tolerance=None, join_tolerance=None, join_x_tolerance=None, join_y_tolerance=None, edge_min_length=3, min_words_vertical=3, min_words_horizontal=1, intersection_tolerance=None, intersection_x_tolerance=None, intersection_y_tolerance=None, text_tolerance=None, text_x_tolerance=None, text_y_tolerance=None, add_lines=None, add_boxes=None, paths=None, use_layout=True) Find tables on the page and return an object with related information. Typically, the default values of the many parameters will be sufficient. Adjustments should ever only be needed in corner case situations. @@ -492,6 +492,8 @@ In a nutshell, this is what you can do with PyMuPDF: :arg list paths: list of vector graphics in the format as returned be :meth:`Page.get_drawings`. Using this parameter will prevent the method to extract vector graphics itself. This is useful if the vector graphics are already available. This can save execution time significantly. + :arg bool use_layout: use layout analysis to gate line-based table candidates. Set to `False` for pure line-based detection. + .. image:: images/img-findtables.* :returns: a `TableFinder` object that has the following significant attributes: diff --git a/setup.py b/setup.py index 07e16a250..f0d7a9f83 100755 --- a/setup.py +++ b/setup.py @@ -623,6 +623,10 @@ def build(): ret.append( (f'{g_root}/src/__main__.py', to_dir) ) ret.append( (f'{g_root}/src/pymupdf.py', to_dir) ) ret.append( (f'{g_root}/src/table.py', to_dir) ) + ret.append( (f'{g_root}/src/_table_refine.py', to_dir) ) + ret.append( (f'{g_root}/src/_table_spans.py', to_dir) ) + ret.append( (f'{g_root}/src/_table_union.py', to_dir) ) + ret.append( (f'{g_root}/src/_table_headers.py', to_dir) ) ret.append( (f'{g_root}/src/utils.py', to_dir) ) ret.append( (f'{g_root}/src/_wxcolors.py', to_dir) ) ret.append( (f'{g_root}/src/_apply_pages.py', to_dir) ) diff --git a/src/_table_headers.py b/src/_table_headers.py new file mode 100644 index 000000000..00ae860ca --- /dev/null +++ b/src/_table_headers.py @@ -0,0 +1,976 @@ +""" +Copyright (C) 2023 Artifex Software, Inc. + +This file is part of PyMuPDF. + +PyMuPDF is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License +along with MuPDF. If not, see + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. + +--------------------------------------------------------------------- + +PyMuPDF table header detection and HTML serialization (opt-in extension). + +Pure text-grid module (no pymupdf import): the header-region rules operate on a +row-major ``[[cell text]]`` grid, and the serializer turns a tagged placement +grid into an HTML ````. Used only by find_tables(refine=True) (via +pymupdf.table) and Table.to_html(); never runs on the default detection path. +""" +from __future__ import annotations +from dataclasses import dataclass +from statistics import mean +from typing import Any +import re + + + +_TOKEN_RE = re.compile(r"[A-Za-z]+|\d+(?:[.,:/-]\d+)*|[%$€£¥()–—-]+") +_MONTH_RE = re.compile( + r"\b(?:jan|feb|mar|apr|may|jun|jul|aug|sep|sept|oct|nov|dec)[a-z]*\.?\b", + re.IGNORECASE, +) +_MONEY_RE = re.compile(r"(?:[$€£¥]\s*[-(]?\d|(?:usd|eur|gbp|jpy)\b)", re.IGNORECASE) +_PERCENT_RE = re.compile(r"[-(]?\d+(?:\.\d+)?\s*%") +_YEAR_RE = re.compile(r"\b(?:19|20)\d{2}\b") +_CODE_RE = re.compile(r"^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9][A-Za-z0-9._/# -]{0,24}$") +_RANGE_RE = re.compile(r"(?:>=|<=|<|>| thru | through | to |\b\d+\s*[-–]\s*\d+\b)", re.IGNORECASE) +_DOTTED_ENUM_RE = re.compile(r"^\s*\d+(?:\.\d+){2,}\b") +_REF_TOKEN_RE = re.compile(r"[\[(]\s*\d{1,2}\s*[\])]|\[\s*\d{1,2}\s*\)") +_REF_FORMULA_ALLOWED_RE = re.compile(r"^[\s\d\[\]\(\)=+\-*/xX.]+$") +_UNIT_ROW_TERMS = ("in million", "in thousand", "in billion", "$ in", "usd", "'000", "%") + +VALUE_TYPES = {"number", "money", "percent"} +LABEL_TYPES = {"text_label", "long_text", "code_id", "date_period", "unit"} + + +def count_alpha(text: str) -> int: + return sum(1 for char in text if char.isalpha()) + + +def count_digit(text: str) -> int: + return sum(1 for char in text if char.isdigit()) + + +def tokens(text: str) -> list[str]: + return _TOKEN_RE.findall(text) + + +def numeric_like(text: str) -> bool: + stripped = text.strip() + if not stripped: + return False + alpha = count_alpha(stripped) + digit = count_digit(stripped) + if digit == 0: + return False + if _MONTH_RE.search(stripped): + return False + return alpha / max(1, alpha + digit) <= 0.30 + + +def date_or_period_like(text: str) -> bool: + stripped = text.strip() + if not stripped: + return False + return bool(_MONTH_RE.search(stripped) or re.search(r"\b(?:q[1-4]|fy|year|month|period)\b", stripped, re.I)) + + +def cell_type(text: str) -> str: + stripped = " ".join(text.strip().split()) + if not stripped: + return "empty" + + alpha = count_alpha(stripped) + digit = count_digit(stripped) + token_count = len(tokens(stripped)) + lowered = stripped.lower() + + if _MONEY_RE.search(stripped): + return "money" + if _PERCENT_RE.fullmatch(stripped) or (stripped.endswith("%") and digit > 0 and alpha == 0): + return "percent" + if date_or_period_like(stripped) or (_YEAR_RE.fullmatch(stripped) and token_count == 1): + return "date_period" + if _RANGE_RE.search(stripped) and digit > 0 and token_count <= 8: + return "range" + if numeric_like(stripped): + return "number" + if lowered in {"%", "$", "$000", "$m", "usd", "eur", "gbp", "amount", "rate", "ratio"}: + return "unit" + if _CODE_RE.match(stripped) and token_count <= 4: + return "code_id" + if alpha > 0 and digit > 0 and token_count <= 6: + return "code_id" + if token_count >= 8: + return "long_text" + return "text_label" + + +def cell_type_ratios(texts: list[str]) -> dict[str, float]: + nonempty_types = [cell_type(text) for text in texts if text.strip()] + if not nonempty_types: + return { + "value_type_ratio": 0.0, + "label_type_ratio": 0.0, + "long_text_type_ratio": 0.0, + "code_type_ratio": 0.0, + "date_period_type_ratio": 0.0, + "range_type_ratio": 0.0, + } + total = len(nonempty_types) + return { + "value_type_ratio": sum(1 for item in nonempty_types if item in VALUE_TYPES) / total, + "label_type_ratio": sum(1 for item in nonempty_types if item in LABEL_TYPES) / total, + "long_text_type_ratio": sum(1 for item in nonempty_types if item == "long_text") / total, + "code_type_ratio": sum(1 for item in nonempty_types if item == "code_id") / total, + "date_period_type_ratio": sum(1 for item in nonempty_types if item == "date_period") / total, + "range_type_ratio": sum(1 for item in nonempty_types if item == "range") / total, + } + + +def row_features(texts: list[str]) -> dict[str, float]: + nonempty = [text for text in texts if text.strip()] + if not texts: + return { + "cell_count": 0.0, + "nonempty_count": 0.0, + "nonempty_ratio": 0.0, + "empty_ratio": 0.0, + "numeric_ratio": 0.0, + "alpha_ratio": 0.0, + "date_period_ratio": 0.0, + "avg_tokens": 0.0, + **cell_type_ratios([]), + } + return { + "cell_count": float(len(texts)), + "nonempty_count": float(len(nonempty)), + "nonempty_ratio": len(nonempty) / len(texts), + "empty_ratio": (len(texts) - len(nonempty)) / len(texts), + "numeric_ratio": sum(1 for text in nonempty if numeric_like(text)) / max(1, len(nonempty)), + "alpha_ratio": sum(1 for text in nonempty if count_alpha(text) > 0) / max(1, len(nonempty)), + "date_period_ratio": sum(1 for text in nonempty if date_or_period_like(text)) / max(1, len(nonempty)), + "avg_tokens": mean([len(tokens(text)) for text in nonempty]) if nonempty else 0.0, + **cell_type_ratios(texts), + } + + +def column_text(row: list[str], col: int) -> str: + return row[col] if col < len(row) else "" + + +def row_vector(texts: list[str]) -> dict[str, float]: + features = row_features(texts) + first = column_text(texts, 0) + rest = [text for text in texts[1:] if text.strip()] + rest_types = [cell_type(text) for text in rest] + first_nonempty = bool(first.strip()) + return { + **features, + "first_empty": 0.0 if first_nonempty else 1.0, + "first_alpha": 1.0 if count_alpha(first) > 0 else 0.0, + "first_numeric": 1.0 if numeric_like(first) else 0.0, + "first_value_type": 1.0 if cell_type(first) in VALUE_TYPES else 0.0, + "first_label_type": 1.0 if cell_type(first) in LABEL_TYPES else 0.0, + "rest_numeric_ratio": sum(1 for text in rest if numeric_like(text)) / max(1, len(rest)), + "rest_alpha_ratio": sum(1 for text in rest if count_alpha(text) > 0) / max(1, len(rest)), + "rest_value_type_ratio": sum(1 for item in rest_types if item in VALUE_TYPES) / max(1, len(rest_types)), + "rest_label_type_ratio": sum(1 for item in rest_types if item in LABEL_TYPES) / max(1, len(rest_types)), + "rest_long_text_type_ratio": sum(1 for item in rest_types if item == "long_text") / max(1, len(rest_types)), + "avg_tokens_norm": min(1.0, features["avg_tokens"] / 8.0), + } + + +def header_candidate_row(row: list[str]) -> bool: + features = row_features(row) + if features["nonempty_count"] == 0: + return False + short_or_sparse = features["avg_tokens"] <= 6.0 or features["empty_ratio"] >= 0.20 + return bool(short_or_sparse) + + +def grouped_header_scaffold(row: list[str]) -> bool: + features = row_features(row) + if features["cell_count"] == 0: + return False + texts = [text.strip() for text in row] + nonempty = [text for text in texts if text] + unique_nonempty = {text.lower() for text in nonempty} + repeated_labels = len(nonempty) >= 3 and len(unique_nonempty) <= max(1, len(nonempty) // 2) + sparse_labels = features["empty_ratio"] >= 0.20 or features["nonempty_ratio"] <= 0.70 + return bool(sparse_labels or repeated_labels) + + +def data_row_like(row: list[str]) -> bool: + vector = row_vector(row) + return bool( + vector["first_alpha"] > 0 + and vector["first_numeric"] == 0 + and vector["rest_numeric_ratio"] >= 0.35 + and vector["numeric_ratio"] >= 0.25 + ) + + +def dense_numeric_row_like(row: list[str]) -> bool: + vector = row_vector(row) + return bool(vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35) + + +def money_or_percent_heavy(row: list[str]) -> bool: + types = [cell_type(text) for text in row if text.strip()] + if not types: + return False + return sum(1 for item in types if item in {"money", "percent"}) / len(types) >= 0.25 + + +def rest_date_period_ratio(row: list[str]) -> float: + rest = [cell for cell in row[1:] if cell.strip()] + if not rest: + return 0.0 + return sum(1 for cell in rest if cell_type(cell) == "date_period") / len(rest) + + +def first_row_period_restore_guard(row: list[str]) -> bool: + features = row_features(row) + return bool(features["date_period_type_ratio"] >= 0.25 or rest_date_period_ratio(row) >= 0.35) + + +def first_row_date_or_unit_restore_guard(row: list[str]) -> bool: + rest = [cell for cell in row[1:] if cell.strip()] + if not rest: + return False + rest_types = [cell_type(cell) for cell in rest] + dp_unit_ratio = sum(1 for item in rest_types if item in {"date_period", "unit"}) / len(rest_types) + money_or_percent = any(item in {"money", "percent"} for item in rest_types) + row_text = " ".join(row).lower() + unit_row = any(term in row_text for term in _UNIT_ROW_TERMS) + return bool((dp_unit_ratio >= 0.50 or unit_row) and not money_or_percent) + + +def first_row_restore_guards(row: list[str]) -> list[str]: + guards = [] + if first_row_period_restore_guard(row): + guards.append("first_row_period_restore") + if first_row_date_or_unit_restore_guard(row): + guards.append("first_row_date_or_unit_restore") + return guards + + +def headerless_dotted_enum_guard(rows: list[list[str]]) -> bool: + if len(rows) < 3: + return False + row = rows[0] + cells = [cell.strip() for cell in row if cell and cell.strip()] + if not cells or not _DOTTED_ENUM_RE.search(cells[0]): + return False + features = row_features(row) + return bool(features["long_text_type_ratio"] >= 0.25 or features["avg_tokens"] >= 5.0) + + + + +def simple_integer_enum_header_like(row: list[str]) -> bool: + vector = row_vector(row) + if vector["first_label_type"] <= 0 or vector["first_value_type"] > 0: + return False + values = [] + for text in [text.strip() for text in row[1:] if text.strip()]: + if not re.fullmatch(r"\d{1,2}", text): + return False + values.append(int(text)) + if len(values) < 2: + return False + ordered = values == sorted(values) + unique = len(set(values)) == len(values) + contiguous = max(values) - min(values) == len(values) - 1 + return bool(ordered and unique and contiguous) + + +def reference_formula_header_row_like(candidate: list[str], following_rows: list[list[str]]) -> bool: + """A bracket/formula reference row under a grouped header, e.g. [3] = [1] * [2].""" + nonempty = [text.strip() for text in candidate if text.strip()] + if len(nonempty) < 3: + return False + if any(any(char.isalpha() for char in text) for text in nonempty): + return False + if any(("$" in text or "%" in text or "," in text) for text in nonempty): + return False + if not all(_REF_FORMULA_ALLOWED_RE.match(text) for text in nonempty): + return False + ref_cells = [text for text in nonempty if _REF_TOKEN_RE.search(text)] + formula_cells = [text for text in nonempty if "=" in text and len(_REF_TOKEN_RE.findall(text)) >= 2] + if len(ref_cells) / len(nonempty) < 0.75: + return False + if not formula_cells and len(ref_cells) < 4: + return False + if len(following_rows) < 2: + return False + + following_profiles = row_profiles(following_rows) + body_features = body_window_features_from_profiles(following_profiles, 0) + if body_window_is_stable(body_features): + return True + if following_profiles[0]["section_label_row"] and len(following_profiles) >= 3: + shifted = body_window_features_from_profiles(following_profiles, 1) + return body_window_is_stable(shifted) + return False + + +def dense_integer_axis_row_like(row: list[str]) -> bool: + values = [] + alpha_cells = 0 + nonempty = [text.strip() for text in row if text.strip()] + for text in nonempty: + if re.fullmatch(r"\d{1,2}", text): + values.append(int(text)) + elif any(char.isalpha() for char in text): + alpha_cells += 1 + else: + return False + if len(values) < 5: + return False + unique = sorted(set(values)) + if unique != list(range(unique[0], unique[-1] + 1)): + return False + if unique[0] not in {0, 1}: + return False + if alpha_cells > max(4, len(nonempty) // 5): + return False + return True + + +def sparse_prefix_extension_prefix_like(row: list[str]) -> bool: + features = row_features(row) + if features["nonempty_count"] == 0: + return False + if features["nonempty_count"] <= 2: + return features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20 + if features["empty_ratio"] < 0.25: + return False + return bool(features["alpha_ratio"] >= 0.50 or features["date_period_ratio"] >= 0.20) + + +def next_header_row_value_guard(candidate: list[str], following_rows: list[list[str]]) -> bool: + features = row_features(candidate) + vector = row_vector(candidate) + if features["nonempty_count"] < 2: + return False + if features["avg_tokens"] > 7.0 or features["long_text_type_ratio"] >= 0.35: + return False + if money_or_percent_heavy(candidate): + return False + if reference_formula_header_row_like(candidate, following_rows): + return True + if features["value_type_ratio"] > 0.85: + return False + + # date_period_type_ratio recognises a bare-year header row that the + # keyword-based date_period_ratio misses; combine them only in this guard. + date_period_signal = max(features["date_period_ratio"], features["date_period_type_ratio"]) + has_label_signal = ( + features["alpha_ratio"] >= 0.30 + or date_period_signal >= 0.20 + or vector["first_label_type"] > 0 + ) + if not has_label_signal: + return False + + if len(following_rows) < 2: + return False + following_profiles = row_profiles(following_rows) + body_features = body_window_features_from_profiles(following_profiles, 0) + if not body_window_is_stable(body_features): + return False + + if features["numeric_ratio"] >= 0.50 or features["value_type_ratio"] >= 0.50: + if vector["first_label_type"] <= 0 and date_period_signal < 0.20: + return False + if vector["first_value_type"] > 0: + return False + + types = [cell_type(text) for text in candidate if text.strip()] + type_count = len(types) + if not type_count: + return False + date_type_ratio = sum(1 for item in types if item == "date_period") / type_count + code_range_number_ratio = sum(1 for item in types if item in {"code_id", "range", "number"}) / type_count + + if features["value_type_ratio"] > 0.25: + return simple_integer_enum_header_like(candidate) + if code_range_number_ratio >= 0.50 and date_type_ratio < 0.30: + return simple_integer_enum_header_like(candidate) + return True + + +def sparse_prefix_conservative_extension_guard(rows: list[list[str]], top_header_rows: int) -> bool: + if top_header_rows != 1 or len(rows) < 3: + return False + prefix = rows[0] + candidate = rows[1] + prefix_features = row_features(prefix) + candidate_features = row_features(candidate) + if prefix_features["empty_ratio"] < 0.50: + return False + if candidate_features["nonempty_count"] < 3 and candidate_features["date_period_ratio"] < 0.50: + return False + if not sparse_prefix_extension_prefix_like(prefix): + return False + if candidate_features["label_type_ratio"] < 0.50: + return False + if candidate_features["value_type_ratio"] >= 0.50: + return False + if section_label_row_like(candidate): + return False + following_profiles = row_profiles(rows[2:]) + body_features = body_window_features_from_profiles(following_profiles, 0) + return body_window_is_stable(body_features) + + +def sparse_prefix_next_header_guard(rows: list[list[str]], top_header_rows: int) -> bool: + """Return True when the first body row should be included as a second header row.""" + if top_header_rows < 1 or top_header_rows >= len(rows) - 1: + return False + return sparse_prefix_conservative_extension_guard(rows, top_header_rows) + + +def rowspan_leaf_header_guard(rows: list[list[str]], top_header_rows: int) -> bool: + """Promote a leaf-label row under a sparse/grouped rowspan header scaffold. + + An empty first cell in the candidate row is the textual shadow of the row-0 + rowspan label; treat it as header only when later rows form a stable body. + """ + if top_header_rows != 1 or len(rows) < 3: + return False + max_cols = max((len(row) for row in rows), default=0) + if max_cols < 3: + return False + + prefix = rows[0] + candidate = rows[1] + prefix_features = row_features(prefix) + candidate_features = row_features(candidate) + candidate_vector = row_vector(candidate) + shifted_by_rowspan = len(candidate) < max_cols + if column_text(candidate, 0).strip() and not shifted_by_rowspan: + return False + sparse_rowspan_prefix = len(prefix) < max_cols and len(prefix) <= max(2, int(max_cols * 0.50)) + if not (grouped_header_scaffold(prefix) or prefix_features["empty_ratio"] >= 0.35 or sparse_rowspan_prefix): + return False + if candidate_features["nonempty_count"] < max(2, int(max_cols * 0.45)): + return False + if candidate_features["value_type_ratio"] > 0.25 or candidate_features["numeric_ratio"] > 0.35: + return False + if candidate_vector["rest_label_type_ratio"] < 0.50 and candidate_features["alpha_ratio"] < 0.50: + return False + if section_label_row_like(candidate) or dense_numeric_row_like(candidate): + return False + + following_profiles = row_profiles(rows[2:]) + body_features = body_window_features_from_profiles(following_profiles, 0) + return body_window_is_stable(body_features) + + +def leading_body_record_like(row: list[str]) -> bool: + vector = row_vector(row) + if vector["nonempty_count"] < 2: + return False + first_is_label = vector["first_label_type"] > 0 and vector["first_value_type"] == 0 + rest_is_values = vector["rest_numeric_ratio"] >= 0.50 or vector["rest_value_type_ratio"] >= 0.50 + value_heavy = vector["numeric_ratio"] >= 0.35 or vector["value_type_ratio"] >= 0.35 + return bool(first_is_label and rest_is_values and value_heavy and not grouped_header_scaffold(row)) + + +def leading_long_form_row_like(row: list[str]) -> bool: + vector = row_vector(row) + return bool( + vector["nonempty_count"] >= 4 + and vector["long_text_type_ratio"] >= 0.30 + and vector["value_type_ratio"] <= 0.20 + and vector["empty_ratio"] <= 0.35 + ) + + +def section_label_row_like(row: list[str]) -> bool: + if len(row) < 2: + return False + features = row_features(row) + first = column_text(row, 0).strip() + nonempty = [text for text in row if text.strip()] + first_only = bool(first and not [text for text in row[1:] if text.strip()]) + sparse_label = ( + features["nonempty_count"] <= 3 + and features["empty_ratio"] >= 0.60 + and features["alpha_ratio"] >= 0.50 + ) + return bool(first_only or (nonempty and sparse_label)) + + +def centered_section_label_row_like(row: list[str]) -> bool: + if not section_label_row_like(row): + return False + nonempty_indices = [idx for idx, text in enumerate(row) if text.strip()] + return bool(len(nonempty_indices) == 1 and nonempty_indices[0] > 0 and len(row) >= 3) + + +def section_header_rows(rows: list[list[str]], top_header_rows: int) -> list[int]: + """Return sparse section-label rows inside the detected top header prefix.""" + section_rows = [] + for row_idx, row in enumerate(rows[:top_header_rows]): + if row_idx == 0 or not centered_section_label_row_like(row): + continue + nonempty = [text.strip() for text in row if text.strip()] + if len(nonempty) != 1: + continue + if len(row) < 2: + continue + section_rows.append(row_idx) + return section_rows + + +def row_profile(row: list[str], index: int) -> dict[str, Any]: + vector = row_vector(row) + coordinate_header_row = dense_integer_axis_row_like(row) + body_value_row = ( + vector["first_alpha"] > 0 + and vector["first_numeric"] == 0 + and vector["rest_numeric_ratio"] >= 0.35 + and vector["numeric_ratio"] >= 0.25 + ) + body_dense_row = vector["numeric_ratio"] >= 0.70 and vector["alpha_ratio"] <= 0.35 + body_long_form_row = ( + vector["first_label_type"] > 0 + and vector["first_value_type"] == 0 + and vector["rest_long_text_type_ratio"] >= 0.50 + and vector["value_type_ratio"] <= 0.25 + and vector["empty_ratio"] <= 0.35 + ) + if coordinate_header_row: + body_value_row = False + body_dense_row = False + body_long_form_row = False + return { + "index": index, + "vector": vector, + "nonempty": vector["nonempty_count"] > 0, + "body_value_row": body_value_row, + "body_dense_row": body_dense_row, + "body_long_form_row": body_long_form_row, + "body_like_row": body_value_row or body_dense_row or body_long_form_row, + "coordinate_header_row": coordinate_header_row, + "leading_body_record_row": leading_body_record_like(row), + "leading_long_form_row": leading_long_form_row_like(row), + "section_label_row": section_label_row_like(row), + "grouped_header_scaffold": grouped_header_scaffold(row), + "header_candidate": header_candidate_row(row), + } + + +def row_profiles(rows: list[list[str]]) -> list[dict[str, Any]]: + return [row_profile(row, index) for index, row in enumerate(rows)] + + +def combine_body_vectors(vectors: list[dict[str, float]]) -> dict[str, float]: + if not vectors: + return { + "count": 0.0, + "first_label_ratio": 0.0, + "rest_numeric_ratio": 0.0, + "numeric_ratio": 0.0, + "alpha_ratio": 0.0, + "avg_tokens_norm": 0.0, + "body_row_ratio": 0.0, + "dense_numeric_row_ratio": 0.0, + "long_form_row_ratio": 0.0, + "value_type_ratio": 0.0, + "rest_value_type_ratio": 0.0, + "rest_long_text_type_ratio": 0.0, + "long_text_type_ratio": 0.0, + } + return { + "count": float(len(vectors)), + "first_label_ratio": sum(1 for item in vectors if item["first_label_type"] > 0 and item["first_value_type"] == 0) + / len(vectors), + "rest_numeric_ratio": sum(item["rest_numeric_ratio"] for item in vectors) / len(vectors), + "numeric_ratio": sum(item["numeric_ratio"] for item in vectors) / len(vectors), + "alpha_ratio": sum(item["alpha_ratio"] for item in vectors) / len(vectors), + "avg_tokens_norm": sum(item["avg_tokens_norm"] for item in vectors) / len(vectors), + "value_type_ratio": sum(item["value_type_ratio"] for item in vectors) / len(vectors), + "rest_value_type_ratio": sum(item["rest_value_type_ratio"] for item in vectors) / len(vectors), + "rest_long_text_type_ratio": sum(item["rest_long_text_type_ratio"] for item in vectors) / len(vectors), + "long_text_type_ratio": sum(item["long_text_type_ratio"] for item in vectors) / len(vectors), + "body_row_ratio": sum( + 1 + for item in vectors + if ( + item["first_alpha"] > 0 + and item["first_numeric"] == 0 + and item["rest_numeric_ratio"] >= 0.35 + and item["numeric_ratio"] >= 0.25 + ) + or (item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35) + ) + / len(vectors), + "dense_numeric_row_ratio": sum( + 1 for item in vectors if item["numeric_ratio"] >= 0.70 and item["alpha_ratio"] <= 0.35 + ) + / len(vectors), + "long_form_row_ratio": sum( + 1 + for item in vectors + if ( + item["first_label_type"] > 0 + and item["first_value_type"] == 0 + and item["rest_long_text_type_ratio"] >= 0.50 + and item["value_type_ratio"] <= 0.25 + and item["empty_ratio"] <= 0.35 + ) + ) + / len(vectors), + } + + +def body_window_features_from_profiles( + profiles: list[dict[str, Any]], start: int, size: int = 4 +) -> dict[str, float]: + vectors = [profile["vector"] for profile in profiles[start : start + size] if profile["nonempty"]] + return combine_body_vectors(vectors) + + +def body_window_is_stable(features: dict[str, float]) -> bool: + if features["count"] < 2: + return False + value_body = ( + features["body_row_ratio"] >= 0.50 + and features["rest_numeric_ratio"] >= 0.45 + and features["first_label_ratio"] >= 0.35 + ) + dense_numeric_body = ( + features["dense_numeric_row_ratio"] >= 0.50 + and features["numeric_ratio"] >= 0.60 + and features["alpha_ratio"] <= 0.55 + ) + long_form_body = ( + features["long_form_row_ratio"] >= 0.50 + and features["first_label_ratio"] >= 0.50 + and features["rest_long_text_type_ratio"] >= 0.50 + and features["value_type_ratio"] <= 0.25 + and features["numeric_ratio"] <= 0.25 + ) + return value_body or dense_numeric_body or long_form_body + + +def boundary_distance(header_row: list[str], body_features: dict[str, float]) -> float: + header = row_vector(header_row) + distance = 0.0 + distance += abs(header["numeric_ratio"] - body_features["numeric_ratio"]) + distance += abs(header["alpha_ratio"] - body_features["alpha_ratio"]) + distance += abs(header["rest_numeric_ratio"] - body_features["rest_numeric_ratio"]) + distance += abs(header["first_alpha"] - body_features["first_label_ratio"]) + distance += 0.5 * abs(header["avg_tokens_norm"] - body_features["avg_tokens_norm"]) + if header["empty_ratio"] >= 0.20: + distance += 0.25 + return distance + + +def header_prefix_ok( + rows: list[list[str]], + profiles: list[dict[str, Any]], + body_start: int, + body_features: dict[str, float], +) -> bool: + reasons = [] + if body_start <= 1: + return True + if body_start > 5: + return False + + candidate_profiles = profiles[1:body_start] + if not candidate_profiles: + return True + + allowed_section_prefix = all(centered_section_label_row_like(rows[profile["index"]]) for profile in candidate_profiles) + coordinate_prefix = any(profile.get("coordinate_header_row", False) for profile in candidate_profiles) + disallowed_section_profiles = [] + for profile in candidate_profiles: + if not profile["section_label_row"]: + continue + idx = int(profile["index"]) + sparse_coordinate_tail = ( + coordinate_prefix + and idx == body_start - 1 + and profile["header_candidate"] + and not profile["body_like_row"] + and profile["vector"]["nonempty_count"] <= 3 + and idx > 1 + and profiles[idx - 1]["header_candidate"] + ) + if not centered_section_label_row_like(rows[idx]) and not sparse_coordinate_tail: + disallowed_section_profiles.append(profile) + if any(profile["section_label_row"] for profile in candidate_profiles) and not allowed_section_prefix: + if disallowed_section_profiles: + reasons.append("section_label_before_body") + elif disallowed_section_profiles: + reasons.append("section_label_before_body") + if any(profile["body_like_row"] for profile in candidate_profiles): + reasons.append("body_like_row_before_body") + if not all(profile["header_candidate"] for profile in candidate_profiles): + reasons.append("non_header_candidate_before_body") + + previous = profiles[body_start - 2] + last_candidate = profiles[body_start - 1] + contrast = boundary_distance(rows[body_start - 1], body_features) + if ( + body_start == 2 + and last_candidate["vector"]["first_empty"] == 0 + and last_candidate["vector"]["numeric_ratio"] > 0.0 + and not previous["grouped_header_scaffold"] + ): + reasons.append("numeric_first_row_without_group_scaffold") + if last_candidate["body_dense_row"] and not previous["grouped_header_scaffold"]: + reasons.append("dense_candidate_without_group_scaffold") + if last_candidate["vector"]["alpha_ratio"] < 0.20 and not previous["grouped_header_scaffold"]: + reasons.append("low_alpha_candidate_without_group_scaffold") + if contrast < 1.05: + reasons.append("weak_header_body_contrast") + + return not reasons + + +def body_start_candidates( + profiles: list[dict[str, Any]], +) -> list[dict[str, Any]]: + candidates = [] + max_start = min(5, len(profiles) - 2) + for start in range(1, max_start + 1): + body_features = body_window_features_from_profiles(profiles, start) + start_row_body_like = profiles[start]["body_like_row"] + stable = start_row_body_like and body_window_is_stable(body_features) + prior_body_seen = any( + profile["body_like_row"] and not profile.get("coordinate_header_row", False) + for profile in profiles[1:start] + ) + candidates.append( + { + "body_start": start, + "body_like": stable, + "start_row_body_like": start_row_body_like, + "prior_body_seen": prior_body_seen, + "body": body_features, + } + ) + return candidates + + +def _nonempty_col_indices(row: list[str], max_cols: int) -> set[int]: + return {idx for idx in range(min(max_cols, len(row))) if column_text(row, idx).strip()} + + +def _header_column_incomplete(rows: list[list[str]], top_header_rows: int) -> bool: + if top_header_rows <= 0: + return False + max_cols = max((len(row) for row in rows), default=0) + if max_cols < 3: + return False + filled: set[int] = set() + for row in rows[:top_header_rows]: + filled.update(_nonempty_col_indices(row, max_cols)) + return len(filled) < max_cols and len(filled) <= max(1, int(max_cols * 0.75)) + + +def _bare_year_value_row_like(row: list[str]) -> bool: + nonempty = [text.strip() for text in row if text.strip()] + if len(nonempty) < 2: + return False + return all(_YEAR_RE.fullmatch(text) for text in nonempty) + + +def _undertag_extension_candidate(row: list[str], following_rows: list[list[str]], max_cols: int) -> bool: + features = row_features(row) + profile = row_profile(row, 0) + filled_cols = sorted(_nonempty_col_indices(row, max_cols)) + nonempty_after_stub = [column_text(row, col) for col in range(1, min(max_cols, len(row))) if column_text(row, col).strip()] + reasons: list[str] = [] + + if features["nonempty_count"] < 2 or len(filled_cols) < 2: + reasons.append("single_or_sparse_row") + if len(nonempty_after_stub) < 1: + reasons.append("no_column_header_cells_after_stub") + if profile["body_like_row"] or data_row_like(row) or dense_numeric_row_like(row): + reasons.append("body_like_row") + if money_or_percent_heavy(row) or features["value_type_ratio"] > 0.35 or features["numeric_ratio"] > 0.50: + reasons.append("strong_value_row") + if _bare_year_value_row_like(row): + reasons.append("bare_year_value_row") + if section_label_row_like(row): + reasons.append("section_label_row") + if features["avg_tokens"] > 8.0 or features["long_text_type_ratio"] >= 0.35: + reasons.append("long_text_row") + if len(following_rows) >= 2: + following_profiles = row_profiles(following_rows) + body_features = body_window_features_from_profiles(following_profiles, 0) + if not body_window_is_stable(body_features): + reasons.append("following_body_not_stable") + + label_signal = features["alpha_ratio"] >= 0.25 or features["date_period_ratio"] >= 0.20 + if not label_signal: + reasons.append("weak_label_signal") + + return not reasons + + +def extend_header_undertag( + rows: list[list[str]], + top_header_rows: int, + *, + cap: int = 3, +) -> int: + """Promote under-tagged column-header rows immediately below the base boundary.""" + max_cols = max((len(row) for row in rows), default=0) + if top_header_rows <= 0: + return top_header_rows + if top_header_rows >= len(rows): + return top_header_rows + if not _header_column_incomplete(rows, top_header_rows): + return top_header_rows + + new_top = top_header_rows + for row_idx in range(top_header_rows, min(len(rows), top_header_rows + cap)): + if row_idx >= len(rows) - 1: + break + ok = _undertag_extension_candidate(rows[row_idx], rows[row_idx + 1 :], max_cols) + if not ok: + break + new_top = row_idx + 1 + + return new_top + + +def find_top_header_rows_by_body_change( + rows: list[list[str]], +) -> int: + if not rows: + return 0 + if len(rows) == 1: + return 1 + + profiles = row_profiles(rows) + candidates = body_start_candidates(profiles) + for candidate in candidates: + body_start = int(candidate["body_start"]) + if not candidate["body_like"] or candidate["prior_body_seen"]: + continue + + if body_start == 1: + if profiles[0]["leading_body_record_row"]: + restore_guards = first_row_restore_guards(rows[0]) + if restore_guards: + return 1 + return 0 + if headerless_dotted_enum_guard(rows): + return 0 + if sparse_prefix_next_header_guard(rows, 1): + return 2 + if rowspan_leaf_header_guard(rows, 1): + return 2 + # A dense period/label header (row 0) over a value-like sub-row (bare + # years, enumerated headers) is not caught by the sparse-prefix guards. + if len(rows) >= 3 and next_header_row_value_guard(rows[1], rows[2:]): + return 2 + return 1 + + if header_prefix_ok(rows, profiles, body_start, candidate["body"]): + return body_start + + if headerless_dotted_enum_guard(rows): + return 0 + + if sparse_prefix_next_header_guard(rows, 1): + return 2 + + if rowspan_leaf_header_guard(rows, 1): + return 2 + + return 1 + + +@dataclass(frozen=True) +class HeaderRegion: + top_header_rows: int + section_header_rows: tuple[int, ...] + + +def find_header_region(rows: list[list[str]]) -> HeaderRegion: + top_header_rows = find_top_header_rows_by_body_change(rows) + # Promote under-tagged column-header rows immediately below the detected + # header/body boundary. + top_header_rows = extend_header_undertag(rows, top_header_rows) + section_rows = tuple(section_header_rows(rows, top_header_rows)) + return HeaderRegion( + top_header_rows=top_header_rows, + section_header_rows=section_rows, + ) + + +# --- HTML serialization: tagged placement grid ->
-------------------- +def collapse_cell_ws(text: str) -> str: + """Whitespace-collapse a cell's text (runs of whitespace/newlines -> one space).""" + return " ".join(text.split()) + + +def escape_html_text(text: str) -> str: + """Escape a cell's text for the serializer: only ``& < >`` (quotes left literal).""" + return text.replace("&", "&").replace("<", "<").replace(">", ">") + + +def _cell_inner(text: str) -> str: + """A cell's inner HTML: escaped non-empty lines joined by ``
``.""" + return "
".join(escape_html_text(part.strip()) for part in text.splitlines() if part.strip()) + + +def render_table_html(rows, section_header_rows=()) -> str: + """Serialize a tagged placement grid to its final ``
`` HTML in one pass. + + ``rows`` is a row-major grid of cells duck-typed with ``text`` / ``colspan`` + / ``rowspan`` / ``tag`` (e.g. :class:`pymupdf.table.SpanCell`): each cell + emits its tag, its ``colspan`` / ``rowspan`` attributes and its ``
``- + joined escaped inner HTML. A row whose index is in ``section_header_rows`` + and that carries a single non-empty label collapses to one + ``
`` spanning the row.""" + section_rows = set(section_header_rows or ()) + parts = [""] + for row_idx, cells in enumerate(rows): + if row_idx in section_rows: + nonempty = [collapse_cell_ws(cell.text) for cell in cells if collapse_cell_ws(cell.text)] + if len(nonempty) == 1 and len(cells) >= 2: + parts.append( + '' + % (len(cells), escape_html_text(nonempty[0])) + ) + continue + parts.append("") + for cell in cells: + attrs = "" + if cell.colspan > 1: + attrs += ' colspan="%d"' % cell.colspan + if cell.rowspan > 1: + attrs += ' rowspan="%d"' % cell.rowspan + parts.append( + "<%s%s>%s" % (cell.tag, attrs, _cell_inner(cell.text), cell.tag) + ) + parts.append("") + parts.append("
%s
") + return "".join(parts) diff --git a/src/_table_refine.py b/src/_table_refine.py new file mode 100644 index 000000000..dc8984cac --- /dev/null +++ b/src/_table_refine.py @@ -0,0 +1,867 @@ +""" +Copyright (C) 2023 Artifex Software, Inc. + +This file is part of PyMuPDF. + +PyMuPDF is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License +along with MuPDF. If not, see + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. + +--------------------------------------------------------------------- + +PyMuPDF table grid refinement (opt-in extension). + +Provides the refine_grid / refine_grid_structure / refine_grid_rows API that +refines a detected table's cell grid using page text and vector-graphics +geometry. Re-exported by pymupdf.table; never runs on the default +find_tables() path. +""" + +import itertools +import re +import pymupdf + + +# Grid refinement runs three splitters, each taking (page, grid) and returning a +# grid without mutating the page: +# * _split_shaded_rows split rows the line grid merged but a cell +# background-shading rectangle separates, +# * _split_undersegmented_columns split a column that jams several values into +# one cell, +# * _split_overmerged_rows split body rows that collapsed several +# records into a single grid row. +# Word selection uses center-point cell membership with rotated/vertical span +# substitution; it is independent of the CHARS/extract_words path in +# pymupdf.table, so refinement needs no CHARS state. + +_REFINE_LINE_GAP = 3.0 # center-y gap (points) that groups body words into lines + + +# --- word selection: center-point membership + rotated-span substitution ----- +def _refine_rawdict_spans(page): + """Flattened rawdict text spans carrying line-direction metadata. + + Only used to locate vertical/rotated text so those words can be replaced by + span-level bboxes in _refine_page_words. Minimal fields (bbox/text/dir/wmode) + -- the only ones the consumers read.""" + spans = [] + raw = page.get_text("rawdict") + for block in raw.get("blocks", []): + if block.get("type") != 0: + continue + for line in block.get("lines", []): + direction = line.get("dir") + wmode = line.get("wmode") + for span in line.get("spans", []): + text = "".join(str(char.get("c", "")) for char in span.get("chars", [])) + if not text.strip(): + continue + spans.append( + { + "bbox": [float(v) for v in span.get("bbox", [])], + "text": text, + "dir": direction, + "wmode": wmode, + } + ) + return spans + + +def _refine_is_vertical_or_rotated(span): + """True when a rawdict span's direction indicates non-horizontal text flow. + + wmode != 0 is vertical; otherwise compare |dir_x| vs |dir_y| (a missing or + unparsable direction counts as horizontal).""" + if span.get("wmode") not in (None, 0): + return True + direction = span.get("dir") + if not isinstance(direction, (list, tuple)) or len(direction) < 2: + return False + try: + dx = abs(float(direction[0])) + dy = abs(float(direction[1])) + except (TypeError, ValueError): + return False + return dy > dx + + +def _refine_page_words(page): + """Center-point word list for grid refinement, cached on the page object. + + Extract page words once (page.get_text("words")), then substitute vertical/ + rotated text: any horizontal word whose center falls inside a rotated span's + bbox is dropped, and each rotated span is appended as a single word.""" + cache = getattr(page, "_refine_words_cache", None) + if cache is not None: + return cache + words = [ + (float(w[0]), float(w[1]), float(w[2]), float(w[3]), str(w[4])) + for w in page.get_text("words") + if str(w[4]).strip() + ] + rotated_spans = [ + span + for span in _refine_rawdict_spans(page) + if _refine_is_vertical_or_rotated(span) and str(span.get("text") or "").strip() + ] + if rotated_spans: + rotated_rects = [pymupdf.Rect(span.get("bbox") or []) for span in rotated_spans] + kept = [] + for word in words: + wx0, wy0, wx1, wy1, _ = word + center = pymupdf.Point((wx0 + wx1) * 0.5, (wy0 + wy1) * 0.5) + if any(rect.contains(center) for rect in rotated_rects if not rect.is_empty): + continue + kept.append(word) + for span, rect in zip(rotated_spans, rotated_rects): + if rect.is_empty: + continue + kept.append( + (float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1), str(span["text"])) + ) + words = sorted(kept, key=lambda item: (item[1], item[0])) + try: + setattr(page, "_refine_words_cache", words) + except Exception: + pass + return words + + +def _refine_word_in_rect(wx0, wy0, wx1, wy1, x0, y0, x1, y1): + """Word-to-cell membership by CENTER point (inclusive), not clip-overlap. + + A word straddling a column line is claimed by exactly one cell.""" + cx = (wx0 + wx1) * 0.5 + cy = (wy0 + wy1) * 0.5 + return x0 <= cx <= x1 and y0 <= cy <= y1 + + +def _refine_words_in_rect(page, rect): + """Page words whose center lies in rect, as (x0, y0, x1, y1, text) tuples.""" + x0, y0, x1, y1 = float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1) + return [ + (wx0, wy0, wx1, wy1, text) + for wx0, wy0, wx1, wy1, text in _refine_page_words(page) + if _refine_word_in_rect(wx0, wy0, wx1, wy1, x0, y0, x1, y1) + ] + + +def _refine_has_digit(text): + """Loose value-like predicate: the text contains any digit.""" + return bool(re.search(r"\d", text.strip())) + + +def _refine_has_digit_no_alpha(text): + """Strict value-like predicate: contains a digit and no letters.""" + stripped = text.strip() + return bool(re.search(r"\d", stripped)) and not any(char.isalpha() for char in stripped) + + +# --- stage 1: split rows separated by cell background shading ---------------- +def _refine_is_white(fill, *, threshold=0.95): + if fill is None: + return True + try: + r, g, b = fill[:3] + except Exception: + return True + return float(r) > threshold and float(g) > threshold and float(b) > threshold + + +def _refine_table_rect(cells, table_bbox): + if table_bbox is not None: + rect = pymupdf.Rect(table_bbox) + if not rect.is_empty: + return rect + rects = [ + pymupdf.Rect(cell) + for row in cells + for cell in row + if cell is not None and not pymupdf.Rect(cell).is_empty + ] + if not rects: + return None + rect = pymupdf.Rect(rects[0]) + for item in rects[1:]: + rect.include_rect(item) + return rect + + +def _refine_raw_shaded_rects(page, table_rect, *, min_dim): + out = [] + page_width = float(page.rect.width) + for drawing in page.get_drawings(): + if _refine_is_white(drawing.get("fill")): + continue + for item in drawing.get("items", []): + if item[0] != "re": + continue + rect = pymupdf.Rect(item[1]) + rect.normalize() + if rect.width >= 0.95 * page_width: + continue + if rect.width < min_dim or rect.height < min_dim: + continue + if (rect & table_rect).is_empty: + continue + out.append(rect) + return out + + +def _refine_cell_background_rects(raw_rects, *, contain_frac=0.9): + keep = [] + for index, rect in enumerate(raw_rects): + area = rect.get_area() + if area <= 0: + continue + nested = any( + other_index != index + and (rect & other).get_area() >= contain_frac * area + and other.get_area() > area * 1.05 + for other_index, other in enumerate(raw_rects) + ) + if not nested: + keep.append(rect) + return keep + + +def _refine_cluster(values, *, tolerance): + clusters = [] + current = [] + for value in sorted(values): + if current and value - current[-1] > tolerance: + clusters.append(sum(current) / len(current)) + current = [] + current.append(value) + if current: + clusters.append(sum(current) / len(current)) + return clusters + + +def _refine_border_lines(page, table_rect): + xs = set() + ys = set() + for drawing in page.get_drawings(): + stroked = drawing.get("type") in ("s", "fs") + for item in drawing.get("items", []): + kind = item[0] + if kind == "l": + p1, p2 = item[1], item[2] + if ( + abs(p1.y - p2.y) < 1.0 + and abs(p1.x - p2.x) > 10.0 + and table_rect.y0 - 2.0 < (p1.y + p2.y) * 0.5 < table_rect.y1 + 2.0 + ): + ys.add(round((p1.y + p2.y) * 0.5, 1)) + elif ( + abs(p1.x - p2.x) < 1.0 + and abs(p1.y - p2.y) > 10.0 + and table_rect.x0 - 2.0 < (p1.x + p2.x) * 0.5 < table_rect.x1 + 2.0 + ): + xs.add(round((p1.x + p2.x) * 0.5, 1)) + continue + if kind != "re": + continue + rect = pymupdf.Rect(item[1]) + rect.normalize() + if (rect & table_rect).is_empty: + continue + if rect.height < 2.0 and rect.width > 10.0: + ys.add(round((rect.y0 + rect.y1) * 0.5, 1)) + elif rect.width < 2.0 and rect.height > 10.0: + xs.add(round((rect.x0 + rect.x1) * 0.5, 1)) + elif stroked: + ys.add(round(rect.y0, 1)) + ys.add(round(rect.y1, 1)) + xs.add(round(rect.x0, 1)) + xs.add(round(rect.x1, 1)) + return sorted(xs), sorted(ys) + + +def _refine_drop_near(edges, borders, *, tolerance): + return [edge for edge in edges if not any(abs(edge - border) <= tolerance for border in borders)] + + +def _refine_has_text(words, x0, y0, x1, y1, *, margin): + for wx0, wy0, wx1, wy1, text in words: + if not str(text).strip(): + continue + cx = (wx0 + wx1) * 0.5 + cy = (wy0 + wy1) * 0.5 + if x0 + margin < cx < x1 - margin and y0 + margin < cy < y1 - margin: + return True + return False + + +def _refine_content_filter_row_edges(words, y0, y1, candidates, x0, x1, *, margin): + edges = sorted(edge for edge in candidates if y0 + margin < edge < y1 - margin) + while edges: + bounds = [y0, *edges, y1] + empty_index = None + for index in range(len(bounds) - 1): + if not _refine_has_text(words, x0, bounds[index], x1, bounds[index + 1], margin=margin): + empty_index = index + break + if empty_index is None: + break + if empty_index < len(edges): + edges.pop(empty_index) + elif empty_index - 1 >= 0: + edges.pop(empty_index - 1) + else: + break + return edges + + +def _split_shaded_rows( + page, + cells, + table_bbox=None, + *, + cluster_tolerance=3.0, + margin=2.0, + min_dim=6.0, + border_tolerance=2.5, +): + """Split rows the line grid merged but a cell background rectangle separates. + + Cell background-shading rectangles give row boundaries the border lines do + not; candidate y-edges from those rectangles (minus edges that coincide with + real borders, minus edges that would cut an empty band) subdivide each row.""" + table_rect = _refine_table_rect(cells, table_bbox) + if table_rect is None or not cells: + return cells + + raw_rects = _refine_raw_shaded_rects(page, table_rect, min_dim=min_dim) + background_rects = _refine_cell_background_rects(raw_rects) + if not background_rects: + return cells + + y_edges = _refine_cluster( + [edge for rect in background_rects for edge in (float(rect.y0), float(rect.y1))], + tolerance=cluster_tolerance, + ) + _border_xs, border_ys = _refine_border_lines(page, table_rect) + y_edges = _refine_drop_near(y_edges, border_ys, tolerance=border_tolerance) + if not y_edges: + return cells + + words = _refine_page_words(page) + new_cells = [] + added_rows = 0 + for row in cells: + live = [pymupdf.Rect(cell) for cell in row if cell is not None] + if not live: + new_cells.append(row) + continue + ry0 = min(float(cell.y0) for cell in live) + ry1 = max(float(cell.y1) for cell in live) + rx0 = min(float(cell.x0) for cell in live) + rx1 = max(float(cell.x1) for cell in live) + cuts = _refine_content_filter_row_edges(words, ry0, ry1, y_edges, rx0, rx1, margin=margin) + if not cuts: + new_cells.append(row) + continue + bounds = [ry0, *cuts, ry1] + for band_index in range(len(bounds) - 1): + band = [] + for cell in row: + if cell is None: + band.append(None) + else: + rect = pymupdf.Rect(cell) + band.append( + [float(rect.x0), bounds[band_index], float(rect.x1), bounds[band_index + 1]] + ) + new_cells.append(band) + added_rows += len(bounds) - 2 + + if added_rows <= 0: + return cells + return new_cells + + +# --- stage 2: split under-segmented columns ---------------------------------- +def _refine_value_like(text): + return _refine_has_digit_no_alpha(text) + + +def _refine_span_value_like(text): + return _refine_has_digit(text) + + +def _refine_cell_span_signal(page, cell, *, gap, line_tolerance): + """True if a cell holds two value-like groups separated by a wide gap on one + text line -- the signal that a single grid column packs several columns.""" + rect = pymupdf.Rect(cell) + words = [] + for x0, y0, x1, y1, text in _refine_words_in_rect(page, rect): + stripped = str(text).strip() + if not stripped: + continue + cx = (float(x0) + float(x1)) / 2.0 + cy = (float(y0) + float(y1)) / 2.0 + if rect.x0 <= cx <= rect.x1 and rect.y0 <= cy <= rect.y1: + words.append((float(x0), float(x1), cy, stripped)) + if len(words) < 2: + return False + + words.sort(key=lambda item: item[2]) + lines = [] + current = [words[0]] + for word in words[1:]: + if word[2] - current[-1][2] > line_tolerance: + lines.append(current) + current = [word] + else: + current.append(word) + lines.append(current) + + for line in lines: + line.sort(key=lambda item: item[0]) + for index in range(len(line) - 1): + if line[index + 1][0] - line[index][1] <= gap: + continue + left = " ".join(word[3] for word in line[: index + 1]) + right = " ".join(word[3] for word in line[index + 1 :]) + if _refine_span_value_like(left) and _refine_span_value_like(right): + return True + return False + + +def _refine_detect_span_columns( + page, cells, *, gap=12.0, line_tolerance=4.0, support_ratio=0.3, min_support=2 +): + """Return the column indices that fire the under-segmentation gate: columns + where enough rows carry the two-value-group signal (a support threshold).""" + rows = len(cells) + support_threshold = max(min_support, int(support_ratio * rows)) + col_hits = {} + for row in cells: + for col_index, cell in enumerate(row): + if cell is None: + continue + if _refine_cell_span_signal(page, cell, gap=gap, line_tolerance=line_tolerance): + col_hits[col_index] = col_hits.get(col_index, 0) + 1 + return sorted(col for col, count in col_hits.items() if count >= support_threshold) + + +def _refine_column_words(page, cells, col): + rows = [] + for row in cells: + words = [] + if col < len(row) and row[col] is not None: + rect = pymupdf.Rect(row[col]) + for x0, y0, x1, y1, text in _refine_words_in_rect(page, rect): + if not str(text).strip(): + continue + cx = (float(x0) + float(x1)) / 2.0 + cy = (float(y0) + float(y1)) / 2.0 + if rect.x0 <= cx <= rect.x1 and rect.y0 <= cy <= rect.y1: + words.append((float(x0), float(x1), str(text))) + rows.append(sorted(words)) + return rows + + +def _refine_cut_xs(rows_words, *, bridge_tolerance=0): + intervals = [(x0, x1, row_index) for row_index, words in enumerate(rows_words) for (x0, x1, _) in words] + if len(intervals) < 2: + return [] + + lo = min(x0 for x0, _, _ in intervals) + hi = max(x1 for _, x1, _ in intervals) + if hi - lo < 1: + return [] + + channels = [] + current = None + x = lo + while x <= hi: + crossing_rows = {row_index for x0, x1, row_index in intervals if x0 < x < x1} + if len(crossing_rows) > bridge_tolerance: + if current is not None: + channels.append((current[0], current[1])) + current = None + else: + current = [x, x] if current is None else [current[0], x] + x += 1.0 + if current is not None: + channels.append((current[0], current[1])) + + cuts = [] + for left_edge, right_edge in channels: + if left_edge <= lo + 1 or right_edge >= hi - 1: + continue + cut_x = (left_edge + right_edge) / 2.0 + crossing_count = sum(1 for words in rows_words if any(x0 < cut_x < x1 for x0, x1, _ in words)) + # Evaluate the symmetry/value guards on non-bridging rows only, so a + # merged label spanning value subcolumns does not block column recovery. + body_rows = [words for words in rows_words if not any(x0 < cut_x < x1 for x0, x1, _ in words)] + left_rows = right_rows = 0 + left_ok = right_ok = 0 + left_n = right_n = 0 + left_values = [] + right_values = [] + for words in body_rows: + left = [text for x0, x1, text in words if x1 <= cut_x] + right = [text for x0, x1, text in words if x0 >= cut_x] + if left: + left_rows += 1 + left_n += 1 + left_ok += _refine_value_like(" ".join(left)) + left_values.extend(left) + if right: + right_rows += 1 + right_n += 1 + right_ok += _refine_value_like(" ".join(right)) + right_values.extend(right) + + if left_rows < 2 or right_rows < 2: + continue + if crossing_count > 0: + if not _refine_value_like(" ".join(left_values)) or not _refine_value_like(" ".join(right_values)): + continue + elif (left_n and left_ok / left_n < 0.5) or (right_n and right_ok / right_n < 0.5): + continue + cuts.append(cut_x) + + return sorted(cuts) + + +def _refine_split_columns(page, cells, *, bridge_tolerance=0, allowed_cols=None): + ncols = max((len(row) for row in cells), default=0) + active_cols = None if allowed_cols is None else set(allowed_cols) + col_cuts = {} + for col in range(ncols): + if active_cols is not None and col not in active_cols: + continue + cuts = _refine_cut_xs(_refine_column_words(page, cells, col), bridge_tolerance=bridge_tolerance) + if cuts: + col_cuts[col] = cuts + + if not col_cuts: + return cells + + new_cells = [] + for row in cells: + new_row = [] + for col, cell in enumerate(row): + cuts = col_cuts.get(col) + if not cuts: + new_row.append(cell) + elif cell is None: + new_row.extend([None] * (len(cuts) + 1)) + else: + x0, y0, x1, y1 = pymupdf.Rect(cell) + xs = [x0] + [cut_x for cut_x in cuts if x0 < cut_x < x1] + [x1] + for index in range(len(xs) - 1): + new_row.append([xs[index], y0, xs[index + 1], y1]) + new_cells.append(new_row) + + return new_cells + + +def _split_undersegmented_columns( + page, + cells, + *, + gap=12.0, + line_tolerance=4.0, + support_ratio=0.3, + min_support=2, + bridge_tolerance=0, +): + """Split under-segmented columns, but only those that fire the value-group + gate -- so ordinary single-value columns are never disturbed.""" + fired_cols = _refine_detect_span_columns( + page, + cells, + gap=gap, + line_tolerance=line_tolerance, + support_ratio=support_ratio, + min_support=min_support, + ) + if not fired_cols: + return cells + return _refine_split_columns( + page, cells, bridge_tolerance=bridge_tolerance, allowed_cols=set(fired_cols) + ) + + +# --- stage 3: split over-merged body rows ------------------------------------ +def _refine_rects_from_cells(cells): + return [pymupdf.Rect(cell) for row in cells for cell in row if cell is not None] + + +def _refine_union_rect(rects): + if not rects: + return None + rect = pymupdf.Rect(rects[0]) + for item in rects[1:]: + rect.include_rect(item) + return rect + + +def _refine_best_column_bounds(cells): + best_row = max( + cells, + key=lambda row: (sum(1 for cell in row if cell is not None), len(row)), + default=[], + ) + bounds = sorted( + (float(rect.x0), float(rect.x1)) + for cell in best_row + if cell is not None + for rect in [pymupdf.Rect(cell)] + if rect.x1 > rect.x0 + ) + return bounds + + +def _refine_cluster_words_by_y(words): + """Group center-point words into body "lines" by a fixed center-y gap, + returning each line's union bbox and joined text.""" + if not words: + return [] + words = sorted(words, key=lambda item: ((item[1] + item[3]) / 2.0, item[0])) + clusters = [[words[0]]] + last_center = (words[0][1] + words[0][3]) / 2.0 + for word in words[1:]: + center = (word[1] + word[3]) / 2.0 + if center - last_center > _REFINE_LINE_GAP: + clusters.append([word]) + else: + clusters[-1].append(word) + last_center = center + + lines = [] + for cluster in clusters: + lines.append( + { + "x0": min(word[0] for word in cluster), + "y0": min(word[1] for word in cluster), + "x1": max(word[2] for word in cluster), + "y1": max(word[3] for word in cluster), + "text": " ".join(word[4] for word in sorted(cluster, key=lambda item: item[0])), + } + ) + return lines + + +def _refine_line_columns(line, col_bounds): + cols = set() + line_x0 = float(line["x0"]) + line_x1 = float(line["x1"]) + center = (line_x0 + line_x1) / 2.0 + for index, (x0, x1) in enumerate(col_bounds): + overlap = min(line_x1, x1) - max(line_x0, x0) + if overlap > 0 or x0 <= center <= x1: + cols.add(index) + return cols + + +def _refine_page_words_in_rect(page, rect): + words = [] + rx0, ry0, rx1, ry1 = float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1) + for x0, y0, x1, y1, text in _refine_page_words(page): + cx = (x0 + x1) * 0.5 + cy = (y0 + y1) * 0.5 + if rx0 <= cx <= rx1 and ry0 <= cy <= ry1: + text = str(text).strip() + if text: + words.append((float(x0), float(y0), float(x1), float(y1), text)) + return words + + +def _refine_merge_overlapping_lines(lines, *, frac): + if not lines: + return [] + items = sorted(lines) + merged = [[float(items[0][0]), float(items[0][1])]] + for y0, y1 in items[1:]: + current = merged[-1] + overlap = current[1] - float(y0) + min_height = min(current[1] - current[0], float(y1) - float(y0)) + if min_height > 0 and overlap / min_height >= frac: + current[0] = min(current[0], float(y0)) + current[1] = max(current[1], float(y1)) + else: + merged.append([float(y0), float(y1)]) + return [(y0, y1) for y0, y1 in merged] + + +def _refine_detect_row_overmerge(page, cells, *, clean_threshold=0.85, header_row_count=1): + """Decide whether the body rows are over-merged. Returns the geometry needed + to re-cut them (body bbox, column bounds, per-record y-bounds) or None. + + Fires only when the body text clusters into clean multi-column "record" lines + (clean_ratio >= clean_threshold) and there are more records than existing body + rows -- i.e. several records collapsed into one grid row.""" + col_bounds = _refine_best_column_bounds(cells) + header_row_count = max(1, min(int(header_row_count), len(cells))) if cells else 0 + existing_body_rows = max(0, len(cells) - header_row_count) + if len(cells) <= header_row_count: + return None + if len(col_bounds) < 2: + return None + + body_rect = _refine_union_rect(_refine_rects_from_cells(cells[header_row_count:])) + if body_rect is None or body_rect.is_empty: + return None + + words = _refine_page_words_in_rect(page, body_rect) + lines = _refine_cluster_words_by_y(words) + if not lines: + return None + + min_cols = max(2, (len(col_bounds) + 1) // 2) # == max(2, ceil(len/2)) + record_lines = [] + for line in lines: + cols = sorted(_refine_line_columns(line, col_bounds)) + if len(cols) >= min_cols: + record_lines.append(line) + + clean_ratio = len(record_lines) / len(lines) + if clean_ratio < clean_threshold: + return None + if len(record_lines) <= existing_body_rows: + return None + + return { + "body_bbox": [body_rect.x0, body_rect.y0, body_rect.x1, body_rect.y1], + "col_bounds": [[round(x0, 2), round(x1, 2)] for x0, x1 in col_bounds], + "record_line_bounds": [[float(line["y0"]), float(line["y1"])] for line in record_lines], + "existing_body_rows": existing_body_rows, + "header_row_count": header_row_count, + } + + +def _split_overmerged_rows( + page, cells, *, clean_threshold=0.85, merge_overlap_frac=0.35, header_row_count=1 +): + """Re-cut over-merged body rows into one grid row per detected record. + + Keeps the header rows intact, then rebuilds the body as evenly-bounded rows + (cut at the midpoints between consecutive record centers) across the detected + column bounds. header_row_count is the number of leading header rows to keep.""" + meta = _refine_detect_row_overmerge( + page, cells, clean_threshold=clean_threshold, header_row_count=header_row_count + ) + if meta is None: + return cells + + body_bbox = pymupdf.Rect(meta["body_bbox"]) + col_bounds = [(float(x0), float(x1)) for x0, x1 in meta["col_bounds"]] + raw_line_bounds = [(float(y0), float(y1)) for y0, y1 in meta["record_line_bounds"]] + line_bounds = _refine_merge_overlapping_lines(raw_line_bounds, frac=merge_overlap_frac) + if len(line_bounds) <= int(meta["existing_body_rows"]): + return cells + + centers = [(y0 + y1) / 2.0 for y0, y1 in line_bounds] + edges = [float(body_bbox.y0)] + for prev, nxt in zip(centers, centers[1:]): + edges.append((prev + nxt) / 2.0) + edges.append(float(body_bbox.y1)) + + header_rows = cells[: int(meta["header_row_count"])] + new_body = [] + for row_index in range(len(line_bounds)): + y0 = edges[row_index] + y1 = edges[row_index + 1] + new_row = [] + for x0, x1 in col_bounds: + rect = pymupdf.Rect(x0, y0, x1, y1) + new_row.append([float(rect.x0), float(rect.y0), float(rect.x1), float(rect.y1)]) + new_body.append(new_row) + + return [*header_rows, *new_body] + + +# --- public seam ------------------------------------------------------------- +def refine_grid_structure(page, cells, *, table_bbox=None): + """Structural half of refine_grid: split shaded rows, then under-segmented + columns. Exposed separately so a caller that must know the header/body + boundary of the post-structure grid can insert that computation between the + two phases. table_bbox, when given, bounds the shaded-rectangle search; + otherwise the cells' union is used.""" + cells = _split_shaded_rows(page, cells, table_bbox) + cells = _split_undersegmented_columns(page, cells) + return cells + + +def refine_grid_rows(page, cells, *, header_row_count=1, clean_threshold=0.85, merge_overlap_frac=0.35): + """Row half of refine_grid: split body rows that collapsed several records + into one grid row. header_row_count is the number of leading header rows to + keep intact; callers that resolve a header region pass it in, and the default + of 1 is a conservative single-header assumption.""" + return _split_overmerged_rows( + page, + cells, + clean_threshold=clean_threshold, + merge_overlap_frac=merge_overlap_frac, + header_row_count=header_row_count, + ) + + +def refine_grid(page, cells, *, table_bbox=None, header_row_count=1): + """Refine a detected table's cell grid and return the refined grid. + + ``cells`` is a row-major grid: a list of rows, each a list of ``[x0, y0, x1, + y1]`` cell rectangles (``None`` for a gap). The grid is refined in three + stages -- split rows that cell background shading separates, split columns + that jam several values into one cell, and split body rows that merged + several records -- using the page's words and vector graphics. The result is + a new grid in the same format (rows may be added and rows may widen). + + ``table_bbox`` optionally bounds the table (else the cells' union is used). + ``header_row_count`` is the number of leading header rows to preserve when + re-cutting body rows (default 1). This is the all-in-one convenience wrapper; + a caller needing the header boundary of the intermediate grid can instead + call :func:`refine_grid_structure` then :func:`refine_grid_rows`. + """ + cells = refine_grid_structure(page, cells, table_bbox=table_bbox) + cells = refine_grid_rows(page, cells, header_row_count=header_row_count) + return cells + + +def _refine_cells_to_grid(cells): + """Convert a Table's flat cell list into the row-major grid refine_grid wants. + + Mirrors Table.rows: columns are the sorted distinct left edges, rows are the + cells grouped by top edge, each padded with None where a column has no cell.""" + if not cells: + return [] + xs = sorted({c[0] for c in cells}) + col_index = {x: i for i, x in enumerate(xs)} + grid = [] + ordered = sorted(cells, key=lambda c: (c[1], c[0])) + for _y, row_cells in itertools.groupby(ordered, key=lambda c: c[1]): + row = [None] * len(xs) + for cell in row_cells: + row[col_index[cell[0]]] = [float(cell[0]), float(cell[1]), float(cell[2]), float(cell[3])] + grid.append(row) + return grid + + +def _refine_grid_to_cells(grid): + """Flatten a refined grid back to a Table's flat (x0, y0, x1, y1) cell list.""" + return [ + (float(cell[0]), float(cell[1]), float(cell[2]), float(cell[3])) + for row in grid + for cell in row + if cell is not None + ] diff --git a/src/_table_spans.py b/src/_table_spans.py new file mode 100644 index 000000000..ff0d89d40 --- /dev/null +++ b/src/_table_spans.py @@ -0,0 +1,596 @@ +""" +Copyright (C) 2023 Artifex Software, Inc. + +This file is part of PyMuPDF. + +PyMuPDF is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License +along with MuPDF. If not, see + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. + +--------------------------------------------------------------------- + +PyMuPDF table cell-span resolution (opt-in extension). + +Provides SpanCell and resolve_spans (plus the _span_* helpers), which +reconstruct a detected table's merged-cell (colspan / rowspan) structure. +Re-exported by pymupdf.table; never runs on the default find_tables() path. +Reuses the word-selection helpers of pymupdf._table_refine. +""" + +import pymupdf + +from pymupdf._table_refine import ( + _refine_is_vertical_or_rotated, + _refine_page_words, +) + + +class SpanCell: + """One reconstructed table cell after span resolution (PyMuPDF extension). + + ``bbox`` is the placement's ``(x0, y0, x1, y1)`` union rect, ``text`` the + page text it claims (lines joined by ``\\n``), and ``colspan``/``rowspan`` + how many grid columns/rows it covers. resolve_spans always sets a real bbox; + a caller padding its own grid may construct SpanCells with ``bbox=None``. + + ``tag`` is the cell's HTML tag (``"td"``/``"th"``), defaulting to ``"td"``. + resolve_spans leaves it at the default; find_tables(refine=True) overwrites + it from the resolved header region so Table.to_html() can serialize the grid + directly, and a caller building its own grid may set it too.""" + + def __init__(self, bbox, text, colspan, rowspan, tag="td"): + self.bbox = bbox + self.text = text + self.colspan = colspan + self.rowspan = rowspan + self.tag = tag + + +# --- slot geometry: cluster cell edges into column/row boundaries ------------ +def _span_clustered_boundaries(values, *, tolerance=3.0): + """Greedy 1-D clustering of edge coordinates into slot boundaries. + + Keeps a sorted value only when it is more than ``tolerance`` from the last + kept boundary, so the first value of each run is the retained boundary.""" + boundaries = [] + for value in sorted(values): + if boundaries and value - boundaries[-1] <= tolerance: + continue + boundaries.append(float(value)) + return boundaries + + +def _span_covered_slot_count(start, end, boundaries, *, tolerance=1.0): + """How many [boundaries[i], boundaries[i+1]] slots the span start..end covers.""" + count = 0 + for index in range(len(boundaries) - 1): + midpoint = (boundaries[index] + boundaries[index + 1]) / 2.0 + if start - tolerance <= midpoint <= end + tolerance: + count += 1 + return max(1, count) + + +def _span_slot_range(rect, x_boundaries, *, tolerance=1.0): + """The half-open column-slot range (first, last+1) a rect's x-extent covers.""" + hits = [] + for index in range(len(x_boundaries) - 1): + midpoint = (x_boundaries[index] + x_boundaries[index + 1]) / 2.0 + if rect.x0 - tolerance <= midpoint <= rect.x1 + tolerance: + hits.append(index) + if not hits: + return (0, 1) + return (min(hits), max(hits) + 1) + + +def _span_ranges_intersect(a, b): + return a[0] < b[1] and b[0] < a[1] + + +def _span_point_in_rect(x, y, rect): + return float(rect.x0) <= x <= float(rect.x1) and float(rect.y0) <= y <= float(rect.y1) + + +def _span_rect_union(rects): + return pymupdf.Rect( + min(rect.x0 for rect in rects), + min(rect.y0 for rect in rects), + max(rect.x1 for rect in rects), + max(rect.y1 for rect in rects), + ) + + +def _span_x_overlap(a, b): + return min(a.x1, b.x1) - max(a.x0, b.x0) + + +def _span_substantial_x_overlap(span_rect, cell_rect): + """True if a text span overlaps a cell by enough x to signal a merge. + + A few points of overlap is noise (a span drifting across a column line with + trailing whitespace or a currency glyph); require >2pt and >=15% of the cell + width before treating it as a merged-cell signal.""" + overlap = _span_x_overlap(span_rect, cell_rect) + if overlap <= 2.0: + return False + cell_width = max(1.0, cell_rect.width) + return overlap / cell_width >= 0.15 + + +def _span_contiguous_ranges(indices): + """Collapse a sorted index list into (start, end) inclusive runs.""" + if not indices: + return [] + ranges = [] + start = end = indices[0] + for index in indices[1:]: + if index == end + 1: + end = index + continue + ranges.append((start, end)) + start = end = index + ranges.append((start, end)) + return ranges + + +def _span_merge_intervals(intervals): + """Merge overlapping (start, end, text) intervals, accumulating their texts.""" + merged = [] + for start, end, text in sorted(intervals, key=lambda item: (item[0], item[1])): + if start == end: + continue + if not merged or start > merged[-1][1]: + merged.append((start, end, [text])) + continue + prev_start, prev_end, texts = merged[-1] + merged[-1] = (prev_start, max(prev_end, end), texts + [text]) + return merged + + +# --- cell text: page words + vertical-text lines claimed by a placement ------- +def _span_line_text(line): + """Space-joined non-empty span texts of a get_text('dict') line.""" + parts = [ + str(span.get("text") or "").strip() + for span in line.get("spans", []) + if str(span.get("text") or "").strip() + ] + return " ".join(parts) + + +def _span_line_rect(line): + """Bounding rect of a dict line (its bbox, else the union of its span bboxes).""" + bbox = line.get("bbox") + if bbox: + rect = pymupdf.Rect(bbox) + if not rect.is_empty: + return rect + rects = [ + pymupdf.Rect(span.get("bbox") or []) + for span in line.get("spans", []) + if span.get("bbox") + ] + rects = [rect for rect in rects if not rect.is_empty] + if not rects: + return None + return _span_rect_union(rects) + + +def _span_vertical_text_lines(page): + """Vertical/rotated text lines as (rect, text), cached on the page. + + Selects non-horizontal lines whose reading order get_text('dict') already + preserves.""" + cached = getattr(page, "_span_vertical_lines_cache", None) + if cached is not None: + return cached + lines = [] + for block in page.get_text("dict").get("blocks", []): + if block.get("type") not in (None, 0): + continue + for line in block.get("lines", []): + if not _refine_is_vertical_or_rotated(line): + continue + text = _span_line_text(line) + if not text: + continue + rect = _span_line_rect(line) + if rect is None: + continue + lines.append((rect, text)) + try: + setattr(page, "_span_vertical_lines_cache", lines) + except Exception: + pass + return lines + + +def _span_vertical_text_for_rect(page, rect, selected_words): + """Text of vertical lines centered in rect, when they dominate the rect. + + Returns the stacked line text only if vertical lines are centered in the rect, + cover >=60% of the rect's selected words, and carry >=2 tokens; else None so + the caller falls back to horizontal line synthesis.""" + if not selected_words: + return None + candidates = [] + for line_rect, text in _span_vertical_text_lines(page): + cx = (float(line_rect.x0) + float(line_rect.x1)) * 0.5 + cy = (float(line_rect.y0) + float(line_rect.y1)) * 0.5 + if _span_point_in_rect(cx, cy, rect): + candidates.append((line_rect, text)) + if not candidates: + return None + vertical_hits = 0 + for _, (wx0, wy0, wx1, wy1, _) in selected_words: + cx = (wx0 + wx1) * 0.5 + cy = (wy0 + wy1) * 0.5 + if any(_span_point_in_rect(cx, cy, line_rect) for line_rect, _ in candidates): + vertical_hits += 1 + if vertical_hits / max(1, len(selected_words)) < 0.6: + return None + if sum(len(text.split()) for _, text in candidates) < 2: + return None + return "\n".join( + text + for _, text in sorted( + candidates, + key=lambda item: (float(item[0].x0), -float((item[0].y0 + item[0].y1) * 0.5)), + ) + ) + + +def _span_words_to_line_text(words): + """Join center-point-selected cell words into text, re-synthesizing lines. + + ``words`` are (y0, x0, y1, text) tuples; they are grouped into lines by an + adaptive median-height nearest-line rule, each line's words ordered by x, and + the lines joined by newlines.""" + if not words: + return "" + heights = [max(0.1, y1 - y0) for y0, _, y1, _ in words] + median_height = sorted(heights)[len(heights) // 2] + line_threshold = max(2.0, median_height * 0.55) + lines = [] + for y0, x0, y1, text in sorted(words, key=lambda item: (item[0], item[1])): + cy = (y0 + y1) / 2.0 + best_line = None + best_distance = line_threshold + for line in lines: + distance = abs(cy - float(line["center_y"])) + if distance <= best_distance: + best_line = line + best_distance = distance + if best_line is None: + lines.append({"center_y": cy, "words": [(x0, text)]}) + continue + best_line["words"].append((x0, text)) + count = len(best_line["words"]) + best_line["center_y"] = (float(best_line["center_y"]) * (count - 1) + cy) / count + text_lines = [] + for line in sorted(lines, key=lambda item: float(item["center_y"])): + text_lines.append(" ".join(text for _, text in sorted(line["words"], key=lambda item: item[0]))) + return "\n".join(text_lines) + + +def _span_word_line_tuple(word): + """Reorder a (x0, y0, x1, y1, text) word to the (y0, x0, y1, text) line tuple.""" + x0, y0, _x1, y1, text = word + return (float(y0), float(x0), float(y1), str(text)) + + +def _span_select_words_in_rect(page_words, rect): + """(index, word) pairs whose center lies in rect, index into ``page_words``. + + The index is what lets resolve_spans claim each page word for exactly one + placement (an earlier cell's word is not re-claimed by a later one).""" + selected = [] + for index, word in enumerate(page_words): + wx0, wy0, wx1, wy1, text = word + if not str(text).strip(): + continue + cx = (wx0 + wx1) * 0.5 + cy = (wy0 + wy1) * 0.5 + if _span_point_in_rect(cx, cy, rect): + selected.append((index, word)) + return selected + + +def _span_words_text_for_rect(page, rect, selected_words): + """Text for a rect: vertical-line text if it dominates, else line synthesis.""" + vertical_text = _span_vertical_text_for_rect(page, rect, selected_words) + if vertical_text is not None: + return vertical_text + return _span_words_to_line_text([_span_word_line_tuple(word) for _, word in selected_words]) + + +def _span_claim_text_in_rect(page, rect, page_words, claimed_words): + """Text of rect's words, skipping words already claimed and claiming the rest.""" + selected = [ + (index, word) + for index, word in _span_select_words_in_rect(page_words, rect) + if index not in claimed_words + ] + for index, _ in selected: + claimed_words.add(index) + return _span_words_text_for_rect(page, rect, selected) + + +def _span_text_spans(page): + """Page text spans as (rect, text), length>=2, cached on the page. + + These drive merged-cell detection: a single span whose x-extent crosses a + grid column line signals cells the line grid split but text joins.""" + cached = getattr(page, "_span_text_spans_cache", None) + if cached is not None: + return cached + spans = [] + for block in page.get_text("dict").get("blocks", []): + for line in block.get("lines", []): + for span in line.get("spans", []): + text = str(span.get("text") or "").strip() + if len(text) < 2: + continue + bbox = span.get("bbox") + if not bbox: + continue + rect = pymupdf.Rect(bbox) + if rect.is_empty: + continue + spans.append((rect, text)) + try: + setattr(page, "_span_text_spans_cache", spans) + except Exception: + pass + return spans + + +def _span_crossing_intervals(entries, text_spans): + """Column ranges within one grid row that a single text span crosses. + + ``entries`` are the row's cell rects (in column order). A text span centered + in the row band and substantially overlapping >=2 adjacent cells marks those + cells for merging; overlapping intervals are merged. Returns + (start, end, texts) with end>start.""" + if len(entries) < 2: + return [] + row_y0 = min(entry.y0 for entry in entries) + row_y1 = max(entry.y1 for entry in entries) + intervals = [] + for span_rect, span_text in text_spans: + center_y = (span_rect.y0 + span_rect.y1) / 2.0 + if center_y < row_y0 - 1.0 or center_y > row_y1 + 1.0: + continue + hit_positions = [ + position + for position, entry in enumerate(entries) + if _span_substantial_x_overlap(span_rect, entry) + ] + if len(hit_positions) < 2: + continue + for start, end in _span_contiguous_ranges(hit_positions): + if end > start: + intervals.append((start, end, span_text)) + return _span_merge_intervals(intervals) + + +# --- strict-colspan gate: reject a merge that fights the header/body split ---- +# When strict_colspan is set, a header cell may only merge across columns if the +# body rows below actually split there (and vice-versa for a body cell against a +# leaf header). This keeps a stray text span from collapsing a real column. +def _span_is_leaf_header_range(row_idx, cols, base, body_start): + if row_idx >= body_start: + return False + for other in base: + if other["role"] not in {"header", "header_leaf", "header_group"}: + continue + if other["row"] <= row_idx or other["row"] >= body_start: + continue + if _span_ranges_intersect(cols, other["cols"]): + return False + return True + + +def _span_build_base_cells(cells, x_boundaries, body_start): + """Describe every grid cell by row/slot-range/role for the strict-colspan gate. + + Header cells split into ``header_leaf`` (no header cell below within body) vs + ``header_group`` (spans over a lower leaf), which the reject rules compare + against the body split.""" + base = [] + for row_idx, row in enumerate(cells): + for position, cell in enumerate([cell for cell in row if cell is not None]): + rect = pymupdf.Rect(cell) + cols = _span_slot_range(rect, x_boundaries) + base.append( + { + "row": row_idx, + "position": position, + "rect": rect, + "cols": cols, + "colspan": cols[1] - cols[0], + "role": "body" if row_idx >= body_start else "header", + } + ) + for item in base: + if item["role"] != "header": + continue + item["role"] = ( + "header_leaf" + if _span_is_leaf_header_range(item["row"], item["cols"], base, body_start) + else "header_group" + ) + return base + + +def _span_body_rows_split_under(cols, base, body_start): + """True if some body row splits the column range ``cols`` into >1 cell.""" + body_rows = sorted({item["row"] for item in base if item["role"] == "body" and item["row"] >= body_start}) + for row_idx in body_rows: + hits = [ + item + for item in base + if item["role"] == "body" and item["row"] == row_idx and _span_ranges_intersect(cols, item["cols"]) + ] + if not hits: + continue + if len(hits) == 1 and hits[0]["cols"] == cols: + continue + covered_start = min(max(cols[0], item["cols"][0]) for item in hits) + covered_end = max(min(cols[1], item["cols"][1]) for item in hits) + if covered_start <= cols[0] and covered_end >= cols[1]: + return True + return False + + +def _span_header_leafs_split_over(cols, base): + """True if leaf-header cells split the column range ``cols`` into >1 cell.""" + hits = [item for item in base if item["role"] == "header_leaf" and _span_ranges_intersect(cols, item["cols"])] + if not hits: + return False + if len(hits) == 1 and hits[0]["cols"] == cols: + return False + covered_start = min(max(cols[0], item["cols"][0]) for item in hits) + covered_end = max(min(cols[1], item["cols"][1]) for item in hits) + return covered_start <= cols[0] and covered_end >= cols[1] + + +def _span_reject_colspan_mismatch_merge(*, row_idx, cols, base, body_start): + """Whether to reject a candidate merge over ``cols`` as a (reason,) mismatch.""" + if cols[1] - cols[0] <= 1: + return False, "" + if row_idx < body_start: + if not _span_is_leaf_header_range(row_idx, cols, base, body_start): + return False, "" + if _span_body_rows_split_under(cols, base, body_start): + return True, "header_leaf_colspan_changed_against_body_split" + return False, "" + if _span_header_leafs_split_over(cols, base): + return True, "body_colspan_changed_against_header_leaf_split" + return False, "" + + +def _span_cell_texts_for_entries(page, entries, start, end, page_words): + texts = [] + for entry in entries[start : end + 1]: + words = _span_select_words_in_rect(page_words, entry) + texts.append(_span_words_text_for_rect(page, entry, words)) + return texts + + +def _span_allow_header_colspan_merge_with_empty_part(*, reason, page, entries, start, end, page_words): + """Allow an otherwise-rejected header merge when a covered part is empty. + + A header leaf that would be rejected against a body split is still merged if + one of the merged parts has no text (an empty header slot the body fills).""" + if reason != "header_leaf_colspan_changed_against_body_split": + return False + part_texts = [text.strip() for text in _span_cell_texts_for_entries(page, entries, start, end, page_words)] + return not all(part_texts) + + +def resolve_spans(page, cells, *, header_row_count=None, strict_colspan=False): + """Resolve a detected table's merged-cell (colspan/rowspan) structure. + + ``cells`` is a row-major grid: a list of rows, each a list of ``[x0, y0, x1, + y1]`` cell rectangles (``None`` for a gap) -- the same grid shape + :func:`refine_grid` accepts. Returns a row-major grid of :class:`SpanCell` + placements, ragged where cells span: each placement carries its union + ``bbox``, the page ``text`` it claims, and its ``colspan``/``rowspan`` (how + many clustered column/row slots it covers). Every page word is claimed by at + most one placement. + + ``header_row_count`` is the number of leading header rows (default a + conservative 1); it only matters together with ``strict_colspan``. + ``strict_colspan`` (default False), when set, refuses a merge whose colspan + would contradict the header/body column split. This is a PyMuPDF extension; + it reads page text/graphics but does not mutate the page. + """ + rows = len(cells) + + x_edges = [] + y_edges = [] + for row in cells: + for cell in row: + if cell is None: + continue + rect = pymupdf.Rect(cell) + x_edges.extend([rect.x0, rect.x1]) + y_edges.extend([rect.y0, rect.y1]) + + x_boundaries = _span_clustered_boundaries(x_edges) + y_boundaries = _span_clustered_boundaries(y_edges) + body_start = max(1, min(int(header_row_count or 1), rows)) if rows else 0 + base_cells = _span_build_base_cells(cells, x_boundaries, body_start) if strict_colspan else [] + + text_spans = _span_text_spans(page) + page_words = _refine_page_words(page) + claimed_words = set() + placements = [] + for row_idx, row in enumerate(cells): + placement_row = [] + entries = [pymupdf.Rect(cell) for cell in row if cell is not None] + intervals = _span_crossing_intervals(entries, text_spans) + intervals_by_start = {start: (end, texts) for start, end, texts in intervals} + position = 0 + while position < len(entries): + interval = intervals_by_start.get(position) + if interval is not None: + end_position, _crossing_texts = interval + rect = _span_rect_union(entries[position : end_position + 1]) + if strict_colspan: + candidate_cols = _span_slot_range(rect, x_boundaries) + reject, reason = _span_reject_colspan_mismatch_merge( + row_idx=row_idx, + cols=candidate_cols, + base=base_cells, + body_start=body_start, + ) + if reject and _span_allow_header_colspan_merge_with_empty_part( + reason=reason, + page=page, + entries=entries, + start=position, + end=end_position, + page_words=page_words, + ): + reject = False + if reject: + rect = entries[position] + position += 1 + else: + position = end_position + 1 + else: + position = end_position + 1 + else: + rect = entries[position] + position += 1 + + if rect.is_empty: + continue + colspan = _span_covered_slot_count(rect.x0, rect.x1, x_boundaries) + rowspan = _span_covered_slot_count(rect.y0, rect.y1, y_boundaries) + placement_row.append( + SpanCell( + bbox=tuple(rect), + text=_span_claim_text_in_rect(page, rect, page_words, claimed_words), + colspan=colspan, + rowspan=rowspan, + ) + ) + placements.append(placement_row) + + return placements diff --git a/src/_table_union.py b/src/_table_union.py new file mode 100644 index 000000000..d7a4d00e0 --- /dev/null +++ b/src/_table_union.py @@ -0,0 +1,364 @@ +""" +Copyright (C) 2023 Artifex Software, Inc. + +This file is part of PyMuPDF. + +PyMuPDF is free software: you can redistribute it and/or modify it under the +terms of the GNU Affero General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. + +PyMuPDF is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more +details. + +You should have received a copy of the GNU Affero General Public License +along with MuPDF. If not, see + +Alternative licensing terms are available from the licensor. +For commercial licensing, see or contact +Artifex Software, Inc., 39 Mesa Street, Suite 108A, San Francisco, +CA 94129, USA, for further information. + +--------------------------------------------------------------------- + +PyMuPDF table union stage, behind find_tables(union=True): fuse the layout +analyzer's table grids with the line-based finder's candidates. Table, +TableFinder and _iou come from pymupdf.table; find_tables is imported lazily. +""" + +import pymupdf + +from pymupdf.table import CHARS, EDGES, Table, TableFinder, _iou + + +# --------------------------------------------------------------------------- +# find_tables(union=True) fuses two table sources on one page: PRIMARY grids +# from the layout analyzer (each "table" group's GridPrediction, read from +# page.get_layout(return_raw=True) by _layout_table_grids in its raw form) and +# CANDIDATE grids from a nested line-based find_tables. A candidate matching a +# primary 1:1 (high IoU) may REPLACE its grid (grid-ref), several candidates +# each contained in one primary may SPLIT it, and candidates owned by no primary +# are APPENDED. The output order -- primaries (kept / grid-ref'd / split) then +# appended candidates -- is contractual: downstream consumers key tables by it. +# +# _layout_table_grids also replaces find_tables' make_table_from_bbox path, +# which silently yields an empty Table because the stext grid block it needs is +# not emitted; make_table_from_bbox is left in place pending a separate cleanup. +# --------------------------------------------------------------------------- +_UNION_STRATEGY = "lines_strict" # candidate detection strategy +_UNION_GRID_REF_IOU = 0.9 # min IoU for a 1:1 grid-ref replacement +_UNION_GRID_REF_SPAN_MULT_GATE = True # reject under-segmented candidate grids +_UNION_GRID_REF_SPAN_MULT_THRESHOLD = 3.0 # max horizontally-separated span groups per cell +_UNION_OWNER_CONTAINMENT = 0.85 # min containment for a split candidate's owner +_UNION_OWNER_AMBIGUOUS_OVERLAP = 0.25 # overlap above which an unowned candidate is suppressed + + +def _layout_table_grids(page): + """Primary table grids from the raw layout analyzer result. + + Reads page.layout_information in its raw (return_raw=True) form and yields a + ``(bbox, grid)`` pair per "table" group -- ``grid`` is the full row-major cell + grid built from the group box plus its interior GridPrediction lines, in + layout (reading) order. Boxes without a usable grid are skipped. + """ + grids = [] + for group in (page.layout_information or []): + if not isinstance(group, dict): + # The union path needs the raw (return_raw=True) layout form; the + # normalized [x0, y0, x1, y1, class] tuples carry no table_grid. + continue + if group.get("class_name") != "table": + continue + group_bbox = group.get("group_bbox") + grid_pred = group.get("table_grid") + if not group_bbox or grid_pred is None: + continue + x0, y0, x1, y1 = group_bbox[0], group_bbox[1], group_bbox[2], group_bbox[3] + h_lines = [y0] + [h + y0 for h in grid_pred.h_lines] + [y1] + v_lines = [x0] + [v + x0 for v in grid_pred.v_lines] + [x1] + grid = [] + for i in range(len(h_lines) - 1): + row = [] + for j in range(len(v_lines) - 1): + row.append((v_lines[j], h_lines[i], v_lines[j + 1], h_lines[i + 1])) + grid.append(row) + grids.append((pymupdf.Rect(group_bbox[:4]), grid)) + return grids + + +def _union_line_candidates(page): + """Line-based table candidates for the union stage as ``(bbox, grid)`` pairs. + + Runs a nested find_tables (strategy=_UNION_STRATEGY, use_layout=False) and + keeps each detected table's bbox and row-major cell grid (Table.rows, None + for a gap), deduped by rounded bbox. Returns ``(candidates, finder)``; the + finder is reused as the returned TableFinder shell. + """ + # Imported here, not at module top, to break the import cycle: this + # module is itself imported lazily by table.find_tables (union path). + from pymupdf.table import find_tables + finder = find_tables(page, strategy=_UNION_STRATEGY, use_layout=False) + candidates = [] + seen = set() + for tab in (getattr(finder, "tables", None) or []): + try: + bbox = pymupdf.Rect(tab.bbox) + except (ValueError, TypeError): + continue + if bbox.is_empty: + continue + grid = [[cell for cell in row.cells] for row in (tab.rows or [])] + if not grid: + continue + key = tuple(round(value) for value in bbox) + if key in seen: + continue + seen.add(key) + candidates.append((bbox, grid)) + return candidates, finder + + +def _union_rect_area(rect): + return max(0.0, float(rect.x1 - rect.x0)) * max(0.0, float(rect.y1 - rect.y0)) + + +def _union_intersection_area(left, right): + x0 = max(float(left.x0), float(right.x0)) + y0 = max(float(left.y0), float(right.y0)) + x1 = min(float(left.x1), float(right.x1)) + y1 = min(float(left.y1), float(right.y1)) + if x1 <= x0 or y1 <= y0: + return 0.0 + return (x1 - x0) * (y1 - y0) + + +def _union_x_overlap(left, right): + return max(0.0, min(float(left.x1), float(right.x1)) - max(float(left.x0), float(right.x0))) + + +def _union_find_owner(candidate_bbox, existing_bboxes): + """The primary a split candidate belongs inside, plus an ambiguity flag. + + Returns ``(owner_index, ambiguous)``: owner is the best-contained primary + (candidate>=_UNION_OWNER_CONTAINMENT inside it), else None; ambiguous is True + when the candidate overlaps some primary enough to be unsafe to append.""" + candidate_area = _union_rect_area(candidate_bbox) + if candidate_area <= 0: + return None, True + best_owner = None + best_containment = 0.0 + ambiguous = False + for index, existing_bbox in enumerate(existing_bboxes): + existing_area = _union_rect_area(existing_bbox) + inter_area = _union_intersection_area(candidate_bbox, existing_bbox) + if inter_area <= 0 or existing_area <= 0: + continue + candidate_containment = inter_area / candidate_area + existing_coverage = inter_area / existing_area + if candidate_containment >= _UNION_OWNER_CONTAINMENT and candidate_containment > best_containment: + best_owner = index + best_containment = candidate_containment + elif candidate_containment >= _UNION_OWNER_AMBIGUOUS_OVERLAP or existing_coverage >= _UNION_OWNER_AMBIGUOUS_OVERLAP: + ambiguous = True + return best_owner, ambiguous + + +def _union_text_span_rects(page): + """Non-empty page text-span rects, cached on the page. + + Drives the grid-ref span-multiplicity gate: every non-blank span as a bare + rect.""" + cached = getattr(page, "_union_text_spans_cache", None) + if cached is not None: + return cached + spans = [] + for block in page.get_text("dict").get("blocks", []) or []: + for line in block.get("lines", []) or []: + for span in line.get("spans", []) or []: + if not str(span.get("text") or "").strip(): + continue + bbox = span.get("bbox") + if not bbox: + continue + rect = pymupdf.Rect(bbox) + if not rect.is_empty: + spans.append(rect) + try: + setattr(page, "_union_text_spans_cache", spans) + except Exception: + pass + return spans + + +def _union_cell_span_group_count(cell, text_spans): + """Max horizontally-separated text-span groups on any single text line in a cell.""" + line_bands = [] + for span in text_spans: + center_y = (float(span.y0) + float(span.y1)) / 2.0 + if center_y < float(cell.y0) - 1.0 or center_y > float(cell.y1) + 1.0: + continue + if _union_x_overlap(span, cell) <= 1.0: + continue + x0 = max(float(cell.x0), float(span.x0)) + x1 = min(float(cell.x1), float(span.x1)) + if x1 <= x0: + continue + for index, (band_y, intervals) in enumerate(line_bands): + if abs(center_y - band_y) <= 4.0: + intervals.append((x0, x1)) + line_bands[index] = ((band_y + center_y) / 2.0, intervals) + break + else: + line_bands.append((center_y, [(x0, x1)])) + best = 0 + for _, intervals in line_bands: + groups = 0 + last_x1 = None + for x0, x1 in sorted(intervals): + if last_x1 is None or x0 - last_x1 > 4.0: + groups += 1 + last_x1 = x1 + else: + last_x1 = max(last_x1, x1) + best = max(best, groups) + return best + + +def _union_span_multiplicity(page, grid): + """Max cell span-group count over a grid (high => under-segmented grid).""" + if page is None: + return None + text_spans = _union_text_span_rects(page) + best = None + for row in grid: + for cell in row: + if cell is None: + continue + rect = pymupdf.Rect(cell) + if rect.is_empty: + continue + count = _union_cell_span_group_count(rect, text_spans) + if count > 0: + best = count if best is None else max(best, count) + return float(best) if best is not None else None + + +def _union_one_to_one_grid_refs(existing, candidates, *, iou_threshold, page, span_mult_gate, span_mult_threshold): + """Primaries a candidate can grid-ref (replace grid with), 1:1 by IoU. + + ``existing``/``candidates`` are ``(bbox, grid)`` lists. Returns ``(refs, + consumed)``: ``refs`` maps a primary index to the candidate supplying its + grid (mutual 1:1 IoU>=threshold matches passing the span-multiplicity gate), + ``consumed`` the candidate indexes used.""" + existing_matches = {} + candidate_matches = {} + for existing_index, (existing_bbox, _grid) in enumerate(existing): + for candidate_index, (candidate_bbox, _cgrid) in enumerate(candidates): + iou = _iou(existing_bbox, candidate_bbox) + if iou >= iou_threshold: + existing_matches.setdefault(existing_index, []).append((candidate_index, float(iou))) + candidate_matches.setdefault(candidate_index, []).append((existing_index, float(iou))) + refs = {} + consumed = set() + for existing_index, matches in existing_matches.items(): + if len(matches) != 1: + continue + candidate_index, _ = matches[0] + if len(candidate_matches.get(candidate_index, [])) != 1: + continue + if span_mult_gate: + span_mult = _union_span_multiplicity(page, candidates[candidate_index][1]) + if span_mult is not None and span_mult >= span_mult_threshold: + continue + refs[existing_index] = candidates[candidate_index] + consumed.add(candidate_index) + return refs, consumed + + +def _union_replace_append(existing, candidates, *, page, grid_ref, grid_ref_iou, span_mult_gate, span_mult_threshold): + """Fuse primary and candidate ``(bbox, grid)`` entries. + + Applies grid-ref replacement, split replacement (>=2 candidates owned by one + primary replace it, ordered by y0/x0) and append of unowned candidates, + returning the fused entry list in the contractual order.""" + existing_bboxes = [entry[0] for entry in existing] + if grid_ref: + grid_refs, consumed = _union_one_to_one_grid_refs( + existing, + candidates, + iou_threshold=grid_ref_iou, + page=page, + span_mult_gate=span_mult_gate, + span_mult_threshold=span_mult_threshold, + ) + else: + grid_refs, consumed = {}, set() + replacements = {} + append_candidates = [] + for candidate_index, candidate in enumerate(candidates): + if candidate_index in consumed: + continue + owner_index, ambiguous = _union_find_owner(candidate[0], existing_bboxes) + if owner_index is not None: + replacements.setdefault(owner_index, []).append(candidate) + elif ambiguous: + continue # overlaps a primary but is not contained -> suppress + else: + append_candidates.append(candidate) + final_replacements = { + index: sorted(items, key=lambda entry: (float(entry[0].y0), float(entry[0].x0))) + for index, items in replacements.items() + if len(items) >= 2 + } + entries = [] + for index, entry in enumerate(existing): + if index in final_replacements: + entries.extend(final_replacements[index]) + elif index in grid_refs: + # Grid-ref: keep the primary's (layout) bbox, take the candidate grid. + entries.append((entry[0], grid_refs[index][1])) + else: + entries.append(entry) + entries.extend(append_candidates) + return entries + + +def _find_tables_union(page): + """Detect a page's tables by fusing layout grids with line-based candidates. + + Ensures the raw layout (computed only when page.layout_information is None, + like the official use_layout path), reads primary grids, detects candidates, + applies grid-ref / split / append, and returns a TableFinder whose .tables + carry the fused grids in contractual order (grid-ref tables keep their + explicit layout bbox).""" + if page.layout_information is None: + page.get_layout(return_raw=True) + primaries = _layout_table_grids(page) + candidates, finder = _union_line_candidates(page) + entries = _union_replace_append( + primaries, + candidates, + page=page, + grid_ref=True, + grid_ref_iou=_UNION_GRID_REF_IOU, + span_mult_gate=_UNION_GRID_REF_SPAN_MULT_GATE, + span_mult_threshold=_UNION_GRID_REF_SPAN_MULT_THRESHOLD, + ) + if finder is None: + # Candidate detection failed mid-way; the nested find_tables may have + # left partially-filled EDGES/CHARS state behind, and TableFinder's + # constructor runs a full detection from that state -- clear it first + # so the fallback really is an empty shell. + EDGES.clear() + CHARS.clear() + finder = TableFinder(page) + tables = [] + for bbox, grid in entries: + flat = [cell for row in grid for cell in row if cell is not None] + if not flat: + continue + tables.append(Table(page, flat, bbox=bbox)) + finder.tables = tables + return finder diff --git a/src/table.py b/src/table.py index c0f8f8981..21e4dd052 100644 --- a/src/table.py +++ b/src/table.py @@ -77,20 +77,92 @@ import string import html from collections.abc import Sequence +from contextvars import ContextVar from dataclasses import dataclass from operator import itemgetter import weakref import pymupdf from pymupdf import mupdf +# The opt-in refine, cell-span and layout-union stages live in sibling modules, +# re-imported here to keep the public pymupdf.table.* surface unchanged. +# _table_union imports find_tables back from this module, so it is imported +# lazily inside find_tables() instead, to avoid an import cycle. +from pymupdf._table_refine import ( + refine_grid_structure, + refine_grid_rows, + _refine_cells_to_grid, + _refine_grid_to_cells, + _refine_page_words, +) +from pymupdf._table_spans import ( + resolve_spans, + SpanCell, + _span_select_words_in_rect, + _span_word_line_tuple, + _span_words_to_line_text, +) +# Header semantics and HTML serialization live in the _table_headers sibling. +from pymupdf._table_headers import ( + find_header_region, + collapse_cell_ws, + render_table_html, +) + +# refine_grid is unused in this module; re-exported for the public +# pymupdf.table.* surface. +from pymupdf._table_refine import refine_grid # noqa: F401 # pylint: disable=unused-import + # ------------------------------------------------------------------- # Start of PyMuPDF interface code # ------------------------------------------------------------------- # pylint: disable=no-name-in-module -EDGES = [] # vector graphics from PyMuPDF -CHARS = [] # text characters from PyMuPDF +_EDGES_VAR = ContextVar("pymupdf_table_edges", default=None) +_CHARS_VAR = ContextVar("pymupdf_table_chars", default=None) + + +class _TableStateList: + """List-like proxy for per-call table extraction state.""" + + def __init__(self, var): + self._var = var + + def _list(self): + value = self._var.get() + if value is None: + value = [] + self._var.set(value) + return value + + def append(self, item): + return self._list().append(item) + + def clear(self): + return self._list().clear() + + def extend(self, items): + return self._list().extend(items) + + def __bool__(self): + return bool(self._list()) + + def __getitem__(self, item): + return self._list()[item] + + def __iter__(self): + return iter(self._list()) + + def __len__(self): + return len(self._list()) + + def __setitem__(self, key, value): + self._list()[key] = value + + +EDGES = _TableStateList(_EDGES_VAR) # vector graphics from PyMuPDF +CHARS = _TableStateList(_CHARS_VAR) # text characters from PyMuPDF TEXTPAGE = None # textpage for cell text extraction TEXT_BOLD = mupdf.FZ_STEXT_BOLD TEXT_STRIKEOUT = mupdf.FZ_STEXT_STRIKEOUT @@ -1521,14 +1593,27 @@ def __init__(self, bbox, cells, names, above): class Table: - def __init__(self, page, cells): + def __init__(self, page, cells, bbox=None): self.page = page self.textpage = None + self._chars = None self.cells = cells + # Explicit reported-bbox override; None means bbox is computed from + # cells. Set only for a union grid-ref table, whose reported region + # (its layout box) is decoupled from its replacement cell grid. + self._bbox = bbox self.header = self._get_header() # PyMuPDF extension + # Filled by find_tables(refine=True): placements is a row-major grid of + # tagged SpanCell colspan/rowspan placements (None otherwise); header_rows + # is the leading header-row count, section_rows the section-label rows. + self.placements = None + self.header_rows = 0 + self.section_rows = () @property def bbox(self): + if self._bbox is not None: + return tuple(self._bbox) c = self.cells return ( min(map(itemgetter(0), c)), @@ -1557,7 +1642,7 @@ def col_count(self) -> int: # PyMuPDF extension return max([len(r.cells) for r in self.rows]) def extract(self, **kwargs) -> list: - chars = CHARS + chars = self._chars if self._chars is not None else CHARS table_arr = [] def char_in_bbox(char, bbox) -> bool: @@ -1664,6 +1749,26 @@ def to_markdown(self, clean=False, fill_empty=True): output += line return output + "\n" + def to_html(self): + """Output the table as an HTML ```` string. + + With span placements (from ``find_tables(refine=True)``) each + placement's colspan/rowspan and th/td tag are honoured and a + section-label row collapses to one spanning ``" in html + assert "" in html and "" in html + finally: + doc.close() + + +def test_render_table_html_section_row_collapse(): + """The core serializer collapses a section-label row (a lone centered label) + to a single
``. Otherwise a flat + td-only table is built from :meth:`extract`. + """ + if self.placements is not None: + return render_table_html(self.placements, self.section_rows) + # No placements: flat td-only grid from extract(). + rows = [ + [ + SpanCell(bbox=None, text=(cell or ""), colspan=1, rowspan=1) + for cell in row + ] + for row in self.extract() + ] + return render_table_html(rows) + def to_pandas(self, **kwargs): """Return a pandas DataFrame version of the table.""" try: @@ -2180,6 +2285,7 @@ def __getitem__(self, i): # ----------------------------------------------------------------------------- def make_chars(page, clip=None): """Extract text as "rawdict" to fill CHARS.""" + chars = CHARS._list() # bind once: avoid per-append proxy overhead below page_number = page.number + 1 page_height = page.rect.height ctm = page.transformation_matrix @@ -2234,7 +2340,7 @@ def make_chars(page, clip=None): "y0": bbox_ctm.y0, "y1": bbox_ctm.y1, } - CHARS.append(char_dict) + chars.append(char_dict) return TEXTPAGE @@ -2244,6 +2350,7 @@ def make_chars(page, clip=None): # else to lines. # ------------------------------------------------------------------------ def make_edges(page, clip=None, tset=None, paths=None, add_lines=None, add_boxes=None): + edges = EDGES._list() # bind once: avoid per-append proxy overhead below snap_x = tset.snap_x_tolerance snap_y = tset.snap_y_tolerance min_length = tset.edge_min_length @@ -2419,7 +2526,7 @@ def make_line(p, p1, p2, clip): p1, p2 = i[1:] line_dict = make_line(p, p1, p2, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) elif i[0] == "re": # A rectangle: decompose into 4 lines, but filter out @@ -2434,7 +2541,7 @@ def make_line(p, p1, p2, clip): p2 = pymupdf.Point(x, rect.y1) line_dict = make_line(p, p1, p2, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) continue if ( @@ -2445,24 +2552,24 @@ def make_line(p, p1, p2, clip): p2 = pymupdf.Point(rect.x1, y) line_dict = make_line(p, p1, p2, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) continue line_dict = make_line(p, rect.tl, rect.bl, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, rect.bl, rect.br, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, rect.br, rect.tr, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, rect.tr, rect.tl, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) else: # must be a quad # we convert it into (up to) 4 lines @@ -2470,37 +2577,37 @@ def make_line(p, p1, p2, clip): line_dict = make_line(p, ul, ll, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, ll, lr, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, lr, ur, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(p, ur, ul, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) path = {"color": (0, 0, 0), "fill": None, "width": 1} for bbox in bboxes: # add the border lines for all enveloping bboxes line_dict = make_line(path, bbox.tl, bbox.tr, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, bbox.bl, bbox.br, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, bbox.tl, bbox.bl, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, bbox.tr, bbox.br, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) if add_lines is not None: # add user-specified lines assert isinstance(add_lines, (tuple, list)) @@ -2511,7 +2618,7 @@ def make_line(p, p1, p2, clip): p2 = pymupdf.Point(p2) line_dict = make_line(path, p1, p2, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) if add_boxes is not None: # add user-specified rectangles assert isinstance(add_boxes, (tuple, list)) @@ -2521,16 +2628,16 @@ def make_line(p, p1, p2, clip): r = pymupdf.Rect(box) line_dict = make_line(path, r.tl, r.bl, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, r.bl, r.br, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, r.br, r.tr, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) line_dict = make_line(path, r.tr, r.tl, clip) if line_dict: - EDGES.append(line_to_edge(line_dict)) + edges.append(line_to_edge(line_dict)) def page_rotation_set0(page): @@ -2590,6 +2697,114 @@ def page_rotation_reset(page, xref, rot, mediabox): return page +# --------------------------------------------------------------------------- +# find_tables(refine=True) reconstruction glue. +# +# Resolve a merged-cell placement grid (falling back to a flat 1x1 grid when +# span resolution changes the column count), determine the header/body split, +# and tag each placement td/th. Built on resolve_spans, find_header_region and +# the refine_* stages; only reached when refine=True. +# --------------------------------------------------------------------------- +def _refine_placement_grid_width(placements): + """Column extent of a placement grid after resolving colspan/rowspan.""" + occupied = set() + max_col = 0 + for row_idx, row in enumerate(placements): + col_idx = 0 + for placement in row: + while (row_idx, col_idx) in occupied: + col_idx += 1 + for dr in range(placement.rowspan): + for dc in range(placement.colspan): + occupied.add((row_idx + dr, col_idx + dc)) + col_idx += placement.colspan + max_col = max(max_col, col_idx) + return max_col + + +def _refine_flat_placement_grid(page, cells, col_count): + """Flat fallback grid: one 1x1 SpanCell per slot, padded to ``col_count``. + + Selects words per cell by center-point and synthesizes each cell's line text + ("" for a gap), using the same word source and line builder as resolve_spans. + Used when span resolution changes the column count.""" + page_words = _refine_page_words(page) + grid = [] + for row in cells: + out = [] + for cell in row: + if cell is None: + out.append(SpanCell(bbox=None, text="", colspan=1, rowspan=1)) + else: + rect = pymupdf.Rect(cell) + line_words = [ + _span_word_line_tuple(word) + for _, word in _span_select_words_in_rect(page_words, rect) + ] + out.append( + SpanCell( + bbox=tuple(rect), + text=_span_words_to_line_text(line_words), + colspan=1, + rowspan=1, + ) + ) + while len(out) < col_count: + out.append(SpanCell(bbox=None, text="", colspan=1, rowspan=1)) + grid.append(out) + return grid + + +def _refine_placement_or_flat_grid(page, cells, *, strict_colspan=False, header_row_count=None): + """The reconstructed cell grid: resolved SpanCell placements, or the flat 1x1 + fallback when span resolution changes the column count (grid width != col + count). ``strict_colspan``/``header_row_count`` pass straight to resolve_spans.""" + col_count = max((len(row) for row in cells), default=0) + placements = resolve_spans( + page, cells, header_row_count=header_row_count, strict_colspan=strict_colspan + ) + if _refine_placement_grid_width(placements) == col_count: + return placements + return _refine_flat_placement_grid(page, cells, col_count) + + +def _refine_placements_text_grid(grid): + """Row-major whitespace-collapsed cell text -- the ``[[text]]`` header rules read.""" + return [[collapse_cell_ws(cell.text) for cell in row] for row in grid] + + +def _refine_tag_grid(grid, top_header_rows): + """Set each placement's HTML tag in place: cells in the top header rows + become ``th``; every other cell becomes ``td``.""" + for row_idx, row in enumerate(grid): + for cell in row: + cell.tag = "th" if row_idx < top_header_rows else "td" + return grid + + +def _refine_body_start_row(page, cells): + """Header/body boundary: resolve the merge-preserved placement grid once and + ask the header finder how many leading rows are header, clamped to [1, rows].""" + try: + model_grid = _refine_placement_or_flat_grid(page, cells) + region = find_header_region(_refine_placements_text_grid(model_grid)) + except Exception: + return 1 + raw = region.top_header_rows + return max(1, min(int(raw), len(cells))) if cells else 0 + + +def _refine_build_placements(page, working, body_start): + """Resolve the final placement grid (strict colspan, header boundary known), + run header rules on its own text grid, tag cells -> (tagged grid, region).""" + grid = _refine_placement_or_flat_grid( + page, working, strict_colspan=True, header_row_count=body_start + ) + region = find_header_region(_refine_placements_text_grid(grid)) + tagged = _refine_tag_grid(grid, region.top_header_rows) + return tagged, region + + def find_tables( page, clip=None, @@ -2616,11 +2831,39 @@ def find_tables( add_lines=None, # user-specified lines add_boxes=None, # user-specified rectangles paths=None, # accept vector graphics as parameter + use_layout: bool = True, # gate line-based tables by layout table boxes + union: bool = False, # opt-in: fuse layout grids with line-based candidates + refine: bool = False, # opt-in: refine each detected table's cell grid ): + """Detect and extract tables on a page. + + use_layout: if True (default), use page.get_layout() to find layout- + identified table regions and use them to restrict/complete line-based + detection; if layout ran and found no tables, return an empty result + immediately. Set False to always run full detection. + + union: if True (requires the layout analyzer), detect tables by fusing the + layout analyzer's table grids with the line-based finder's candidates -- a + candidate matching a layout table 1:1 can replace its grid, candidates + contained in one layout table can split it, and unowned candidates are + appended -- returning the fused tables in layout order then appended order. + Off by default. Needs the raw layout form: when page.layout_information is + None it is computed with page.get_layout(return_raw=True); when already + populated it is reused as-is. + + refine: if True, refine each detected table's cell grid before building the + Table -- splitting rows/columns the line grid merged (using page text and + background shading) -- then reconstruct its merged-cell structure and header + semantics. The tagged grid is attached as Table.placements (a row-major grid + of SpanCell colspan/rowspan placements, each carrying its td/th tag; None on + the default path), the header meta as Table.header_rows/section_rows, and + Table.to_html() serializes it. Off by default. A grid whose column count the + span resolution cannot preserve falls back to a flat one-cell-per-slot + placement grid. + """ pymupdf._warn_layout_once() - CHARS.clear() - EDGES.clear() - TEXTPAGE = None + _CHARS_VAR.set([]) + _EDGES_VAR.set([]) old_small = bool(pymupdf.TOOLS.set_small_glyph_heights()) # save old value pymupdf.TOOLS.set_small_glyph_heights(True) # we need minimum bboxes if page.rotation != 0: @@ -2668,55 +2911,93 @@ def find_tables( old_quad_corrections = pymupdf.TOOLS.unset_quad_corrections() try: - page.get_layout() - if page.layout_information: - pymupdf.TOOLS.unset_quad_corrections(True) - boxes = [ - pymupdf.Rect(b[:4]) for b in page.layout_information if b[-1] == "table" - ] + if union: + # Union path: fuse layout grids with line-based candidates instead + # of the ordinary use_layout gating. It builds its own tables (and, + # via a nested find_tables, its own char list), so the make_chars/ + # make_edges/TableFinder setup below is skipped; TEXTPAGE comes from + # the nested finder. Imported here, not at module top, to avoid an + # import cycle: _table_union imports find_tables back from this module. + from pymupdf._table_union import _find_tables_union + tbf = _find_tables_union(page) + TEXTPAGE = tbf.textpage else: boxes = [] + if use_layout: + page.get_layout() + if page.layout_information: + pymupdf.TOOLS.unset_quad_corrections(True) + boxes = [ + pymupdf.Rect(b[:4]) for b in page.layout_information if b[-1] == "table" + ] - if boxes: # layout did find some tables - pass - elif page.layout_information is not None: - # layout was executed but found no tables - # make sure we exit quickly with an empty TableFinder - tbf = TableFinder(page) - return tbf - - tset = TableSettings.resolve(settings=settings) - page.table_settings = tset - - TEXTPAGE = make_chars(page, clip=clip) # create character list of page - make_edges( - page, - clip=clip, - tset=tset, - paths=paths, - add_lines=add_lines, - add_boxes=add_boxes, - ) # create lines and curves - - tbf = TableFinder(page, settings=tset) - tbf.textpage = TEXTPAGE # store textpage for later use - if boxes: - # only keep Finder tables that match a layout box - tbf.tables = [ - tab - for tab in tbf.tables - if any(_iou(tab.bbox, r) >= 0.6 for r in boxes) + if not boxes and page.layout_information is not None: + # layout was executed but found no tables + # make sure we exit quickly with an empty TableFinder + tbf = TableFinder(page) + return tbf + + tset = TableSettings.resolve(settings=settings) + page.table_settings = tset + + TEXTPAGE = make_chars(page, clip=clip) # create character list of page + make_edges( + page, + clip=clip, + tset=tset, + paths=paths, + add_lines=add_lines, + add_boxes=add_boxes, + ) # create lines and curves + + tbf = TableFinder(page, settings=tset) + tbf.textpage = TEXTPAGE # store textpage for later use + if boxes: + # only keep Finder tables that match a layout box + tbf.tables = [ + tab + for tab in tbf.tables + if any(_iou(tab.bbox, r) >= 0.6 for r in boxes) + ] + # build the complementary list of layout table boxes + my_boxes = [ + r for r in boxes if all(_iou(r, tab.bbox) < 0.6 for tab in tbf.tables) ] - # build the complementary list of layout table boxes - my_boxes = [ - r for r in boxes if all(_iou(r, tab.bbox) < 0.6 for tab in tbf.tables) - ] - if my_boxes: - word_rects = [pymupdf.Rect(w[:4]) for w in TEXTPAGE.extractWORDS()] - tp2 = page.get_textpage(flags=TABLE_DETECTOR_FLAGS) - for rect in my_boxes: - cells = make_table_from_bbox(tp2, word_rects, rect) # pylint: disable=E0606 - tbf.tables.append(Table(page, cells)) + if my_boxes: + word_rects = [pymupdf.Rect(w[:4]) for w in TEXTPAGE.extractWORDS()] + tp2 = page.get_textpage(flags=TABLE_DETECTOR_FLAGS) + for rect in my_boxes: + cells = make_table_from_bbox(tp2, word_rects, rect) # pylint: disable=E0606 + tbf.tables.append(Table(page, cells)) + if refine: + # Grid refinement + reconstruction. Runs while the page is still + # derotated (before the finally block resets rotation) so word and + # vector coordinates match the detected cells. Order: structural + # split (shaded rows + under-segmented columns), resolve the header/ + # body boundary, split over-merged body rows, resolve the final + # merged-cell placement grid (strict colspan) and tag each placement + # td/th. The tagged grid is attached as .placements, the header meta + # as .header_rows/.section_rows. + refined_tables = [] + for tab in tbf.tables: + grid = _refine_cells_to_grid(tab.cells) + # The reported bbox (a union grid-ref table's layout box, else + # the cells' union) bounds the shaded-rectangle search. + working = refine_grid_structure(page, grid, table_bbox=tab.bbox) + body_start = _refine_body_start_row(page, working) + working = refine_grid_rows(page, working, header_row_count=body_start) + flat = _refine_grid_to_cells(working) + # Preserve an explicit reported-bbox override (union grid-ref + # tables): the refined grid must not change the reported region. + new_tab = Table(page, flat, bbox=tab._bbox) if flat else tab + # Build the tagged model on `working` directly, not on the + # re-gridded new_tab.cells, so placements match the refined grid. + placements, region = _refine_build_placements(page, working, body_start) + new_tab.placements = placements + new_tab.header_rows = region.top_header_rows + new_tab.section_rows = region.section_header_rows + refined_tables.append(new_tab) + tbf.tables = refined_tables except Exception as e: pymupdf.message("find_tables: exception occurred: %s" % str(e)) return None @@ -2725,6 +3006,12 @@ def find_tables( if old_xref is not None: page = page_rotation_reset(page, old_xref, old_rot, old_mediabox) pymupdf.TOOLS.unset_quad_corrections(old_quad_corrections) + + # Snapshot the page's characters onto each table so a later find_tables() + # call's CHARS reset cannot leak into this table's extract(). One shallow + # copy shared per call. + chars = list(CHARS) for table in tbf.tables: table.textpage = TEXTPAGE + table._chars = chars return tbf diff --git a/tests/test_pylint.py b/tests/test_pylint.py index b2cd215d0..edf01d8dd 100644 --- a/tests/test_pylint.py +++ b/tests/test_pylint.py @@ -118,6 +118,10 @@ def test_pylint(): '__init__.py', '__main__.py', '_apply_pages.py', + '_table_headers.py', + '_table_refine.py', + '_table_spans.py', + '_table_union.py', '_wxcolors.py', 'fitz___init__.py', 'fitz_table.py', diff --git a/tests/test_tables.py b/tests/test_tables.py index d42f9e652..66ce4c761 100644 --- a/tests/test_tables.py +++ b/tests/test_tables.py @@ -4,6 +4,7 @@ import textwrap import pickle import platform +from concurrent.futures import ThreadPoolExecutor import pymupdf @@ -249,6 +250,83 @@ def test_add_lines(): assert tab2.row_count == 5 +def _make_find_tables_state_doc(): + doc = pymupdf.open() + page = doc.new_page(width=360, height=220) + rect = pymupdf.Rect(40, 40, 320, 180) + cells = pymupdf.make_table(rect, rows=3, cols=3) + for row_index, row in enumerate(cells): + for col_index, cell in enumerate(row): + page.draw_rect(cell) + page.insert_textbox( + cell, + f"r{row_index}c{col_index}", + align=pymupdf.TEXT_ALIGN_CENTER, + ) + return doc.tobytes() + + +def _find_tables_use_layout_false_signature(pdf_bytes): + doc = pymupdf.open("pdf", pdf_bytes) + try: + table = doc[0].find_tables(strategy="lines_strict", use_layout=False)[0] + return table.row_count, table.col_count, table.extract() + finally: + doc.close() + + +def test_find_tables_use_layout_false_does_not_call_get_layout(): + """use_layout=False must keep find_tables on the pure line-based path.""" + pdf_bytes = _make_find_tables_state_doc() + doc = pymupdf.open("pdf", pdf_bytes) + page = doc[0] + page_cls = type(page) + original_get_layout = page_cls.get_layout + + def fail_get_layout(self, *args, **kwargs): + raise AssertionError("get_layout() should not be called") + + page_cls.get_layout = fail_get_layout + try: + table = page.find_tables(strategy="lines_strict", use_layout=False)[0] + assert table.row_count == 3 + assert table.col_count == 3 + assert table.extract()[1][1] == "r1c1" + finally: + page_cls.get_layout = original_get_layout + doc.close() + + +def test_find_tables_state_is_call_local_for_threads(): + """Concurrent find_tables calls must not mix text/vector extraction state.""" + if platform.python_implementation() == "GraalVM": + print("test_find_tables_state_is_call_local_for_threads(): not running because slow on GraalVM.") + return + + pdf_bytes = _make_find_tables_state_doc() + expected = _find_tables_use_layout_false_signature(pdf_bytes) + + # find_tables() saves and restores the process-global TOOLS flags (small + # glyph heights, quad corrections) around each call; concurrent calls can + # race that save/restore and leave a flag set. Snapshot the flags here and + # restore them explicitly so the suite's global-state checks stay clean. + small_before = bool(pymupdf.TOOLS.set_small_glyph_heights()) + quad_before = bool(pymupdf.TOOLS.unset_quad_corrections()) + try: + with ThreadPoolExecutor(max_workers=8) as executor: + results = list( + executor.map( + lambda _: _find_tables_use_layout_false_signature(pdf_bytes), + range(32), + ) + ) + finally: + pymupdf.TOOLS.set_small_glyph_heights(small_before) + pymupdf.TOOLS.unset_quad_corrections(quad_before) + + assert results == [expected] * 32 + + def test_3148(): """Ensure correct extraction text of rotated text.""" doc = pymupdf.open() @@ -447,3 +525,414 @@ def test_md_styles(): tabs = page.find_tables()[0] text = """|Column 1|Column 2|Column 3|\n|---|---|---|\n|Zelle (0,0)|**Bold (0,1)**|Zelle (0,2)|\n|~~Strikeout (1,0), Zeile 1~~
~~Hier kommt Zeile 2.~~|Zelle (1,1)|~~Strikeout (1,2)~~|\n|**`Bold-monospaced`**
**`(2,0)`**|_Italic (2,1)_|**_Bold-italic_**
**_(2,2)_**|\n|Zelle (3,0)|~~**Bold-strikeout**~~
~~**(3,1)**~~|Zelle (3,2)|\n\n""" assert tabs.to_markdown() == text + + +def _make_marker_table_doc(marker): + """Build a 1-page doc with a small drawn table whose cells embed `marker`.""" + doc = pymupdf.open() + page = doc.new_page(width=360, height=220) + rect = pymupdf.Rect(40, 40, 320, 180) + cells = pymupdf.make_table(rect, rows=2, cols=2) + for row_index, row in enumerate(cells): + for col_index, cell in enumerate(row): + page.draw_rect(cell) + page.insert_textbox( + cell, + f"{marker}-{row_index}{col_index}", + align=pymupdf.TEXT_ALIGN_CENTER, + ) + page.clean_contents() + return doc + + +def test_table_extract_stable_after_second_find_tables(): + """Regression test for the stale-CHARS bug. + + find_tables() snapshots table._chars right after each call so that an + already-returned Table's extract() cannot silently pick up a later, + unrelated find_tables() call's live (ContextVar-backed) CHARS content. + """ + doc1 = _make_marker_table_doc("PAGE1") + doc2 = _make_marker_table_doc("PAGE2") + try: + table1 = doc1[0].find_tables(strategy="lines_strict")[0] + first = table1.extract() + + # An unrelated find_tables() call on a different page/doc resets and + # repopulates the shared CHARS state used during text extraction. + doc2[0].find_tables(strategy="lines_strict") + + assert table1.extract() == first + flat_text = " ".join(cell for row in first for cell in row if cell) + assert "PAGE1" in flat_text # guard against both-empty passes + assert "PAGE2" not in flat_text + finally: + doc1.close() + doc2.close() + + +def test_find_tables_use_layout_true_without_layout_is_line_based(): + """use_layout=True (the default) must gracefully degrade to the pure + line-based detection path when the optional layout wheel/model is not + available: get_layout() becomes a no-op, page.layout_information stays + None, and results must match use_layout=False exactly.""" + pdf_bytes = _make_find_tables_state_doc() + doc = pymupdf.open("pdf", pdf_bytes) + page = doc[0] + original_get_layout_fn = pymupdf._get_layout + pymupdf._get_layout = None # simulate: pymupdf.layout wheel not installed + try: + tables_true = page.find_tables(strategy="lines_strict", use_layout=True) + assert page.layout_information is None + + tables_false = page.find_tables(strategy="lines_strict", use_layout=False) + + assert len(tables_true.tables) == 1 + assert [t.extract() for t in tables_true] == [ + t.extract() for t in tables_false + ] + table = tables_true[0] + assert table.row_count == 3 + assert table.col_count == 3 + assert table.extract()[1][1] == "r1c1" + finally: + pymupdf._get_layout = original_get_layout_fn + doc.close() + + +def _make_overmerged_page(): + """A page whose line grid detects one tall body row that actually holds + three record lines -- an under-segmented (over-merged) grid the refinement + is meant to repair. Needs no layout, so it exercises the standalone benefit. + """ + doc = pymupdf.open() + page = doc.new_page(width=400, height=300) + # 2-column grid: header row 100-120, a single tall body row 120-200. + for y in (100, 120, 200): + page.draw_line((100, y), (300, y)) + for x in (100, 200, 300): + page.draw_line((x, 100), (x, 200)) + page.insert_text((130, 114), "A") + page.insert_text((230, 114), "B") + # Three record lines crammed into the one body row. + for i, y in enumerate((140, 160, 180), start=1): + page.insert_text((130, y), str(i)) + page.insert_text((230, y), str(i * 10)) + return doc, page + + +def test_refine_grid_splits_overmerged_body(): + """refine_grid() splits an over-merged body row into one row per record. + + *** PyMuPDF extension (opt-in grid refinement). *** + """ + doc, page = _make_overmerged_page() + try: + grid = [ + [[100, 100, 200, 120], [200, 100, 300, 120]], # header row + [[100, 120, 200, 200], [200, 120, 300, 200]], # one over-merged body row + ] + refined = pymupdf.table.refine_grid(page, grid, header_row_count=1) + # header kept, body row split into the three record rows + assert len(grid) == 2 # input untouched + assert len(refined) == 4 + assert refined[0] == grid[0] # header preserved verbatim + assert all(len(r) == 2 for r in refined) + finally: + doc.close() + + +def test_find_tables_refine_splits_rows_default_unchanged(): + """find_tables(refine=True) repairs the over-merged grid; the default result + is unchanged -- refinement is strictly opt-in. + + *** PyMuPDF extension. *** + """ + doc, page = _make_overmerged_page() + try: + default = page.find_tables(use_layout=False) + refined = page.find_tables(use_layout=False, refine=True) + assert len(default.tables) == 1 + assert len(refined.tables) == 1 + + # Default detects the merged 2-row grid (unchanged behaviour). + assert default.tables[0].row_count == 2 + assert default.tables[0].col_count == 2 + + # refine=True splits the body into three record rows. + t = refined.tables[0] + assert t.row_count == 4 + assert t.col_count == 2 + assert t.extract() == [ + ["A", "B"], + ["1", "10"], + ["2", "20"], + ["3", "30"], + ] + finally: + doc.close() + + +def _make_merged_header_page(): + """A page whose line grid detects a header cell that spans both body columns. + + The middle vertical divider is drawn only in the body (below the header + separator), so the top row is one wide cell over two columns while each body + row has two cells -- a merged header cell find_tables detects on its own. + Needs no layout, so it exercises the standalone benefit. + """ + doc = pymupdf.open() + page = doc.new_page(width=400, height=300) + for y in (100, 120, 140, 160): + page.draw_line((100, y), (300, y)) + page.draw_line((100, 100), (100, 160)) # left border + page.draw_line((300, 100), (300, 160)) # right border + page.draw_line((200, 120), (200, 160)) # middle divider: body only + page.insert_text((150, 114), "Merged Header") + page.insert_text((120, 134), "a") + page.insert_text((220, 134), "b") + page.insert_text((120, 154), "c") + page.insert_text((220, 154), "d") + return doc, page + + +def test_resolve_spans_merged_header(): + """resolve_spans() surfaces a merged header cell as a colspan-2 SpanCell. + + *** PyMuPDF extension (opt-in span resolution). *** + """ + doc, page = _make_merged_header_page() + try: + grid = [ + [[100, 100, 300, 120]], # header spanning both columns + [[100, 120, 200, 140], [200, 120, 300, 140]], + [[100, 140, 200, 160], [200, 140, 300, 160]], + ] + placements = pymupdf.table.resolve_spans(page, grid) + assert len(placements) == 3 + # header is one placement spanning both columns + assert len(placements[0]) == 1 + head = placements[0][0] + assert (head.colspan, head.rowspan) == (2, 1) + assert head.bbox == (100.0, 100.0, 300.0, 120.0) + assert "Merged Header" in head.text + # resolve_spans leaves the HTML tag at its default; tagging td/th is the + # caller's job (find_tables(refine=True) / the engine model builder). + assert head.tag == "td" + # body cells stay 1x1 + assert [(c.colspan, c.rowspan) for c in placements[1]] == [(1, 1), (1, 1)] + assert [c.text for c in placements[2]] == ["c", "d"] + finally: + doc.close() + + +def test_find_tables_refine_exposes_placements_default_none(): + """find_tables(refine=True) attaches Table.placements with the colspan/rowspan + structure; the default result exposes no placements and is otherwise unchanged. + + *** PyMuPDF extension. *** + """ + doc, page = _make_merged_header_page() + try: + default = page.find_tables(use_layout=False) + refined = page.find_tables(use_layout=False, refine=True) + assert len(default.tables) == 1 + assert len(refined.tables) == 1 + + # Default detects the merged-header grid but resolves no spans. + dt = default.tables[0] + assert (dt.row_count, dt.col_count) == (3, 2) + assert dt.placements is None + + # refine=True exposes the header cell's colspan via .placements. + t = refined.tables[0] + assert t.placements is not None + assert t.placements[0][0].colspan == 2 + assert t.placements[0][0].rowspan == 1 + assert [c.colspan for c in t.placements[1]] == [1, 1] + # placements are tagged: the top header row is th, body rows are td. + assert t.placements[0][0].tag == "th" + assert [c.tag for c in t.placements[1]] == ["td", "td"] + assert [c.tag for c in t.placements[2]] == ["td", "td"] + finally: + doc.close() + + +def test_find_tables_refine_to_html_merged_header(): + """find_tables(refine=True) + Table.to_html() serialize a merged header as a + 		, with body rows as . + + *** PyMuPDF extension (opt-in header tagging + HTML serialization). *** + """ + doc, page = _make_merged_header_page() + try: + t = page.find_tables(use_layout=False, refine=True).tables[0] + html = t.to_html() + assert html == ( + "" + '' + "" + "" + "
Merged Header
ab
cd
" + ) + finally: + doc.close() + + +def test_find_tables_refine_header_meta(): + """find_tables(refine=True) exposes the header meta (header_rows/section_rows) + on the Table; the default path leaves the conservative defaults. + + *** PyMuPDF extension. *** + """ + doc, page = _make_merged_header_page() + try: + default = page.find_tables(use_layout=False).tables[0] + assert (default.header_rows, default.section_rows) == (0, ()) + + t = page.find_tables(use_layout=False, refine=True).tables[0] + assert isinstance(t.header_rows, int) and t.header_rows == 1 + assert isinstance(t.section_rows, tuple) and t.section_rows == () + finally: + doc.close() + + +def test_table_to_html_fallback_flat(): + """Table.to_html() on a default (non-refined) table returns a well-formed, + td-only flat built from extract() -- no placements needed. + + *** PyMuPDF extension. *** + """ + doc, page = _make_merged_header_page() + try: + t = page.find_tables(use_layout=False).tables[0] + assert t.placements is None + html = t.to_html() + assert html.startswith("
") and html.endswith("
") + assert "") == t.row_count + # every cell is a plain 		; the merged header text lands in one cell + assert "Merged Headerad, and honours per-cell td/th tags + colspan. + + *** PyMuPDF extension (HTML serialization). *** + """ + from pymupdf._table_headers import render_table_html + from pymupdf.table import SpanCell + + def cell(text, colspan=1, rowspan=1, tag="td"): + return SpanCell(bbox=None, text=text, colspan=colspan, rowspan=rowspan, tag=tag) + + rows = [ + [cell("Group", colspan=3, tag="th")], + [cell(""), cell("Section", tag="th"), cell("")], # centered section label + [cell("x"), cell("1"), cell("2")], + ] + html = render_table_html(rows, section_header_rows=(1,)) + assert html == ( + "" + '' + '' + "" + "
Group
Section
x12
" + ) + # escaping (& < >) and
line joins, quotes left literal + assert render_table_html([[cell('a & b < c > "d"\nsecond')]]) == ( + '
a & b < c > "d"
second
' + ) + + +def _make_bordered_table(page, x0, y0, texts): + """Draw a bordered 2x2 table (cells 100 wide, 20 tall) at (x0, y0), with the + 2x2 ``texts`` grid inserted into its cells; returns nothing (mutates page).""" + x1, x2 = x0 + 100, x0 + 200 + y1, y2 = y0 + 20, y0 + 40 + for y in (y0, y1, y2): + page.draw_line((x0, y), (x2, y)) + for x in (x0, x1, x2): + page.draw_line((x, y0), (x, y2)) + for r, ry in enumerate((y0, y1)): + for c, cx in enumerate((x0, x1)): + page.insert_text((cx + 5, ry + 14), texts[r][c]) + + +def test_find_tables_union_fuses_layout_grid_with_line_candidate(): + """find_tables(union=True) fuses the layout analyzer's GNN table grids with + the line-based finder's candidates: a layout table with no matching line + candidate is kept from its GNN grid, and a disjoint line-detected table is + appended -- layout order first, then appended candidates. + + *** PyMuPDF extension (opt-in layout/candidate union). *** + """ + import types + + doc = pymupdf.open() + page = doc.new_page(width=500, height=500) + # Table B: a real bordered 2x2 table the line finder detects (disjoint from A). + _make_bordered_table(page, 80, 300, [["b00", "b01"], ["b10", "b11"]]) + try: + # Table A: only a layout (GNN) grid, no drawn lines. Inject the raw layout + # form union reads (return_raw=True shape): a "table" group whose + # table_grid carries interior h_lines/v_lines offsets. + grid_pred = types.SimpleNamespace(h_lines=[20.0], v_lines=[100.0]) + page.layout_information = [ + { + "class_name": "table", + "group_bbox": [80.0, 80.0, 280.0, 120.0], + "table_grid": grid_pred, + } + ] + tf = page.find_tables(use_layout=True, union=True) + tables = tf.tables + assert len(tables) == 2 + + # A first (layout order): a 2x2 grid built from group_bbox + interior lines. + a = tables[0] + assert (a.row_count, a.col_count) == (2, 2) + assert tuple(a.bbox) == (80.0, 80.0, 280.0, 120.0) + a_cells = [[cell for cell in row.cells] for row in a.rows] + assert a_cells[0][0] == (80.0, 80.0, 180.0, 100.0) + assert a_cells[1][1] == (180.0, 100.0, 280.0, 120.0) + + # B appended after the layout table: the line-detected grid, extractable. + b = tables[1] + assert (b.row_count, b.col_count) == (2, 2) + assert b.extract()[0][0] == "b00" + finally: + doc.close() + + +def test_find_tables_union_no_layout_degrades_to_line_candidates(): + """union=True degrades to the pure line-based candidates when the layout + analyzer is unavailable: get_layout() is a no-op, layout_information stays + None, there are no primary grids, so every line-detected table is appended -- + matching the plain line-based find_tables result. + + *** PyMuPDF extension. *** + """ + doc = pymupdf.open() + page = doc.new_page(width=400, height=400) + _make_bordered_table(page, 80, 80, [["a", "b"], ["c", "d"]]) + original_get_layout_fn = pymupdf._get_layout + pymupdf._get_layout = None # simulate: pymupdf.layout wheel not installed + try: + union = page.find_tables(use_layout=True, union=True) + assert page.layout_information is None + + line = page.find_tables(strategy="lines_strict", use_layout=False) + assert len(union.tables) == 1 + # Same table as the pure line-based path, just routed through the union. + assert [t.extract() for t in union.tables] == [t.extract() for t in line.tables] + t = union.tables[0] + assert (t.row_count, t.col_count) == (2, 2) + assert t.extract()[0][0] == "a" + finally: + pymupdf._get_layout = original_get_layout_fn + doc.close() From 804b0de95a464374b98e16352f4d19bc2a0c8ed0 Mon Sep 17 00:00:00 2001 From: "youchang.kim" Date: Wed, 22 Jul 2026 20:18:21 +0900 Subject: [PATCH 2/2] table: probe candidate tables against CHARS instead of get_textbox() cells_to_tables()'s no-text filter passed the module-global TEXTPAGE to page.get_textbox(), but that global is never assigned anywhere -- its comment ('textpage for cell text extraction') suggests a global statement was lost at some point; make_chars() and find_tables() only ever bind locals of the same name. Every probe therefore built a fresh full-page TextPage and Python-walked all of its characters (JM_copy_rectangle), ~24ms per candidate table -- the largest single find_tables() cost after character extraction (~30% of wall time on a 503-page table corpus). The same characters are already available in CHARS in identical page space, so scan those instead (built lazily, only when a candidate survives the geometric checks), with the same strict-overlap rule as JM_rects_overlap() and the same whitespace-only rejection. Drop the now-unreferenced TEXTPAGE global. Measured on the 503-page corpus: every produced table's (bbox, cells) is hash-identical; find_tables() mean wall time drops 106.1 -> 75.9 ms/page (-28%). --- src/table.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/src/table.py b/src/table.py index 21e4dd052..68a771966 100644 --- a/src/table.py +++ b/src/table.py @@ -163,7 +163,6 @@ def __setitem__(self, key, value): EDGES = _TableStateList(_EDGES_VAR) # vector graphics from PyMuPDF CHARS = _TableStateList(_CHARS_VAR) # text characters from PyMuPDF -TEXTPAGE = None # textpage for cell text extraction TEXT_BOLD = mupdf.FZ_STEXT_BOLD TEXT_STRIKEOUT = mupdf.FZ_STEXT_STRIKEOUT FLAGS = ( @@ -1540,7 +1539,28 @@ def bbox_to_corners(bbox) -> tuple: tables.append(list(current_cells)) # PyMuPDF modification: - # Remove tables without text or having only 1 column + # Remove tables without text or having only 1 column. + # The text probe scans the current call's CHARS: the module-global + # TEXTPAGE it used to pass to page.get_textbox() was never assigned, so + # every probe built and walked a fresh full-page TextPage per candidate. + # The overlap test mirrors JM_rects_overlap() (strict), and a candidate + # counts as texty iff it overlaps a non-whitespace character. + text_bboxes = None + + def has_text(r): + nonlocal text_bboxes + if text_bboxes is None: # built once, only if a candidate survives + text_bboxes = [ + (c["x0"], c["top"], c["x1"], c["bottom"]) + for c in CHARS + if c["text"] not in white_spaces + ] + rx0, rtop, rx1, rbottom = r + for x0, top, x1, bottom in text_bboxes: + if x0 < rx1 and x1 > rx0 and top < rbottom and bottom > rtop: + return True + return False + for i in range(len(tables) - 1, -1, -1): r = pymupdf.EMPTY_RECT() x1_vals = set() @@ -1552,12 +1572,7 @@ def bbox_to_corners(bbox) -> tuple: if ( len(x1_vals) < 2 or len(x0_vals) < 2 - or white_spaces.issuperset( - page.get_textbox( - r, - textpage=TEXTPAGE, - ) - ) + or not has_text(r) ): del tables[i]