rst2kb.py

#!/usr/bin/env python3
"""
Export RST documentation files for Open WebUI RAG ingestion.

This script discovers .rst files under a source directory and optionally
converts them to plain text for ingestion into Open WebUI Knowledge Bases.

Usage:
    rst2kb.py export --source source --output /tmp/owui-rag
    rst2kb.py export --dry-run --source source --output /tmp/owui-rag
"""

import argparse
import hashlib
import http.client
import io
import json
import os
import sys
import time
import urllib.error
import urllib.request
import uuid
from collections.abc import Iterable
from datetime import UTC, datetime
from pathlib import Path
from typing import Protocol, cast

from docutils import nodes  # pyright: ignore[reportMissingModuleSource]
from docutils.core import publish_doctree  # pyright: ignore[reportMissingModuleSource,reportUnknownVariableType]


class _NodeParent(Protocol):
    def remove(self, node: object) -> None: ...


class _SystemMessageLike(Protocol):
    parent: _NodeParent | None


class _DocTreeLike(Protocol):
    def findall(self, condition: object) -> Iterable[_SystemMessageLike]: ...

    def astext(self) -> str: ...


# Directories to ignore during discovery
IGNORED_DIRS = {"_build", "build", ".venv", "venv", ".git"}


def _resolve_sphinx_path(path_text: str, current_file: Path, source_root: Path) -> Path:
    if path_text.startswith("/"):
        candidate = source_root / path_text.lstrip("/")
    else:
        candidate = current_file.parent / path_text
    return candidate.resolve()


def _is_within_root(candidate: Path, source_root: Path) -> bool:
    try:
        _ = candidate.relative_to(source_root)
        return True
    except ValueError:
        return False


def _collect_directive_block(
    lines: list[str], start_index: int
) -> tuple[list[str], int]:
    block: list[str] = []
    index = start_index

    while index < len(lines):
        line = lines[index]
        if line.strip() == "":
            block.append(line)
            index += 1
            continue

        if line.startswith((" ", "\t")):
            block.append(line)
            index += 1
            continue

        break

    return block, index


def _extract_option(block_lines: list[str], option_name: str) -> str | None:
    prefix = f":{option_name}:"
    for line in block_lines:
        stripped = line.strip()
        if stripped.startswith(prefix):
            value = stripped[len(prefix) :].strip()
            if value:
                return value
    return None


def _indent_lines(text: str, spaces: int = 3) -> list[str]:
    indentation = " " * spaces
    split_lines = text.splitlines()
    if not split_lines:
        return [indentation]
    return [f"{indentation}{line}" if line else "" for line in split_lines]


def _preprocess_rst_recursive(
    text: str,
    current_file: Path,
    source_root: Path,
    allow_include_outside_source: bool,
    depth: int,
    max_depth: int,
) -> tuple[str, list[str]]:
    warnings: list[str] = []
    output_lines: list[str] = []
    lines = text.splitlines()
    index = 0

    while index < len(lines):
        line = lines[index]
        stripped = line.lstrip()

        if stripped.startswith(".. toctree::"):
            _, index = _collect_directive_block(lines, index + 1)
            continue

        if stripped.startswith(".. raw::"):
            _, index = _collect_directive_block(lines, index + 1)
            continue

        if stripped.startswith(".. include::"):
            include_path_text = stripped[len(".. include::") :].strip()
            _, next_index = _collect_directive_block(lines, index + 1)

            if depth >= max_depth:
                warning = (
                    f"include depth limit ({max_depth}) reached at "
                    f"{current_file}: {include_path_text}"
                )
                warnings.append(warning)
                output_lines.append(
                    f"[include skipped: recursion depth limit reached for {include_path_text}]"
                )
                index = next_index
                continue

            resolved_include = _resolve_sphinx_path(
                include_path_text, current_file, source_root
            )
            within_root = _is_within_root(resolved_include, source_root)

            if not allow_include_outside_source and not within_root:
                warning = (
                    f"include blocked outside source_root: {include_path_text} "
                    f"(from {current_file})"
                )
                warnings.append(warning)
                output_lines.append(
                    f"[include blocked outside source_root: {include_path_text}]"
                )
                index = next_index
                continue

            if not resolved_include.exists() or not resolved_include.is_file():
                warning = (
                    f"include file not found: {include_path_text} "
                    f"(resolved to {resolved_include})"
                )
                warnings.append(warning)
                output_lines.append(f"[include missing: {include_path_text}]")
                index = next_index
                continue

            included_text = resolved_include.read_text(
                encoding="utf-8", errors="replace"
            )
            processed_include, include_warnings = _preprocess_rst_recursive(
                text=included_text,
                current_file=resolved_include,
                source_root=source_root,
                allow_include_outside_source=allow_include_outside_source,
                depth=depth + 1,
                max_depth=max_depth,
            )
            output_lines.extend(processed_include.splitlines())
            warnings.extend(include_warnings)
            index = next_index
            continue

        if stripped.startswith(".. literalinclude::"):
            literal_path_text = stripped[len(".. literalinclude::") :].strip()
            block_lines, next_index = _collect_directive_block(lines, index + 1)

            resolved_literal = _resolve_sphinx_path(
                literal_path_text, current_file, source_root
            )
            within_root = _is_within_root(resolved_literal, source_root)

            if not allow_include_outside_source and not within_root:
                warning = (
                    f"literalinclude blocked outside source_root: {literal_path_text} "
                    f"(from {current_file})"
                )
                warnings.append(warning)
                output_lines.append(
                    f"[literalinclude blocked outside source_root: {literal_path_text}]"
                )
                index = next_index
                continue

            if not resolved_literal.exists() or not resolved_literal.is_file():
                warning = (
                    f"literalinclude file not found: {literal_path_text} "
                    f"(resolved to {resolved_literal})"
                )
                warnings.append(warning)
                output_lines.append(f"[literalinclude missing: {literal_path_text}]")
                index = next_index
                continue

            language = _extract_option(block_lines, "language")
            output_lines.append(
                ".. code-block::" + (f" {language}" if language else "")
            )
            output_lines.append("")

            literal_text = resolved_literal.read_text(
                encoding="utf-8", errors="replace"
            )
            output_lines.extend(_indent_lines(literal_text, spaces=3))
            index = next_index
            continue

        output_lines.append(line)
        index += 1

    return "\n".join(output_lines), warnings


def preprocess_rst(
    text: str,
    current_file: str | Path,
    source_root: str | Path,
    allow_include_outside_source: bool = False,
) -> tuple[str, list[str]]:
    current_file_path = Path(current_file).resolve()
    source_root_path = Path(source_root).resolve()

    return _preprocess_rst_recursive(
        text=text,
        current_file=current_file_path,
        source_root=source_root_path,
        allow_include_outside_source=allow_include_outside_source,
        depth=0,
        max_depth=10,
    )


def discover_rst_files(source_dir: str) -> list[str]:
    """
    Discover all .rst files under source_dir, returning sorted relative paths.

    Ignores: _build/, build/, .venv/, venv/, .git/
    Does not follow symlinks.

    Args:
        source_dir: Root directory to search for .rst files

    Returns:
        Sorted list of relative paths to .rst files (relative to source_dir)
    """
    source_path = Path(source_dir).resolve()

    if not source_path.exists():
        raise ValueError(f"Source directory does not exist: {source_dir}")

    if not source_path.is_dir():
        raise ValueError(f"Source path is not a directory: {source_dir}")

    rst_files: list[str] = []

    for root, dirs, files in os.walk(source_path, followlinks=False):
        # Filter out ignored directories in-place to prevent descending into them
        dirs[:] = [d for d in dirs if d not in IGNORED_DIRS]

        root_path = Path(root)

        for filename in files:
            if filename.endswith(".rst"):
                file_path = root_path / filename
                # Get relative path from source_dir
                rel_path = file_path.relative_to(source_path)
                rst_files.append(str(rel_path))

    # Return sorted for deterministic ordering
    return sorted(rst_files)


def _normalize_plaintext(text: str) -> str:
    lines = [line.rstrip() for line in text.splitlines()]
    normalized = "\n".join(lines).strip()
    while "\n\n\n" in normalized:
        normalized = normalized.replace("\n\n\n", "\n\n")
    return normalized + "\n" if normalized else ""


def _convert_rst_to_text(rst_text: str) -> str:
    doctree = cast(
        _DocTreeLike,
        publish_doctree(
            rst_text,
            settings_overrides={"warning_stream": io.StringIO(), "halt_level": 6},
        ),
    )

    for system_message in list(doctree.findall(cast(object, nodes.system_message))):
        if system_message.parent is not None:
            system_message.parent.remove(system_message)
    return _normalize_plaintext(doctree.astext())


def _build_multipart_file_body(
    field_name: str, filename: str, content: bytes
) -> tuple[bytes, str]:
    safe_filename = os.path.basename(filename)
    safe_filename = safe_filename.replace("\r", "").replace("\n", "").replace('"', "")
    if not safe_filename:
        safe_filename = "document.txt"

    boundary = f"----OpenWebUIBoundary{uuid.uuid4().hex}"
    body = b"".join(
        [
            f"--{boundary}\r\n".encode("utf-8"),
            (
                f'Content-Disposition: form-data; name="{field_name}"; '
                f'filename="{safe_filename}"\r\n'
            ).encode("utf-8"),
            b"Content-Type: text/plain\r\n\r\n",
            content,
            b"\r\n",
            f"--{boundary}--\r\n".encode("utf-8"),
        ]
    )
    return body, boundary


def _http_request(
    url: str,
    method: str,
    headers: dict[str, str],
    body: bytes | None = None,
) -> tuple[int, dict[str, object] | None, str]:
    request = urllib.request.Request(url, data=body, headers=headers, method=method)
    try:
        response_handle = cast(
            http.client.HTTPResponse, urllib.request.urlopen(request)
        )
        with response_handle as response:
            status_code = int(response.status)
            response_text = response.read().decode("utf-8", errors="replace")
    except urllib.error.HTTPError as exc:
        status_code = exc.code
        response_text = exc.read().decode("utf-8", errors="replace")
    except urllib.error.URLError as exc:
        raise RuntimeError(f"Request failed for {url}: {exc.reason}") from exc

    payload: dict[str, object] | None = None
    if response_text.strip():
        try:
            decoded_obj = cast(object, json.loads(response_text))
            if isinstance(decoded_obj, dict):
                payload = cast(dict[str, object], decoded_obj)
        except json.JSONDecodeError:
            payload = None

    return status_code, payload, response_text


def _extract_file_id(payload: dict[str, object] | None) -> str | None:
    if payload is None:
        return None

    direct_id = payload.get("id")
    if isinstance(direct_id, str):
        return direct_id

    data_obj = payload.get("data")
    if isinstance(data_obj, dict):
        data_dict = cast(dict[str, object], data_obj)
        nested_id = data_dict.get("id")
        if isinstance(nested_id, str):
            return nested_id
    return None


def _extract_processing_status(payload: dict[str, object] | None) -> str | None:
    if payload is None:
        return None

    direct_status = payload.get("status")
    if isinstance(direct_status, str):
        return direct_status.lower()

    data_obj = payload.get("data")
    if isinstance(data_obj, dict):
        data_dict = cast(dict[str, object], data_obj)
        nested_status = data_dict.get("status")
        if isinstance(nested_status, str):
            return nested_status.lower()
    return None


def main():
    parser = argparse.ArgumentParser(
        description="Export RST files for Open WebUI RAG ingestion"
    )

    subparsers = parser.add_subparsers(dest="command", help="Available commands")

    # Export subcommand
    export_parser = subparsers.add_parser("export", help="Export RST files")
    _ = export_parser.add_argument(
        "--dry-run",
        action="store_true",
        help="Perform discovery without writing any files",
    )
    _ = export_parser.add_argument(
        "--source",
        type=str,
        default="source",
        help="Source directory containing RST files (default: source)",
    )
    _ = export_parser.add_argument(
        "--output",
        type=str,
        default="/tmp/owui-rag",
        help="Output directory for exported files",
    )
    _ = export_parser.add_argument(
        "--max-files", type=int, default=None, help="Maximum number of files to process"
    )
    _ = export_parser.add_argument(
        "--copy-raw",
        action="store_true",
        help="Copy original RST files to output/raw",
    )
    _ = export_parser.add_argument(
        "--fail-fast",
        action="store_true",
        help="Stop on first file conversion failure",
    )

    upload_parser = subparsers.add_parser(
        "upload", help="Upload exported TXT files to Open WebUI"
    )
    _ = upload_parser.add_argument(
        "--output",
        type=str,
        required=True,
        help="Export output directory containing manifest.json and txt/",
    )
    _ = upload_parser.add_argument(
        "--openwebui-url",
        type=str,
        default="http://localhost:3000",
        help="Open WebUI base URL (default: http://localhost:3000)",
    )
    _ = upload_parser.add_argument(
        "--knowledge-id",
        type=str,
        required=True,
        help="Target Open WebUI knowledge base ID",
    )
    _ = upload_parser.add_argument(
        "--api-key-env",
        type=str,
        default="OPENWEBUI_API_KEY",
        help="Environment variable name for API key (default: OPENWEBUI_API_KEY)",
    )
    _ = upload_parser.add_argument(
        "--poll-interval",
        type=float,
        default=2.0,
        help="Polling interval in seconds (default: 2)",
    )
    _ = upload_parser.add_argument(
        "--poll-timeout",
        type=float,
        default=300.0,
        help="Per-file polling timeout in seconds (default: 300)",
    )
    _ = upload_parser.add_argument(
        "--fail-fast",
        action="store_true",
        help="Stop on first upload failure",
    )

    args = parser.parse_args()
    command = cast(str | None, getattr(args, "command", None))

    if command == "upload":
        output_dir = Path(cast(str, getattr(args, "output", ""))).resolve()
        openwebui_url = cast(str, getattr(args, "openwebui_url", "")).rstrip("/")
        knowledge_id = cast(str, getattr(args, "knowledge_id", ""))
        api_key_env = cast(str, getattr(args, "api_key_env", "OPENWEBUI_API_KEY"))
        poll_interval = cast(float, getattr(args, "poll_interval", 2.0))
        poll_timeout = cast(float, getattr(args, "poll_timeout", 300.0))
        fail_fast = cast(bool, getattr(args, "fail_fast", False))

        if poll_interval <= 0:
            print("ERROR: --poll-interval must be greater than 0", file=sys.stderr)
            sys.exit(1)

        if poll_timeout <= 0:
            print("ERROR: --poll-timeout must be greater than 0", file=sys.stderr)
            sys.exit(1)

        api_key = os.getenv(api_key_env)
        if not api_key:
            print(
                f"ERROR: Missing API key in environment variable {api_key_env}",
                file=sys.stderr,
            )
            sys.exit(1)

        manifest_path = output_dir / "manifest.json"
        txt_root = output_dir / "txt"
        report_path = output_dir / "upload_report.json"

        if not manifest_path.exists() or not manifest_path.is_file():
            print(f"ERROR: manifest not found at {manifest_path}", file=sys.stderr)
            sys.exit(1)

        try:
            manifest_raw = manifest_path.read_text(encoding="utf-8")
            manifest_data_obj = cast(object, json.loads(manifest_raw))
        except (OSError, json.JSONDecodeError) as exc:
            print(f"ERROR: failed to read manifest: {exc}", file=sys.stderr)
            sys.exit(1)

        if not isinstance(manifest_data_obj, dict):
            print("ERROR: manifest root must be an object", file=sys.stderr)
            sys.exit(1)
        manifest_data = cast(dict[str, object], manifest_data_obj)

        files_data_obj = manifest_data.get("files")
        if not isinstance(files_data_obj, list):
            print("ERROR: manifest missing 'files' list", file=sys.stderr)
            sys.exit(1)
        files_data = cast(list[object], files_data_obj)

        upload_url = f"{openwebui_url}/api/v1/files/"
        attach_url = f"{openwebui_url}/api/v1/knowledge/{knowledge_id}/file/add"

        attached = 0
        failed = 0
        report_files: list[dict[str, object]] = []
        seen_content_hashes: dict[str, str] = {}

        for entry in files_data:
            txt_rel = ""
            source_rel = None
            if isinstance(entry, dict):
                entry_dict = cast(dict[str, object], entry)
                txt_rel_value = entry_dict.get("txt_rel")
                if isinstance(txt_rel_value, str):
                    txt_rel = txt_rel_value
                source_rel_value = entry_dict.get("source_rel")
                if isinstance(source_rel_value, str):
                    source_rel = source_rel_value
                manifest_error = entry_dict.get("error")
                if manifest_error is not None and manifest_error != "":
                    skipped_result: dict[str, object] = {
                        "txt_rel": txt_rel,
                        **(
                            {"source_rel": source_rel} if source_rel is not None else {}
                        ),
                        "status": "skipped",
                        "upload_http_status": None,
                        "status_http_status": None,
                        "attach_http_status": None,
                        "file_id": None,
                        "attach_ok": None,
                        "error": manifest_error,
                    }
                    report_files.append(skipped_result)
                    continue

            result: dict[str, object] = {
                "txt_rel": txt_rel,
                **({"source_rel": source_rel} if source_rel is not None else {}),
                "status": "error",
                "upload_http_status": None,
                "status_http_status": None,
                "attach_http_status": None,
                "file_id": None,
                "attach_ok": None,
                "error": None,
            }

            if not txt_rel:
                failed += 1
                result["error"] = "manifest entry missing txt_rel"
                report_files.append(result)
                if fail_fast:
                    break
                continue

            txt_path = (txt_root / txt_rel).resolve()
            try:
                _ = txt_path.relative_to(txt_root)
            except ValueError:
                failed += 1
                result["error"] = "txt_rel resolves outside txt root"
                report_files.append(result)
                if fail_fast:
                    break
                continue

            if not txt_path.exists() or not txt_path.is_file():
                failed += 1
                result["error"] = f"text file not found for txt_rel: {txt_rel}"
                report_files.append(result)
                if fail_fast:
                    break
                continue

            try:
                file_content = txt_path.read_bytes()

                if len(file_content) == 0:
                    empty_result: dict[str, object] = {
                        "txt_rel": txt_rel,
                        **(
                            {"source_rel": source_rel} if source_rel is not None else {}
                        ),
                        "status": "skipped",
                        "upload_http_status": None,
                        "status_http_status": None,
                        "attach_http_status": None,
                        "file_id": None,
                        "attach_ok": None,
                        "error": "empty text content",
                    }
                    report_files.append(empty_result)
                    continue

                content_hash = hashlib.sha256(file_content).hexdigest()
                first_txt_rel = seen_content_hashes.get(content_hash)
                if first_txt_rel is not None:
                    duplicate_result: dict[str, object] = {
                        "txt_rel": txt_rel,
                        **(
                            {"source_rel": source_rel} if source_rel is not None else {}
                        ),
                        "status": "skipped",
                        "upload_http_status": None,
                        "status_http_status": None,
                        "attach_http_status": None,
                        "file_id": None,
                        "attach_ok": None,
                        "error": (
                            "duplicate content in upload batch; "
                            f"matches {first_txt_rel}"
                        ),
                    }
                    report_files.append(duplicate_result)
                    continue
                seen_content_hashes[content_hash] = txt_rel

                multipart_body, boundary = _build_multipart_file_body(
                    field_name="file",
                    filename=txt_path.name,
                    content=file_content,
                )
                upload_headers = {
                    "Authorization": f"Bearer {api_key}",
                    "Content-Type": f"multipart/form-data; boundary={boundary}",
                }

                upload_status, upload_payload, _ = _http_request(
                    upload_url,
                    method="POST",
                    headers=upload_headers,
                    body=multipart_body,
                )
                result["upload_http_status"] = upload_status
                if upload_status < 200 or upload_status >= 300:
                    failed += 1
                    result["error"] = f"upload failed with HTTP {upload_status}"
                    report_files.append(result)
                    if fail_fast:
                        break
                    continue

                file_id = _extract_file_id(upload_payload)
                if file_id is None:
                    failed += 1
                    result["error"] = (
                        "upload succeeded but response did not include file id"
                    )
                    report_files.append(result)
                    if fail_fast:
                        break
                    continue
                result["file_id"] = file_id

                status_url = f"{openwebui_url}/api/v1/files/{file_id}/process/status"
                poll_deadline = time.monotonic() + poll_timeout
                while True:
                    status_code, status_payload, _ = _http_request(
                        status_url,
                        method="GET",
                        headers={"Authorization": f"Bearer {api_key}"},
                    )
                    result["status_http_status"] = status_code

                    if status_code < 200 or status_code >= 300:
                        failed += 1
                        result["error"] = f"poll failed with HTTP {status_code}"
                        report_files.append(result)
                        if fail_fast:
                            break
                        break

                    processing_status = _extract_processing_status(status_payload)
                    if processing_status in {"completed", "processed"}:
                        break

                    if processing_status in {"failed", "error"}:
                        failed += 1
                        result["error"] = (
                            f"processing failed with status '{processing_status}'"
                        )
                        report_files.append(result)
                        if fail_fast:
                            break
                        break

                    if time.monotonic() >= poll_deadline:
                        failed += 1
                        result["error"] = "processing status polling timed out"
                        report_files.append(result)
                        if fail_fast:
                            break
                        break

                    time.sleep(poll_interval)

                if result["error"] is not None:
                    if fail_fast:
                        break
                    continue

                attach_status, _, _ = _http_request(
                    attach_url,
                    method="POST",
                    headers={
                        "Authorization": f"Bearer {api_key}",
                        "Content-Type": "application/json",
                    },
                    body=json.dumps({"file_id": file_id}).encode("utf-8"),
                )
                result["attach_http_status"] = attach_status
                result["attach_ok"] = 200 <= attach_status < 300

                if attach_status < 200 or attach_status >= 300:
                    failed += 1
                    result["error"] = f"attach failed with HTTP {attach_status}"
                    report_files.append(result)
                    if fail_fast:
                        break
                    continue

                result["status"] = "ok"
                attached += 1
                report_files.append(result)
            except (OSError, RuntimeError) as exc:
                failed += 1
                result["error"] = str(exc)
                report_files.append(result)
                if fail_fast:
                    break

        report = {
            "summary": {
                "attached": attached,
                "failed": failed,
            },
            "files": report_files,
        }
        output_dir.mkdir(parents=True, exist_ok=True)
        _ = report_path.write_text(
            json.dumps(report, indent=2, sort_keys=True), encoding="utf-8"
        )

        print(f"UPLOAD attached={attached} failed={failed}")
        sys.exit(0 if failed == 0 else 1)

    source = cast(str, getattr(args, "source", "source"))
    output_dir = cast(str, getattr(args, "output", "/tmp/owui-rag"))
    max_files = cast(int | None, getattr(args, "max_files", None))
    dry_run = cast(bool, getattr(args, "dry_run", False))
    copy_raw = cast(bool, getattr(args, "copy_raw", False))
    fail_fast = cast(bool, getattr(args, "fail_fast", False))

    if command != "export":
        parser.print_help()
        sys.exit(1)

    # Discover RST files
    try:
        rst_files = discover_rst_files(source)
    except ValueError as e:
        print(f"ERROR: {e}", file=sys.stderr)
        sys.exit(1)

    discovered = len(rst_files)
    selected_files = rst_files

    # Apply max-files limit if specified
    if max_files is not None and max_files < discovered:
        selected_files = rst_files[:max_files]

    will_write = 0 if dry_run else len(selected_files)

    if dry_run:
        print(f"DRYRUN discovered={discovered} will_write={will_write}")
        sys.exit(0)

    source_root = Path(source).resolve()
    output_root = Path(output_dir).resolve()
    txt_root = output_root / "txt"
    raw_root = output_root / "raw"

    converted = 0
    failed = 0
    skipped = discovered - len(selected_files)
    file_results: list[dict[str, object]] = []
    started_at = datetime.now(UTC).isoformat()

    for rel_path in selected_files:
        source_file = source_root / rel_path
        txt_rel = str(Path(rel_path).with_suffix(".txt"))
        txt_path = txt_root / txt_rel
        raw_rel = str(Path(rel_path)) if copy_raw else None

        warnings: list[str] = []
        error_text: str | None = None
        sha256_text: str | None = None
        bytes_in = 0
        bytes_out = 0

        try:
            input_text = source_file.read_text(encoding="utf-8", errors="replace")
            bytes_in = len(input_text.encode("utf-8"))

            preprocessed_text, preprocess_warnings = preprocess_rst(
                text=input_text,
                current_file=source_file,
                source_root=source_root,
            )
            warnings.extend(preprocess_warnings)

            rendered_text = _convert_rst_to_text(preprocessed_text)
            rendered_bytes = rendered_text.encode("utf-8")
            bytes_out = len(rendered_bytes)
            sha256_text = hashlib.sha256(rendered_bytes).hexdigest()

            txt_path.parent.mkdir(parents=True, exist_ok=True)
            _ = txt_path.write_text(rendered_text, encoding="utf-8")

            if copy_raw:
                raw_path = raw_root / rel_path
                raw_path.parent.mkdir(parents=True, exist_ok=True)
                _ = raw_path.write_text(input_text, encoding="utf-8")

            converted += 1
        except Exception as exc:
            failed += 1
            error_text = str(exc)
            if fail_fast:
                file_results.append(
                    {
                        "source_rel": rel_path,
                        "txt_rel": txt_rel,
                        **({"raw_rel": raw_rel} if raw_rel is not None else {}),
                        "sha256": sha256_text,
                        "bytes_in": bytes_in,
                        "bytes_out": bytes_out,
                        "warnings": warnings,
                        "error": error_text,
                    }
                )
                break

        file_results.append(
            {
                "source_rel": rel_path,
                "txt_rel": txt_rel,
                **({"raw_rel": raw_rel} if raw_rel is not None else {}),
                "sha256": sha256_text,
                "bytes_in": bytes_in,
                "bytes_out": bytes_out,
                "warnings": warnings,
                "error": error_text,
            }
        )

    if fail_fast and len(file_results) < len(selected_files):
        skipped += len(selected_files) - len(file_results)

    finished_at = datetime.now(UTC).isoformat()
    summary = {
        "started_at": started_at,
        "finished_at": finished_at,
        "discovered": discovered,
        "converted": converted,
        "skipped": skipped,
        "failed": failed,
    }
    manifest = {
        "summary": summary,
        "files": file_results,
    }

    output_root.mkdir(parents=True, exist_ok=True)
    manifest_path = output_root / "manifest.json"
    _ = manifest_path.write_text(
        json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8"
    )

    print(
        f"DONE discovered={discovered} converted={converted} skipped={skipped} failed={failed}"
    )
    sys.exit(0 if failed == 0 else 1)


if __name__ == "__main__":
    main()