Generate llms output in one Sphinx build

pyproject.toml

@@ -38,6 +38,7 @@ dependencies = [
     "sphinx-autoapi",
     "sphinx-copybutton",
     "sphinx-design",
+    "sphinx-markdown-builder>=0.6.9",
     "sphinxcontrib-mermaid>=2.0.0rc0,<2.2",
     "sphinx-reredirects",
     "toml",
@@ -58,9 +59,6 @@ wiki = [
     "sphinx-markdown-builder>=0.6.9",
 
 ]
-llms = [
-    "sphinx-markdown-builder>=0.6.9",
-]
 themes = [
     "shibuya",
     "sphinxawesome-theme",
@@ -84,7 +82,6 @@ develop = [
     "breathe>=4.35.0",
     "sphinx-rust",
     "sphinx-js>=5.0.0",
-    "sphinx-markdown-builder>=0.6.9",
     # Themes
     "shibuya",
     "sphinxawesome-theme",

yardang/cli.py

@@ -35,8 +35,6 @@ def build(
     config_base: str | None = "tool.yardang",
     previous_versions: bool | None = False,
 ):
-    llms_config_base = f"{config_base or 'tool.yardang'}.llms"
-    use_llms = get_config(section="enabled", base=llms_config_base) is True
     with generate_docs_configuration(
         project=project,
         title=title,
@@ -57,25 +55,21 @@ def build(
         config_base=config_base,
         previous_versions=previous_versions,
     ) as file:
-        build_commands = [[executable, "-m", "sphinx", ".", output, "-c", file]]
-        if use_llms:
-            build_commands.append([executable, "-m", "sphinx", "-b", "yardang-llms", ".", output, "-c", file])
-
-        for build_cmd in build_commands:
-            if debug:
-                print(" ".join(build_cmd))
-            if quiet:
-                process = Popen(build_cmd)
-            else:
-                process = Popen(build_cmd, stderr=stderr, stdout=stdout)
-            while process.poll() is None:
-                sleep(0.1)
-            if process.returncode != 0:
-                if pdb:
-                    import pdb  # noqa: T100
-
-                    pdb.set_trace()  # noqa: T100
-                raise Exit(process.returncode)
+        build_cmd = [executable, "-m", "sphinx", ".", output, "-c", file]
+        if debug:
+            print(" ".join(build_cmd))
+        if quiet:
+            process = Popen(build_cmd)
+        else:
+            process = Popen(build_cmd, stderr=stderr, stdout=stdout)
+        while process.poll() is None:
+            sleep(0.1)
+        if process.returncode != 0:
+            if pdb:
+                import pdb  # noqa: T100
+
+                pdb.set_trace()  # noqa: T100
+            raise Exit(process.returncode)
 
 
 def debug():

yardang/sphinx/llms.py

@@ -1,13 +1,60 @@
 from pathlib import Path
+from urllib.parse import urlsplit, urlunsplit
 
 from docutils import nodes
+from sphinx.application import Sphinx
 from sphinx_markdown_builder.builder import MarkdownBuilder
+from sphinx_markdown_builder.translator import MarkdownTranslator
 
 from yardang import __version__
 
+_DOCNAMES_ATTRIBUTE = "_yardang_llms_docnames"
+
+
+def _target_uri(docname: str) -> str:
+    return f"{docname}.html.md"
+
+
+class _LlmsMarkdownTranslator(MarkdownTranslator):
+    def visit_toctree(self, node: nodes.Node) -> None:
+        raise nodes.SkipNode
+
+    def _fetch_ref_uri(self, node: nodes.Node) -> str:
+        refuri = super()._fetch_ref_uri(node)
+        parsed = urlsplit(refuri)
+        if parsed.scheme or parsed.netloc or not parsed.path.endswith(".html"):
+            return refuri
+        return urlunsplit(parsed._replace(path=f"{parsed.path}.md"))
+
+
+class _MarkdownRenderer:
+    """Render resolved Sphinx doctrees without running a second builder."""
+
+    name = "yardang-llms"
+    format = "markdown"
+    default_translator_class = _LlmsMarkdownTranslator
+
+    def __init__(self, app: Sphinx):
+        self.app = app
+        self.config = app.config
+        self.current_doc_name = ""
+
+    def create_translator(self, *args):
+        return self.app.registry.create_translator(self, *args)
+
+    @staticmethod
+    def get_target_uri(docname: str, typ: str | None = None) -> str:
+        return _target_uri(docname)
+
+    def render(self, docname: str, doctree: nodes.document) -> str:
+        self.current_doc_name = docname
+        translator = self.create_translator(doctree, self)
+        doctree.walkabout(translator)
+        return translator.astext()
+
 
 class LlmsBuilder(MarkdownBuilder):
-    """Build LLM-friendly Markdown and index files."""
+    """Retain an explicit builder for direct Sphinx use."""
 
     name = "yardang-llms"
     epilog = "The LLM-friendly documentation is in %(outdir)s."
@@ -17,83 +64,127 @@ def init(self):
         self.out_suffix = ".html.md"
 
     def get_outdated_docs(self):
-        for docname in self._ordered_docnames():
+        for docname in self.env.collect_relations():
             source_mtime = self._get_source_mtime(docname)
             target_mtime = self._get_target_mtime(docname)
             if docname not in self.env.all_docs or source_mtime is None or target_mtime is None or source_mtime > target_mtime:
                 yield docname
 
-    def get_target_uri(self, docname: str, typ: str | None = None):
-        return f"{docname}{self.out_suffix}"
-
-    def finish(self):
-        docnames = self._ordered_docnames()
-        self._write_sitemap(docnames)
-
-        full_path = Path(self.outdir) / "llms-full.txt"
-        if self.config.yardang_llms_full_build:
-            content = []
-            for docname in docnames:
-                target = Path(self.outdir) / self.get_target_uri(docname)
-                content.append(f"<!-- {self.get_target_uri(docname)} -->\n\n{target.read_text(encoding='utf-8').strip()}")
-            full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8")
-        else:
-            full_path.unlink(missing_ok=True)
-
-    def _ordered_docnames(self) -> list[str]:
-        return list(self.env.collect_relations())
-
-    def _write_sitemap(self, docnames: list[str]) -> None:
-        lines = [f"# {self.config.yardang_llms_title}", ""]
-        description = self.config.yardang_llms_description.strip()
-        if description:
-            lines.extend(f"> {line}" for line in description.splitlines())
-            lines.append("")
-
-        lines.extend(["## Pages", ""])
-        for docname in docnames:
-            lines.append(f"- [{self._title(docname)}]({self.get_target_uri(docname)}): {self._description(docname)}")
-
-        if self.config.yardang_llms_full_build:
-            lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."])
-
-        (Path(self.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
-
-    def _title(self, docname: str) -> str:
-        if docname == self.config.root_doc:
-            return self.config.yardang_llms_title
-        title = self.env.titles.get(docname)
-        return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title()
-
-    def _description(self, docname: str) -> str:
-        metadata_description = self.env.metadata.get(docname, {}).get("description")
-        if metadata_description:
-            return self._shorten(metadata_description)
-
-        doctree = self.env.get_doctree(docname)
-        for node in doctree.findall(nodes.meta):
-            if node.get("name") == "description" and node.get("content"):
-                return self._shorten(node["content"])
-        for node in doctree.findall(nodes.paragraph):
-            if any(node.findall(nodes.image)):
-                continue
-            if text := node.astext().strip():
-                return self._shorten(text)
-        return "Documentation page."
+    def get_target_uri(self, docname: str, typ: str | None = None) -> str:
+        return _target_uri(docname)
+
+    def finish(self) -> None:
+        docnames = list(self.env.collect_relations())
+        _write_sitemap(self.app, docnames)
+        _write_full_document(self.app, docnames)
+
+
+def _prepare_markdown(app: Sphinx, builder) -> None:
+    if builder.name == "html":
+        setattr(app, _DOCNAMES_ATTRIBUTE, frozenset(app.env.collect_relations()))
+
+
+def _write_markdown(app: Sphinx, doctree: nodes.document, docname: str) -> None:
+    docnames = getattr(app, _DOCNAMES_ATTRIBUTE, None)
+    if docnames is None:
+        docnames = frozenset(app.env.collect_relations())
+        setattr(app, _DOCNAMES_ATTRIBUTE, docnames)
+    if app.builder.name != "html" or docname not in docnames:
+        return
+
+    renderer = _MarkdownRenderer(app)
+    target = Path(app.outdir) / renderer.get_target_uri(docname)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(renderer.render(docname, doctree), encoding="utf-8")
+
+
+def _build_llms(app: Sphinx, exception: Exception | None) -> None:
+    if exception is not None or app.builder.name != "html":
+        return
+
+    renderer = _MarkdownRenderer(app)
+    docnames = list(app.env.collect_relations())
+    for docname in docnames:
+        target = Path(app.outdir) / renderer.get_target_uri(docname)
+        if target.is_file():
+            continue
+        doctree = app.env.get_and_resolve_doctree(docname, app.builder)
+        if not target.is_file():
+            _write_markdown(app, doctree, docname)
+    _write_sitemap(app, docnames)
+    _write_full_document(app, docnames)
+
+
+def _write_sitemap(app: Sphinx, docnames: list[str]) -> None:
+    lines = [f"# {app.config.yardang_llms_title}", ""]
+    description = app.config.yardang_llms_description.strip()
+    if description:
+        lines.extend(f"> {line}" for line in description.splitlines())
+        lines.append("")
+
+    lines.extend(["## Pages", ""])
+    for docname in docnames:
+        lines.append(f"- [{_title(app, docname)}]({_target_uri(docname)}): {_description(app, docname)}")
+
+    if app.config.yardang_llms_full_build:
+        lines.extend(["", "## Full documentation", "", "- [llms-full.txt](llms-full.txt): All pages in one document."])
+
+    (Path(app.outdir) / "llms.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _write_full_document(app: Sphinx, docnames: list[str]) -> None:
+    full_path = Path(app.outdir) / "llms-full.txt"
+    if not app.config.yardang_llms_full_build:
+        full_path.unlink(missing_ok=True)
+        return
+
+    content = []
+    for docname in docnames:
+        uri = _target_uri(docname)
+        page = (Path(app.outdir) / uri).read_text(encoding="utf-8").strip()
+        content.append(f"<!-- {uri} -->\n\n{page}")
+    full_path.write_text("\n\n".join(content) + "\n", encoding="utf-8")
+
+
+def _title(app: Sphinx, docname: str) -> str:
+    if docname == app.config.root_doc:
+        return app.config.yardang_llms_title
+    title = app.env.titles.get(docname)
+    return title.astext() if title is not None else docname.rsplit("/", 1)[-1].replace("_", " ").title()
+
+
+def _description(app: Sphinx, docname: str) -> str:
+    metadata_description = app.env.metadata.get(docname, {}).get("description")
+    if metadata_description:
+        return _shorten(metadata_description)
+
+    doctree = app.env.get_doctree(docname)
+    for node in doctree.findall(nodes.meta):
+        if node.get("name") == "description" and node.get("content"):
+            return _shorten(node["content"])
+    for node in doctree.findall(nodes.paragraph):
+        if any(node.findall(nodes.image)):
+            continue
+        if text := node.astext().strip():
+            return _shorten(text)
+    return "Documentation page."
 
-    @staticmethod
-    def _shorten(value: str, limit: int = 160) -> str:
-        text = " ".join(value.split())
-        return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
+
+def _shorten(value: str, limit: int = 160) -> str:
+    text = " ".join(value.split())
+    return text if len(text) <= limit else f"{text[: limit - 3].rstrip()}..."
 
 
-def setup(app) -> dict[str, object]:
-    """Register Yardang's LLM-friendly Markdown builder."""
+def setup(app: Sphinx) -> dict[str, object]:
+    """Generate LLM-friendly files after a successful HTML build."""
     app.setup_extension("sphinx_markdown_builder")
     app.add_config_value("yardang_llms_title", "Documentation", "")
     app.add_config_value("yardang_llms_description", "", "")
     app.add_config_value("yardang_llms_full_build", True, "")
     app.add_builder(LlmsBuilder)
+    app.connect("write-started", _prepare_markdown)
+    app.connect("doctree-resolved", _write_markdown, priority=900)
+    app.connect("build-finished", _build_llms)
     return {
         "version": __version__,
         "parallel_read_safe": True,

yardang/tests/test_llms.py

@@ -1,5 +1,6 @@
 import re
 from pathlib import Path
+from subprocess import Popen
 
 from yardang.build import generate_docs_configuration
 from yardang.cli import build
@@ -24,8 +25,8 @@ def _write_project(tmp_path: Path, *, full_build: bool = True) -> None:
 full-build = {str(full_build).lower()}
 """
     )
-    (tmp_path / "README.md").write_text("# Test Project\n\nProject overview.\n")
-    (tmp_path / "guide.md").write_text("# Guide\n\nGuide summary for language models.\n")
+    (tmp_path / "README.md").write_text("# Test Project\n\nProject overview.\n\nSee the [guide](guide.md).\n")
+    (tmp_path / "guide.md").write_text("# Guide\n\nGuide summary for language models.\n\nReturn to the [overview](index.md).\n")
     (tmp_path / "orphan.md").write_text("# Orphan\n\nThis page is not in the toctree.\n")
 
 
@@ -44,19 +45,29 @@ def test_generated_configuration_enables_yardang_llms(tmp_path, monkeypatch):
     assert not (tmp_path / "conf.py").exists()
 
 
-def test_build_generates_llms_outputs(tmp_path, monkeypatch):
+def test_build_generates_llms_outputs_in_one_sphinx_process(tmp_path, monkeypatch):
     _write_project(tmp_path)
     monkeypatch.chdir(tmp_path)
     output = tmp_path / "html"
+    commands = []
+
+    def tracked_popen(command, *args, **kwargs):
+        commands.append(command)
+        return Popen(command, *args, **kwargs)
+
+    monkeypatch.setattr("yardang.cli.Popen", tracked_popen)
 
     build(quiet=True, output=str(output))
 
+    assert len(commands) == 1
     assert (output / "index.html").is_file()
     assert (output / "guide.html").is_file()
     assert (output / "index.html.md").is_file()
     assert (output / "guide.html.md").is_file()
     assert not (output / "orphan.html.md").exists()
     assert (output / "llms-full.txt").is_file()
+    assert "(guide.html.md)" in (output / "index.html.md").read_text()
+    assert "(index.html.md)" in (output / "guide.html.md").read_text()
 
     sitemap = (output / "llms.txt").read_text()
     assert sitemap.startswith("# Test Project\n\n> A project for LLMs\n")