From 7f67b2fb8b6b3a4450cafae8cc780d1356df973f Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Thu, 6 Aug 2026 12:53:21 -0500 Subject: [PATCH 1/4] Add md_to_json.py: convert kotlin-web-site docs to JSON (ADFA-5039) Converts kotlin-web-site/docs (Writerside-flavored Markdown) into the JSON block schema this project's templating engine renders - one JSON file per topic, plus theme.json and a copy of images/. Split out of the larger Kotlin-docs pipeline PR so this ticket's scope (producing the JSON) can be reviewed independently of the database-insertion side (ADFA-4739). Includes review_build_json.sh, a throwaway helper that clones kotlin-web-site and runs the converter against it, for reviewers to see real output without any other setup - not part of the actual pipeline. Co-Authored-By: Claude Sonnet 5 --- .../ProcessKotlinWebsiteJSON/README.md | 62 ++ .../ProcessKotlinWebsiteJSON/config.json | 4 + .../ProcessKotlinWebsiteJSON/md_to_json.py | 619 ++++++++++++++++++ .../review_build_json.sh | 24 + requirements.txt | 1 + 5 files changed, 710 insertions(+) create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json create mode 100644 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py create mode 100755 ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/review_build_json.sh diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md new file mode 100644 index 000000000..4627fff72 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -0,0 +1,62 @@ +# Process Kotlin Website JSON + +Converts a `kotlin-web-site/docs` checkout (JetBrains Writerside-flavored +Markdown) into the JSON block schema this project's templating engine +renders. + +This PR (ADFA-5039) covers only [`md_to_json.py`](md_to_json.py) — the +conversion step itself. Building the sidebar nav from `kr.tree` +(`build_nav.py`), QA-ing the source tree for broken links/images +(`find_missing_assets.py`), and loading any of this into `documentation.db` +(`populate_db.py`, `insert_optimized_media.py`) are a separate ticket +(ADFA-4739) and land in a later PR. + +## Requirements + +- Python 3.10+ +- `markdown-it-py` (now in the repo's root `requirements.txt`) + +## Usage + +```bash +python3 md_to_json.py [--topics-subdir topics] +``` + +- `` — a checkout of `kotlin-web-site/docs` (contains `v.list`, `topics/`, `images/`). +- `` — a JSON file with theming colors, e.g. [`config.json`](config.json): + ```json + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} + ``` + +`` ends up containing: +- `topics/**/*.json` — one page per source `.md` file (schema below) +- `theme.json` — the two theming colors, carried through from `` +- `images/` — copied straight from `/images/` + +### Page JSON schema + +```json +{ + "id": "enum-classes", + "sourceFile": "topics/enum-classes.md", + "title": "Enum classes", + "blocks": [ { "type": "heading", "level": 2, "id": "...", "html": "..." }, "..." ] +} +``` + +Block types: `heading`, `paragraph`, `code`, `blockquote`, `list`, `table`, +`image`, `hr`, `tabs`, `note`/`tip`/`warning`, `html` (raw passthrough). See +the module docstring in [`md_to_json.py`](md_to_json.py) for full shapes and +known limitations (nested tabs, `` resolution, variable +substitution). + +## Trying it out + +[`review_build_json.sh`](review_build_json.sh) is a throwaway helper for +reviewers — it installs `markdown-it-py`, clones `kotlin-web-site`, and runs +`md_to_json.py` against it so you can look at real output without any other +setup. It's not part of the actual pipeline (that's ADFA-4739): + +```bash +./review_build_json.sh +``` diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json new file mode 100644 index 000000000..b69baed62 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/config.json @@ -0,0 +1,4 @@ +{ + "broken-ext-link-color": "#cc0000", + "menu-no-link-color": "#999999" +} diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py new file mode 100644 index 000000000..352ac15f7 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py @@ -0,0 +1,619 @@ +#!/usr/bin/env python3 +""" +Converts JetBrains Writerside-flavored Markdown (as used by kotlin-web-site/docs) +into a simple JSON block schema suitable for a templating engine. + +Usage: + python3 md_to_json.py [--topics-subdir topics] + + is the checkout of kotlin-web-site/docs (contains v.list, topics/, ...). +One JSON file is written per input .md file, mirroring its relative path under +. + + is a JSON file with: + {"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"} +"broken-ext-link-color" colors tags in the rendered content that are +either off-site (any http(s)/mailto: link) or a same-tree ".md" reference +that doesn't resolve to a real page. "menu-no-link-color" isn't used here - +it's carried through to /theme.json for build_nav.py (which +builds the sidebar from kr.tree, a separate input this script doesn't read) +to pick up. + +Output schema (one object per page): +{ + "id": "enum-classes", + "sourceFile": "topics/enum-classes.md", + "title": "Enum classes", + "blocks": [ , ... ] +} + +Block shapes: + {"type": "heading", "level": 2, "id": "anonymous-classes", "html": "..."} + {"type": "paragraph", "html": "..."} + {"type": "code", "lang": "kotlin", "code": "...", "attrs": {"kotlin-runnable": "true"}} + {"type": "blockquote", "attrs": {"style": "note"}, "blocks": [...]} + {"type": "list", "ordered": false, "items": [{"blocks": [...]}]} + {"type": "table", "headers": ["a", "b"], "rows": [["1", "2"]]} + {"type": "image", "src": "...", "alt": "..."} + {"type": "hr"} + {"type": "tabs", "attrs": {"group": "build-system"}, + "tabs": [{"title": "Gradle", "attrs": {"group-key": "gradle"}, "blocks": [...]}]} + {"type": "html", "html": ""} + +Cross-page links (`[text](other-page.md#anchor)`) and images (`![alt](foo.png)`) +use Writerside's bare-filename convention - the referenced file is looked up +by name anywhere under / (links) or images/ (images), the same +way kr.tree's topic="..." references are resolved. Rendered HTML rewrites +these to root-relative URLs that only resolve once this JSON has been passed +through templates/page.peb and rendered by RenderDocs: links become +"/.html#anchor", and images become "/images/". +The images/ directory itself is copied to /images/ so the +resolved paths have something to point at. + +Known limitations (fine for a first pass, worth revisiting before production use): + - / grouping and attribute-line merging (the "{...}" line after a + fence/blockquote) only happen at the top level of a page and inside list + items/blockquotes/table cells one level deep; deeply nested tabs-in-tabs + are not handled. + - // admonitions are recognized as block-level tags. The + inline, single-line form seen inside HTML tables (e.g. roadmap.md) is passed + through as raw "html" blocks instead of being unpacked, since those pages + are basically hand-written HTML tables rather than prose. + - elements are passed through as raw "html" blocks; resolving them + to the referenced snippet is not implemented. + - %variables% (defined in v.list) are substituted textually in rendered HTML + and code, using simple %name% -> value replacement. + - A handful of images/ filenames collide across subdirectories (leftover + duplicates in the source tree); resolution keeps the first match in sorted + order and prints a warning rather than guessing which one is "correct". +""" +import argparse +import json +import re +import shutil +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +from markdown_it import MarkdownIt +from markdown_it.token import Token + +TITLE_RE = re.compile(r"^\[//\]:\s*#\s*\(title:\s*(.*?)\)\s*$", re.MULTILINE) +ATTR_LINE_RE = re.compile(r"^\{(.*)\}$") +ATTR_PAIR_RE = re.compile(r'([\w-]+)=(?:"([^"]*)"|(\S+))') +TAG_RE = re.compile(r"^<(/?)(tabs|tab|note|tip|warning)([^>]*)/?>$", re.I) +VAR_RE = re.compile(r"%([\w.-]+)%") +MD_LINK_RE = re.compile(r"^([\w.-]+)\.md(#.*)?$") +EXTERNAL_HREF_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+.-]*:)?//|^mailto:", re.I) +LINK_TAG_RE = re.compile(r']*\bhref="([^"]*)"[^>]*>') + +CONTAINER_TAGS = {"tabs", "tab", "note", "tip", "warning"} + + +def build_topic_index(topics_dir: Path) -> dict: + """Bare filename stem (e.g. "enum-classes") -> page id (e.g. "kotlin-tour/enum-classes").""" + index = {} + for md_path in sorted(topics_dir.rglob("*.md")): + page_id = str(md_path.relative_to(topics_dir).with_suffix("")).replace("\\", "/") + index.setdefault(md_path.stem, page_id) + return index + + +def build_image_index(images_dir: Path): + """Bare filename (e.g. "mascot-main.png") -> path relative to images_dir. + + Returns (index, collisions), where collisions is a list of + (filename, [candidate relative paths]) for filenames that exist in more + than one place under images_dir - index keeps the first (sorted) one.""" + index = {} + candidates = {} + if not images_dir.is_dir(): + return index, [] + for img_path in sorted(images_dir.rglob("*")): + if not img_path.is_file(): + continue + rel = img_path.relative_to(images_dir).as_posix() + candidates.setdefault(img_path.name, []).append(rel) + index.setdefault(img_path.name, rel) + collisions = [(name, rels) for name, rels in sorted(candidates.items()) if len(rels) > 1] + for name, rels in collisions: + print(f"warning: ambiguous image filename {name!r}: " + f"using images/{rels[0]}, ignoring {', '.join('images/' + r for r in rels[1:])}", file=sys.stderr) + return index, collisions + + +def load_variables(docs_root: Path) -> dict: + v_list = docs_root / "v.list" + if not v_list.exists(): + return {} + tree = ET.parse(v_list) + return {el.get("name"): el.get("value") for el in tree.getroot().findall("var")} + + +def substitute_vars(text: str, variables: dict) -> str: + if not text: + return text + return VAR_RE.sub(lambda m: variables.get(m.group(1), m.group(0)), text) + + +def parse_attrs(attr_str: str) -> dict: + attrs = {} + for name, quoted, bare in ATTR_PAIR_RE.findall(attr_str or ""): + attrs[name] = quoted if quoted != "" or '="' in (attr_str or "") else bare + return attrs + + +def extract_title(raw_text: str): + m = TITLE_RE.search(raw_text) + if not m: + return None, raw_text + title = m.group(1).strip() + remaining = raw_text[: m.start()] + raw_text[m.end():] + return title, remaining + + +def slugify(text: str) -> str: + slug = re.sub(r"[^\w\s-]", "", text.lower()).strip() + return re.sub(r"[\s_]+", "-", slug) + + +class Node: + """Generic open/close tree built from markdown-it's flat token stream.""" + + __slots__ = ("token", "children") + + def __init__(self, token: Token): + self.token = token + self.children = [] + + +def build_tree(tokens) -> list: + root = [] + stack = [root] + for tok in tokens: + if tok.nesting == 1: + node = Node(tok) + stack[-1].append(node) + stack.append(node.children) + elif tok.nesting == -1: + stack.pop() + else: + stack[-1].append(Node(tok)) + return root + + +class Converter: + def __init__(self, md: MarkdownIt, variables: dict, topic_index: dict = None, image_index: dict = None, + broken_ext_link_color: str = None, image_url_prefix: str = "/images/"): + self.md = md + self.variables = variables + self.topic_index = topic_index or {} + self.image_index = image_index or {} + self.broken_ext_link_color = broken_ext_link_color + # Overridable so a different deployment target (e.g. populate_db.py's + # database-backed site, which serves images from "/k/html/images/" + # rather than a bare "/images/") can retarget every image src without + # a separate rewrite pass - resolve_image_src just uses this prefix + # directly. + self.image_url_prefix = image_url_prefix + self.current_source = None + # Populated as a side effect of resolve_href/resolve_image_src failing + # to resolve a reference; find_missing_assets.py reuses this same + # resolution logic (rather than re-parsing links with regexes) by + # running convert_file over every page and reading this list back. + self.warnings = [] + + def resolve_href(self, href: str): + """"other-page.md#anchor" -> "/.html#anchor", or None to leave href untouched.""" + m = MD_LINK_RE.match(href or "") + if not m: + return None + stem, anchor = m.groups() + page_id = self.topic_index.get(stem) + if page_id is None: + print(f"warning: link to unknown topic {href!r}", file=sys.stderr) + self.warnings.append({"kind": "link", "source": self.current_source, "reference": href}) + return None + return f"/{page_id}.html{anchor or ''}" + + def resolve_image_src(self, src: str): + """"foo.png" -> "", or None to leave src untouched.""" + if not src or "://" in src or src.startswith("/") or "/" in src: + return None + rel = self.image_index.get(src) + if rel is None: + print(f"warning: image not found: {src!r}", file=sys.stderr) + self.warnings.append({"kind": "image", "source": self.current_source, "reference": src}) + return None + return f"{self.image_url_prefix}{rel}" + + def rewrite_urls(self, html: str) -> str: + """Rewrites every href="...md" / src="foo.png" attribute found in a + blob of rendered/raw HTML. Applied to markdown-rendered HTML *and* to + Writerside's raw / passthrough HTML (which markdown-it never + tokenizes as links/images at all, so token-level rewriting alone + would miss it); resolve_href/resolve_image_src already leave anything + that isn't a bare same-tree ".md"/image reference untouched, so this + is safe to run unconditionally on any HTML string.""" + if not html: + return html + + def href_repl(m): + new_href = self.resolve_href(m.group(1)) + return f'href="{new_href}"' if new_href is not None else m.group(0) + + def src_repl(m): + new_src = self.resolve_image_src(m.group(1)) + return f'src="{new_src}"' if new_src is not None else m.group(0) + + html = re.sub(r'href="([^"]*)"', href_repl, html) + html = re.sub(r'src="([^"]*)"', src_repl, html) + return self.style_broken_and_external_links(html) + + def classify_href(self, href: str): + """Classifies an *already rewritten* href: 'external' for off-site + (or mailto:) links, 'broken' for a same-tree ".md" reference that's + still literally "foo.md" because resolve_href couldn't find it, or + None for anything that resolved fine (now "/id.html...") or is a + same-page "#anchor" link.""" + if not href or href.startswith("#"): + return None + if EXTERNAL_HREF_RE.match(href): + return "external" + if MD_LINK_RE.match(href): + return "broken" + return None + + def style_broken_and_external_links(self, html: str) -> str: + """Colors broken and off-site tags with --broken-ext-link-color + (from the config passed on the command line) so readers can tell at a + glance which links leave this site or don't go anywhere at all.""" + if not self.broken_ext_link_color: + return html + + def repl(m): + tag = m.group(0) + if self.classify_href(m.group(1)) is None: + return tag + return tag[:-1] + f' style="color: {self.broken_ext_link_color};">' + + return LINK_TAG_RE.sub(repl, html) + + def fold_image_attrs(self, tokens) -> list: + """Folds a `{...}` attribute-text token immediately following an + image (e.g. `![alt](x.png){width="500"}` - CommonMark has no syntax + for this, so it otherwise tokenizes as a literal "{width=..}" text + run right after the image) into that image's own HTML attributes + instead of leaving it as visible text.""" + result = [] + for tok in tokens: + if tok.type == "text" and result and result[-1].type == "image": + m = ATTR_LINE_RE.match(tok.content.strip()) + if m: + result[-1].attrs.update(parse_attrs(m.group(1))) + continue + if tok.children: + tok.children = self.fold_image_attrs(tok.children) + result.append(tok) + return result + + def render_inline(self, inline_token: Token) -> str: + children = self.fold_image_attrs(inline_token.children) + return self.rewrite_urls(self.md.renderer.render(children, self.md.options, {})) + + def convert_nodes(self, nodes: list) -> list: + blocks = [] + for n in nodes: + result = self.convert_node(n) + if result is None: + continue + if isinstance(result, list): + blocks.extend(result) + else: + blocks.append(result) + blocks = self.merge_attr_lines(blocks) + blocks = self.group_containers(blocks) + return blocks + + def convert_node(self, node: Node): + t = node.token + ttype = t.type + + if ttype == "paragraph_open": + inline = node.children[0].token + # "_raw" carries the unescaped markdown source so merge_attr_lines + # can detect/parse a trailing "{...}" attribute line; it is + # stripped out of the final block before output. + return {"type": "paragraph", "html": self.render_inline(inline), "_raw": inline.content} + + if ttype == "heading_open": + inline = node.children[0].token + level = int(t.tag[1:]) + text = inline.content + return { + "type": "heading", + "level": level, + "id": slugify(text), + "html": self.render_inline(inline), + } + + if ttype == "fence": + return { + "type": "code", + "lang": (t.info or "").strip() or None, + "code": t.content, + "attrs": {}, + } + + if ttype == "blockquote_open": + return { + "type": "blockquote", + "attrs": {}, + "blocks": self.convert_nodes(node.children), + } + + if ttype in ("bullet_list_open", "ordered_list_open"): + items = [] + for item_node in node.children: + if item_node.token.type == "list_item_open": + items.append({"blocks": self.convert_nodes(item_node.children)}) + return {"type": "list", "ordered": ttype == "ordered_list_open", "items": items} + + if ttype == "table_open": + headers = [] + rows = [] + for section in node.children: + if section.token.type == "thead_open": + for tr in section.children: + row = [self.render_inline(td.children[0].token) for td in tr.children] + headers = row + elif section.token.type == "tbody_open": + for tr in section.children: + row = [self.render_inline(td.children[0].token) for td in tr.children] + rows.append(row) + return {"type": "table", "headers": headers, "rows": rows} + + if ttype == "hr": + return {"type": "hr"} + + if ttype == "html_block": + # CommonMark merges consecutive non-blank-line-separated HTML + # lines into a single html_block token, so a lone and + # the that immediately follows it commonly land in the + # same token. Split back into lines so each tag can be matched + # (and grouped into a container) independently. + result = [] + raw_run = [] + + def flush_raw(): + if raw_run: + result.append({"type": "html", "html": self.rewrite_urls("\n".join(raw_run))}) + raw_run.clear() + + for line in t.content.splitlines(): + m = TAG_RE.match(line.strip()) + if m: + flush_raw() + closing, tag, attrstr = m.groups() + result.append({ + "type": "tag_marker", + "closing": bool(closing), + "tag": tag.lower(), + "attrs": parse_attrs(attrstr), + }) + elif line.strip(): + raw_run.append(line) + flush_raw() + return result + + if ttype == "inline": + # top-level bare inline content (e.g. an attribute line with no + # surrounding paragraph); treat like a paragraph. + return {"type": "paragraph", "html": self.render_inline(t), "_raw": t.content} + + # Fallback: anything not explicitly handled (images are inline-only, + # so plain "image" blocks don't occur at block level; captured via + # paragraph HTML instead). + return {"type": "html", "html": self.rewrite_urls(str(t.content or ""))} + + @staticmethod + def merge_attr_lines(blocks: list) -> list: + """Fold a standalone `{key="value"}` paragraph into the preceding block's attrs.""" + merged = [] + for b in blocks: + if b["type"] == "paragraph": + raw = b.pop("_raw", "").strip() + m = ATTR_LINE_RE.match(raw) + if m and merged: + merged[-1].setdefault("attrs", {}).update(parse_attrs(m.group(1))) + continue + merged.append(b) + return merged + + @staticmethod + def group_containers(blocks: list) -> list: + """Turn //// tag_marker pairs into nested blocks.""" + root: dict = {"blocks": []} + stack = [root] + for b in blocks: + if b["type"] == "tag_marker": + if not b["closing"]: + node = {"type": b["tag"], "attrs": b["attrs"], "blocks": []} + stack[-1]["blocks"].append(node) + stack.append(node) + elif len(stack) > 1 and stack[-1]["type"] == b["tag"]: + stack.pop() + continue + stack[-1]["blocks"].append(b) + + return Converter._finalize_list(root["blocks"]) + + @staticmethod + def _finalize_list(blocks: list) -> list: + """Maps _finalize_container over a list, splicing in any block it + unwraps back into a plain list instead of a "tabs" block (see below) + in place, rather than nesting a list-within-a-list.""" + result = [] + for b in blocks: + out = Converter._finalize_container(b) + if isinstance(out, list): + result.extend(out) + else: + result.append(out) + return result + + @staticmethod + def _finalize_container(b: dict): + if b.get("type") == "tabs" and "blocks" in b: + children = Converter._finalize_list(b["blocks"]) + tab_children = [c for c in children if c.get("type") == "tab"] + if tab_children: + # Well-formed ....... + return { + "type": "tabs", + "attrs": b["attrs"], + "tabs": [ + {"title": c["attrs"].get("title"), "attrs": c["attrs"], "blocks": c["blocks"]} + for c in tab_children + ], + } + if len(children) >= 2 and all(c.get("type") == "code" for c in children): + # Bare wrapping only fenced code blocks with no + # tags at all (seen in some compatibility guide pages, e.g. + # a Kotlin and a Groovy fence back to back) - Writerside + # authors apparently rely on adjacency here instead of + # writing explicitly, so treat each code block's own + # language as its tab instead of rendering a tabs shell with + # no tabs in it (which silently dropped every one of these + # code blocks - see the "type": "tabs" but no "tabs" key + # schema mismatch this used to produce). + return { + "type": "tabs", + "attrs": b["attrs"], + "tabs": [ + { + "title": (c["lang"] or "").capitalize() or f"Tab {i + 1}", + "attrs": {"group-key": (c["lang"] or "").lower() or str(i + 1)}, + "blocks": [c], + } + for i, c in enumerate(children) + ], + } + # Bare wrapper with no children and no recognizable + # tab structure to synthesize - there's nothing tab-like left to + # preserve, so drop the wrapper and splice its content in place + # instead of emitting a "tabs" block with no "tabs" key (which + # templates/page.peb can't render - it silently produces an + # empty
and drops the content entirely). + return children + if "blocks" in b: + b["blocks"] = Converter._finalize_list(b["blocks"]) + return b + + def convert_file(self, path: Path, page_id: str, source_rel: str) -> dict: + self.current_source = source_rel + raw = path.read_text(encoding="utf-8") + # Substitute %variables% in the raw source, before markdown-it ever + # sees it. Doing this post-render instead would (a) miss the title, + # which is extracted straight from the raw source, and (b) run into + # markdown-it percent-encoding "%" inside link URLs, which turns + # "%kotlinEapVersion%" into "%25kotlinEapVersion%25" before we'd get + # a chance to match it. + raw = substitute_vars(raw, self.variables) + title, body = extract_title(raw) + tokens = self.md.parse(body) + tree = build_tree(tokens) + blocks = self.convert_nodes(tree) + return { + "id": page_id, + "sourceFile": source_rel, + "title": title, + "blocks": blocks, + } + + +def make_markdown_it() -> MarkdownIt: + md = MarkdownIt("commonmark") + md.enable("table") + return md + + +CONFIG_KEYS = ("broken-ext-link-color", "menu-no-link-color") + + +def load_config(config_path: Path) -> dict: + """Loads the {"broken-ext-link-color": ..., "menu-no-link-color": ...} + theming config. Missing keys just disable that particular styling (a + warning is printed) rather than being a hard error, since neither is + required for the JSON conversion itself to be correct.""" + config = json.loads(config_path.read_text(encoding="utf-8")) + for key in CONFIG_KEYS: + if key not in config: + print(f"warning: config {config_path} is missing {key!r}; that styling will be skipped", file=sys.stderr) + return config + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("docs_root", type=Path, help="Path to kotlin-web-site/docs") + parser.add_argument("output_dir", type=Path, help="Directory to write JSON files into") + parser.add_argument("config", type=Path, + help='Path to a JSON config with "broken-ext-link-color" and "menu-no-link-color"') + parser.add_argument("--topics-subdir", default="topics", help="Subdirectory of docs_root holding .md files") + parser.add_argument("--images-subdir", default="images", help="Subdirectory of docs_root holding image files") + args = parser.parse_args() + + docs_root: Path = args.docs_root + topics_dir = docs_root / args.topics_subdir + images_dir = docs_root / args.images_subdir + if not topics_dir.is_dir(): + print(f"error: {topics_dir} is not a directory", file=sys.stderr) + sys.exit(1) + if not args.config.is_file(): + print(f"error: {args.config} does not exist", file=sys.stderr) + sys.exit(1) + + config = load_config(args.config) + variables = load_variables(docs_root) + topic_index = build_topic_index(topics_dir) + image_index, _image_collisions = build_image_index(images_dir) + md = make_markdown_it() + converter = Converter(md, variables, topic_index, image_index, + broken_ext_link_color=config.get("broken-ext-link-color")) + + md_files = sorted(topics_dir.rglob("*.md")) + args.output_dir.mkdir(parents=True, exist_ok=True) + + # menu-no-link-color applies to the sidebar, which build_nav.py builds + # separately from nav.json/kr.tree - it reads this back from the output + # directory it's already given, rather than needing its own copy of the + # config on its own command line. + (args.output_dir / "theme.json").write_text( + json.dumps({key: config.get(key) for key in CONFIG_KEYS}, indent=2), encoding="utf-8" + ) + + if images_dir.is_dir(): + shutil.copytree(images_dir, args.output_dir / "images", dirs_exist_ok=True) + else: + print(f"warning: {images_dir} not found; image references will 404", file=sys.stderr) + + count = 0 + for md_path in md_files: + rel = md_path.relative_to(topics_dir) + page_id = str(rel.with_suffix("")) + source_rel = str(Path(args.topics_subdir) / rel) + try: + page = converter.convert_file(md_path, page_id, source_rel) + except Exception as exc: # noqa: BLE001 - surface which file broke + print(f"error converting {md_path}: {exc}", file=sys.stderr) + continue + out_path = args.output_dir / args.topics_subdir / rel.with_suffix(".json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(page, separators=(",", ":"), ensure_ascii=False), encoding="utf-8") + count += 1 + + print(f"Converted {count}/{len(md_files)} files into {args.output_dir}") + + +if __name__ == "__main__": + main() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/review_build_json.sh b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/review_build_json.sh new file mode 100755 index 000000000..04d11ef30 --- /dev/null +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/review_build_json.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Throwaway helper for reviewers of ADFA-5039 - installs requirements, clones +# kotlin-web-site, and runs md_to_json.py against it so you can look at real +# JSON output without any other setup. Not part of the actual pipeline +# (that's ADFA-4739, a separate ticket/PR). +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKDIR="$(mktemp -d)" + +echo "== Installing requirements ==" +python3 -m pip install markdown-it-py + +echo "== Cloning kotlin-web-site into $WORKDIR/kotlin-web-site ==" +git clone --depth 1 https://github.com/JetBrains/kotlin-web-site.git "$WORKDIR/kotlin-web-site" + +echo "== Running md_to_json.py ==" +python3 "$SCRIPT_DIR/md_to_json.py" \ + "$WORKDIR/kotlin-web-site/docs" \ + "$WORKDIR/json-output" \ + "$SCRIPT_DIR/config.json" + +echo +echo "Done. JSON output at $WORKDIR/json-output" diff --git a/requirements.txt b/requirements.txt index 265d3c394..db54dac57 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ brotli Pillow openpyxl>=3.1.0 tqdm-loggable>=0.1.0 +markdown-it-py From 6b25e263e0e1642e9584b408112d23458c586471 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Fri, 7 Aug 2026 15:20:07 -0500 Subject: [PATCH 2/4] Address PR review feedback and add regression test suite Fixes 12 correctness bugs found by review (verified against a live kotlin-web-site corpus, not just read-the-code): - TAG_RE matched "tab" as a prefix of "table", silently eating every raw HTML table in the corpus (38 occurrences / 18 files). - merge_attr_lines deleted any {...}-shaped paragraph even when it parsed to zero attrs, dropping real content. - blocks discarded any non- sibling (intro/trailing prose). - A mismatched closing tag_marker was silently ignored, leaving the wrong frame open and relocating later content into it. - ATTR_PAIR_RE didn't allow whitespace around "=", and parse_attrs' whole- string quoted-vs-bare check blanked bare values whenever any sibling in the same group was quoted. - fold_image_attrs only handled exactly one trailing {...} group and required the whole text token to be nothing else, so a second attribute group or trailing prose leaked as literal visible text. - broken-ext-link-color was interpolated into a style="..." attribute unvalidated (markup-injection hole) and unconditionally appended even when the already had a style=, producing a silently-ignored duplicate attribute. - main() swallowed per-file conversion failures and still exited 0. - The image-src regex rewrote src= on any element, not just , producing false "image not found" warnings for ') + assert 'src="x.js">' in html + assert 'src="x.js">' in html + assert '/images/sub/x.js' in html + assert not any(w["kind"] == "image" for w in conv.warnings) + + +# --- load_config: reject an unsafe/invalid color ------------------------ + +def test_load_config_rejects_markup_injection_payload(tmp_path): + """broken-ext-link-color is interpolated directly into a style="..." + HTML attribute; an unvalidated value is a markup-injection hole.""" + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({ + "broken-ext-link-color": 'red">', + "menu-no-link-color": "#999999", + })) + with pytest.raises(SystemExit) as exc_info: + m.load_config(config_path) + assert exc_info.value.code == 1 + + +def test_load_config_accepts_hex_and_named_colors(tmp_path): + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({ + "broken-ext-link-color": "#cc0000", + "menu-no-link-color": "gray", + })) + config = m.load_config(config_path) + assert config["broken-ext-link-color"] == "#cc0000" + assert config["menu-no-link-color"] == "gray" + + +# --- heading ids: de-dup + explicit {id=...} override ------------------- + +def test_duplicate_heading_text_gets_deduped_ids(): + """Two headings with identical text used to share a slug, so an anchor + to the second one landed on the first.""" + conv = make_converter() + first = conv.unique_heading_id("Overview") + second = conv.unique_heading_id("Overview") + assert first == "overview" + assert second == "overview-2" + + +def test_heading_trailing_id_attr_overrides_slug_and_is_stripped_from_html(): + """## Checks with `is`/`!is` operators {id="is-and-is-operators"} used + to render the literal "{id="is-and-is-operators"}" as visible + heading text, with no id override and no attribute handling at all.""" + md = m.make_markdown_it() + conv = m.Converter(md, {}) + tokens = md.parse('## Checks with `is` {id="is-and-is-operators"}\n') + tree = m.build_tree(tokens) + block = conv.convert_node(tree[0]) + assert block["id"] == "is-and-is-operators" + assert "{id=" not in block["html"] + assert block["attrs"] == {"id": "is-and-is-operators"} + + +def test_heading_without_trailing_attrs_unaffected(): + md = m.make_markdown_it() + conv = m.Converter(md, {}) + tokens = md.parse("## Plain heading\n") + tree = m.build_tree(tokens) + block = conv.convert_node(tree[0]) + assert block["id"] == "plain-heading" + assert "attrs" not in block + + +# --- build_topic_index: collision warning mirrors build_image_index ----- + +def test_build_topic_index_warns_on_duplicate_stem(tmp_path, capsys): + """Two topics sharing a filename stem used to resolve first-wins with + no diagnostic at all, unlike the equivalent image-filename collision.""" + topics = tmp_path / "topics" + (topics / "native").mkdir(parents=True) + (topics / "js").mkdir(parents=True) + (topics / "native" / "basics.md").write_text("native") + (topics / "js" / "basics.md").write_text("js") + + index = m.build_topic_index(topics) + assert index["basics"] in ("native/basics", "js/basics") + assert "warning: ambiguous topic filename 'basics'" in capsys.readouterr().err + + +def test_build_topic_index_no_warning_without_collision(tmp_path, capsys): + topics = tmp_path / "topics" + topics.mkdir() + (topics / "a.md").write_text("a") + m.build_topic_index(topics) + assert capsys.readouterr().err == "" + + +# --- main(): exit code reflects partial failure ------------------------- + +def _write_minimal_docs_root(tmp_path): + docs_root = tmp_path / "docs" + (docs_root / "topics").mkdir(parents=True) + (docs_root / "topics" / "good.md").write_text("# Good\n\nHello.\n") + config = tmp_path / "config.json" + config.write_text(json.dumps({"broken-ext-link-color": "#cc0000", "menu-no-link-color": "#999999"})) + return docs_root, config + + +def _run_main(*args): + script = Path(__file__).resolve().parent.parent / "md_to_json.py" + return subprocess.run([sys.executable, str(script), *map(str, args)], capture_output=True, text=True) + + +def test_main_exits_zero_on_full_success(tmp_path): + docs_root, config = _write_minimal_docs_root(tmp_path) + out_dir = tmp_path / "out" + result = _run_main(docs_root, out_dir, config) + assert result.returncode == 0 + assert "Converted 1/1" in result.stdout + + +def test_main_exits_nonzero_when_a_file_fails(tmp_path, monkeypatch): + """A run that converts nothing (or partially fails) used to print + "Converted 0/N files" and still exit 0 - a CI step calling this + couldn't distinguish a complete run from a total failure.""" + docs_root, config = _write_minimal_docs_root(tmp_path) + # A .md file that isn't valid UTF-8 makes convert_file's read_text raise. + (docs_root / "topics" / "bad.md").write_bytes(b"\xff\xfe not utf-8") + out_dir = tmp_path / "out" + result = _run_main(docs_root, out_dir, config) + assert result.returncode == 1 + assert "Converted 1/2" in result.stdout + + +def test_main_allow_failures_exits_zero_despite_failure(tmp_path): + docs_root, config = _write_minimal_docs_root(tmp_path) + (docs_root / "topics" / "bad.md").write_bytes(b"\xff\xfe not utf-8") + out_dir = tmp_path / "out" + result = _run_main(docs_root, out_dir, config, "--allow-failures") + assert result.returncode == 0 + + +def test_main_uses_posix_separators_for_nested_page_ids(tmp_path): + """page_id/sourceFile used str(Path(...)) instead of .as_posix(), which + would disagree with build_topic_index's forward-slashed ids on Windows.""" + docs_root, config = _write_minimal_docs_root(tmp_path) + (docs_root / "topics" / "tour").mkdir() + (docs_root / "topics" / "tour" / "hello.md").write_text("# Hello\n") + out_dir = tmp_path / "out" + result = _run_main(docs_root, out_dir, config) + assert result.returncode == 0 + page = json.loads((out_dir / "topics" / "tour" / "hello.json").read_text()) + assert page["id"] == "tour/hello" + assert page["sourceFile"] == "topics/tour/hello.md" + + +def test_main_prunes_stale_topic_json(tmp_path): + """A topic removed upstream used to keep shipping its stale JSON + forever, since nothing ever cleared the output topics directory.""" + docs_root, config = _write_minimal_docs_root(tmp_path) + out_dir = tmp_path / "out" + assert _run_main(docs_root, out_dir, config).returncode == 0 + assert (out_dir / "topics" / "good.json").exists() + + (docs_root / "topics" / "good.md").unlink() + (docs_root / "topics" / "new.md").write_text("# New\n") + assert _run_main(docs_root, out_dir, config).returncode == 0 + assert not (out_dir / "topics" / "good.json").exists() + assert (out_dir / "topics" / "new.json").exists() From 5ec1f60a41c4f55af85623daac179f2d12e9af8f Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Mon, 10 Aug 2026 13:00:20 -0500 Subject: [PATCH 3/4] Fix 5 regressions from round-2 PR review group_containers crashed (KeyError) on a top-level unmatched closing tag since the len(stack) > 1 guard was dropped during the round-1 fix. The topics-dir pruning rmtree could delete the source docs when output_dir resolves to the same tree as docs_root. The duplicate-style merge could produce invalid (dropped) CSS when the existing style had no trailing ";". fold_image_attrs re-stripped the remainder after an image, eating the leading space before trailing prose. load_config raised a bare TypeError instead of its own clean error on a non-string color value. Co-Authored-By: Claude Sonnet 5 --- .../ProcessKotlinWebsiteJSON/md_to_json.py | 15 +++-- .../tests/test_md_to_json.py | 60 ++++++++++++++++++- 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py index 633e1bb31..76cdb829b 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py @@ -349,7 +349,8 @@ def repl(m): # second one. existing = STYLE_ATTR_RE.search(tag) if existing: - return tag[:existing.start(1)] + existing.group(1) + " " + rule + tag[existing.end(1):] + sep = "" if not existing.group(1).strip() or existing.group(1).rstrip().endswith(";") else "; " + return tag[:existing.start(1)] + existing.group(1) + sep + " " + rule + tag[existing.end(1):] return tag[:-1] + f' style="{rule}">' return LINK_TAG_RE.sub(repl, html) @@ -379,8 +380,8 @@ def fold_image_attrs(self, tokens) -> list: pos = m.end() folded_any = True if folded_any: - remainder = content[pos:].strip() - if remainder: + remainder = content[pos:] + if remainder.strip(): tok.content = remainder result.append(tok) continue @@ -585,7 +586,7 @@ def group_containers(self, blocks: list) -> list: node = {"type": b["tag"], "attrs": b["attrs"], "blocks": []} stack[-1]["blocks"].append(node) stack.append(node) - elif stack[-1]["type"] == b["tag"]: + elif len(stack) > 1 and stack[-1]["type"] == b["tag"]: stack.pop() else: # A closing marker that doesn't match the top of the @@ -738,7 +739,7 @@ def load_config(config_path: Path) -> dict: for key in CONFIG_KEYS: if key not in config: print(f"warning: config {config_path} is missing {key!r}; that styling will be skipped", file=sys.stderr) - elif not COLOR_RE.match(config[key]): + elif not isinstance(config[key], str) or not COLOR_RE.match(config[key]): print(f"error: config {config_path} has an invalid {key!r} value {config[key]!r} " f"(expected a hex color like \"#cc0000\" or a CSS color name)", file=sys.stderr) sys.exit(1) @@ -809,6 +810,10 @@ def main(): # forever, since nothing else here ever removes a file on its own. topics_out_dir = args.output_dir / args.topics_subdir if topics_out_dir.is_dir(): + if topics_out_dir.resolve() == topics_dir.resolve(): + print(f"error: output topics dir {topics_out_dir} is the source topics dir {topics_dir}", + file=sys.stderr) + sys.exit(1) shutil.rmtree(topics_out_dir) count = 0 diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py index 956281624..2a4e5b507 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py @@ -110,7 +110,11 @@ def test_fold_image_attrs_preserves_trailing_prose(): out = make_converter().fold_image_attrs([img, text]) assert img.attrs == {"width": "25", "type": "joined"} assert len(out) == 2 - assert out[1].content == "Slack:" + # Asserting the exact string (not .strip()'d) matters here: the + # remainder used to be re-stripped after slicing, silently eating the + # leading space that separates the image from this text - "Slack:" + # would pass that bug just as well as " Slack:" does. + assert out[1].content == " Slack:" def test_fold_image_attrs_single_group_still_leaves_no_leftover_token(): @@ -207,6 +211,22 @@ def test_well_formed_open_close_does_not_warn(): assert conv.warnings == [] +def test_top_level_unmatched_closer_does_not_crash(): + """A closing tag_marker with no matching opener anywhere on the stack - + stack is back down to the bare {"blocks": []} root, which has no "type" + key at all - used to raise KeyError('type') instead of hitting the + already-correct "no matching frame" warning path.""" + blocks = [ + {"type": "tag_marker", "closing": True, "tag": "note", "attrs": {}}, + {"type": "paragraph", "html": "After."}, + ] + conv = make_converter() + conv.current_source = "test.md" + result = conv.group_containers(blocks) + assert [b["html"] for b in result if b.get("type") == "paragraph"] == ["After."] + assert any(w["kind"] == "tag" for w in conv.warnings) + + # --- style_broken_and_external_links: no duplicate style= --------------- def test_broken_link_style_merges_into_existing_style_attr(): @@ -220,6 +240,19 @@ def test_broken_link_style_merges_into_existing_style_attr(): assert "font-weight:bold" in html +def test_broken_link_style_merge_inserts_semicolon_separator(): + """The merge used to concatenate the existing style value and the + injected rule with just a space when the existing value had no trailing + ";" - "font-weight: bold color: red;" is ONE malformed CSS declaration, + which browsers discard entirely, losing both rules.""" + conv = make_converter(broken_ext_link_color="#cc0000") + html = conv.style_broken_and_external_links( + 'x') + assert html.count('style="') == 1 + assert "font-weight: bold;" in html + assert "color: #cc0000;" in html + + def test_non_broken_link_untouched(): conv = make_converter(broken_ext_link_color="#cc0000", topic_index={}) html = conv.style_broken_and_external_links('x') @@ -255,6 +288,21 @@ def test_load_config_rejects_markup_injection_payload(tmp_path): assert exc_info.value.code == 1 +def test_load_config_rejects_non_string_color_value(tmp_path): + """A JSON number (an unquoted hex-like value is a plausible hand-edit + slip, e.g. writing cc0000 instead of "#cc0000") used to raise a bare + TypeError from COLOR_RE.match(int) instead of the intended clean + "invalid ... value" error the line below is meant to produce.""" + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({ + "broken-ext-link-color": 123, + "menu-no-link-color": "#999999", + })) + with pytest.raises(SystemExit) as exc_info: + m.load_config(config_path) + assert exc_info.value.code == 1 + + def test_load_config_accepts_hex_and_named_colors(tmp_path): config_path = tmp_path / "config.json" config_path.write_text(json.dumps({ @@ -398,3 +446,13 @@ def test_main_prunes_stale_topic_json(tmp_path): assert _run_main(docs_root, out_dir, config).returncode == 0 assert not (out_dir / "topics" / "good.json").exists() assert (out_dir / "topics" / "new.json").exists() + + +def test_main_refuses_when_output_dir_is_docs_root(tmp_path): + """output_dir == docs_root makes topics_out_dir the very same directory + as the source topics/ - the pruning rmtree used to delete it outright, + with every already-globbed source file then failing to convert.""" + docs_root, config = _write_minimal_docs_root(tmp_path) + result = _run_main(docs_root, docs_root, config) + assert result.returncode == 1 + assert (docs_root / "topics" / "good.md").exists() From 5336ee5c7d2d9fa7aa8cda2b2c98a66e543034b6 Mon Sep 17 00:00:00 2001 From: Alex Miller Date: Wed, 12 Aug 2026 16:07:25 -0500 Subject: [PATCH 4/4] Fix 3 issues from Hal's latest code review Self-closing container tags () had their trailing "/" eaten by TAG_RE's greedy attrs group, so they were recorded as openers that never close - silently nesting the rest of the page inside them. TAG_RE now captures the self-close marker in its own group, and the html_block dispatch emits an immediate open/close pair for one. Indented (4-space) code blocks produced markdown-it's "code_block" token, which convert_node had no case for - they fell through to the generic fallback and rendered as unescaped raw HTML instead of a typed code block. Added a code_block case alongside the existing fence one. The documented "image" block type was never actually emitted - images are inline-only in markdown-it, always folded into their containing block's own html. Fixed the docstring/README to match reality instead of documenting a block shape that can't occur. Co-Authored-By: Claude Sonnet 5 --- .../ProcessKotlinWebsiteJSON/README.md | 8 ++- .../ProcessKotlinWebsiteJSON/md_to_json.py | 49 +++++++++++---- .../tests/test_md_to_json.py | 60 +++++++++++++++++++ 3 files changed, 104 insertions(+), 13 deletions(-) diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md index 97e2a1c87..67cb27dad 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/README.md @@ -45,10 +45,12 @@ python3 md_to_json.py [--topics-subdir topics] ``` Block types: `heading`, `paragraph`, `code`, `blockquote`, `list`, `table`, -`image`, `hr`, `tabs`, `note`/`tip`/`warning`, `html` (raw passthrough). See -the module docstring in [`md_to_json.py`](md_to_json.py) for full shapes and +`hr`, `tabs`, `note`/`tip`/`warning`, `html` (raw passthrough). See the +module docstring in [`md_to_json.py`](md_to_json.py) for full shapes and known limitations (nested tabs, `` resolution, variable -substitution). +substitution). There is no standalone `image` block type - an image is +always inline content inside whatever block contains it (typically +`paragraph`), rendered straight into that block's own `html` string. A heading's `id` is `slugify()`'d from its text, unless the source line has an explicit `{id="..."}` (which overrides it directly). Cross-page links diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py index 76cdb829b..26067a1b4 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/md_to_json.py @@ -38,12 +38,17 @@ {"type": "blockquote", "attrs": {"style": "note"}, "blocks": [...]} {"type": "list", "ordered": false, "items": [{"blocks": [...]}]} {"type": "table", "headers": ["a", "b"], "rows": [["1", "2"]]} - {"type": "image", "src": "...", "alt": "..."} {"type": "hr"} {"type": "tabs", "attrs": {"group": "build-system"}, "tabs": [{"title": "Gradle", "attrs": {"group-key": "gradle"}, "blocks": [...]}]} {"type": "html", "html": ""} +There is no standalone "image" block type - CommonMark only ever produces +"image" as an inline token nested inside a paragraph/heading/etc., so a +`![alt](foo.png)` in the source always ends up as an inside that +block's own "html" string (via render_inline), never as a top-level block +of its own. + Cross-page links (`[text](other-page.md#anchor)`) and images (`![alt](foo.png)`) use Writerside's bare-filename convention - the referenced file is looked up by name anywhere under / (links) or images/ (images), the same @@ -114,8 +119,15 @@ # twice) so the two can't silently diverge. The (?![\w-]) after the # alternation is load-bearing: without it, "tab" matches as a prefix of # "table", consuming "" as tag "tab" with attrs "le". +# +# Group 3 (attrs) is non-greedy and group 4 (self-closing "/") is anchored +# right before the final ">" - with a single greedy "([^>]*)/?>$" instead, +# the attrs group swallows a self-closing tag's trailing "/" before the +# optional "/?" ever gets a chance to match it, so "" came out +# indistinguishable from "": an opener with no matching closer, +# silently nesting the rest of the page inside it. CONTAINER_TAGS = {"tabs", "tab", "note", "tip", "warning"} -TAG_RE = re.compile(r"^<(/?)(" + "|".join(CONTAINER_TAGS) + r")(?![\w-])([^>]*)/?>$", re.I) +TAG_RE = re.compile(r"^<(/?)(" + "|".join(CONTAINER_TAGS) + r")(?![\w-])([^>]*?)\s*(/?)>$", re.I) def build_topic_index(topics_dir: Path) -> dict: @@ -476,7 +488,14 @@ def convert_node(self, node: Node): block["attrs"] = heading_attrs return block - if ttype == "fence": + if ttype in ("fence", "code_block"): + # code_block is markdown-it's token for indented (4-space) code, + # as opposed to fence (```-delimited) - it carries no language + # info, but otherwise wants the same "code" block shape. Without + # this case it fell through to the generic fallback below and + # came out as an unescaped "html" block instead - raw <, & etc. + # in the code would be interpreted as markup rather than shown + # as text, and it lost its code typing entirely. return { "type": "code", "lang": (t.info or "").strip() or None, @@ -533,13 +552,23 @@ def flush_raw(): m = TAG_RE.match(line.strip()) if m: flush_raw() - closing, tag, attrstr = m.groups() - result.append({ - "type": "tag_marker", - "closing": bool(closing), - "tag": tag.lower(), - "attrs": parse_attrs(attrstr), - }) + closing, tag, attrstr, self_closing = m.groups() + attrs = parse_attrs(attrstr) + tag = tag.lower() + if self_closing: + # "" has no children and no separate + # closer - emit the open/close pair immediately + # rather than treating it as an opener that leaves + # the container open for the rest of the page. + result.append({"type": "tag_marker", "closing": False, "tag": tag, "attrs": attrs}) + result.append({"type": "tag_marker", "closing": True, "tag": tag, "attrs": {}}) + else: + result.append({ + "type": "tag_marker", + "closing": bool(closing), + "tag": tag, + "attrs": attrs, + }) elif line.strip(): raw_run.append(line) flush_raw() diff --git a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py index 2a4e5b507..763c044b7 100644 --- a/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py +++ b/ProcessDocs/ProcessKotlinDocs/ProcessKotlinWebsiteJSON/tests/test_md_to_json.py @@ -46,6 +46,66 @@ def test_tag_re_still_matches_real_container_tags(): assert m.TAG_RE.match("") is not None +# --- TAG_RE / html_block: self-closing tags must not stay open ---------- + +def test_tag_re_captures_self_closing_marker_separately(): + """The attrs group used to be greedy, so it swallowed a self-closing + tag's trailing "/" before the optional "/?" at the end ever got a + chance to match it - "" came out with an empty closing + group, i.e. indistinguishable from a plain opener.""" + closing, tag, attrs, self_closing = m.TAG_RE.match('').groups() + assert (closing, tag, self_closing) == ("", "tab", "/") + assert attrs.strip() == 'title="A"' + + closing, tag, attrs, self_closing = m.TAG_RE.match('').groups() + assert self_closing == "" + + +def test_html_block_self_closing_tag_emits_open_and_close_markers(): + """convert_node's html_block dispatch must turn a self-closing tag into + an immediate open/close pair rather than a bare opener - otherwise the + container never closes and silently nests the rest of the page.""" + node = m.Node(FakeToken("html_block", content='')) + conv = make_converter() + result = conv.convert_node(node) + assert [(b["type"], b["closing"], b["tag"]) for b in result] == [ + ("tag_marker", False, "tab"), + ("tag_marker", True, "tab"), + ] + + +def test_self_closing_tag_does_not_swallow_trailing_content(): + """End-to-end repro: a self-closing immediately followed by a + paragraph must not leave that paragraph nested inside the tab.""" + node = m.Node(FakeToken("html_block", content='')) + conv = make_converter() + conv.current_source = "test.md" + markers = conv.convert_node(node) + blocks = markers + [{"type": "paragraph", "html": "After."}] + result = conv.group_containers(blocks) + assert [b["html"] for b in result if b.get("type") == "paragraph"] == ["After."] + assert conv.warnings == [] + + +# --- convert_node: indented code_block must render as code, not html ---- + +def test_indented_code_block_renders_as_code_not_html(): + """convert_node only handled markdown-it's "fence" (```-delimited) + token; an indented (4-space) code block produces a different token + type, "code_block", which fell through to the generic fallback and + came out as an unescaped "html" block instead - raw "<"/"&" in the + code would be interpreted as markup rather than shown as text, and the + block lost its code typing entirely.""" + md = m.make_markdown_it() + conv = m.Converter(md, {}) + tokens = md.parse(' List x = listOf("a", "b");\n') + tree = m.build_tree(tokens) + block = conv.convert_node(tree[0]) + assert block["type"] == "code" + assert block["lang"] is None + assert "List" in block["code"] + + # --- ATTR_PAIR_RE / parse_attrs ----------------------------------------- def test_parse_attrs_allows_whitespace_around_equals():