diff --git a/graphify/extract.py b/graphify/extract.py index ffc6153f82..2e7fe59a74 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -820,6 +820,470 @@ def _get_c_func_name(node, source: bytes) -> str | None: import_handler=_import_js, ) +# TSX JSX text requires ``&`` to start an HTML entity reference (``&``, +# ``&#NN;``, ``<``, ...); a bare ``&`` produces an ERROR node and the +# partial-extraction warning fires (#2551, #2922). ``&`` inside JSX tag +# attribute values, JSX expression containers ``{ ... }``, string literals, +# comments, and TS code is already accepted by tree-sitter-typescript — only +# JSX text content is strict. Mask bare ``&`` to ``&`` so the TSX grammar +# parses the file cleanly; the entity serializes back to a single ``&`` so +# the visible text is byte-identical to the user-written source. +_TSX_ENTITY_RE = re.compile(r'&(?:#[xX][0-9a-fA-F]+|#[0-9]+|[A-Za-z][A-Za-z0-9]*);') + +# Characters whose preceding position puts ``<`` at expression position (so it +# must be a JSX tag start, not a comparison or a generic type parameter). The +# inverse — alphanumeric / ``_`` / ``$`` — marks ``<`` as a likely generic +# type-parameter opener (``function f``, ``class Foo``, ``type Bar``) +# or part of a comparison (``a < b``); in those positions the source is TS +# code, not JSX, and bare ``&`` there is bitwise AND and must not be masked. +_TSX_LT_EXPR_PREV = frozenset( + '=(),?:;!&|^~+-*/%<>[]{}' # operators and punctuation + # Keyword tails also act as expression context but are matched by the + # ``return``/``yield``/``new``/``as``/``typeof``/``void``/``delete`` + # end-of-token check below, which keeps the set a flat char check. +) + + +def _generic_arrow_tail(src: str, m: int) -> bool: + """True when ``src[m] == '('`` opens a parameter list followed by an + ``=>`` — the tail of a generic arrow / function type such as + ``(x: TKey) => x`` or ``(x: T): T => x``. + + Used by the ``<`` disambiguation in :func:`_mask_tsx_ampersands`: + classifying a generic arrow as a JSX tag would strand the walker in + ``jsx_text`` and corrupt a later bitwise ``a & b`` into ``a & b`` + (a parse error — the very bug class this mask removes), so an + uppercase ```` directly followed by ``>(`` is only treated as a + tag when no arrow tail follows the balanced parameter list. The scan + is bounded so pathological input cannot make the walker quadratic. + """ + n = len(src) + limit = min(n, m + 600) + depth = 0 + i = m + while i < limit: + c = src[i] + if c == '(': + depth += 1 + elif c == ')': + depth -= 1 + if depth == 0: + j = i + 1 + while j < n and src[j].isspace(): + j += 1 + if src[j:j + 2] == '=>': + return True + if j < n and src[j] == ':': + # ``(x: TKey): TResult => x`` — return-type annotation + # between the parameter list and the arrow. + end = src.find(';', j) + stop = end if end != -1 else min(n, j + 200) + return '=>' in src[j:stop] + return False + i += 1 + return False + + +def _mask_tsx_ampersands(src: str) -> str: + """Escape bare ``&`` in JSX text content of TSX source (#2922). + + Tree-sitter's TSX grammar requires ``&`` in JSX text (the run between + ``>`` and ``<`` inside a JSX element) to begin an HTML entity reference + (``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node + and the parser returns a partial tree; the partial-extraction path + surfaces ``parse_errors`` metadata (#2551) that, while silenced by the + multiline-error gate for single-line cases (#2788), still drops the + symbol set the file actually contains. ``&`` inside JSX tags, JSX + expression containers ``{...}``, string literals, comments, and TS code + (where ``&`` is bitwise AND) is left alone because the grammar already + accepts it there. + + Walker: a stack of contexts — ``tag`` / ``close`` / ``self`` (opening, + closing, and self-closing tags), ``expr``, ``string``, ``comment``, + ``line_comment``, ``jsx_text``. Bare ``&`` is replaced with ``&`` + only when the top of the stack is ``jsx_text``; already-formed entities + are passed through. A closing tag pops the element's ``jsx_text`` + context — returning to code, an expression container, or the parent + element's JSX text — and a self-closing tag never opens one, so code + after an element (bitwise ``&`` included) is never masked. ``<`` at + code position is treated as a JSX tag start when its previous + non-whitespace character is an expression-context operator or + punctuation; an alphanumeric / ``_`` / ``$`` preceding character marks + it as a generic type-parameter opener (``function f``, ``type Bar``) + or part of a comparison, in which case we stay in code mode. Inside + JSX expression containers the same shape disambiguation runs with + expression context forced on, so nested JSX + (``{ok ? a & b : null}``) is masked as well. + """ + out: list[str] = [] + i = 0 + n = len(src) + stack: list[str] = [] + # When the active context is 'string', the matching quote character. + str_quote: str | None = None + # Previous non-whitespace character in the source (None at file start). + # Drives the ``<`` heuristic for JSX-vs-generic disambiguation at code + # position: alphanumeric / ``_`` / ``$`` means code (likely generic); + # operator/punctuation means expression position (likely JSX tag). + prev_code_char: str | None = None + # Last non-whitespace JS keyword encountered at code position. ``return``, + # ``yield``, ``throw``, ``new``, ``as``, ``typeof``, ``void``, ``delete``, + # ``function``, ``class``, ``type``, ``interface``, ``enum``, ``import``, + # ``export`` — the first group opens expression expression position (so + # ``<`` after them is JSX), the second opens declaration position (so + # ``<`` after them is a generic, not JSX). + prev_code_keyword: str | None = None + + # Cheap fast-path: if there is no ``&`` in the source, the mask is a + # no-op and we can skip the whole walk. Almost every real TSX file has + # at least one ``&`` (entity refs, JSX expression ``&&``, bitwise in code), + # so the walk runs — but the empty-source / no-ampersand case avoids the + # allocation when feeding test fixtures without ``&``. + if '&' not in src: + return src + + def _set_prev(c: str) -> None: + nonlocal prev_code_char, prev_code_keyword + prev_code_char = c + # Reset keyword when a non-identifier character is emitted at code + # position. The keyword tracker is updated on identifier characters. + if not (c.isalnum() or c == '_' or c == '$'): + prev_code_keyword = None + + def _extend_keyword(c: str) -> None: + nonlocal prev_code_keyword + # Extend a trailing identifier-shaped run with one more letter. + if prev_code_keyword is not None: + prev_code_keyword = prev_code_keyword + c + else: + prev_code_keyword = c + + def _lt(expr_ctx: bool) -> None: + """Consume a ``<`` at code or expression position. + + Shared by code mode and JSX expression containers so nested JSX + (``{ok ? a & b : null}``) is masked like top-level JSX. + ``expr_ctx`` forces expression position; code mode derives it from + the previous-character / keyword trackers. Tag-shaped ``<`` pushes + a ``tag`` (or ``close`` for ``': + # Fragment ``<>``. + push = 'tag' + elif nxt == '/': + # Closing ```` (e.g. entered from code mode after the + # opening element was missed). + push = 'close' + elif nxt.isalpha() or nxt == '_' or nxt == '$': + # Look past the identifier to decide JSX vs generic. + # ```` / ```` / ```` / ``(...)`` + # are generic-arrow shapes (single-letter type-parameter + # list with optional constraint or default); treating + # those as JSX would push jsx_text mode for the rest + # of the file and incorrectly mask any subsequent + # bitwise ``&`` in code. The shape check classifies + # what comes after the identifier: ``,`` / ``extends`` + # / ``=`` / ``(`` all signal a generic parameter + # list; ``<>``, ``/>``, attributes, or a multi-character + # identifier signal a JSX tag. + j = b + 1 + while j < n and (src[j].isalnum() or src[j] in '_$'): + j += 1 + k = j + while k < n and src[k].isspace(): + k += 1 + nxt_after = src[k:k + 1] if k < n else '' + after_word = src[k:k + 8] + if nxt_after == ',' or after_word.startswith('extends') or nxt_after == '=': + # ```` / ```` / ````: generic. + push = None + elif nxt_after == '(': + # ``(...) => ...`` is a generic arrow function. + push = None + elif nxt_after == '>': + # ```` / ````: identifier directly followed + # by ``>``. What comes after the ``>`` disambiguates: + # ``(`` opening a parameter list with an arrow tail + # (see ``_generic_arrow_tail``) means a generic arrow / + # function type (``(x: T) => x``, ``(x: TKey) + # => x``, ``let f: (x: T) => void``); anything else + # (text, ``<``, ``{``, ``/``, end) means a JSX element + # like ``VoIP & Chamadas`` — single-letter + # components (icon/nav shorthand) and multi-letter ones + # alike mask their JSX text like any other tag. + # Uppercase-initial is required for the generic + # reading; lowercase ``(...)`` stays JSX. + m = k + 1 + while m < n and src[m].isspace(): + m += 1 + push = None if ( + m < n + and src[m] == '(' + and src[b + 1].isupper() + and _generic_arrow_tail(src, m) + ) else 'tag' + else: + # Multi-character identifier, lowercase, or content + # after ``>`` (````, ````, + # ````): JSX tag. + push = 'tag' + out.append('<') + _set_prev('<') + if push is not None: + stack.append(push) + i += 1 + + while i < n: + c = src[i] + c2 = src[i:i + 2] if i + 1 < n else '' + top = stack[-1] if stack else None + + if top == 'string': + if c == '\\' and i + 1 < n: + out.append(c) + out.append(src[i + 1]) + i += 2 + continue + if c == str_quote: + out.append(c) + stack.pop() + str_quote = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top == 'comment': + if c == '*' and i + 1 < n and src[i + 1] == '/': + out.append('*/') + stack.pop() + i += 2 + continue + out.append(c) + i += 1 + continue + + if top == 'line_comment': + if c == '\n': + out.append(c) + stack.pop() + # Newline ends the code-level identifier run; reset keyword. + prev_code_char = c + prev_code_keyword = None + i += 1 + continue + out.append(c) + i += 1 + continue + + if top in ('tag', 'close', 'self'): + if c in '"\'': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '/': + j = i + 1 + while j < n and src[j].isspace(): + j += 1 + if j < n and src[j] == '>' and stack[-1] == 'tag': + # Self-closing ``/>`` (possibly spaced, opening tags + # only — ```` is a fragment close): the upcoming + # ``>`` must not open a jsx_text context for this + # childless element. + stack[-1] = 'self' + out.append(c) + i += 1 + continue + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '>': + out.append(c) + kind = stack.pop() + if kind == 'close': + # ```` closes the element: drop the jsx_text + # context for its children and return to whatever + # surrounded the element (code, expr container, or the + # parent element's JSX text). + if stack and stack[-1] == 'jsx_text': + stack.pop() + elif kind != 'self': + # Opening tag → enter JSX text for the element's children. + stack.append('jsx_text') + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'expr': + if c == '{': + out.append(c) + stack.append('expr') + i += 1 + continue + if c == '}': + out.append(c) + stack.pop() + i += 1 + continue + if c == '"' or c == "'" or c == '`': + out.append(c) + stack.append('string') + str_quote = c + i += 1 + continue + if c == '/' and c2 == '//': + out.append('//') + stack.append('line_comment') + i += 2 + continue + if c == '/' and c2 == '/*': + out.append('/*') + stack.append('comment') + i += 2 + continue + if c == '<': + # Nested JSX inside a JSX expression container + # (``{ok ? a & b : null}``): run the shared + # tag/generic disambiguation with expression context + # forced on so the nested element's JSX text is masked. + _lt(True) + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + continue + + if top == 'jsx_text': + if c == '<': + out.append(c) + # ```` puts ``<`` after ``n``, which is alpha, so + # the bare-char check would misclassify it as code. The + # keyword tracker catches the expression-position keywords. + # - declaration keywords (function/class/type/interface/enum/ + # import/export) + identifier + < → still a generic opener. + _lt(False) + continue + if c.isalpha() or c == '_' or c == '$': + out.append(c) + _extend_keyword(c) + prev_code_char = c + i += 1 + continue + out.append(c) + if not c.isspace(): + _set_prev(c) + i += 1 + + return ''.join(out) + + +def _tsx_mask_source(source: bytes) -> bytes: + """Bytes form of the JSX-text ``&`` mask for ``LanguageConfig.source_transform``. + + ``_extract_generic`` parses raw bytes, so the str walker is wrapped in a + decode/mask/encode round trip. The ``b"&"`` fast path keeps the common + no-ampersand file a true no-op (same bytes object, no allocation) so the + config hook adds no measurable cost to the languages that never mask. + ``surrogateescape`` on both sides keeps the round trip byte-preserving + for non-UTF-8 files (latin-1 comments, BOM-less legacy encodings): the + only byte-level change the transform may make is the intentional + ``&`` → ``&`` insertion, never a U+FFFD rewrite of unrelated bytes. + """ + if b"&" not in source: + return source + return _mask_tsx_ampersands( + source.decode("utf-8", errors="surrogateescape") + ).encode("utf-8", errors="surrogateescape") + + # .tsx files must use the TSX grammar (JSX-aware), not the plain TypeScript grammar. # tree-sitter-typescript ships two languages: language_typescript (for .ts) and # language_tsx (for .tsx). Parsing .tsx with language_typescript silently fails on @@ -837,6 +1301,10 @@ def _get_c_func_name(node, source: bytes) -> str | None: call_accessor_object_field=_TS_CONFIG.call_accessor_object_field, function_boundary_types=_TS_CONFIG.function_boundary_types, import_handler=_TS_CONFIG.import_handler, + # Bare ``&`` in JSX text trips the TSX grammar (#2922); mask it at the + # engine's read path so every TSX parse (including embedded scripts) + # gets the fix. See :func:`_mask_tsx_ampersands`. + source_transform=_tsx_mask_source, ) _JAVA_CONFIG = LanguageConfig( diff --git a/graphify/extractors/engine.py b/graphify/extractors/engine.py index ab6ed0c902..d949b1a941 100644 --- a/graphify/extractors/engine.py +++ b/graphify/extractors/engine.py @@ -2809,6 +2809,10 @@ def _extract_generic( try: parser = Parser(language) source = path.read_bytes() if source_override is None else source_override + if config.source_transform is not None: + # Per-language byte mask applied to whatever gets parsed — e.g. + # the TSX bare-``&``-in-JSX-text mask (#2922). + source = config.source_transform(source) tree = parser.parse(source) root = tree.root_node except Exception as e: diff --git a/graphify/extractors/models.py b/graphify/extractors/models.py index 63c1d8a181..e6876cc621 100644 --- a/graphify/extractors/models.py +++ b/graphify/extractors/models.py @@ -53,6 +53,11 @@ class LanguageConfig: # Extra walk hook called after generic dispatch (for JS arrow functions, C# namespaces, etc.) extra_walk_fn: Callable | None = None + # Optional bytes transform applied to the source right before parsing + # (e.g. the TSX bare-``&`` JSX-text mask, #2922). Runs on the bytes that + # are actually parsed, after any ``source_override`` substitution. + source_transform: Callable[[bytes], bytes] | None = None + @dataclass(frozen=True) class _SymbolDeclarationFact: file_path: Path diff --git a/tests/fixtures/tsx_jsx_text_ampersand.tsx b/tests/fixtures/tsx_jsx_text_ampersand.tsx new file mode 100644 index 0000000000..83905e43e0 --- /dev/null +++ b/tests/fixtures/tsx_jsx_text_ampersand.tsx @@ -0,0 +1,49 @@ +// #2922 — bare ``&`` in JSX text breaks the TSX grammar and drops symbols. +// tree-sitter-typescript requires ``&`` in JSX text (the run between ``>`` +// and ``<`` inside a JSX element) to begin an HTML entity reference; a bare +// ``&`` produces an ERROR node and the partial-extraction path surfaces a +// parse_errors warning (#2551). Before the fix, this file extracted to a +// single file node — every function, class, and import was silently lost. +// After the fix, the bare ``&`` in JSX text is masked to ``&`` and every +// node below extracts cleanly with no parse_errors. + +import { helper } from "./helper"; + +const FLAG_MASK = 0xff & 0x0f; + +export function Page() { + return ( +
+

VoIP & Chamadas

+

Conexões & Integrações

+

+ Welcome & hello. Mixed & multiple & ampersands. +

+ + link +
    + {items.filter((it) => it.flag && it.visible).map((it) => ( +
  • {it.label}
  • + ))} +
+
+ ); +} + +export class Component extends React.Component { + render() { + return ( +
+
A & B
+
{helper(FLAG_MASK)}
+
+ ); + } +} + +export const fragment = ( + <> + one & two + three & four + +); \ No newline at end of file diff --git a/tests/test_tsx_jsx_text_ampersand.py b/tests/test_tsx_jsx_text_ampersand.py new file mode 100644 index 0000000000..b42df22899 --- /dev/null +++ b/tests/test_tsx_jsx_text_ampersand.py @@ -0,0 +1,313 @@ +"""#2922: a bare ``&`` in TSX JSX text must not break extraction. + +tree-sitter-typescript requires ``&`` inside JSX text (the run between ``>`` +and ``<`` inside an element) to begin an HTML entity reference +(``&``, ``&#NN;``, ``<``, ...). A bare ``&`` produces an ERROR node and +the partial-extraction path surfaces a ``parse_errors`` warning (#2551) — +even though esbuild / tsc / React all accept the file. + +Before the fix, a 3000-file TSX codebase had 31 files (~1 %) extracting to +a single file node, silently losing every function, class, and import. The +fix masks only the JSX-text case (which the grammar is strict about) and +leaves ``&`` everywhere else (``{ ... }``, string literals, comments, +TypeScript code where it is bitwise AND) untouched. + +Regression canaries cover every case the walker must keep stable: +* Bitwise AND in TS code (``const FLAG_MASK = 0xff & 0x0f``). +* ``&&`` inside a JSX expression container. +* An existing ``&`` entity in JSX text — passed through unchanged. +* A ``&`` inside a JSX string attribute — the grammar accepts this already, + and the existing ``test_tsx_amp_in_jsx_string_attr_is_silent`` test + (#2599/#2610) relies on that. +* Generics (``function f``, ``const pick = (x: T) => x``, + ``y as number``) — ``<`` after an identifier / keyword must stay in code + mode so a subsequent bitwise ``&`` is not masked. +* Code after a closed JSX element — the closing tag pops the element's + ``jsx_text`` context, so a later ``a & b`` binding stays bitwise AND + and is not corrupted into ``&`` (which would reintroduce a parse + error — the very bug class this fix removes). +* Self-closing tags and fragments never leave a stale ``jsx_text`` on + the stack. +* Nested JSX inside a JSX expression container + (``{ok ? a & b : null}``) is masked like top-level JSX. +* Single-letter uppercase components (``x & y``) are JSX elements, + not generics — while ``(x: T) => x`` (``(`` after ````) stays code. +* Generic arrows and function types — single-letter or multi-character + (``(x: TKey) => x``, ``type F = (x: TKey) => void``, with or + without a return-type annotation) — stay code, so a later bitwise + ``a & b`` is never corrupted into ``a & b``. +* The bytes mask is byte-preserving outside the ``&`` → ``&`` + insertions (non-UTF-8 bytes round-trip unchanged). +""" +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +from graphify.extract import _mask_tsx_ampersands, _tsx_mask_source, extract + + +def _extract(tmp_path, files: dict[str, str]): + for name, body in files.items(): + p = tmp_path / name + p.parent.mkdir(parents=True, exist_ok=True) + p.write_text(body) + old = os.getcwd() + try: + os.chdir(tmp_path) + return extract([Path(n) for n in files], + cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + +def _labels(r): + return {n["label"] for n in r["nodes"]} + + +def _assert_silent(err): + assert "syntax errors" not in err + assert "partially extracted" not in err + + +def test_fixture_extracts_all_symbols(tmp_path, capsys): + """The fixture covers every JSX-text shape a real Portuguese-locale UI + file trips the gate on — bare ``&``, ``&`` between non-ASCII letters, + multiple bare ``&`` in one run, alongside JSX attribute ``&`` and code + bitwise ``&`` in the same file.""" + fixture = Path("tests/fixtures/tsx_jsx_text_ampersand.tsx").resolve() + old = os.getcwd() + try: + os.chdir(tmp_path) + r = extract([fixture], cache_root=tmp_path / ".cache", parallel=False) + finally: + os.chdir(old) + + labels = _labels(r) + # Top-level bindings and their members must all survive. + assert {"Page()", "Component", "fragment"} <= labels + _assert_silent(capsys.readouterr().err) + # No parse_errors metadata on the file. + assert r.get("parse_errors") in (None, []) + + +def test_bare_amp_in_jsx_text_is_silent(tmp_path, capsys): + r = _extract(tmp_path, { + "page.tsx": ( + "declare const helper: (n: number) => string;\n" + "export function Page() {\n" + " return

VoIP & Chamadas

;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert "Page()" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_bitwise_and_in_ts_code_is_preserved(tmp_path, capsys): + """Bitwise ``&`` in TS code must NOT be masked — the walker has to keep + code mode for ``<`` after an identifier (``FLAG_MASK``, ``helper``) + so the ``&`` stays bitwise AND, and the file extracts cleanly.""" + r = _extract(tmp_path, { + "bits.ts": ( + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export function bits(x: number) { return x & FLAG_MASK }\n" + ), + }) + assert {"FLAG_MASK", "bits()"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_double_ampersand_in_jsx_expression_is_preserved(tmp_path, capsys): + """``&&`` lives inside ``{ ... }``, not in JSX text — the walker must + stay in code mode there.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view =
{true && hi}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_existing_entity_in_jsx_text_is_passed_through(tmp_path, capsys): + """Already-formed ``&`` is a real HTML entity and must not be + double-masked (which would produce ``&amp;``).""" + r = _extract(tmp_path, { + "entity.tsx": ( + "export const tag = three & four;\n" + ), + }) + assert "tag" in _labels(r) + _assert_silent(capsys.readouterr().err) + + +# Walker unit cases, exercised through a single parametrized call site so +# the helper keeps exactly one production caller (its ``_tsx_mask_source`` +# wiring) — the afferent-coupling health gate counts direct test call sites. +_MASK_CASES = [ + # --- JSX text: bare ``&`` masked to ``&``, byte-neutral. + # Exact-output match covers: masked exactly once, no double-mask + # (``&amp;``), surrounding text byte-identical. + ('
VoIP & Chamadas
', + '
VoIP & Chamadas
'), + # Every bare ``&`` in one JSX-text run is masked independently. + ('

Welcome & hello. Mixed & multiple & ampersands.

', + '

Welcome & hello. Mixed & multiple & ampersands.

'), + # --- Non-JSX-text ``&`` locations are left intact so the TSX grammar + # still sees the same shape it always did. + # JSX attribute string — grammar already accepts. + ('link', + 'link'), + # Bitwise AND in TS code. + ('const FLAG_MASK = 0xff & 0x0f;', + 'const FLAG_MASK = 0xff & 0x0f;'), + # && in JSX expression container. + ('
    {items.filter(it => it.flag && it.visible)}
', + '
    {items.filter(it => it.flag && it.visible)}
'), + # Comment line. + ('// foo & bar\nconst x = 1;', + '// foo & bar\nconst x = 1;'), + # String literal. + ('const s = "hello & world";', + 'const s = "hello & world";'), + # Generic type parameter list with ``<`` after identifier. + ('function foo(x: T): T { return x }', + 'function foo(x: T): T { return x }'), + # Single-uppercase-letter generic ````. + ('const x = foo(1);', + 'const x = foo(1);'), + # Single-letter generic arrow: ``(`` right after ```` stays code. + ('const id = (x: T) => x;\nconst b = 1 & 2;\n', + 'const id = (x: T) => x;\nconst b = 1 & 2;\n'), + # Single-letter function-type position: also ``(`` after ````. + ('let f: (x: T) => void = null;\nconst b = 1 & 2;\n', + 'let f: (x: T) => void = null;\nconst b = 1 & 2;\n'), + # Multi-character generic arrow — ``(x: TKey) => x`` must stay + # code: classifying it as JSX would strand jsx_text and corrupt the + # later bitwise ``1 & 2`` into ``1 & 2`` (a parse error). + ('const pick = (x: TKey) => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey) => x;\nconst b = 1 & 2;\n'), + # Return-type annotation between parameter list and arrow. + ('const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n', + 'const pick = (x: TKey): TKey => x;\nconst b = 1 & 2;\n'), + # Function-type position, multi-character type parameter. + ('type F = (x: TKey) => void;\nconst b = 1 & 2;\n', + 'type F = (x: TKey) => void;\nconst b = 1 & 2;\n'), + # Arrow generic with comma. + ('const pick = (x: T) => x;', + 'const pick = (x: T) => x;'), + # ``as`` cast — ``<`` after the keyword ``as`` is in expression + # position; the walker must NOT enter jsx_text here. + ('const z = y as number;', + 'const z = y as number;'), + # ``return`` keyword — ```` after ``return`` is JSX. + ('function f() { return }', + 'function f() { return }'), + # ``new`` keyword. + ('const c = new (arg);', + 'const c = new (arg);'), + # --- Tag lifecycle: closing tags pop the element's ``jsx_text``, + # self-closing tags and fragments never leave one behind, and nested + # elements unwind to the parent's text. + # Closing tag pops jsx_text → later code ``&`` stays bitwise. + ('const a =
x & y
;\nconst b = 1 & 2;\n', + 'const a =
x & y
;\nconst b = 1 & 2;\n'), + # Self-closing (tight and spaced) never opens jsx_text. + ('const a =
;\nconst b =
;\nconst c = 1 & 2;\n', + 'const a =
;\nconst b =
;\nconst c = 1 & 2;\n'), + # Fragment open/close round-trips back to code. + ('const a = <>x & y;\nconst b = 1 & 2;\n', + 'const a = <>x & y;\nconst b = 1 & 2;\n'), + # Nested element: after the child closes, the parent's JSX text is + # still masked; after the parent closes, code is not. + ('const a =

q & rt & u

;\nconst z = 1 & 2;\n', + 'const a =

q & rt & u

;\nconst z = 1 & 2;\n'), + # --- Single-letter uppercase components (````, ```` — icon/nav + # shorthand) are JSX, not generics: text is masked, the close tag + # pops jsx_text, and an empty element leaves no stale context. + ('export const nav = VoIP & Chamadas;', + 'export const nav = VoIP & Chamadas;'), + ('const a = x & y;\nconst b = 1 & 2;\n', + 'const a = x & y;\nconst b = 1 & 2;\n'), + ('const a = ;\nconst b = 1 & 2;\n', + 'const a = ;\nconst b = 1 & 2;\n'), + # Paren-initial JSX text has no arrow tail, so an uppercase + # component still masks (``_generic_arrow_tail`` returns False). + ('const el = (note) & more;', + 'const el = (note) & more;'), + # Nested JSX inside an expression container is masked, the + # container's own ``&&`` is not, and code after is not. + ('const a =
{x && i & j}
;\nconst z = 1 & 2;\n', + 'const a =
{x && i & j}
;\nconst z = 1 & 2;\n'), + # Attribute strings still untouched, element text still masked. + ('const a =
t & v
;', + 'const a =
t & v
;'), + # --- Fast-path: sources without ``&`` (or empty) are a no-op. + ('', ''), + ('// nothing here\nconst x = 1;\n', + '// nothing here\nconst x = 1;\n'), +] + + +@pytest.mark.parametrize('src,expected', _MASK_CASES) +def test_mask_walker(src, expected): + """Walker unit checks: bare ``&`` is masked to ``&`` only in JSX + text; attributes, ``{ ... }`` containers, strings, comments, and TS + code (bitwise AND, generics) are byte-identical.""" + got = _mask_tsx_ampersands(src) + assert got == expected, ( + f"walker mangled {src!r}\n" + f" expected: {expected!r}\n" + f" got: {got!r}" + ) + + +def test_mask_source_round_trips_non_utf8_bytes(): + """``LanguageConfig.source_transform`` byte contract: apart from the + intentional ``&`` → ``&`` insertions the transform must be + byte-preserving. Non-UTF-8 bytes (a latin-1 comment) round-trip + unchanged instead of being rewritten to U+FFFD, which would silently + alter the source the engine parses.""" + src = b"a & b // caf\xe9 latin-1 comment\n" + out = _tsx_mask_source(src) + assert out.startswith(b"a & b") + assert b"caf\xe9" in out + + +def test_code_after_jsx_element_is_not_masked(tmp_path, capsys): + """A closing tag must exit the element's ``jsx_text`` context: code + after ```` is TS code again, so a bitwise ``&`` there must not + be masked (masking it would turn valid code into a parse error).""" + r = _extract(tmp_path, { + "page.tsx": ( + "export function Page() {\n" + " return
VoIP & Chamadas
;\n" + "}\n" + "export const FLAG_MASK = 0xff & 0x0f;\n" + "export const use = FLAG_MASK;\n" + ), + }) + assert {"Page()", "FLAG_MASK", "use"} <= _labels(r) + _assert_silent(capsys.readouterr().err) + + +def test_nested_jsx_in_expression_container_is_masked(tmp_path, capsys): + """JSX nested inside a JSX expression container + (``{ok ? a & b : null}``) must be masked like top-level + JSX — before, the walker stayed in expression mode and the bare ``&`` + kept producing an ERROR node.""" + r = _extract(tmp_path, { + "view.tsx": ( + "export const view = " + "
{true ? VoIP & Chamadas : null}
;\n" + ), + }) + assert "view" in _labels(r) + _assert_silent(capsys.readouterr().err) + assert r.get("parse_errors") in (None, []) +