Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 3 additions & 58 deletions graphify/extract.py
Original file line number Diff line number Diff line change
Expand Up @@ -1290,50 +1290,6 @@ def walk_docstrings(node, parent_nid: str) -> None:
_add_rationale(stripped, lineno, file_nid)


# ── TypeScript import type normalization (#3154) ──────────────────────────────
# tree-sitter-typescript misparses `import(...)` types used inside explicit call-
# expression type arguments (e.g. `f<typeof import("mod")>()` or
# `f<import("mod").Foo>()`) as binary comparison expressions (`<` and `>`).
# The trailing `();` generates an ERROR node, leaving an open `binary_expression`
# that absorbs subsequent declarations as anonymous `function_expression` or `class`
# expressions, silently dropping them from extraction. Normalizing `import(...)`
# within generic call type arguments `<...>(...)` to a valid type identifier of
# identical byte length keeps AST parsing clean while preserving source offsets.

_TS_IMPORT_CALL_RE = re.compile(
rb"\bimport\s*\(\s*['\"][^'\"\r\n]+['\"]\s*\)"
)
_TS_IMPORT_TYPE_CALL_RE = re.compile(
rb"<((?:[^;{}]*?\bimport\s*\([^()]+\)[^;{}]*?)+)>(?=\s*\()"
)


def _normalize_ts_import_types(source: bytes) -> bytes | None:
"""Rewrite TypeScript `import(...)` type arguments in call expressions
to standard type identifiers of identical byte length (#3154).

Preserves byte length, newlines, and source offsets so all downstream node
source_location metadata remains 100% accurate.
"""
if rb"import(" not in source and rb"import (" not in source:
return None

def repl_type_args(m: "re.Match[bytes]") -> bytes:
type_arg_content = m.group(1)

def repl_import(im: "re.Match[bytes]") -> bytes:
matched = im.group(0)
return b"T" + re.sub(rb"[^\r\n]", b" ", matched[1:])

new_content = _TS_IMPORT_CALL_RE.sub(repl_import, type_arg_content)
return b"<" + new_content + b">"

norm = _TS_IMPORT_TYPE_CALL_RE.sub(repl_type_args, source)
return norm if norm != source else None


# ── Public API ────────────────────────────────────────────────────────────────

def extract_python(path: Path) -> dict:
"""Extract classes, functions, and imports from a .py file via tree-sitter AST."""
result = _extract_generic(path, _PYTHON_CONFIG)
Expand All @@ -1345,21 +1301,13 @@ def extract_python(path: Path) -> dict:
def extract_js(path: Path) -> dict:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Health regressionextract_js()

85 callers depend on it (afferent coupling).

Grounded coupling-delta finding (deterministic), not an LLM guess.

"""Extract classes, functions, arrow functions, and imports from a .js/.ts/.tsx/.mts/.cts file."""
suffix = path.suffix.lower()
is_ts = suffix in (".ts", ".tsx", ".mts", ".cts")
if suffix == ".tsx":
config = _TSX_CONFIG
elif suffix in (".ts", ".mts", ".cts"):
config = _TS_CONFIG
else:
config = _JS_CONFIG
source_override = None
if is_ts:
try:
source = path.read_bytes()
source_override = _normalize_ts_import_types(source)
except OSError:
pass
result = _extract_generic(path, config, source_override=source_override)
result = _extract_generic(path, config)
if "error" not in result:
_extract_js_rationale(path, result)
_rescue_js_dynamic_imports(path, result)
Expand Down Expand Up @@ -1394,7 +1342,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None:
try:
import re as _re
src = path.read_text(encoding="utf-8", errors="replace")
if "import(" not in src: # cheap bail — most files have none
if not _re.search(r"(?<!\w)import(?:\s|\\\r?\n)*\(", src): # cheap bail — most files have none
return
existing_ids = {n["id"] for n in result.get("nodes", [])}
file_node_id = _make_id(str(path))
Expand Down Expand Up @@ -1436,7 +1384,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None:
# handling: a literal `import(`./x`)` resolves, `${`-substituted ones
# are excluded (no `$` in the class) as statically unresolvable.
for m in _re.finditer(
r"""(?<!\w)import\(\s*(?:'([^'\n]+)'|"([^"\n]+)"|`([^`$\n]+)`)\s*\)""",
r"""(?<!\w)import(?:\s|\\\r?\n)*\(\s*(?:'([^'\n]+)'|"([^"\n]+)"|`([^`$\n]+)`)\s*\)""",
src,
):
raw = m.group(1) or m.group(2) or m.group(3)
Expand Down Expand Up @@ -1825,9 +1773,6 @@ def extract_vue(path: Path) -> dict:
else: # "ts" or unspecified — default to the TS grammar (superset of JS)
config = _TS_CONFIG
masked_bytes = masked.encode("utf-8")
if config in (_TS_CONFIG, _TSX_CONFIG):
masked_bytes = _normalize_ts_import_types(masked_bytes) or masked_bytes

result = _extract_generic(path, config, source_override=masked_bytes)

# Dynamic `import('…')` calls aren't edged by the AST pass; recover by regex,
Expand Down
87 changes: 87 additions & 0 deletions graphify/extractors/engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import hashlib
import importlib
import re
from graphify.extractors.base import _LANGUAGE_BUILTIN_GLOBALS, _file_stem, _make_id, _read_text
from graphify.ids import normalize_id
from graphify.extractors.models import LanguageConfig
Expand Down Expand Up @@ -2875,6 +2876,82 @@ def _ruby_extra_walk(node, source: bytes, file_nid: str, stem: str, str_path: st
del ruby_namespace[-len(const_segments):]
return True

def _recover_3154_ts_import_types(tree, source: bytes) -> bytes | None:
"""Rewrite TypeScript `import(...)` type arguments in malformed generic call expressions
to standard type identifiers of identical byte length (#3154 / #3210).

tree-sitter-typescript misparses generic call expressions whose type arguments start with
`typeof import(...)` or `import(...).Foo` (e.g. `f<typeof import("mod")>()`) as relational
binary comparison expressions (`<` and `>`), generating an ERROR node for `();` that absorbs
subsequent declarations as anonymous function/class expressions.

AST-first: only runs when `tree.root_node.has_error` is True, identifying exact `import(...)`
call expressions nested inside such malformed call expressions, and leaves runtime dynamic
imports completely untouched.

Preserves byte length, newlines, and source offsets so all downstream node source_location
metadata remains 100% accurate.
"""
if b"import(" not in source and b"import (" not in source:
return None
if not tree.root_node.has_error:
return None

def find_candidates(node):
candidates = []
if node.type == "call_expression":
if node.children and node.children[0].type == "import":
curr = node.parent
while curr is not None and curr.type not in (
"expression_statement",
"lexical_declaration",
"statement_block",
"program",
):
curr = curr.parent
if curr is not None and curr.type == "expression_statement":
main_expr = curr.children[0] if curr.children else None
if main_expr and main_expr.type in ("binary_expression", "sequence_expression"):
def leftmost_leaf(n):
while n.children:
n = n.children[0]
return n

first_leaf = leftmost_leaf(main_expr)
if first_leaf.type in ("identifier", "property_identifier"):
def has_call_error(n):
if n.type == "ERROR":
raw = source[n.start_byte:n.end_byte].lstrip()
if (
raw.startswith(b"(")
or raw.startswith(b">(")
or any(
c.type in ("formal_parameters", "arguments")
for c in n.children
)
):
return True
return any(has_call_error(c) for c in n.children)

if has_call_error(curr):
candidates.append((node.start_byte, node.end_byte))
for c in node.children:
candidates.extend(find_candidates(c))
return candidates

import_ranges = find_candidates(tree.root_node)
if not import_ranges:
return None

rewritten = bytearray(source)
for start, end in sorted(import_ranges, key=lambda r: r[0], reverse=True):
orig_slice = source[start:end]
repl = b"T" + re.sub(rb"[^\r\n]", b" ", orig_slice[1:])
rewritten[start:end] = repl

return bytes(rewritten)


def _extract_generic(

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Health regression_extract_generic()

fans out to 26 callees (efferent coupling); 18 callers depend on it (afferent coupling).

Grounded coupling-delta finding (deterministic), not an LLM guess.

path: Path, config: LanguageConfig, *, source_override: bytes | None = None
) -> dict:
Expand Down Expand Up @@ -2913,6 +2990,16 @@ def _extract_generic(
source = path.read_bytes() if source_override is None else source_override
tree = parser.parse(source)
root = tree.root_node
if (
root.has_error
and config.ts_module in ("tree_sitter_typescript",)
and (b"import(" in source or b"import (" in source)
):
rewritten = _recover_3154_ts_import_types(tree, source)
if rewritten is not None:
source = rewritten
tree = parser.parse(source)
root = tree.root_node
except Exception as e:
return {"nodes": [], "edges": [], "error": str(e)}

Expand Down
109 changes: 109 additions & 0 deletions tests/test_ts_import_type_arguments.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,112 @@ def test_ts_multiline_import_type_arguments(tmp_path: Path, capsys):
assert "after()" in labels
assert "mod" in labels
_assert_silent(capsys.readouterr().err)


def test_ts_call_multiple_import_type_arguments(tmp_path: Path, capsys):
r = _extract(tmp_path, {
"multi.ts": (
"function before() {}\n"
"load<typeof import('m1'), typeof import('m2')>();\n"
"load<string, typeof import('m3')>();\n"
"function after() {}\n"
)
})
labels = _labels(r)
assert "before()" in labels
assert "after()" in labels
assert "m1" in labels
assert "m2" in labels
assert "m3" in labels
_assert_silent(capsys.readouterr().err)


def test_ts_member_call_import_type_arguments(tmp_path: Path, capsys):
r = _extract(tmp_path, {
"member.ts": (
"function before() {}\n"
"obj.load<typeof import('mod1')>();\n"
"obj.foo.bar<typeof import('mod2')>();\n"
"function after() {}\n"
)
})
labels = _labels(r)
assert "before()" in labels
assert "after()" in labels
assert "mod1" in labels
assert "mod2" in labels
_assert_silent(capsys.readouterr().err)


def test_ts_asi_comparison_does_not_corrupt_runtime_import_3210(tmp_path: Path, capsys):
"""#3210: Relational comparisons in semicolon-less TypeScript must not erase runtime imports."""
r = _extract(tmp_path, {
"service.ts": (
"async function load(a: number, b: number) {\n"
" const flag = a < b\n"
" const m = await import('./mod')\n"
" const ok = b > (a)\n"
" return [flag, m, ok]\n"
"}\n"
)
})
labels = _labels(r)
assert "load()" in labels
# Function-level deferred edge from load()
load_nid = next(n["id"] for n in r["nodes"] if n["label"] == "load()")
deferred_edges = [
e for e in r["edges"]
if e.get("source") == load_nid and e.get("deferred") and e.get("relation") == "imports_from"
]
assert len(deferred_edges) == 1
assert "mod" in deferred_edges[0]["target"] or deferred_edges[0]["target"].endswith("mod")
_assert_silent(capsys.readouterr().err)


def test_ts_whitespace_runtime_dynamic_imports(tmp_path: Path, capsys):
"""Ensure dynamic imports with varying whitespace before '(' are properly extracted/rescued."""
r = _extract(tmp_path, {
"whitespace.ts": (
"async function load() {\n"
" const m1 = await import ('./m1')\n"
" const m2 = await import ('./m2')\n"
" const m3 = await import(\n"
" './m3'\n"
" )\n"
" return [m1, m2, m3]\n"
"}\n"
)
})
labels = _labels(r)
assert "load()" in labels
assert "./m1" in labels
assert "./m2" in labels
assert "./m3" in labels
_assert_silent(capsys.readouterr().err)


def test_ts_relational_expressions_safety(tmp_path: Path, capsys):
"""Check relational expressions spanning lines and mixed comparisons."""
r = _extract(tmp_path, {
"relational.ts": (
"const x = a < b\n"
"const m = await import('./mod')\n"
"const y = c > d\n"
)
})
labels = _labels(r)
assert "./mod" in labels


def test_ts_unrelated_syntax_error_preserves_runtime_import(tmp_path: Path):
"""Unrelated syntax error must not destroy runtime dynamic import rescue."""
r = _extract(tmp_path, {
"broken.ts": (
"async function f() {\n"
" const m = await import('./mod')\n"
" const x =\n"
"}\n"
)
})
labels = _labels(r)
assert "./mod" in labels
Loading