Coverage for src/renaissance/integrations/python/ast/cst_node.py: 97%
133 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-11-16 16:04 +0000
« prev ^ index » next coverage.py v7.17.0, created at 2026-11-16 16:04 +0000
1# Copyright (C) 2024-2026 TNO
2#
3# Licensed under the terms specified in the LICENSE file in the root directory
4# of this repository, also available at:
5# https://github.com/TNO/Renaissance.Py/blob/main/LICENSE
7"""AI: ASTNode implementation backed by libcst's concrete syntax tree."""
9import io
10import itertools
11import sys
12from collections import defaultdict
13from pathlib import Path
14from typing import Self
16import libcst
17from libcst import BaseCompoundStatement, BaseSmallStatement, ClassDef, CSTNode, FunctionDef, MetadataWrapper
18from libcst.metadata import Access, Assignment, CodePosition, PositionProvider, ScopeProvider
20from renaissance.integrations.python.ast.kinds import PYTHON_KIND_MAP
21from renaissance.syntax_tree.ast_node import ASTReference
22from renaissance.syntax_tree.semantic_kind import SemanticKind
23from renaissance.utils.ast_utils import next_sibling, preceding_sibling
25type _Link = tuple[CSTNode, str, Access]
28class PythonCstTranslationUnit:
29 """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping."""
31 def __init__(self, content: str, file_name: str) -> None:
32 """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping."""
33 # TODO(pjljvandelaar): see docs/developer/architecture/position-consistency.md - exposes no buffer, so the rewriter
34 # derives its own.
35 # https://github.com/TNO/Renaissance.Py/issues/153
36 # TODO(FrancescoPezzella): content is to become the file's UTF-8 bytes, with every offset a UTF-8 byte offset into
37 # them (decided for AST and regular expression matching; no str). Until then content is the str decoded by load
38 # and offsets count bytes of self.encoding, the filesystem encoding, so never slice content with them (#153).
39 # https://github.com/TNO/Renaissance.Py/issues/188
40 self.content = content
41 self.encoding = sys.getfilesystemencoding()
42 # newline="" splits on \n, \r\n and \r only, like libcst, and keeps each line ending.
43 self._lines = io.StringIO(content, newline="").readlines()
44 self._line_starts = list(itertools.accumulate((len(line.encode(self.encoding)) for line in self._lines), initial=0))
45 self.file_name = file_name
46 self.wrapper = MetadataWrapper(libcst.parse_module(content))
47 self.atu = self.wrapper.module
48 self.spans = self.wrapper.resolve(PositionProvider)
49 self.wrappers: dict[CSTNode, PythonCstNode] = {}
50 self._uses_of: dict[CSTNode, list[_Link]] | None = None
51 self._definitions_of: dict[CSTNode, list[_Link]] = {}
53 def _index_references(self) -> dict[CSTNode, list[_Link]]:
54 """Resolve libcst's scopes once, linking every definition to its uses; return the definition-to-uses map.
56 Also fills self._definitions_of, the use-to-definitions map. Builtins, which have no defining node, are skipped.
57 An annotation or type hint above a definition in the same scope is linked to it too, since it is evaluated
58 lazily (PEP 563, PEP 649); a value read there at once is not.
59 """
60 if self._uses_of is not None:
61 return self._uses_of
62 uses_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list)
63 definitions_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list)
64 scopes = {scope for scope in self.wrapper.resolve(ScopeProvider).values() if scope is not None}
65 for scope in scopes:
66 for assignment in scope.assignments:
67 if not isinstance(assignment, Assignment):
68 continue
69 lazy_accesses_above = {
70 access
71 for access in scope.accesses[assignment.name]
72 if not access.referents and (access.is_annotation or access.is_type_hint)
73 }
74 # libcst only links an access to the definitions above it in the same scope.
75 for access in [*assignment.references, *lazy_accesses_above]:
76 uses_of[assignment.node].append((access.node, assignment.name, access))
77 definitions_of[access.node].append((assignment.node, assignment.name, access))
78 self._uses_of = dict(uses_of)
79 self._definitions_of = dict(definitions_of)
80 return self._uses_of
82 def _to_references(self, links: list[_Link], ref_kind: str) -> list[ASTReference[PythonCstNode]]:
83 """Wrap links as references, ordered by the position of the node each points at.
85 Each reference's properties hold the bound "name" and whether the use sits in an annotation ("is_annotation")
86 or another type hint such as `cast("T", x)` ("is_type_hint").
87 """
88 references = [
89 ASTReference(
90 self.wrappers[node],
91 ref_kind,
92 {"name": name, "is_annotation": access.is_annotation, "is_type_hint": access.is_type_hint},
93 )
94 for node, name, access in links
95 if node in self.wrappers
96 ]
97 return sorted(references, key=lambda reference: reference.node.offset)
99 def uses_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]:
100 """Return the uses of the name(s) that node defines; empty if node defines nothing."""
101 return self._to_references(self._index_references().get(node, []), "use")
103 def definitions_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]:
104 """Return the definitions that node, a use of a name, resolves to; empty if node is not a resolved use."""
105 self._index_references()
106 return self._to_references(self._definitions_of.get(node, []), "definition")
108 def _byte_offset(self, position: CodePosition) -> int:
109 """Return the offset, in bytes of self.encoding, of a libcst position whose column counts characters.
111 A position on the line after the last one (the end of a node that ends with the file) maps to the file's length.
112 """
113 line_index = position.line - 1
114 if line_index >= len(self._lines):
115 return self._line_starts[-1]
116 return self._line_starts[line_index] + len(self._lines[line_index][: position.column].encode(self.encoding))
118 def start_of(self, node: CSTNode) -> int:
119 """Return the byte offset where node begins in the source text, or 0 if it has no position.
121 A decorated function or class begins at its first decorator.
122 """
123 if isinstance(node, (ClassDef, FunctionDef)) and node.decorators:
124 node = node.decorators[0]
125 span = self.spans.get(node)
126 return self._byte_offset(span.start) if span else 0
128 def end_of(self, node: CSTNode) -> int:
129 """Return the byte offset where node's code ends in the source text, or 0 if it has no position."""
130 span = self.spans.get(node)
131 return self._byte_offset(span.end) if span else 0
133 def signature_of(self, node: CSTNode) -> str:
134 """AI: Return the source code text corresponding to node, or an empty string on failure."""
135 try:
136 return self.atu.code_for_node(node)
137 except Exception:
138 return ""
141class PythonCstNode:
142 """AI: ASTNode implementation backed by libcst's concrete syntax tree."""
144 def __init__(self, node: CSTNode, translation_unit: PythonCstTranslationUnit, parent: Self | None = None) -> None:
145 """Wrap node and, recursively, its children as nodes of this class within the given translation unit."""
146 self.parent = parent
147 self.root: Self = parent.root if parent and parent.root else self
148 self.translation_unit = translation_unit
149 self.node = node
150 translation_unit.wrappers[node] = self
152 self.is_statement = isinstance(self.node, (BaseSmallStatement, BaseCompoundStatement))
153 self.parser_kind = type(node).__name__
154 self.semantic_kind = PYTHON_KIND_MAP.get(self.parser_kind, SemanticKind.NODE)
156 self.children: list[Self] = [type(self)(child, translation_unit, self) for child in node.children]
157 self.properties = {}
159 # for shower
160 self.is_implicit = True
161 self.show_props = False
163 # for rewriter
164 self.text = self.signature
166 def __str__(self) -> str:
167 """AI: Return the string representation of the wrapped CST node."""
168 return str(self.node)
170 def __repr__(self) -> str:
171 """AI: Return the repr of the wrapped CST node."""
172 return repr(self.node)
174 @property
175 def signature(self):
176 """AI: Return the source code text of the wrapped CST node."""
177 return self.translation_unit.signature_of(self.node)
179 @property
180 def offset(self):
181 """Return the offset, in bytes of translation_unit.encoding, where this node begins in the source text."""
182 return self.translation_unit.start_of(self.node)
184 @property
185 def length(self):
186 """Return the length, in bytes of translation_unit.encoding, of this node's source text."""
187 return self.end_offset - self.offset
189 @property
190 def end_offset(self):
191 """Return the offset, in bytes of translation_unit.encoding, where this node ends in the source text."""
192 return self.translation_unit.end_of(self.node)
194 @property
195 def extended_end_offset(self) -> int:
196 """Return the end offset; trailing comments are found by the rewriter, as for the RST backend."""
197 return self.end_offset
199 @property
200 def filename(self):
201 """AI: Return the source file name of this node's translation unit."""
202 return self.translation_unit.file_name
204 @property
205 def name(self):
206 """AI: Return this node's class/function name, or an empty string if not applicable."""
207 if isinstance(self.node, (ClassDef, FunctionDef)):
208 return self.node.name.value
209 return ""
211 @property
212 def references(self) -> list[ASTReference[PythonCstNode]]:
213 """Return the definitions this node uses, resolved with libcst's scope analysis (empty if it uses none)."""
214 return self.translation_unit.definitions_of(self.node)
216 @property
217 def referenced_by(self) -> list[ASTReference[PythonCstNode]]:
218 """Return the uses of the name(s) this node defines, resolved with libcst's scope analysis (empty if none)."""
219 return self.translation_unit.uses_of(self.node)
221 @property
222 def next_sibling(self) -> Self | None:
223 """AI: Return the sibling node immediately following this one, or None."""
224 return next_sibling(self)
226 @property
227 def preceding_sibling(self) -> Self | None:
228 """AI: Return the sibling node immediately preceding this one, or None."""
229 return preceding_sibling(self)
231 @classmethod
232 def load(cls, file_path: Path) -> Self:
233 """Parse the Python source file at file_path into a tree of nodes of this class."""
234 # TODO(FrancescoPezzella): drop the explicit encoding with Python 3.15, where UTF-8 is the default.
235 # https://github.com/TNO/Renaissance.Py/issues/188
236 with Path(file_path).open(encoding="utf-8") as file:
237 content = file.read()
238 return cls.load_from_text(content, str(file_path))
240 @classmethod
241 def load_from_text(
242 cls,
243 text: str,
244 file_name: str = "cst_snippet.py",
245 ) -> Self:
246 """Parse Python source text into a tree of nodes of this class, attributed to file_name."""
247 translation_unit = PythonCstTranslationUnit(text, file_name=str(file_name))
248 return cls(translation_unit.atu, translation_unit)