Coverage for src/renaissance/integrations/python/ast/cst_node.py: 97%

133 statements  

« prev     ^ index     » next       coverage.py v7.17.0, created at 2026-11-16 16:04 +0000

1# Copyright (C) 2024-2026 TNO 

2# 

3# Licensed under the terms specified in the LICENSE file in the root directory 

4# of this repository, also available at: 

5# https://github.com/TNO/Renaissance.Py/blob/main/LICENSE 

6 

7"""AI: ASTNode implementation backed by libcst's concrete syntax tree.""" 

8 

9import io 

10import itertools 

11import sys 

12from collections import defaultdict 

13from pathlib import Path 

14from typing import Self 

15 

16import libcst 

17from libcst import BaseCompoundStatement, BaseSmallStatement, ClassDef, CSTNode, FunctionDef, MetadataWrapper 

18from libcst.metadata import Access, Assignment, CodePosition, PositionProvider, ScopeProvider 

19 

20from renaissance.integrations.python.ast.kinds import PYTHON_KIND_MAP 

21from renaissance.syntax_tree.ast_node import ASTReference 

22from renaissance.syntax_tree.semantic_kind import SemanticKind 

23from renaissance.utils.ast_utils import next_sibling, preceding_sibling 

24 

25type _Link = tuple[CSTNode, str, Access] 

26 

27 

28class PythonCstTranslationUnit: 

29 """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping.""" 

30 

31 def __init__(self, content: str, file_name: str) -> None: 

32 """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping.""" 

33 # TODO(pjljvandelaar): see docs/developer/architecture/position-consistency.md - exposes no buffer, so the rewriter 

34 # derives its own. 

35 # https://github.com/TNO/Renaissance.Py/issues/153 

36 # TODO(FrancescoPezzella): content is to become the file's UTF-8 bytes, with every offset a UTF-8 byte offset into 

37 # them (decided for AST and regular expression matching; no str). Until then content is the str decoded by load 

38 # and offsets count bytes of self.encoding, the filesystem encoding, so never slice content with them (#153). 

39 # https://github.com/TNO/Renaissance.Py/issues/188 

40 self.content = content 

41 self.encoding = sys.getfilesystemencoding() 

42 # newline="" splits on \n, \r\n and \r only, like libcst, and keeps each line ending. 

43 self._lines = io.StringIO(content, newline="").readlines() 

44 self._line_starts = list(itertools.accumulate((len(line.encode(self.encoding)) for line in self._lines), initial=0)) 

45 self.file_name = file_name 

46 self.wrapper = MetadataWrapper(libcst.parse_module(content)) 

47 self.atu = self.wrapper.module 

48 self.spans = self.wrapper.resolve(PositionProvider) 

49 self.wrappers: dict[CSTNode, PythonCstNode] = {} 

50 self._uses_of: dict[CSTNode, list[_Link]] | None = None 

51 self._definitions_of: dict[CSTNode, list[_Link]] = {} 

52 

53 def _index_references(self) -> dict[CSTNode, list[_Link]]: 

54 """Resolve libcst's scopes once, linking every definition to its uses; return the definition-to-uses map. 

55 

56 Also fills self._definitions_of, the use-to-definitions map. Builtins, which have no defining node, are skipped. 

57 An annotation or type hint above a definition in the same scope is linked to it too, since it is evaluated 

58 lazily (PEP 563, PEP 649); a value read there at once is not. 

59 """ 

60 if self._uses_of is not None: 

61 return self._uses_of 

62 uses_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list) 

63 definitions_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list) 

64 scopes = {scope for scope in self.wrapper.resolve(ScopeProvider).values() if scope is not None} 

65 for scope in scopes: 

66 for assignment in scope.assignments: 

67 if not isinstance(assignment, Assignment): 

68 continue 

69 lazy_accesses_above = { 

70 access 

71 for access in scope.accesses[assignment.name] 

72 if not access.referents and (access.is_annotation or access.is_type_hint) 

73 } 

74 # libcst only links an access to the definitions above it in the same scope. 

75 for access in [*assignment.references, *lazy_accesses_above]: 

76 uses_of[assignment.node].append((access.node, assignment.name, access)) 

77 definitions_of[access.node].append((assignment.node, assignment.name, access)) 

78 self._uses_of = dict(uses_of) 

79 self._definitions_of = dict(definitions_of) 

80 return self._uses_of 

81 

82 def _to_references(self, links: list[_Link], ref_kind: str) -> list[ASTReference[PythonCstNode]]: 

83 """Wrap links as references, ordered by the position of the node each points at. 

84 

85 Each reference's properties hold the bound "name" and whether the use sits in an annotation ("is_annotation") 

86 or another type hint such as `cast("T", x)` ("is_type_hint"). 

87 """ 

88 references = [ 

89 ASTReference( 

90 self.wrappers[node], 

91 ref_kind, 

92 {"name": name, "is_annotation": access.is_annotation, "is_type_hint": access.is_type_hint}, 

93 ) 

94 for node, name, access in links 

95 if node in self.wrappers 

96 ] 

97 return sorted(references, key=lambda reference: reference.node.offset) 

98 

99 def uses_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]: 

100 """Return the uses of the name(s) that node defines; empty if node defines nothing.""" 

101 return self._to_references(self._index_references().get(node, []), "use") 

102 

103 def definitions_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]: 

104 """Return the definitions that node, a use of a name, resolves to; empty if node is not a resolved use.""" 

105 self._index_references() 

106 return self._to_references(self._definitions_of.get(node, []), "definition") 

107 

108 def _byte_offset(self, position: CodePosition) -> int: 

109 """Return the offset, in bytes of self.encoding, of a libcst position whose column counts characters. 

110 

111 A position on the line after the last one (the end of a node that ends with the file) maps to the file's length. 

112 """ 

113 line_index = position.line - 1 

114 if line_index >= len(self._lines): 

115 return self._line_starts[-1] 

116 return self._line_starts[line_index] + len(self._lines[line_index][: position.column].encode(self.encoding)) 

117 

118 def start_of(self, node: CSTNode) -> int: 

119 """Return the byte offset where node begins in the source text, or 0 if it has no position. 

120 

121 A decorated function or class begins at its first decorator. 

122 """ 

123 if isinstance(node, (ClassDef, FunctionDef)) and node.decorators: 

124 node = node.decorators[0] 

125 span = self.spans.get(node) 

126 return self._byte_offset(span.start) if span else 0 

127 

128 def end_of(self, node: CSTNode) -> int: 

129 """Return the byte offset where node's code ends in the source text, or 0 if it has no position.""" 

130 span = self.spans.get(node) 

131 return self._byte_offset(span.end) if span else 0 

132 

133 def signature_of(self, node: CSTNode) -> str: 

134 """AI: Return the source code text corresponding to node, or an empty string on failure.""" 

135 try: 

136 return self.atu.code_for_node(node) 

137 except Exception: 

138 return "" 

139 

140 

141class PythonCstNode: 

142 """AI: ASTNode implementation backed by libcst's concrete syntax tree.""" 

143 

144 def __init__(self, node: CSTNode, translation_unit: PythonCstTranslationUnit, parent: Self | None = None) -> None: 

145 """Wrap node and, recursively, its children as nodes of this class within the given translation unit.""" 

146 self.parent = parent 

147 self.root: Self = parent.root if parent and parent.root else self 

148 self.translation_unit = translation_unit 

149 self.node = node 

150 translation_unit.wrappers[node] = self 

151 

152 self.is_statement = isinstance(self.node, (BaseSmallStatement, BaseCompoundStatement)) 

153 self.parser_kind = type(node).__name__ 

154 self.semantic_kind = PYTHON_KIND_MAP.get(self.parser_kind, SemanticKind.NODE) 

155 

156 self.children: list[Self] = [type(self)(child, translation_unit, self) for child in node.children] 

157 self.properties = {} 

158 

159 # for shower 

160 self.is_implicit = True 

161 self.show_props = False 

162 

163 # for rewriter 

164 self.text = self.signature 

165 

166 def __str__(self) -> str: 

167 """AI: Return the string representation of the wrapped CST node.""" 

168 return str(self.node) 

169 

170 def __repr__(self) -> str: 

171 """AI: Return the repr of the wrapped CST node.""" 

172 return repr(self.node) 

173 

174 @property 

175 def signature(self): 

176 """AI: Return the source code text of the wrapped CST node.""" 

177 return self.translation_unit.signature_of(self.node) 

178 

179 @property 

180 def offset(self): 

181 """Return the offset, in bytes of translation_unit.encoding, where this node begins in the source text.""" 

182 return self.translation_unit.start_of(self.node) 

183 

184 @property 

185 def length(self): 

186 """Return the length, in bytes of translation_unit.encoding, of this node's source text.""" 

187 return self.end_offset - self.offset 

188 

189 @property 

190 def end_offset(self): 

191 """Return the offset, in bytes of translation_unit.encoding, where this node ends in the source text.""" 

192 return self.translation_unit.end_of(self.node) 

193 

194 @property 

195 def extended_end_offset(self) -> int: 

196 """Return the end offset; trailing comments are found by the rewriter, as for the RST backend.""" 

197 return self.end_offset 

198 

199 @property 

200 def filename(self): 

201 """AI: Return the source file name of this node's translation unit.""" 

202 return self.translation_unit.file_name 

203 

204 @property 

205 def name(self): 

206 """AI: Return this node's class/function name, or an empty string if not applicable.""" 

207 if isinstance(self.node, (ClassDef, FunctionDef)): 

208 return self.node.name.value 

209 return "" 

210 

211 @property 

212 def references(self) -> list[ASTReference[PythonCstNode]]: 

213 """Return the definitions this node uses, resolved with libcst's scope analysis (empty if it uses none).""" 

214 return self.translation_unit.definitions_of(self.node) 

215 

216 @property 

217 def referenced_by(self) -> list[ASTReference[PythonCstNode]]: 

218 """Return the uses of the name(s) this node defines, resolved with libcst's scope analysis (empty if none).""" 

219 return self.translation_unit.uses_of(self.node) 

220 

221 @property 

222 def next_sibling(self) -> Self | None: 

223 """AI: Return the sibling node immediately following this one, or None.""" 

224 return next_sibling(self) 

225 

226 @property 

227 def preceding_sibling(self) -> Self | None: 

228 """AI: Return the sibling node immediately preceding this one, or None.""" 

229 return preceding_sibling(self) 

230 

231 @classmethod 

232 def load(cls, file_path: Path) -> Self: 

233 """Parse the Python source file at file_path into a tree of nodes of this class.""" 

234 # TODO(FrancescoPezzella): drop the explicit encoding with Python 3.15, where UTF-8 is the default. 

235 # https://github.com/TNO/Renaissance.Py/issues/188 

236 with Path(file_path).open(encoding="utf-8") as file: 

237 content = file.read() 

238 return cls.load_from_text(content, str(file_path)) 

239 

240 @classmethod 

241 def load_from_text( 

242 cls, 

243 text: str, 

244 file_name: str = "cst_snippet.py", 

245 ) -> Self: 

246 """Parse Python source text into a tree of nodes of this class, attributed to file_name.""" 

247 translation_unit = PythonCstTranslationUnit(text, file_name=str(file_name)) 

248 return cls(translation_unit.atu, translation_unit)