Coverage for src/renaissance/integrations/clang/clang_ast_node.py: 84%
340 statements
« prev ^ index » next coverage.py v7.16.1, created at 2026-10-02 13:12 +0000
1"""AI: ASTNode implementation backed by libclang's Python bindings."""
3import sys
4from collections.abc import Sequence
5from functools import cache
6from operator import attrgetter
7from pathlib import Path
8from typing import Any, override
10import clang.native
11from clang.cindex import Config, Cursor, CursorKind, Index, TypeKind
12from clang.cindex import TranslationUnit as ClangCindexTranslationUnit
14from renaissance.integrations.clang.cpp_utils import matches_node_kind
15from renaissance.integrations.clang.kinds import CLANG_KIND_MAP
16from renaissance.integrations.clang.predicates import is_clang_compound_statement, is_clang_macro_definition
17from renaissance.syntax_tree import ASTFinder, ASTNode, ASTReference
18from renaissance.syntax_tree.pattern_kind import PatternKind
19from renaissance.syntax_tree.semantic_kind import SemanticKind
20from renaissance.utils.ast_utils import match_children, match_props
22EMPTY_DICT = {}
23EMPTY_STR = ""
24EMPTY_LIST = []
26DECLARATION_EXPRESSION_PARSER_KINDS = {"DeclRefExpr", "DECL_REF_EXPR"}
27IRRELEVANT_PROPS = {"comment"}
28IRRELEVANT_NODES = {"comment"}
29PRINT_ALL_NODES = False
32class Clangastreference:
33 """AI: Represent a reference from one clang AST node to another by id."""
35 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None:
36 """AI: Represent a reference from one clang AST node to another by id."""
37 self.node_id = node_id
38 self.ref_kind = ref_kind
39 self.properties = properties
42class ClangTranslationUnit:
43 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
45 cache = []
47 def __init__(self, clang_atu: ClangCindexTranslationUnit, file_name: str):
48 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
49 self.clang_atu = clang_atu
50 self.file_name = file_name
51 self.references_initialized = False
52 # print_node_kind(clang_atu.cursor)
53 self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu)
54 # references are used as a cache to store the references of a node
55 # they are stored as id for lazy creation
56 self._references: dict[str, list[Clangastreference]] = {}
57 self._referenced_by: dict[str, list[Clangastreference]] = {}
58 self._nodes: dict[str, ClangASTNode] = {}
60 def lazy_create_references(self, node: ClangASTNode) -> None:
61 """AI: Build the translation unit's reference cache on first use, then no-op on subsequent calls."""
62 if self.references_initialized:
63 return
64 node.root.process(ReferenceHelper.create_references)
65 self.references_initialized = True
67 @staticmethod
68 def _collect_expansions(
69 translation_unit: ClangCindexTranslationUnit,
70 ) -> set[tuple[str, int, int]]:
71 result: set[tuple[str, int, int]] = set()
72 for child in translation_unit.cursor.get_children():
73 if child.kind.name == "MACRO_INSTANTIATION":
74 result.add(
75 (
76 child.extent.start.file.name,
77 child.extent.start.offset,
78 child.extent.end.offset,
79 ),
80 )
81 return result
84class ClangASTNode(ASTNode[Cursor, ClangTranslationUnit]):
85 """AI: ASTNode implementation backed by libclang cursors."""
87 @staticmethod
88 def set_library_path() -> None:
89 """AI: Configure libclang's native library path from the bundled clang.native package."""
90 try:
91 Config.set_library_path(Path(clang.native.__file__).parent)
92 except Exception as e:
93 print(e)
95 set_library_path()
96 index = Index.create()
97 parse_args = [
98 "-fparse-all-comments",
99 "-ferror-limit=0",
100 "-Xclang",
101 "-detailed-preprocessing-record",
102 "-fsyntax-only",
103 ]
105 def __init__(
106 self,
107 node: Cursor,
108 translation_unit: ClangTranslationUnit,
109 parent=None,
110 start_offset: int | None = None,
111 length: int | None = None,
112 insert_kind: str | None = None,
113 ):
114 """AI: Wrap a clang cursor as an AST node within the given translation unit."""
115 super().__init__(self if parent is None else parent.root)
116 self.node = node
117 self._children = None
118 self._parent = parent
119 self.translation_unit = translation_unit
120 self.inserted = insert_kind is not None
121 self.show_props = False
122 self._filename = self._get_containing_filename()
123 self._name = self._derive_name()
124 # if the node has not been added to the translation unit, add it
125 # a node might already be added if it is split into multiple nodes
126 # an example is for base types like int, char, etc. which are split into multiple nodes
127 if self.node.hash not in self.translation_unit._nodes:
128 self.translation_unit._nodes[node.hash] = self
129 self._offset = start_offset if start_offset is not None else self.__derive_start_offset()
130 self._length = length if length is not None else self.__derive_length()
131 self._kind = insert_kind if insert_kind is not None else self.__derive_kind()
132 self.parser_kind = self._kind
133 self.semantic_kind = CLANG_KIND_MAP.get(self.parser_kind, SemanticKind.NODE)
134 self.pattern_kind = {
135 "MatchOne": PatternKind.MATCH_ONE,
136 "MatchAll": PatternKind.MATCH_ALL,
137 }.get(self.parser_kind)
138 self.indent = ""
139 # TODO: TextUtils.get_indent(self.content, self._offset)
140 # an fake child is introduced to handle the case where the type of a declaration is not found
141 # for example in the case of a base type.
142 # without the fake child pattern matching on types will be difficult
143 self.__inserted_children = []
144 # NOTE: clang.cindex.{TypeKind,CursorKind} assign their named members via
145 # runtime attribute assignment after the class body (e.g. `TypeKind.INVALID =
146 # TypeKind(0)`), so pyright cannot see them as declared class attributes from
147 # this module, hence the `pyright: ignore[reportAttributeAccessIssue]` below.
148 if (
149 insert_kind is None
150 and not self.node.location.is_in_system_header
151 and self.node.kind.is_declaration()
152 and self.node.type.kind != TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
153 ):
154 loc_offset: int = self.node.location.offset
155 length = len(self.node.spelling.encode(sys.getdefaultencoding()))
156 insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC")
157 insert_child._children = []
158 self.__inserted_children.append(insert_child)
159 if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # pyright: ignore[reportAttributeAccessIssue]
160 my_type = (
161 self.node.type
162 if self.node.result_type.kind == TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
163 else self.node.result_type
164 )
165 length_ref = len(my_type.spelling.encode(sys.getdefaultencoding()))
166 insert_child = ClangASTNode(
167 self.node,
168 self.translation_unit,
169 self,
170 self._offset,
171 length_ref,
172 CursorKind.TYPE_REF.name, # pyright: ignore[reportAttributeAccessIssue]
173 )
174 insert_child._children = []
175 self.__inserted_children.append(insert_child)
177 self._children = []
178 for n in self.__inserted_children:
179 self._children.append(n)
180 for n in self.node.get_children():
181 if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION":
182 self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self))
184 self._properties = self._derive_properties()
185 if self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
186 self._properties["name"] = self._name
188 @property
189 def kind_key(self) -> SemanticKind | str:
190 """AI: Return the semantic kind, or the raw parser kind when no semantic kind applies."""
191 return self.semantic_kind if self.semantic_kind is not SemanticKind.NODE else self.parser_kind
193 def __eq__(self, other):
194 """AI: Return whether this node is structurally equal to `other`, ignoring irrelevant properties/children."""
195 return (
196 other
197 and isinstance(other, type(self))
198 and self.kind_key == other.kind_key
199 and match_props(self.properties, other.properties, IRRELEVANT_PROPS)
200 and match_children(self.children, other.children, IRRELEVANT_NODES)
201 )
203 def __hash__(self):
204 """AI: Return a hash based on the node's kind key and properties."""
205 return hash((self.kind_key, frozenset(self.properties.items())))
207 @override
208 @staticmethod
209 def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ClangASTNode:
210 args = [*extra_args, *ClangASTNode.parse_args]
211 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:])
212 ClangASTNode.check_diagnostics(translation_unit, file_path.name)
213 root_node = ClangASTNode(
214 translation_unit.cursor,
215 ClangTranslationUnit(translation_unit, file_name=str(file_path)),
216 None,
217 )
218 return root_node
220 @override
221 @staticmethod
222 def load_from_text(
223 text: str,
224 file_name: str,
225 extra_args: Sequence[str] = None,
226 working_dir: Path = None,
227 ) -> ClangASTNode:
228 # Convert file_content to bytes
229 file_content_bytes = text.encode(sys.getfilesystemencoding())
230 # add to cache to avoid reading the file again
231 ASTNode.cache[file_name] = file_content_bytes
232 args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args]
233 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args)
234 ClangASTNode.check_diagnostics(translation_unit, file_name)
235 try:
236 root_node = ClangASTNode(
237 translation_unit.cursor,
238 ClangTranslationUnit(translation_unit, file_name=str(file_name)),
239 None,
240 )
241 except Exception as e:
242 print(e)
243 raise e
244 ClangASTNode.check_diagnostics(translation_unit, file_name)
245 return root_node
247 @staticmethod
248 def check_diagnostics(translation_unit: ClangCindexTranslationUnit, file_name: str) -> None:
249 """AI: Raise an Exception if the translation unit's diagnostics contain any errors."""
250 has_error = False
251 errors = ""
252 for d in translation_unit.diagnostics:
253 if d.severity >= 3:
254 has_error = True
255 errors += f"{d.severity}: {d.spelling} at {d.location}\n"
256 print(f"{d.severity}: {d.spelling} at {d.location}")
257 if has_error:
258 raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}")
260 def _derive_name(self) -> str:
261 try:
262 # NOTE: see the clang.cindex enum note near __init__ above.
263 if self.node.type.kind == TypeKind.RECORD: # pyright: ignore[reportAttributeAccessIssue]
264 return self.node.type.spelling
265 except Exception as e:
266 print(e)
267 try:
268 return self.node.spelling
269 except Exception as e:
270 print(e)
271 return EMPTY_STR
273 def _get_containing_filename(self) -> str:
274 if self is self.root:
275 return self.translation_unit.clang_atu.spelling
276 try:
277 return self.node.location.file.name
278 except Exception:
279 return EMPTY_STR
281 @override
282 @property
283 def extended_end_offset(self) -> int:
284 try:
285 end_offset = self._offset + self._length
286 if (
287 (not self._is_statement_or_declaration())
288 and (
289 self.parent and (is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT)
290 )
291 and not is_clang_macro_definition(self)
292 ):
293 content = self.root.binary_file_content()
294 while end_offset < len(content) and content[end_offset - 1] not in b";":
295 end_offset += 1
296 return end_offset
297 except Exception:
298 return 0
300 def _is_statement_or_declaration(self):
301 return self.semantic_kind in {
302 SemanticKind.STATEMENT,
303 SemanticKind.DECLARATION,
304 SemanticKind.DEFINITION,
305 SemanticKind.FUNCTION,
306 SemanticKind.CLASS,
307 SemanticKind.CONDITIONAL,
308 SemanticKind.LOOP,
309 SemanticKind.RETURN,
310 SemanticKind.IMPORT,
311 SemanticKind.TRANSLATION_UNIT,
312 }
314 @override
315 def matches_kind(self, node: ASTNode[Cursor, ClangTranslationUnit]) -> bool:
316 return matches_node_kind(self, node)
318 def _derive_properties(self) -> dict[str, int | str]:
319 result = {}
320 offsets = (self.filename, self.offset, self.end_offset)
321 if offsets in self.translation_unit.macro_expansions:
322 result["macro_expansion"] = self.text
324 if self.semantic_kind is SemanticKind.BINARY_OPERATION:
325 # TODO remove below code after clang release that supports the getOpCode() statement
326 children = self.children
327 start_offset = children[0].offset + children[0].length
328 end_offset = children[1].offset
329 operator = self.content(start_offset, end_offset)
330 result["operator"] = operator.strip()
331 # next statement works in C++ but not in Python (yet) will be released later
332 # result['operator'] = self.node.getOpCode()
333 elif self.semantic_kind is SemanticKind.UNARY_OPERATION:
334 # TODO remove below code after clang release that supports the getOpCode() statement
335 child = self.children[0]
336 # list all attributes of self.node excluding the once starting with _
338 if child.offset > self.offset:
339 start_offset = self.offset
340 end_offset = child.offset
341 prefix_operator = True
342 else:
343 start_offset = child.offset + child.length
344 end_offset = self.offset + self.length
345 prefix_operator = False
347 operator = self.content(start_offset, end_offset)
348 result["operator"] = operator.strip()
349 result["prefixOperator"] = prefix_operator
350 # next statement works in C++ but not in Python (yet) will be released later
351 # result['operator'] = self.node.getOpCode()
352 elif self.semantic_kind is SemanticKind.LITERAL or self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
353 self._add_tokens(result, "LITERAL")
355 is_all = {
356 attr[len("is_") :]: True
357 for attr in dir(self.node)
358 if attr.startswith("is_") and callable(getattr(self.node, attr)) and getattr(self.node, attr)()
359 }
360 result.update(is_all)
361 return result
363 @override
364 @property
365 def is_statement(self) -> bool:
366 """Pretty good definition."""
367 return self.parent is not None and (
368 is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT
369 )
371 @override
372 @property
373 def referenced_by(self) -> Sequence[ASTReference[Cursor, ClangTranslationUnit]]:
374 self.translation_unit.lazy_create_references(self)
375 node_id = self.node.hash
376 ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST)
377 # if both the function declaration and function definition are available
378 # the references are stored in the function definition,
379 # but we want them to also show up in the declaration
380 if len(ref_by) == 0:
381 definition = self._get_function_definition()
382 if definition:
383 ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST)
384 return list(
385 ASTReference(
386 self.translation_unit._nodes[ref.node_id],
387 ref.ref_kind,
388 ref.properties,
389 )
390 for ref in ref_by
391 )
393 def _get_function_definition(self):
394 # NOTE: see the clang.cindex enum note near __init__ above.
395 if self.node.type.kind == TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
396 signature = self.node.displayname
397 semantic_parent = self.node.semantic_parent.hash
399 def has_body(node):
400 return any(
401 c.kind == CursorKind.COMPOUND_STMT # pyright: ignore[reportAttributeAccessIssue]
402 for c in node.node.get_children()
403 )
405 def is_match(node):
406 if node._kind != self._kind:
407 return False
408 if node.node.type.kind != TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
409 return False
410 if node.node.semantic_parent.hash != semantic_parent:
411 return False
412 if node.node.displayname != signature:
413 return False
414 return has_body(node)
416 if has_body(self):
417 return None
418 body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None)
419 if isinstance(body, ClangASTNode):
420 return body
421 return None
423 @override
424 @property
425 def references(self) -> Sequence[ASTReference[Cursor, ClangTranslationUnit]]:
426 self.translation_unit.lazy_create_references(self)
427 return list(
428 ASTReference(
429 self.translation_unit._nodes[ref.node_id],
430 ref.ref_kind,
431 ref.properties,
432 )
433 for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST)
434 )
436 def _add_tokens(self, result: dict[str, str], *token_kind):
437 for token in self.node.get_tokens():
438 # find all attr of token that are of type str or int
439 kind = str(token.kind).split(".")[-1]
440 if kind in token_kind:
441 result[kind] = token.spelling
443 def __derive_start_offset(self) -> int:
444 try:
445 if self.node.kind.name == "MACRO_DEFINITION":
446 return self.node.extent.start.offset - 8
448 return self.node.extent.start.offset
450 except Exception:
451 return 0
453 def __derive_length(self) -> int:
454 try:
455 if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]:
456 end_offset = self.node.extent.end.offset + 1
457 elif self.node.kind.name == "MACRO_DEFINITION":
458 end_offset = self.node.extent.end.offset
459 else:
460 end_offset = self.node.extent.end.offset
461 return end_offset - self.__derive_start_offset()
462 except Exception:
463 return 0
465 def __derive_kind(self) -> str:
466 try:
467 if self.node.kind.name == "MACRO_DEFINITION":
468 return str(self.node.kind.name)
469 if self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]:
470 if self.node.displayname.startswith("$$") and " " not in self.node.displayname:
471 return "MatchAll"
472 if self.node.displayname.startswith("$") and " " not in self.node.displayname:
473 return "MatchOne"
474 return str(self.node.kind.name)
475 except Exception:
476 return EMPTY_STR
478 @staticmethod
479 def remove_wrapper(cursor):
480 """AI: Unwrap a cursor through single-child unexposed wrapper nodes to reach the real node."""
481 try:
482 if ClangASTNode._is_wrapped(cursor):
483 return ClangASTNode.remove_wrapper(list(cursor.children)[0])
484 except Exception:
485 pass
486 return cursor
488 @staticmethod
489 def _is_reference(node):
490 # refactor this
491 try:
492 print(type(node))
493 print(vars(node))
494 print(dir(node))
495 print(node.__dict__)
496 node.__dict__["id"]
497 return True
498 except Exception:
499 return False
501 @staticmethod
502 @cache
503 def __is_property(key, value):
504 return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"])
506 @staticmethod
507 def _is_wrapped(cursor):
508 return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1
510 @property
511 def is_implicit(self):
512 """AI: Return whether this node is implicitly generated (part of the translation unit)."""
513 return self.is_part_of_translation_unit()
516# def get_ancestor(self, types ):
517# return get_ancestor(self, types)
520SYSTEM_MACROS = {
521 "linux",
522 "unix",
523 "_LP64",
524 "_WIN32",
525 "_WIN64",
526 "_ISO_VOLATILE",
527 "_INTEGRAL_MAX_BITS",
528 "_CRT_USE_BUILTIN_OFFSETOF",
529}
532def is_system_macro(n):
533 """AI: Return True if n is a predefined/builtin compiler or system macro that should be filtered from output."""
534 return n.kind.name == "MACRO_DEFINITION" and (
535 n.displayname.startswith("__")
536 or n.displayname.startswith("_MS")
537 or n.displayname.startswith("_M_")
538 or n.displayname in SYSTEM_MACROS
539 )
542class ReferenceHelper:
543 """AI: Static helpers that populate reference caches between clang AST nodes."""
545 @staticmethod
546 def create_references(ast_node: ClangASTNode) -> None:
547 """AI: Populate the reference and referenced-by caches for ast_node's translation unit."""
548 assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}"
549 references = []
550 node_id: str = ast_node.node.hash
551 ast_node.translation_unit._references[node_id] = references
552 ref_fields = ["referenced"] # , 'type.get_declaration()']
553 for field in ref_fields:
554 try:
555 element = attrgetter(field)(ast_node.node)
556 if element.kind.name == "NO_DECL_FOUND":
557 continue
558 ref_id = element.hash
559 ref_kind = field.split(".")[0]
560 properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"}
561 if node_id == ref_id:
562 return
563 reference = Clangastreference(ref_id, ref_kind, properties)
564 referenced_by = Clangastreference(
565 node_id,
566 ref_kind,
567 {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"},
568 )
569 try:
570 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)
571 except Exception:
572 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]
573 references.append(reference)
574 except Exception:
575 pass