Coverage for src/renaissance/integrations/clang/clang_ast_node.py: 84%
340 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-10-15 10:34 +0000
1# Copyright (C) 2024-2026 TNO
2#
3# Licensed under the terms specified in the LICENSE file in the root directory
4# of this repository, also available at:
5# https://github.com/TNO/Renaissance.Py/blob/main/LICENSE
7"""AI: ASTNode implementation backed by libclang's Python bindings."""
9import sys
10from collections.abc import Sequence
11from functools import cache
12from operator import attrgetter
13from pathlib import Path
14from typing import Any, override
16import clang.native
17from clang.cindex import Config, Cursor, CursorKind, Index, TypeKind
18from clang.cindex import TranslationUnit as ClangCindexTranslationUnit
20from renaissance.integrations.clang.cpp_utils import matches_node_kind
21from renaissance.integrations.clang.kinds import CLANG_KIND_MAP
22from renaissance.integrations.clang.predicates import is_clang_compound_statement, is_clang_macro_definition
23from renaissance.syntax_tree import ASTFinder, ASTNode, ASTReference
24from renaissance.syntax_tree.pattern_kind import PatternKind
25from renaissance.syntax_tree.semantic_kind import SemanticKind
26from renaissance.utils.ast_utils import match_children, match_props
28EMPTY_DICT = {}
29EMPTY_STR = ""
30EMPTY_LIST = []
32DECLARATION_EXPRESSION_PARSER_KINDS = {"DeclRefExpr", "DECL_REF_EXPR"}
33IRRELEVANT_PROPS = {"comment"}
34IRRELEVANT_NODES = {"comment"}
35PRINT_ALL_NODES = False
38class Clangastreference:
39 """AI: Represent a reference from one clang AST node to another by id."""
41 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None:
42 """AI: Represent a reference from one clang AST node to another by id."""
43 self.node_id = node_id
44 self.ref_kind = ref_kind
45 self.properties = properties
48class ClangTranslationUnit:
49 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
51 cache = []
53 def __init__(self, clang_atu: ClangCindexTranslationUnit, file_name: str) -> None:
54 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
55 self.clang_atu = clang_atu
56 self.file_name = file_name
57 self.references_initialized = False
58 # print_node_kind(clang_atu.cursor)
59 self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu)
60 # references are used as a cache to store the references of a node
61 # they are stored as id for lazy creation
62 self._references: dict[str, list[Clangastreference]] = {}
63 self._referenced_by: dict[str, list[Clangastreference]] = {}
64 self._nodes: dict[str, ClangASTNode] = {}
66 def lazy_create_references(self, node: ClangASTNode) -> None:
67 """AI: Build the translation unit's reference cache on first use, then no-op on subsequent calls."""
68 if self.references_initialized:
69 return
70 node.root.process(ReferenceHelper.create_references)
71 self.references_initialized = True
73 @staticmethod
74 def _collect_expansions(
75 translation_unit: ClangCindexTranslationUnit,
76 ) -> set[tuple[str, int, int]]:
77 result: set[tuple[str, int, int]] = set()
78 for child in translation_unit.cursor.get_children():
79 if child.kind.name == "MACRO_INSTANTIATION":
80 result.add(
81 (
82 child.extent.start.file.name,
83 child.extent.start.offset,
84 child.extent.end.offset,
85 ),
86 )
87 return result
90class ClangASTNode(ASTNode[Cursor, ClangTranslationUnit]):
91 """AI: ASTNode implementation backed by libclang cursors."""
93 @staticmethod
94 def set_library_path() -> None:
95 """AI: Configure libclang's native library path from the bundled clang.native package."""
96 try:
97 Config.set_library_path(Path(clang.native.__file__).parent)
98 except Exception as e:
99 print(e)
101 set_library_path()
102 index = Index.create()
103 parse_args = [
104 "-fparse-all-comments",
105 "-ferror-limit=0",
106 "-Xclang",
107 "-detailed-preprocessing-record",
108 "-fsyntax-only",
109 ]
111 def __init__(
112 self,
113 node: Cursor,
114 translation_unit: ClangTranslationUnit,
115 parent=None,
116 start_offset: int | None = None,
117 length: int | None = None,
118 insert_kind: str | None = None,
119 ) -> None:
120 """AI: Wrap a clang cursor as an AST node within the given translation unit."""
121 super().__init__(self if parent is None else parent.root)
122 self.node = node
123 self._children = None
124 self._parent = parent
125 self.translation_unit = translation_unit
126 self.inserted = insert_kind is not None
127 self.show_props = False
128 self._filename = self._get_containing_filename()
129 self._name = self._derive_name()
130 # if the node has not been added to the translation unit, add it
131 # a node might already be added if it is split into multiple nodes
132 # an example is for base types like int, char, etc. which are split into multiple nodes
133 if self.node.hash not in self.translation_unit._nodes:
134 self.translation_unit._nodes[node.hash] = self
135 self._offset = start_offset if start_offset is not None else self.__derive_start_offset()
136 self._length = length if length is not None else self.__derive_length()
137 self._kind = insert_kind if insert_kind is not None else self.__derive_kind()
138 self.parser_kind = self._kind
139 self.semantic_kind = CLANG_KIND_MAP.get(self.parser_kind, SemanticKind.NODE)
140 self.pattern_kind = {
141 "MatchOne": PatternKind.MATCH_ONE,
142 "MatchAll": PatternKind.MATCH_ALL,
143 }.get(self.parser_kind)
144 self.indent = ""
145 # TODO: TextUtils.get_indent(self.content, self._offset)
146 # an fake child is introduced to handle the case where the type of a declaration is not found
147 # for example in the case of a base type.
148 # without the fake child pattern matching on types will be difficult
149 self.__inserted_children = []
150 # NOTE: clang.cindex.{TypeKind,CursorKind} assign their named members via
151 # runtime attribute assignment after the class body (e.g. `TypeKind.INVALID =
152 # TypeKind(0)`), so pyright cannot see them as declared class attributes from
153 # this module, hence the `pyright: ignore[reportAttributeAccessIssue]` below.
154 if (
155 insert_kind is None
156 and not self.node.location.is_in_system_header
157 and self.node.kind.is_declaration()
158 and self.node.type.kind != TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
159 ):
160 loc_offset: int = self.node.location.offset
161 length = len(self.node.spelling.encode(sys.getdefaultencoding()))
162 insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC")
163 insert_child._children = []
164 self.__inserted_children.append(insert_child)
165 if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # pyright: ignore[reportAttributeAccessIssue]
166 my_type = (
167 self.node.type
168 if self.node.result_type.kind == TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
169 else self.node.result_type
170 )
171 length_ref = len(my_type.spelling.encode(sys.getdefaultencoding()))
172 insert_child = ClangASTNode(
173 self.node,
174 self.translation_unit,
175 self,
176 self._offset,
177 length_ref,
178 CursorKind.TYPE_REF.name, # pyright: ignore[reportAttributeAccessIssue]
179 )
180 insert_child._children = []
181 self.__inserted_children.append(insert_child)
183 self._children = []
184 for n in self.__inserted_children:
185 self._children.append(n)
186 for n in self.node.get_children():
187 if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION":
188 self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self))
190 self._properties = self._derive_properties()
191 if self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
192 self._properties["name"] = self._name
194 @property
195 def kind_key(self) -> SemanticKind | str:
196 """AI: Return the semantic kind, or the raw parser kind when no semantic kind applies."""
197 return self.semantic_kind if self.semantic_kind is not SemanticKind.NODE else self.parser_kind
199 def __eq__(self, other) -> bool:
200 """AI: Return whether this node is structurally equal to `other`, ignoring irrelevant properties/children."""
201 return (
202 isinstance(other, type(self))
203 and self.kind_key == other.kind_key
204 and match_props(self.properties, other.properties, IRRELEVANT_PROPS)
205 and match_children(self.children, other.children, IRRELEVANT_NODES)
206 )
208 def __hash__(self) -> int:
209 """AI: Return a hash based on the node's kind key and properties."""
210 return hash((self.kind_key, frozenset(self.properties.items())))
212 @override
213 @staticmethod
214 def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ClangASTNode:
215 args = [*extra_args, *ClangASTNode.parse_args]
216 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:])
217 ClangASTNode.check_diagnostics(translation_unit, file_path.name)
218 root_node = ClangASTNode(
219 translation_unit.cursor,
220 ClangTranslationUnit(translation_unit, file_name=str(file_path)),
221 None,
222 )
223 return root_node
225 @override
226 @staticmethod
227 def load_from_text(
228 text: str,
229 file_name: str,
230 extra_args: Sequence[str] = None,
231 working_dir: Path = None,
232 ) -> ClangASTNode:
233 # Convert file_content to bytes
234 # TODO: see docs/developer/architecture/position-consistency.md - clang decides the encoding, not the host.
235 file_content_bytes = text.encode(sys.getfilesystemencoding())
236 # add to cache to avoid reading the file again
237 ASTNode.cache[file_name] = file_content_bytes
238 args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args]
239 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args)
240 ClangASTNode.check_diagnostics(translation_unit, file_name)
241 try:
242 root_node = ClangASTNode(
243 translation_unit.cursor,
244 ClangTranslationUnit(translation_unit, file_name=str(file_name)),
245 None,
246 )
247 except Exception as e:
248 print(e)
249 raise e
250 ClangASTNode.check_diagnostics(translation_unit, file_name)
251 return root_node
253 @staticmethod
254 def check_diagnostics(translation_unit: ClangCindexTranslationUnit, file_name: str) -> None:
255 """AI: Raise an Exception if the translation unit's diagnostics contain any errors."""
256 has_error = False
257 errors = ""
258 for d in translation_unit.diagnostics:
259 if d.severity >= 3:
260 has_error = True
261 errors += f"{d.severity}: {d.spelling} at {d.location}\n"
262 print(f"{d.severity}: {d.spelling} at {d.location}")
263 if has_error:
264 raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}")
266 def _derive_name(self) -> str:
267 try:
268 # NOTE: see the clang.cindex enum note near __init__ above.
269 if self.node.type.kind == TypeKind.RECORD: # pyright: ignore[reportAttributeAccessIssue]
270 return self.node.type.spelling
271 except Exception as e:
272 print(e)
273 try:
274 return self.node.spelling
275 except Exception as e:
276 print(e)
277 return EMPTY_STR
279 def _get_containing_filename(self) -> str:
280 if self is self.root:
281 return self.translation_unit.clang_atu.spelling
282 try:
283 return self.node.location.file.name
284 except Exception:
285 return EMPTY_STR
287 @override
288 @property
289 def extended_end_offset(self) -> int:
290 try:
291 end_offset = self._offset + self._length
292 if (
293 (not self._is_statement_or_declaration())
294 and (
295 self.parent and (is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT)
296 )
297 and not is_clang_macro_definition(self)
298 ):
299 content = self.root.binary_file_content()
300 while end_offset < len(content) and content[end_offset - 1] not in b";":
301 end_offset += 1
302 return end_offset
303 except Exception:
304 return 0
306 def _is_statement_or_declaration(self):
307 return self.semantic_kind in {
308 SemanticKind.STATEMENT,
309 SemanticKind.DECLARATION,
310 SemanticKind.DEFINITION,
311 SemanticKind.FUNCTION,
312 SemanticKind.CLASS,
313 SemanticKind.CONDITIONAL,
314 SemanticKind.LOOP,
315 SemanticKind.RETURN,
316 SemanticKind.IMPORT,
317 SemanticKind.TRANSLATION_UNIT,
318 }
320 @override
321 def matches_kind(self, node: ASTNode[Cursor, ClangTranslationUnit]) -> bool:
322 return matches_node_kind(self, node)
324 def _derive_properties(self) -> dict[str, int | str]:
325 result = {}
326 offsets = (self.filename, self.offset, self.end_offset)
327 if offsets in self.translation_unit.macro_expansions:
328 result["macro_expansion"] = self.text
330 if self.semantic_kind is SemanticKind.BINARY_OPERATION:
331 # TODO remove below code after clang release that supports the getOpCode() statement
332 children = self.children
333 start_offset = children[0].offset + children[0].length
334 end_offset = children[1].offset
335 operator = self.content(start_offset, end_offset)
336 result["operator"] = operator.strip()
337 # next statement works in C++ but not in Python (yet) will be released later
338 # result['operator'] = self.node.getOpCode()
339 elif self.semantic_kind is SemanticKind.UNARY_OPERATION:
340 # TODO remove below code after clang release that supports the getOpCode() statement
341 child = self.children[0]
342 # list all attributes of self.node excluding the once starting with _
344 if child.offset > self.offset:
345 start_offset = self.offset
346 end_offset = child.offset
347 prefix_operator = True
348 else:
349 start_offset = child.offset + child.length
350 end_offset = self.offset + self.length
351 prefix_operator = False
353 operator = self.content(start_offset, end_offset)
354 result["operator"] = operator.strip()
355 result["prefixOperator"] = prefix_operator
356 # next statement works in C++ but not in Python (yet) will be released later
357 # result['operator'] = self.node.getOpCode()
358 elif self.semantic_kind is SemanticKind.LITERAL or self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
359 self._add_tokens(result, "LITERAL")
361 is_all = {
362 attr[len("is_") :]: True
363 for attr in dir(self.node)
364 if attr.startswith("is_") and callable(getattr(self.node, attr)) and getattr(self.node, attr)()
365 }
366 result.update(is_all)
367 return result
369 @override
370 @property
371 def is_statement(self) -> bool:
372 """Pretty good definition."""
373 return self.parent is not None and (
374 is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT
375 )
377 @override
378 @property
379 def referenced_by(self) -> Sequence[ASTReference[Cursor, ClangTranslationUnit]]:
380 self.translation_unit.lazy_create_references(self)
381 node_id = self.node.hash
382 ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST)
383 # if both the function declaration and function definition are available
384 # the references are stored in the function definition,
385 # but we want them to also show up in the declaration
386 if len(ref_by) == 0:
387 definition = self._get_function_definition()
388 if definition:
389 ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST)
390 return list(
391 ASTReference(
392 self.translation_unit._nodes[ref.node_id],
393 ref.ref_kind,
394 ref.properties,
395 )
396 for ref in ref_by
397 )
399 def _get_function_definition(self):
400 # NOTE: see the clang.cindex enum note near __init__ above.
401 if self.node.type.kind == TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
402 signature = self.node.displayname
403 semantic_parent = self.node.semantic_parent.hash
405 def has_body(node):
406 return any(
407 c.kind == CursorKind.COMPOUND_STMT # pyright: ignore[reportAttributeAccessIssue]
408 for c in node.node.get_children()
409 )
411 def is_match(node):
412 if node._kind != self._kind:
413 return False
414 if node.node.type.kind != TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
415 return False
416 if node.node.semantic_parent.hash != semantic_parent:
417 return False
418 if node.node.displayname != signature:
419 return False
420 return has_body(node)
422 if has_body(self):
423 return None
424 body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None)
425 if isinstance(body, ClangASTNode):
426 return body
427 return None
429 @override
430 @property
431 def references(self) -> Sequence[ASTReference[Cursor, ClangTranslationUnit]]:
432 self.translation_unit.lazy_create_references(self)
433 return list(
434 ASTReference(
435 self.translation_unit._nodes[ref.node_id],
436 ref.ref_kind,
437 ref.properties,
438 )
439 for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST)
440 )
442 def _add_tokens(self, result: dict[str, str], *token_kind: str) -> None:
443 for token in self.node.get_tokens():
444 # find all attr of token that are of type str or int
445 kind = str(token.kind).split(".")[-1]
446 if kind in token_kind:
447 result[kind] = token.spelling
449 def __derive_start_offset(self) -> int:
450 try:
451 if self.node.kind.name == "MACRO_DEFINITION":
452 return self.node.extent.start.offset - 8
454 return self.node.extent.start.offset
456 except Exception:
457 return 0
459 def __derive_length(self) -> int:
460 try:
461 if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]:
462 end_offset = self.node.extent.end.offset + 1
463 elif self.node.kind.name == "MACRO_DEFINITION":
464 end_offset = self.node.extent.end.offset
465 else:
466 end_offset = self.node.extent.end.offset
467 return end_offset - self.__derive_start_offset()
468 except Exception:
469 return 0
471 def __derive_kind(self) -> str:
472 try:
473 if self.node.kind.name == "MACRO_DEFINITION":
474 return str(self.node.kind.name)
475 if self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]:
476 if self.node.displayname.startswith("$$") and " " not in self.node.displayname:
477 return "MatchAll"
478 if self.node.displayname.startswith("$") and " " not in self.node.displayname:
479 return "MatchOne"
480 return str(self.node.kind.name)
481 except Exception:
482 return EMPTY_STR
484 @staticmethod
485 def remove_wrapper(cursor) -> Cursor:
486 """AI: Unwrap a cursor through single-child unexposed wrapper nodes to reach the real node."""
487 try:
488 if ClangASTNode._is_wrapped(cursor):
489 return ClangASTNode.remove_wrapper(list(cursor.children)[0])
490 except Exception:
491 pass
492 return cursor
494 @staticmethod
495 def _is_reference(node) -> bool | None:
496 # refactor this
497 try:
498 print(type(node))
499 print(vars(node))
500 print(dir(node))
501 print(node.__dict__)
502 node.__dict__["id"]
503 return True
504 except Exception:
505 return False
507 @staticmethod
508 @cache
509 def __is_property(key, value) -> bool:
510 return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"])
512 @staticmethod
513 def _is_wrapped(cursor) -> bool:
514 return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1
516 @property
517 def is_implicit(self):
518 """AI: Return whether this node is implicitly generated (part of the translation unit)."""
519 return self.is_part_of_translation_unit()
522# def get_ancestor(self, types ):
523# return get_ancestor(self, types)
526SYSTEM_MACROS = {
527 "linux",
528 "unix",
529 "_LP64",
530 "_WIN32",
531 "_WIN64",
532 "_ISO_VOLATILE",
533 "_INTEGRAL_MAX_BITS",
534 "_CRT_USE_BUILTIN_OFFSETOF",
535}
538def is_system_macro(n):
539 """AI: Return True if n is a predefined/builtin compiler or system macro that should be filtered from output."""
540 return n.kind.name == "MACRO_DEFINITION" and (
541 n.displayname.startswith("__")
542 or n.displayname.startswith("_MS")
543 or n.displayname.startswith("_M_")
544 or n.displayname in SYSTEM_MACROS
545 )
548class ReferenceHelper:
549 """AI: Static helpers that populate reference caches between clang AST nodes."""
551 @staticmethod
552 def create_references(ast_node: ClangASTNode) -> None:
553 """AI: Populate the reference and referenced-by caches for ast_node's translation unit."""
554 assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}"
555 references = []
556 node_id: str = ast_node.node.hash
557 ast_node.translation_unit._references[node_id] = references
558 ref_fields = ["referenced"] # , 'type.get_declaration()']
559 for field in ref_fields:
560 try:
561 element = attrgetter(field)(ast_node.node)
562 if element.kind.name == "NO_DECL_FOUND":
563 continue
564 ref_id = element.hash
565 ref_kind = field.split(".")[0]
566 properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"}
567 if node_id == ref_id:
568 return
569 reference = Clangastreference(ref_id, ref_kind, properties)
570 referenced_by = Clangastreference(
571 node_id,
572 ref_kind,
573 {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"},
574 )
575 try:
576 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)
577 except Exception:
578 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]
579 references.append(reference)
580 except Exception:
581 pass