Coverage for src/renaissance/integrations/clang/clang_ast_node.py: 84%
339 statements
« prev ^ index » next coverage.py v7.17.0, created at 2026-11-16 16:04 +0000
« prev ^ index » next coverage.py v7.17.0, created at 2026-11-16 16:04 +0000
1# Copyright (C) 2024-2026 TNO
2#
3# Licensed under the terms specified in the LICENSE file in the root directory
4# of this repository, also available at:
5# https://github.com/TNO/Renaissance.Py/blob/main/LICENSE
7"""AI: ASTNode implementation backed by libclang's Python bindings."""
9import sys
10from collections.abc import Sequence
11from functools import cache
12from operator import attrgetter
13from pathlib import Path
14from typing import Any, override
16import clang.native
17from clang.cindex import Config, Cursor, CursorKind, Index, TypeKind
18from clang.cindex import TranslationUnit as ClangCindexTranslationUnit
20from renaissance.integrations.clang.cpp_utils import matches_node_kind
21from renaissance.integrations.clang.kinds import CLANG_KIND_MAP
22from renaissance.integrations.clang.predicates import is_clang_compound_statement, is_clang_macro_definition
23from renaissance.syntax_tree import ASTFinder, ASTNode, ASTReference
24from renaissance.syntax_tree.pattern_kind import PatternKind
25from renaissance.syntax_tree.semantic_kind import SemanticKind
26from renaissance.utils.ast_utils import match_children, match_props
28EMPTY_DICT = {}
29EMPTY_STR = ""
30EMPTY_LIST = []
32DECLARATION_EXPRESSION_PARSER_KINDS = {"DeclRefExpr", "DECL_REF_EXPR"}
33IRRELEVANT_PROPS = {"comment"}
34IRRELEVANT_NODES = {"comment"}
37class Clangastreference:
38 """AI: Represent a reference from one clang AST node to another by id."""
40 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None:
41 """AI: Represent a reference from one clang AST node to another by id."""
42 self.node_id = node_id
43 self.ref_kind = ref_kind
44 self.properties = properties
47class ClangTranslationUnit:
48 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
50 cache = []
52 def __init__(self, clang_atu: ClangCindexTranslationUnit, file_name: str) -> None:
53 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches."""
54 self.clang_atu = clang_atu
55 self.file_name = file_name
56 self.references_initialized = False
57 # print_node_kind(clang_atu.cursor)
58 self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu)
59 # references are used as a cache to store the references of a node
60 # they are stored as id for lazy creation
61 self._references: dict[str, list[Clangastreference]] = {}
62 self._referenced_by: dict[str, list[Clangastreference]] = {}
63 self._nodes: dict[str, ClangASTNode] = {}
65 def lazy_create_references(self, node: ClangASTNode) -> None:
66 """AI: Build the translation unit's reference cache on first use, then no-op on subsequent calls."""
67 if self.references_initialized:
68 return
69 node.root.process(ReferenceHelper.create_references)
70 self.references_initialized = True
72 @staticmethod
73 def _collect_expansions(
74 translation_unit: ClangCindexTranslationUnit,
75 ) -> set[tuple[str, int, int]]:
76 result: set[tuple[str, int, int]] = set()
77 for child in translation_unit.cursor.get_children():
78 if child.kind.name == "MACRO_INSTANTIATION":
79 result.add(
80 (
81 child.extent.start.file.name,
82 child.extent.start.offset,
83 child.extent.end.offset,
84 ),
85 )
86 return result
89class ClangASTNode(ASTNode[Cursor, ClangTranslationUnit]):
90 """AI: ASTNode implementation backed by libclang cursors."""
92 @staticmethod
93 def set_library_path() -> None:
94 """AI: Configure libclang's native library path from the bundled clang.native package."""
95 try:
96 Config.set_library_path(Path(clang.native.__file__).parent)
97 except Exception as e:
98 print(e)
100 set_library_path()
101 index = Index.create()
102 parse_args = [
103 "-fparse-all-comments",
104 "-ferror-limit=0",
105 "-Xclang",
106 "-detailed-preprocessing-record",
107 "-fsyntax-only",
108 ]
110 def __init__(
111 self,
112 node: Cursor,
113 translation_unit: ClangTranslationUnit,
114 parent=None,
115 start_offset: int | None = None,
116 length: int | None = None,
117 insert_kind: str | None = None,
118 ) -> None:
119 """AI: Wrap a clang cursor as an AST node within the given translation unit."""
120 super().__init__(self if parent is None else parent.root)
121 self.node = node
122 self._children = None
123 self._parent = parent
124 self.translation_unit = translation_unit
125 self.inserted = insert_kind is not None
126 self.show_props = False
127 self._filename = self._get_containing_filename()
128 self._name = self._derive_name()
129 # if the node has not been added to the translation unit, add it
130 # a node might already be added if it is split into multiple nodes
131 # an example is for base types like int, char, etc. which are split into multiple nodes
132 if self.node.hash not in self.translation_unit._nodes:
133 self.translation_unit._nodes[node.hash] = self
134 self._offset = start_offset if start_offset is not None else self.__derive_start_offset()
135 self._length = length if length is not None else self.__derive_length()
136 self._kind = insert_kind if insert_kind is not None else self.__derive_kind()
137 self.parser_kind = self._kind
138 self.semantic_kind = CLANG_KIND_MAP.get(self.parser_kind, SemanticKind.NODE)
139 self.pattern_kind = {
140 "MatchOne": PatternKind.MATCH_ONE,
141 "MatchAll": PatternKind.MATCH_ALL,
142 }.get(self.parser_kind)
143 self.indent = ""
144 # TODO: TextUtils.get_indent(self.content, self._offset)
145 # an fake child is introduced to handle the case where the type of a declaration is not found
146 # for example in the case of a base type.
147 # without the fake child pattern matching on types will be difficult
148 self.__inserted_children = []
149 # NOTE: clang.cindex.{TypeKind,CursorKind} assign their named members via
150 # runtime attribute assignment after the class body (e.g. `TypeKind.INVALID =
151 # TypeKind(0)`), so pyright cannot see them as declared class attributes from
152 # this module, hence the `pyright: ignore[reportAttributeAccessIssue]` below.
153 if (
154 insert_kind is None
155 and not self.node.location.is_in_system_header
156 and self.node.kind.is_declaration()
157 and self.node.type.kind != TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
158 ):
159 loc_offset: int = self.node.location.offset
160 length = len(self.node.spelling.encode(sys.getdefaultencoding()))
161 insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC")
162 insert_child._children = []
163 self.__inserted_children.append(insert_child)
164 if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # pyright: ignore[reportAttributeAccessIssue]
165 my_type = (
166 self.node.type
167 if self.node.result_type.kind == TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]
168 else self.node.result_type
169 )
170 length_ref = len(my_type.spelling.encode(sys.getdefaultencoding()))
171 insert_child = ClangASTNode(
172 self.node,
173 self.translation_unit,
174 self,
175 self._offset,
176 length_ref,
177 CursorKind.TYPE_REF.name, # pyright: ignore[reportAttributeAccessIssue]
178 )
179 insert_child._children = []
180 self.__inserted_children.append(insert_child)
182 self._children = []
183 for n in self.__inserted_children:
184 self._children.append(n)
185 for n in self.node.get_children():
186 if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION":
187 self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self))
189 self._properties = self._derive_properties()
190 if self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
191 self._properties["name"] = self._name
193 @property
194 def kind_key(self) -> SemanticKind | str:
195 """AI: Return the semantic kind, or the raw parser kind when no semantic kind applies."""
196 return self.semantic_kind if self.semantic_kind is not SemanticKind.NODE else self.parser_kind
198 def __eq__(self, other) -> bool:
199 """AI: Return whether this node is structurally equal to `other`, ignoring irrelevant properties/children."""
200 return (
201 isinstance(other, type(self))
202 and self.kind_key == other.kind_key
203 and match_props(self.properties, other.properties, IRRELEVANT_PROPS)
204 and match_children(self.children, other.children, IRRELEVANT_NODES)
205 )
207 def __hash__(self) -> int:
208 """AI: Return a hash based on the node's kind key and properties."""
209 return hash((self.kind_key, frozenset(self.properties.items())))
211 @override
212 @staticmethod
213 def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ClangASTNode:
214 args = [*extra_args, *ClangASTNode.parse_args]
215 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:])
216 ClangASTNode.check_diagnostics(translation_unit, file_path.name)
217 root_node = ClangASTNode(
218 translation_unit.cursor,
219 ClangTranslationUnit(translation_unit, file_name=str(file_path)),
220 None,
221 )
222 return root_node
224 @override
225 @staticmethod
226 def load_from_text(
227 text: str,
228 file_name: str,
229 extra_args: Sequence[str] = None,
230 working_dir: Path = None,
231 ) -> ClangASTNode:
232 # Convert file_content to bytes
233 # TODO: see docs/developer/architecture/position-consistency.md - clang decides the encoding, not the host.
234 file_content_bytes = text.encode(sys.getfilesystemencoding())
235 # add to cache to avoid reading the file again
236 ASTNode.cache[file_name] = file_content_bytes
237 args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args]
238 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args)
239 ClangASTNode.check_diagnostics(translation_unit, file_name)
240 try:
241 root_node = ClangASTNode(
242 translation_unit.cursor,
243 ClangTranslationUnit(translation_unit, file_name=str(file_name)),
244 None,
245 )
246 except Exception as e:
247 print(e)
248 raise e
249 ClangASTNode.check_diagnostics(translation_unit, file_name)
250 return root_node
252 @staticmethod
253 def check_diagnostics(translation_unit: ClangCindexTranslationUnit, file_name: str) -> None:
254 """AI: Raise an Exception if the translation unit's diagnostics contain any errors."""
255 has_error = False
256 errors = ""
257 for d in translation_unit.diagnostics:
258 if d.severity >= 3:
259 has_error = True
260 errors += f"{d.severity}: {d.spelling} at {d.location}\n"
261 print(f"{d.severity}: {d.spelling} at {d.location}")
262 if has_error:
263 raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}")
265 def _derive_name(self) -> str:
266 try:
267 # NOTE: see the clang.cindex enum note near __init__ above.
268 if self.node.type.kind == TypeKind.RECORD: # pyright: ignore[reportAttributeAccessIssue]
269 return self.node.type.spelling
270 except Exception as e:
271 print(e)
272 try:
273 return self.node.spelling
274 except Exception as e:
275 print(e)
276 return EMPTY_STR
278 def _get_containing_filename(self) -> str:
279 if self is self.root:
280 return self.translation_unit.clang_atu.spelling
281 try:
282 return self.node.location.file.name
283 except Exception:
284 return EMPTY_STR
286 @override
287 @property
288 def extended_end_offset(self) -> int:
289 try:
290 end_offset = self._offset + self._length
291 if (
292 (not self._is_statement_or_declaration())
293 and (
294 self.parent and (is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT)
295 )
296 and not is_clang_macro_definition(self)
297 ):
298 content = self.root.binary_file_content()
299 while end_offset < len(content) and content[end_offset - 1] not in b";":
300 end_offset += 1
301 return end_offset
302 except Exception:
303 return 0
305 def _is_statement_or_declaration(self):
306 return self.semantic_kind in {
307 SemanticKind.STATEMENT,
308 SemanticKind.DECLARATION,
309 SemanticKind.DEFINITION,
310 SemanticKind.FUNCTION,
311 SemanticKind.CLASS,
312 SemanticKind.CONDITIONAL,
313 SemanticKind.LOOP,
314 SemanticKind.RETURN,
315 SemanticKind.IMPORT,
316 SemanticKind.TRANSLATION_UNIT,
317 }
319 @override
320 def matches_kind(self, node: ASTNode[Cursor, ClangTranslationUnit]) -> bool:
321 return matches_node_kind(self, node)
323 def _derive_properties(self) -> dict[str, int | str]:
324 result = {}
325 offsets = (self.filename, self.offset, self.end_offset)
326 if offsets in self.translation_unit.macro_expansions:
327 result["macro_expansion"] = self.text
329 if self.semantic_kind is SemanticKind.BINARY_OPERATION:
330 # TODO remove below code after clang release that supports the getOpCode() statement
331 children = self.children
332 start_offset = children[0].offset + children[0].length
333 end_offset = children[1].offset
334 operator = self.content(start_offset, end_offset)
335 result["operator"] = operator.strip()
336 # next statement works in C++ but not in Python (yet) will be released later
337 # result['operator'] = self.node.getOpCode()
338 elif self.semantic_kind is SemanticKind.UNARY_OPERATION:
339 # TODO remove below code after clang release that supports the getOpCode() statement
340 child = self.children[0]
341 # list all attributes of self.node excluding the once starting with _
343 if child.offset > self.offset:
344 start_offset = self.offset
345 end_offset = child.offset
346 prefix_operator = True
347 else:
348 start_offset = child.offset + child.length
349 end_offset = self.offset + self.length
350 prefix_operator = False
352 operator = self.content(start_offset, end_offset)
353 result["operator"] = operator.strip()
354 result["prefixOperator"] = prefix_operator
355 # next statement works in C++ but not in Python (yet) will be released later
356 # result['operator'] = self.node.getOpCode()
357 elif self.semantic_kind is SemanticKind.LITERAL or self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS:
358 self._add_tokens(result, "LITERAL")
360 is_all = {
361 attr[len("is_") :]: True
362 for attr in dir(self.node)
363 if attr.startswith("is_") and callable(getattr(self.node, attr)) and getattr(self.node, attr)()
364 }
365 result.update(is_all)
366 return result
368 @override
369 @property
370 def is_statement(self) -> bool:
371 """Pretty good definition."""
372 return self.parent is not None and (
373 is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT
374 )
376 @override
377 @property
378 def referenced_by(self) -> Sequence[ASTReference[ClangASTNode]]:
379 self.translation_unit.lazy_create_references(self)
380 node_id = self.node.hash
381 ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST)
382 # if both the function declaration and function definition are available
383 # the references are stored in the function definition,
384 # but we want them to also show up in the declaration
385 if len(ref_by) == 0:
386 definition = self._get_function_definition()
387 if definition:
388 ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST)
389 return list(
390 ASTReference(
391 self.translation_unit._nodes[ref.node_id],
392 ref.ref_kind,
393 ref.properties,
394 )
395 for ref in ref_by
396 )
398 def _get_function_definition(self):
399 # NOTE: see the clang.cindex enum note near __init__ above.
400 if self.node.type.kind == TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
401 signature = self.node.displayname
402 semantic_parent = self.node.semantic_parent.hash
404 def has_body(node):
405 return any(
406 c.kind == CursorKind.COMPOUND_STMT # pyright: ignore[reportAttributeAccessIssue]
407 for c in node.node.get_children()
408 )
410 def is_match(node):
411 if node._kind != self._kind:
412 return False
413 if node.node.type.kind != TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]
414 return False
415 if node.node.semantic_parent.hash != semantic_parent:
416 return False
417 if node.node.displayname != signature:
418 return False
419 return has_body(node)
421 if has_body(self):
422 return None
423 body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None)
424 if isinstance(body, ClangASTNode):
425 return body
426 return None
428 @override
429 @property
430 def references(self) -> Sequence[ASTReference[ClangASTNode]]:
431 self.translation_unit.lazy_create_references(self)
432 return list(
433 ASTReference(
434 self.translation_unit._nodes[ref.node_id],
435 ref.ref_kind,
436 ref.properties,
437 )
438 for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST)
439 )
441 def _add_tokens(self, result: dict[str, str], *token_kind: str) -> None:
442 for token in self.node.get_tokens():
443 # find all attr of token that are of type str or int
444 kind = str(token.kind).split(".")[-1]
445 if kind in token_kind:
446 result[kind] = token.spelling
448 def __derive_start_offset(self) -> int:
449 try:
450 if self.node.kind.name == "MACRO_DEFINITION":
451 return self.node.extent.start.offset - 8
453 return self.node.extent.start.offset
455 except Exception:
456 return 0
458 def __derive_length(self) -> int:
459 try:
460 if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]:
461 end_offset = self.node.extent.end.offset + 1
462 elif self.node.kind.name == "MACRO_DEFINITION":
463 end_offset = self.node.extent.end.offset
464 else:
465 end_offset = self.node.extent.end.offset
466 return end_offset - self.__derive_start_offset()
467 except Exception:
468 return 0
470 def __derive_kind(self) -> str:
471 try:
472 if self.node.kind.name == "MACRO_DEFINITION":
473 return str(self.node.kind.name)
474 if self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]:
475 if self.node.displayname.startswith("$$") and " " not in self.node.displayname:
476 return "MatchAll"
477 if self.node.displayname.startswith("$") and " " not in self.node.displayname:
478 return "MatchOne"
479 return str(self.node.kind.name)
480 except Exception:
481 return EMPTY_STR
483 @staticmethod
484 def remove_wrapper(cursor) -> Cursor:
485 """AI: Unwrap a cursor through single-child unexposed wrapper nodes to reach the real node."""
486 try:
487 if ClangASTNode._is_wrapped(cursor):
488 return ClangASTNode.remove_wrapper(list(cursor.children)[0])
489 except Exception:
490 pass
491 return cursor
493 @staticmethod
494 def _is_reference(node) -> bool | None:
495 # refactor this
496 try:
497 print(type(node))
498 print(vars(node))
499 print(dir(node))
500 print(node.__dict__)
501 node.__dict__["id"]
502 return True
503 except Exception:
504 return False
506 @staticmethod
507 @cache
508 def __is_property(key, value) -> bool:
509 return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"])
511 @staticmethod
512 def _is_wrapped(cursor) -> bool:
513 return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1
515 @property
516 def is_implicit(self):
517 """AI: Return whether this node is implicitly generated (part of the translation unit)."""
518 return self.is_part_of_translation_unit()
521# def get_ancestor(self, types ):
522# return get_ancestor(self, types)
525SYSTEM_MACROS = {
526 "linux",
527 "unix",
528 "_LP64",
529 "_WIN32",
530 "_WIN64",
531 "_ISO_VOLATILE",
532 "_INTEGRAL_MAX_BITS",
533 "_CRT_USE_BUILTIN_OFFSETOF",
534}
537def is_system_macro(n):
538 """AI: Return True if n is a predefined/builtin compiler or system macro that should be filtered from output."""
539 return n.kind.name == "MACRO_DEFINITION" and (
540 n.displayname.startswith("__")
541 or n.displayname.startswith("_MS")
542 or n.displayname.startswith("_M_")
543 or n.displayname in SYSTEM_MACROS
544 )
547class ReferenceHelper:
548 """AI: Static helpers that populate reference caches between clang AST nodes."""
550 @staticmethod
551 def create_references(ast_node: ClangASTNode) -> None:
552 """AI: Populate the reference and referenced-by caches for ast_node's translation unit."""
553 assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}"
554 references = []
555 node_id: str = ast_node.node.hash
556 ast_node.translation_unit._references[node_id] = references
557 ref_fields = ["referenced"] # , 'type.get_declaration()']
558 for field in ref_fields:
559 try:
560 element = attrgetter(field)(ast_node.node)
561 if element.kind.name == "NO_DECL_FOUND":
562 continue
563 ref_id = element.hash
564 ref_kind = field.split(".")[0]
565 properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"}
566 if node_id == ref_id:
567 return
568 reference = Clangastreference(ref_id, ref_kind, properties)
569 referenced_by = Clangastreference(
570 node_id,
571 ref_kind,
572 {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"},
573 )
574 try:
575 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)
576 except Exception:
577 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]
578 references.append(reference)
579 except Exception:
580 pass