Coverage for src / renaissance / integrations / clang / clang_json_ast_node.py: 89%
385 statements
« prev ^ index » next coverage.py v7.13.4, created at 2026-09-09 14:04 +0000
1# create a class that inherits syntax tree ASTNode
3import json
4import re
5import subprocess
6import sys
7import tempfile
8from collections.abc import Sequence
9from functools import cache
10from pathlib import Path
11from typing import Any, Self, override
13from renaissance.integrations.clang.cpp_utils import CPPUtils, matches_kind
14from renaissance.integrations.types import (
15 KIND_MAP,
16 Call,
17 Comment,
18 CompoundStatement,
19 Constructor,
20 ConstructorExpression,
21 DeclarationExpression,
22 FullComment,
23 MacroDef,
24 MatchAll,
25 MatchOne,
26 Namespace,
27 RecordDef,
28 Statement,
29 TranslationUnit,
30 UnknownType,
31)
32from renaissance.syntax_tree import ASTNode, ASTReference
33from renaissance.utils.ast_utils import match_children, match_props
35EMPTY_DICT = {}
36EMPTY_STR = ""
37EMPTY_LIST: list[ClangJsonASTReference] = []
38ON_NODE_ID_TAGS = ["previousDecl", "parentDeclContextId"]
39ID_TAGS = [
40 "id",
41 "typeAliasDeclId",
42 "templateDeclId",
43 "templateSpecializationDeclId",
44 "referencedDeclId",
45 *ON_NODE_ID_TAGS,
46]
48STMT_PARENTS = [CompoundStatement, TranslationUnit]
49IRRELEVANT_PROPS = {"macro_expansion", "start_point", "end_point", "source_code", "location", "type"}
50IRRELEVANT_NODES = {Comment, MacroDef, FullComment}
51VERBOSE = False
54class ClangJsonASTReference:
55 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None:
56 self.node_id = node_id
57 self.ref_kind = ref_kind
58 self.properties = properties
61class ClangJsonTranslationUnit:
62 def __init__(self, json_root: dict[str, Any], file_name: str):
63 self.json_root = json_root
64 self.filename = file_name
65 self.references_initialized = False
66 # references are used as a cache to store the references of a node
67 # they are stored as id for lazy creation
68 self._references: dict[str, list[ClangJsonASTReference]] = {}
69 self._referenced_by: dict[str, list[ClangJsonASTReference]] = {}
70 self._nodes: dict[str, ClangJsonASTNode] = {}
72 def lazy_create_references(self, node: ClangJsonASTNode) -> None:
73 # TODO: Do I correctly assume that the usage of this function must be synchronized?
74 if self.references_initialized:
75 return
76 node.root.process(ReferenceHelper.create_references)
77 node.root.process(ReferenceHelper.add_record_references)
78 self.references_initialized = True
81class ClangJsonASTNode(ASTNode):
82 parse_args = [
83 "-fparse-all-comments",
84 "-ferror-limit=0",
85 "-Xclang",
86 "-ast-dump=json",
87 "-fsyntax-only",
88 ]
90 def __init__(
91 self,
92 node: dict[str, Any],
93 translation_unit: ClangJsonTranslationUnit,
94 parent: Self | None = None,
95 start_offset: int | None = None,
96 length: int | None = None,
97 insert_kind: str | None = None,
98 insert_name: str | None = None,
99 ) -> None:
100 super().__init__(self if parent is None else parent.root)
101 self.node: dict[str, Any] = node
102 self._children: Sequence[ClangJsonASTNode] | None = None
103 self._parent = parent
104 self.translation_unit = translation_unit
105 self._filename = translation_unit.filename
106 self.inserted = insert_kind is not None
107 self.show_props = False
108 # if the node has not been added to the translation unit, add it
109 # a node might already be added if it is split into multiple nodes
110 # an example is for base types like int, char, etc. which are split into multiple nodes
111 if "id" in node and self.translation_unit._nodes.get(node["id"]) is None:
112 self.translation_unit._nodes[node["id"]] = self
113 self._offset = start_offset if start_offset is not None else self.__derive_start_offset()
114 self._end_offset = self._offset + length if length is not None else self.__derive_end_offset()
115 self._length = self._end_offset - self._offset
116 self._kind = insert_kind if insert_kind is not None else self.__derive_kind()
117 self.ast_type = KIND_MAP.get(self._kind, UnknownType)
118 self._name = insert_name if insert_name is not None else self._derive_name()
119 # a fake child is introduced to handle the case where the type of declaration is not found
120 # for example in the case of a base type.
121 # without the fake child pattern matching on types will be difficult
122 self.__inserted_children: list[ClangJsonASTNode] = []
123 type = self.node.get("type")
124 if insert_kind is None and type and not self.node.get("implicit") and re.fullmatch("(Var|Function|CxxMethod)Decl", self._kind):
125 declared_type = type["qualType"].replace("(", "").replace(")", "").strip()
126 if self.node.get("loc"):
127 loc = self.node["loc"]
128 offset = loc["offset"] if loc.get("offset") else self._get(["loc", "expansionLoc", "offset"], 0)
129 tok_len = loc["tokLen"] if loc.get("tokLen") else self._get(["loc", "expansionLoc", "tokLen"], 0)
130 if tok_len != 0:
131 insert_child = ClangJsonASTNode(
132 self.node,
133 self.translation_unit,
134 self,
135 offset,
136 tok_len,
137 "DeclLoc",
138 )
139 insert_child._children = []
140 self.__inserted_children.append(insert_child)
141 if "TypeRef" not in [inner["kind"] for inner in self.node.get("inner", [])]:
142 # deep clone the type node and remove the parentheses
143 base_type = type.get("desugaredQualType", declared_type).replace("(", "").replace(")", "").strip()
144 if base_type in CPPUtils.RESERVED_KEYWORDS:
145 length_ref = len(declared_type.encode(sys.getdefaultencoding()))
146 insert_child = ClangJsonASTNode(
147 self.node,
148 self.translation_unit,
149 self,
150 self._offset,
151 length_ref,
152 "TypeRef",
153 declared_type,
154 )
155 insert_child._children = []
156 self.__inserted_children.append(insert_child)
157 # add the declaration as node
158 # deep clone the type node and remove the parentheses
159 elif self.ast_type in [DeclarationExpression]:
160 if self.name.startswith("$$"):
161 self._kind = MatchAll.__name__
162 self.ast_type = MatchAll
163 elif self.name.startswith("$"):
164 self._kind = MatchOne.__name__
165 self.ast_type = MatchOne
166 self._children = self.__inserted_children + [
167 ClangJsonASTNode(
168 ClangJsonASTNode._remove_wrapper(n),
169 translation_unit=self.translation_unit,
170 parent=self,
171 )
172 for n in self.node.get("inner", [])
173 if not n.get("isImplicit", False)
174 ]
175 self._children = [n for n in self._children if n.ast_type not in IRRELEVANT_NODES]
177 def __eq__(self, other):
178 return (
179 isinstance(other, type(self))
180 and self.kind == other.kind
181 and match_props(self.properties, other.properties, IRRELEVANT_PROPS)
182 and match_children(self.children, other.children, IRRELEVANT_NODES)
183 )
185 @override
186 @staticmethod
187 def load(
188 file_path: Path,
189 extra_args: Sequence[str],
190 working_dir: Path,
191 code: str | None = None,
192 ) -> Self:
193 # in a shell process compile the file_path with clang compiler
194 try:
195 # remove the compiler name if it is the first argument
196 if len(extra_args) > 0 and re.match(r".*(g\+\+|gcc|cl\.exe).*", extra_args[0]):
197 extra_args = extra_args[1:]
198 # add clang compiler if it is not in the arguments
199 if len(extra_args) == 0 or "clang" not in extra_args[0]:
200 clang = "clang++" if file_path.suffix == ".cpp" else "clang"
201 extra_args = [clang, *extra_args]
203 command = [*extra_args, *ClangJsonASTNode.parse_args]
204 if code:
205 if str(file_path) in command:
206 command.remove(str(file_path))
207 compile = "-xc++" if file_path.suffix == ".cpp" else "-xc"
208 if compile not in command:
209 command.append(compile)
210 if "-" not in command:
211 command.append("-")
212 # command.append('-main-file-name=' + str(file_path))
213 input = code
214 result = subprocess.run(
215 command,
216 input=input,
217 capture_output=True,
218 text=True,
219 cwd=working_dir,
220 )
221 length = len(input)
222 else:
223 if str(file_path) not in command:
224 command.append(str(file_path))
225 result = subprocess.run(
226 command,
227 capture_output=True,
228 text=True,
229 cwd=working_dir,
230 )
231 length = Path(working_dir / file_path).stat().st_size
232 json_dump = result.stdout.replace("<stdin>", str(file_path))
233 error = result.stderr
235 if VERBOSE:
236 temp_dir = tempfile.gettempdir()
237 temp_file_name = Path(temp_dir) / (file_path.name + ".ast.json")
238 with temp_file_name.open("w") as std_out_file:
239 print("result stored in " + str(temp_file_name))
240 std_out_file.write(json_dump)
241 print(error, file=sys.stderr)
242 json_atu = json.loads(json_dump)
243 atu = ClangJsonASTNode(
244 json_atu,
245 translation_unit=ClangJsonTranslationUnit(json_atu, file_name=str(file_path)),
246 length=length,
247 )
248 if code:
249 atu.cache[str(file_path)] = code.encode(sys.getfilesystemencoding())
250 else:
251 with Path(working_dir / file_path).open("rb") as f:
252 atu.cache[str(file_path)] = f.read()
253 # cache the result of the temp file before deleting it
254 atu.content(0, 0)
255 return atu
257 except Exception as e:
258 print("Call to clang failed. Did you install clang?, is it on the env path?")
259 raise e
261 @override
262 @staticmethod
263 def load_from_text(text: str, file_name: str, extra_args: Sequence[str], working_dir: Path) -> Self:
264 return ClangJsonASTNode.load(Path(file_name), extra_args, working_dir, code=text)
266 @cache
267 def _get_containing_filename(self) -> str:
268 if self.node.get("isImplicit", False):
269 return ""
270 if self.node.get("implicit", False):
271 return ""
272 if not self.parent:
273 return self.translation_unit.file_name
274 # return the file name of the node if it exists else return the file name of the parent node
275 containing_file = self._get(["loc", "file"], EMPTY_STR)
276 if containing_file:
277 return containing_file
278 included_file = self._get(["loc", "includedFrom", "file"], "")
279 if included_file: # included but no file location is provided in the node so we don't know the file name
280 return ""
281 included_file = self._get(["loc", "spellingLoc", "includedFrom", "file"], "")
282 if included_file: # included but no file location is provided in the node so we don't know the file name
283 return ""
284 # not included and no file location so it is the same as the parent
285 if self.parent:
286 return self.parent.filename
287 return EMPTY_STR
289 @override
290 @property
291 def extended_end_offset(self) -> int:
292 try:
293 # TODO: Do I correctly assume this is for Expression Statements like
294 end_offset = self._end_offset
295 # "f(x,y);" and "a = f(3);" that are according to clang NOT statements,
296 # but expressions (without the semicolon)
297 if (not self._is_statement_or_declaration()) and (self.parent and self.parent.ast_type in STMT_PARENTS):
298 content = self.root.binary_file_content()
299 while (
300 end_offset < len(content) and content[end_offset - 1] not in b";"
301 ): # Why use 'in' when list has one element, i.e. ';'?
302 end_offset += 1
303 return end_offset
304 except Exception:
305 return 0
307 def _is_statement_or_declaration(self):
308 return re.match("(?i).*(Stmt|Decl)", self.kind)
309 return isinstance(self.ast_type(), (Statement))
311 @override
312 @property
313 def matches_kind(self, node: ASTNode) -> bool:
314 return matches_kind(self.ast_type, node.ast_type)
316 @override
317 @property
318 def properties(self) -> dict[str, Any]:
319 # get all the attributes of self.node except the inner nodes, id, location, range, kind and name and
320 # all reference nodes (that is children with 'id)
321 properties = {
322 k: ClangJsonASTNode._remove_ids(v)
323 for k, v in self.node.items()
324 if ClangJsonASTNode.__is_property(k) and ClangJsonASTNode._is_reference(v) is not None
325 }
326 if self._get(["range", "end", "expansionLoc", "offset"], -1) != -1: # dealing with a macro expansion
327 properties["macro_expansion"] = self.text
328 # matching name through props
329 if self.ast_type == DeclarationExpression:
330 properties["name"] = self.name
332 return properties
334 @override
335 @property
336 def referenced_by(self) -> Sequence[ASTReference]:
337 if self.inserted:
338 return []
339 self.translation_unit.lazy_create_references(self)
340 ref_by = self.translation_unit._referenced_by.get(self.node["id"], EMPTY_LIST)
341 definition_node_id = self._get_function_definition()
342 if definition_node_id:
343 # try to find the definition which might have references
344 ref_by += self.translation_unit._referenced_by.get(definition_node_id, EMPTY_LIST)
345 return [
346 ASTReference(self.translation_unit._nodes[ref.node_id], ref.ref_kind, ref.properties)
347 for ref in ref_by
348 if ref.node_id != self.node["id"]
349 ]
351 def _get_function_definition(self):
352 refs = self.translation_unit._referenced_by.get(self.node["id"], EMPTY_LIST)
353 for ref in refs:
354 if ref.ref_kind == "previousDecl":
355 return ref.node_id
356 return None
358 @override
359 @property
360 def references(self) -> list[ASTReference]:
361 if self.inserted:
362 return []
363 self.translation_unit.lazy_create_references(self)
365 refs = self.translation_unit._references.get(self.node["id"], EMPTY_LIST)
366 definition_node_id = self._get_function_definition()
367 # TODO: also class definitions, type definitions, ...
368 if definition_node_id:
369 # try to find the definition which might have references
370 refs += self.translation_unit._references.get(definition_node_id, EMPTY_LIST)
371 # remove duplicates
372 refs = list({ref.node_id: ref for ref in refs}.values())
374 return [
375 ASTReference(self.translation_unit._nodes[ref.node_id], ref.ref_kind, ref.properties)
376 for ref in refs
377 if ref.node_id != self.node["id"]
378 ]
380 @override
381 @property
382 def is_statement(self) -> bool:
383 return (
384 self.parent is not None and self.parent.ast_type in STMT_PARENTS
385 ) # TODO: Why look at the kind of your parent and not at your own kind?
387 def _derive_name(self) -> str:
388 name = self.node.get("name")
389 if name:
390 return name
391 kind = self.node.get("kind")
392 decl_ref_name_path = ["referencedDecl", "name"]
393 if kind == "CallExpr":
394 # equalize with libclang
395 decl_ref_child = [inner["kind"] for inner in self.node.get("inner", []) if inner.get("kind") == "DeclRefExpr"]
396 if decl_ref_child:
397 return self._get_property(decl_ref_child[0], decl_ref_name_path, default=EMPTY_STR)
398 if kind == "DeclRefExpr":
399 return self._get(decl_ref_name_path, default=EMPTY_STR)
400 if kind == "StringLiteral":
401 return self._get(["value"], default=EMPTY_STR)
402 return self.node.get("name", EMPTY_STR)
404 def __derive_start_offset(self) -> int:
405 offset = self._get(["range", "begin", "offset"], default=-1)
406 if offset == -1:
407 # we might be dealing with a macro in that case use the expansion location
408 offset = self._get(["range", "begin", "expansionLoc", "offset"], default=0)
409 return offset
411 def __derive_end_offset(self) -> int:
412 if self.__derive_kind() == "TranslationUnitDecl":
413 return len(self.binary_file_content(self.filename))
414 offset = self._get(["range", "end", "offset"], default=-1)
415 tok_len = self._get(["range", "end", "tokLen"], default=-1)
416 if offset == -1:
417 # we might be dealing with a macro in that case use the expansion location
418 offset = self._get(["range", "end", "expansionLoc", "offset"], default=0)
419 tok_len = self._get(["range", "end", "expansionLoc", "tokLen"], default=0)
421 return offset + tok_len
423 def __derive_kind(self) -> str:
424 return self.node.get("kind", EMPTY_STR)
426 @staticmethod
427 def _remove_wrapper(node):
428 try:
429 if ClangJsonASTNode._is_wrapped(node):
430 return ClangJsonASTNode._remove_wrapper(list(node["inner"])[0])
431 except Exception:
432 pass
433 return node
435 @staticmethod
436 def _remove_ids(json_node):
437 if not isinstance(json_node, dict):
438 return json_node
439 return {k: v for k, v in json_node.items() if k not in ID_TAGS}
441 @staticmethod
442 def _is_reference(json_node):
443 return len(ReferenceHelper._get_reference_ids(json_node)) > 0
445 @staticmethod
446 @cache
447 def __is_property(key):
448 return key not in [
449 "id",
450 "inner",
451 "loc",
452 "range",
453 "kind",
454 "name",
455 "isUsed",
456 "isReferenced",
457 "referencedDecl",
458 "mangledName",
459 *ON_NODE_ID_TAGS,
460 ]
462 @staticmethod
463 def _is_wrapped(node):
464 """Check if a node is wrapped.
466 A node is considered wrapped if it meets the following conditions:
467 1. The node does not have an 'id' or its 'kind' starts with "Implicit".
468 2. The node has exactly one inner node.
469 """
470 return (not node.get("id") or node["kind"].startswith("Implicit")) and len(list(node["inner"])) == 1
472 def _get[T](self, path: Sequence[str], default: T) -> T:
473 return self._get_property(self.node, path, default)
475 @staticmethod
476 def _get_property[T](target: dict[str, Any], path: Sequence[str], default: T) -> T:
477 assert default is not None, "default value must be provided"
478 try:
479 for p in path:
480 target = target[p]
481 # TODO: Is this code really correct when path contains multiple strings?
482 # Doesn't target become an Any, and hence might not support __get_item__ any more?
483 return target if isinstance(target, type(default)) else default
484 except Exception:
485 return default
487 @property
488 def is_implicit(self):
489 self.is_part_of_translation_unit()
492class ReferenceHelper:
493 @staticmethod
494 def create_references(ast_node: ClangJsonASTNode) -> None:
495 assert isinstance(ast_node, ClangJsonASTNode), (
496 f"Expected ClangJsonASTNode but got {type(ast_node)}"
497 ) # TODO: still needed when using type hints?
498 if ast_node.inserted:
499 return
500 references = []
501 node_id = ast_node.node["id"]
502 ast_node.translation_unit._references[node_id] = references
503 refs = {k: v for k, v in ast_node.node.items() if not ReferenceHelper._is_child_node(k) and ClangJsonASTNode._is_reference(v)}
504 for k in [k for k in ast_node.node if k in ON_NODE_ID_TAGS]:
505 refs[k] = ast_node.node
506 # add the node if it contains a reference for example in case of previousDecl
508 # to make clang json compatible with clang python, we add the reference of the DeclRefExpr child to the CallExpr
509 if ast_node.ast_type == Call:
510 for n in ast_node.children:
511 if n.ast_type == DeclarationExpression:
512 ref_child = {
513 k: v for k, v in n.node.items() if not ReferenceHelper._is_child_node(k) and ClangJsonASTNode._is_reference(v)
514 }
515 refs.update(ref_child)
517 for kind, ref in refs.items():
518 for ref_id in ReferenceHelper._get_reference_ids(ref):
519 if ref_id == node_id:
520 continue
521 properties = {k: p for k, p in ref.items() if k != ref_id} if ref != ast_node.node else EMPTY_DICT
522 reference = ClangJsonASTReference(ref_id, kind, properties)
523 referenced_by = ClangJsonASTReference(node_id, kind, properties)
524 try:
525 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)
526 except Exception:
527 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]
528 references.append(reference)
530 @staticmethod
531 def add_record_references(ast_node: ClangJsonASTNode) -> None:
532 """JSON does not contain direct references between classes and their base classes.
534 Hence these references are created in this method.
536 This method checks if the given AST node is of kind 'CXXRecordDecl' and has a tag 'class'.
537 If so, it processes the base classes of the node and creates references for them.
539 Args:
540 ast_node (ClangJsonASTNode): The AST node to process.
542 Raises:
543 AssertionError: If the provided ast_node is not an instance of ClangJsonASTNode.
545 """
546 assert isinstance(ast_node, ClangJsonASTNode), (
547 f"Expected ClangJsonASTNode but got {type(ast_node)}"
548 ) # TODO: still needed when using type hints?
549 if ast_node.inserted:
550 return
552 bases = ast_node._get(["bases"], [])
553 if not bases:
554 bases = [ast_node.node] if ast_node.node.get("type") else None
555 if not bases:
556 return
557 node_id = ast_node.node["id"]
558 for base in bases:
559 ref_ids = ReferenceHelper._get_record_decl(ast_node, base)
560 for kind, ref_id in ref_ids:
561 properties = {k: p for k, p in base.items() if k != "type"}
562 reference = ClangJsonASTReference(ref_id, kind, properties)
563 referenced_by = ClangJsonASTReference(node_id, kind, properties)
564 try:
565 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)
566 except Exception:
567 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]
568 try:
569 ast_node.translation_unit._references[node_id].append(reference)
570 except Exception:
571 ast_node.translation_unit._references[node_id] = [reference]
573 @staticmethod
574 def _get_record_decl(ast_node, base) -> Sequence[str]:
575 try:
576 tp = base["type"]
577 if "desugaredQualType" in tp and "::" in tp["desugaredQualType"]:
578 # split desugaredQualType to derive the parent namespaces
579 namespaces = tp["desugaredQualType"].split("::")[:-1][::-1]
580 else:
581 namespaces = []
582 qual_type = tp["qualType"]
583 ids = []
584 ctor_type = EMPTY_STR
585 if ast_node.ast_type == ConstructorExpression:
586 ctor_type = ast_node._get(["ctorType", "qualType"], EMPTY_STR)
588 for id, node in ast_node.translation_unit._nodes.items():
589 if node.ast_type == RecordDef and node.name == qual_type:
590 parent = node.parent
591 matches = True
592 for ns in namespaces:
593 if ns != parent.name or parent.ast_type != Namespace:
594 matches = False
595 parent = parent.parent
596 if matches:
597 ids.append((node.ast_type, id))
598 if ctor_type != EMPTY_STR and node.ast_type == Constructor:
599 # link all matching
600 matches = node._get(["type", "qualType"], EMPTY_STR) == ctor_type
601 if matches:
602 ids.append((node.ast_type, id))
603 return ids
604 except Exception:
605 pass
606 return []
608 @staticmethod
609 def _get_reference_ids(json_node):
610 result = []
611 if not isinstance(json_node, dict):
612 return result
613 for key in ID_TAGS:
614 value = json_node.get(key)
615 if value is not None:
616 result.append(value)
617 return result
619 @staticmethod
620 @cache
621 def _is_child_node(key):
622 return key in ["inner"]