Coverage for src/renaissance/integrations/clang/clang_ast_node.py: 84%

339 statements  

« prev     ^ index     » next       coverage.py v7.17.0, created at 2026-11-16 16:04 +0000

1# Copyright (C) 2024-2026 TNO 

2# 

3# Licensed under the terms specified in the LICENSE file in the root directory 

4# of this repository, also available at: 

5# https://github.com/TNO/Renaissance.Py/blob/main/LICENSE 

6 

7"""AI: ASTNode implementation backed by libclang's Python bindings.""" 

8 

9import sys 

10from collections.abc import Sequence 

11from functools import cache 

12from operator import attrgetter 

13from pathlib import Path 

14from typing import Any, override 

15 

16import clang.native 

17from clang.cindex import Config, Cursor, CursorKind, Index, TypeKind 

18from clang.cindex import TranslationUnit as ClangCindexTranslationUnit 

19 

20from renaissance.integrations.clang.cpp_utils import matches_node_kind 

21from renaissance.integrations.clang.kinds import CLANG_KIND_MAP 

22from renaissance.integrations.clang.predicates import is_clang_compound_statement, is_clang_macro_definition 

23from renaissance.syntax_tree import ASTFinder, ASTNode, ASTReference 

24from renaissance.syntax_tree.pattern_kind import PatternKind 

25from renaissance.syntax_tree.semantic_kind import SemanticKind 

26from renaissance.utils.ast_utils import match_children, match_props 

27 

28EMPTY_DICT = {} 

29EMPTY_STR = "" 

30EMPTY_LIST = [] 

31 

32DECLARATION_EXPRESSION_PARSER_KINDS = {"DeclRefExpr", "DECL_REF_EXPR"} 

33IRRELEVANT_PROPS = {"comment"} 

34IRRELEVANT_NODES = {"comment"} 

35 

36 

37class Clangastreference: 

38 """AI: Represent a reference from one clang AST node to another by id.""" 

39 

40 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None: 

41 """AI: Represent a reference from one clang AST node to another by id.""" 

42 self.node_id = node_id 

43 self.ref_kind = ref_kind 

44 self.properties = properties 

45 

46 

47class ClangTranslationUnit: 

48 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches.""" 

49 

50 cache = [] 

51 

52 def __init__(self, clang_atu: ClangCindexTranslationUnit, file_name: str) -> None: 

53 """AI: Wrap a parsed clang translation unit with lazily-built reference and macro-expansion caches.""" 

54 self.clang_atu = clang_atu 

55 self.file_name = file_name 

56 self.references_initialized = False 

57 # print_node_kind(clang_atu.cursor) 

58 self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu) 

59 # references are used as a cache to store the references of a node 

60 # they are stored as id for lazy creation 

61 self._references: dict[str, list[Clangastreference]] = {} 

62 self._referenced_by: dict[str, list[Clangastreference]] = {} 

63 self._nodes: dict[str, ClangASTNode] = {} 

64 

65 def lazy_create_references(self, node: ClangASTNode) -> None: 

66 """AI: Build the translation unit's reference cache on first use, then no-op on subsequent calls.""" 

67 if self.references_initialized: 

68 return 

69 node.root.process(ReferenceHelper.create_references) 

70 self.references_initialized = True 

71 

72 @staticmethod 

73 def _collect_expansions( 

74 translation_unit: ClangCindexTranslationUnit, 

75 ) -> set[tuple[str, int, int]]: 

76 result: set[tuple[str, int, int]] = set() 

77 for child in translation_unit.cursor.get_children(): 

78 if child.kind.name == "MACRO_INSTANTIATION": 

79 result.add( 

80 ( 

81 child.extent.start.file.name, 

82 child.extent.start.offset, 

83 child.extent.end.offset, 

84 ), 

85 ) 

86 return result 

87 

88 

89class ClangASTNode(ASTNode[Cursor, ClangTranslationUnit]): 

90 """AI: ASTNode implementation backed by libclang cursors.""" 

91 

92 @staticmethod 

93 def set_library_path() -> None: 

94 """AI: Configure libclang's native library path from the bundled clang.native package.""" 

95 try: 

96 Config.set_library_path(Path(clang.native.__file__).parent) 

97 except Exception as e: 

98 print(e) 

99 

100 set_library_path() 

101 index = Index.create() 

102 parse_args = [ 

103 "-fparse-all-comments", 

104 "-ferror-limit=0", 

105 "-Xclang", 

106 "-detailed-preprocessing-record", 

107 "-fsyntax-only", 

108 ] 

109 

110 def __init__( 

111 self, 

112 node: Cursor, 

113 translation_unit: ClangTranslationUnit, 

114 parent=None, 

115 start_offset: int | None = None, 

116 length: int | None = None, 

117 insert_kind: str | None = None, 

118 ) -> None: 

119 """AI: Wrap a clang cursor as an AST node within the given translation unit.""" 

120 super().__init__(self if parent is None else parent.root) 

121 self.node = node 

122 self._children = None 

123 self._parent = parent 

124 self.translation_unit = translation_unit 

125 self.inserted = insert_kind is not None 

126 self.show_props = False 

127 self._filename = self._get_containing_filename() 

128 self._name = self._derive_name() 

129 # if the node has not been added to the translation unit, add it 

130 # a node might already be added if it is split into multiple nodes 

131 # an example is for base types like int, char, etc. which are split into multiple nodes 

132 if self.node.hash not in self.translation_unit._nodes: 

133 self.translation_unit._nodes[node.hash] = self 

134 self._offset = start_offset if start_offset is not None else self.__derive_start_offset() 

135 self._length = length if length is not None else self.__derive_length() 

136 self._kind = insert_kind if insert_kind is not None else self.__derive_kind() 

137 self.parser_kind = self._kind 

138 self.semantic_kind = CLANG_KIND_MAP.get(self.parser_kind, SemanticKind.NODE) 

139 self.pattern_kind = { 

140 "MatchOne": PatternKind.MATCH_ONE, 

141 "MatchAll": PatternKind.MATCH_ALL, 

142 }.get(self.parser_kind) 

143 self.indent = "" 

144 # TODO: TextUtils.get_indent(self.content, self._offset) 

145 # an fake child is introduced to handle the case where the type of a declaration is not found 

146 # for example in the case of a base type. 

147 # without the fake child pattern matching on types will be difficult 

148 self.__inserted_children = [] 

149 # NOTE: clang.cindex.{TypeKind,CursorKind} assign their named members via 

150 # runtime attribute assignment after the class body (e.g. `TypeKind.INVALID = 

151 # TypeKind(0)`), so pyright cannot see them as declared class attributes from 

152 # this module, hence the `pyright: ignore[reportAttributeAccessIssue]` below. 

153 if ( 

154 insert_kind is None 

155 and not self.node.location.is_in_system_header 

156 and self.node.kind.is_declaration() 

157 and self.node.type.kind != TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue] 

158 ): 

159 loc_offset: int = self.node.location.offset 

160 length = len(self.node.spelling.encode(sys.getdefaultencoding())) 

161 insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC") 

162 insert_child._children = [] 

163 self.__inserted_children.append(insert_child) 

164 if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # pyright: ignore[reportAttributeAccessIssue] 

165 my_type = ( 

166 self.node.type 

167 if self.node.result_type.kind == TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue] 

168 else self.node.result_type 

169 ) 

170 length_ref = len(my_type.spelling.encode(sys.getdefaultencoding())) 

171 insert_child = ClangASTNode( 

172 self.node, 

173 self.translation_unit, 

174 self, 

175 self._offset, 

176 length_ref, 

177 CursorKind.TYPE_REF.name, # pyright: ignore[reportAttributeAccessIssue] 

178 ) 

179 insert_child._children = [] 

180 self.__inserted_children.append(insert_child) 

181 

182 self._children = [] 

183 for n in self.__inserted_children: 

184 self._children.append(n) 

185 for n in self.node.get_children(): 

186 if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION": 

187 self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self)) 

188 

189 self._properties = self._derive_properties() 

190 if self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS: 

191 self._properties["name"] = self._name 

192 

193 @property 

194 def kind_key(self) -> SemanticKind | str: 

195 """AI: Return the semantic kind, or the raw parser kind when no semantic kind applies.""" 

196 return self.semantic_kind if self.semantic_kind is not SemanticKind.NODE else self.parser_kind 

197 

198 def __eq__(self, other) -> bool: 

199 """AI: Return whether this node is structurally equal to `other`, ignoring irrelevant properties/children.""" 

200 return ( 

201 isinstance(other, type(self)) 

202 and self.kind_key == other.kind_key 

203 and match_props(self.properties, other.properties, IRRELEVANT_PROPS) 

204 and match_children(self.children, other.children, IRRELEVANT_NODES) 

205 ) 

206 

207 def __hash__(self) -> int: 

208 """AI: Return a hash based on the node's kind key and properties.""" 

209 return hash((self.kind_key, frozenset(self.properties.items()))) 

210 

211 @override 

212 @staticmethod 

213 def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ClangASTNode: 

214 args = [*extra_args, *ClangASTNode.parse_args] 

215 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:]) 

216 ClangASTNode.check_diagnostics(translation_unit, file_path.name) 

217 root_node = ClangASTNode( 

218 translation_unit.cursor, 

219 ClangTranslationUnit(translation_unit, file_name=str(file_path)), 

220 None, 

221 ) 

222 return root_node 

223 

224 @override 

225 @staticmethod 

226 def load_from_text( 

227 text: str, 

228 file_name: str, 

229 extra_args: Sequence[str] = None, 

230 working_dir: Path = None, 

231 ) -> ClangASTNode: 

232 # Convert file_content to bytes 

233 # TODO: see docs/developer/architecture/position-consistency.md - clang decides the encoding, not the host. 

234 file_content_bytes = text.encode(sys.getfilesystemencoding()) 

235 # add to cache to avoid reading the file again 

236 ASTNode.cache[file_name] = file_content_bytes 

237 args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args] 

238 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args) 

239 ClangASTNode.check_diagnostics(translation_unit, file_name) 

240 try: 

241 root_node = ClangASTNode( 

242 translation_unit.cursor, 

243 ClangTranslationUnit(translation_unit, file_name=str(file_name)), 

244 None, 

245 ) 

246 except Exception as e: 

247 print(e) 

248 raise e 

249 ClangASTNode.check_diagnostics(translation_unit, file_name) 

250 return root_node 

251 

252 @staticmethod 

253 def check_diagnostics(translation_unit: ClangCindexTranslationUnit, file_name: str) -> None: 

254 """AI: Raise an Exception if the translation unit's diagnostics contain any errors.""" 

255 has_error = False 

256 errors = "" 

257 for d in translation_unit.diagnostics: 

258 if d.severity >= 3: 

259 has_error = True 

260 errors += f"{d.severity}: {d.spelling} at {d.location}\n" 

261 print(f"{d.severity}: {d.spelling} at {d.location}") 

262 if has_error: 

263 raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}") 

264 

265 def _derive_name(self) -> str: 

266 try: 

267 # NOTE: see the clang.cindex enum note near __init__ above. 

268 if self.node.type.kind == TypeKind.RECORD: # pyright: ignore[reportAttributeAccessIssue] 

269 return self.node.type.spelling 

270 except Exception as e: 

271 print(e) 

272 try: 

273 return self.node.spelling 

274 except Exception as e: 

275 print(e) 

276 return EMPTY_STR 

277 

278 def _get_containing_filename(self) -> str: 

279 if self is self.root: 

280 return self.translation_unit.clang_atu.spelling 

281 try: 

282 return self.node.location.file.name 

283 except Exception: 

284 return EMPTY_STR 

285 

286 @override 

287 @property 

288 def extended_end_offset(self) -> int: 

289 try: 

290 end_offset = self._offset + self._length 

291 if ( 

292 (not self._is_statement_or_declaration()) 

293 and ( 

294 self.parent and (is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT) 

295 ) 

296 and not is_clang_macro_definition(self) 

297 ): 

298 content = self.root.binary_file_content() 

299 while end_offset < len(content) and content[end_offset - 1] not in b";": 

300 end_offset += 1 

301 return end_offset 

302 except Exception: 

303 return 0 

304 

305 def _is_statement_or_declaration(self): 

306 return self.semantic_kind in { 

307 SemanticKind.STATEMENT, 

308 SemanticKind.DECLARATION, 

309 SemanticKind.DEFINITION, 

310 SemanticKind.FUNCTION, 

311 SemanticKind.CLASS, 

312 SemanticKind.CONDITIONAL, 

313 SemanticKind.LOOP, 

314 SemanticKind.RETURN, 

315 SemanticKind.IMPORT, 

316 SemanticKind.TRANSLATION_UNIT, 

317 } 

318 

319 @override 

320 def matches_kind(self, node: ASTNode[Cursor, ClangTranslationUnit]) -> bool: 

321 return matches_node_kind(self, node) 

322 

323 def _derive_properties(self) -> dict[str, int | str]: 

324 result = {} 

325 offsets = (self.filename, self.offset, self.end_offset) 

326 if offsets in self.translation_unit.macro_expansions: 

327 result["macro_expansion"] = self.text 

328 

329 if self.semantic_kind is SemanticKind.BINARY_OPERATION: 

330 # TODO remove below code after clang release that supports the getOpCode() statement 

331 children = self.children 

332 start_offset = children[0].offset + children[0].length 

333 end_offset = children[1].offset 

334 operator = self.content(start_offset, end_offset) 

335 result["operator"] = operator.strip() 

336 # next statement works in C++ but not in Python (yet) will be released later 

337 # result['operator'] = self.node.getOpCode() 

338 elif self.semantic_kind is SemanticKind.UNARY_OPERATION: 

339 # TODO remove below code after clang release that supports the getOpCode() statement 

340 child = self.children[0] 

341 # list all attributes of self.node excluding the once starting with _ 

342 

343 if child.offset > self.offset: 

344 start_offset = self.offset 

345 end_offset = child.offset 

346 prefix_operator = True 

347 else: 

348 start_offset = child.offset + child.length 

349 end_offset = self.offset + self.length 

350 prefix_operator = False 

351 

352 operator = self.content(start_offset, end_offset) 

353 result["operator"] = operator.strip() 

354 result["prefixOperator"] = prefix_operator 

355 # next statement works in C++ but not in Python (yet) will be released later 

356 # result['operator'] = self.node.getOpCode() 

357 elif self.semantic_kind is SemanticKind.LITERAL or self.parser_kind in DECLARATION_EXPRESSION_PARSER_KINDS: 

358 self._add_tokens(result, "LITERAL") 

359 

360 is_all = { 

361 attr[len("is_") :]: True 

362 for attr in dir(self.node) 

363 if attr.startswith("is_") and callable(getattr(self.node, attr)) and getattr(self.node, attr)() 

364 } 

365 result.update(is_all) 

366 return result 

367 

368 @override 

369 @property 

370 def is_statement(self) -> bool: 

371 """Pretty good definition.""" 

372 return self.parent is not None and ( 

373 is_clang_compound_statement(self.parent) or self.parent.semantic_kind is SemanticKind.TRANSLATION_UNIT 

374 ) 

375 

376 @override 

377 @property 

378 def referenced_by(self) -> Sequence[ASTReference[ClangASTNode]]: 

379 self.translation_unit.lazy_create_references(self) 

380 node_id = self.node.hash 

381 ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST) 

382 # if both the function declaration and function definition are available 

383 # the references are stored in the function definition, 

384 # but we want them to also show up in the declaration 

385 if len(ref_by) == 0: 

386 definition = self._get_function_definition() 

387 if definition: 

388 ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST) 

389 return list( 

390 ASTReference( 

391 self.translation_unit._nodes[ref.node_id], 

392 ref.ref_kind, 

393 ref.properties, 

394 ) 

395 for ref in ref_by 

396 ) 

397 

398 def _get_function_definition(self): 

399 # NOTE: see the clang.cindex enum note near __init__ above. 

400 if self.node.type.kind == TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue] 

401 signature = self.node.displayname 

402 semantic_parent = self.node.semantic_parent.hash 

403 

404 def has_body(node): 

405 return any( 

406 c.kind == CursorKind.COMPOUND_STMT # pyright: ignore[reportAttributeAccessIssue] 

407 for c in node.node.get_children() 

408 ) 

409 

410 def is_match(node): 

411 if node._kind != self._kind: 

412 return False 

413 if node.node.type.kind != TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue] 

414 return False 

415 if node.node.semantic_parent.hash != semantic_parent: 

416 return False 

417 if node.node.displayname != signature: 

418 return False 

419 return has_body(node) 

420 

421 if has_body(self): 

422 return None 

423 body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None) 

424 if isinstance(body, ClangASTNode): 

425 return body 

426 return None 

427 

428 @override 

429 @property 

430 def references(self) -> Sequence[ASTReference[ClangASTNode]]: 

431 self.translation_unit.lazy_create_references(self) 

432 return list( 

433 ASTReference( 

434 self.translation_unit._nodes[ref.node_id], 

435 ref.ref_kind, 

436 ref.properties, 

437 ) 

438 for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST) 

439 ) 

440 

441 def _add_tokens(self, result: dict[str, str], *token_kind: str) -> None: 

442 for token in self.node.get_tokens(): 

443 # find all attr of token that are of type str or int 

444 kind = str(token.kind).split(".")[-1] 

445 if kind in token_kind: 

446 result[kind] = token.spelling 

447 

448 def __derive_start_offset(self) -> int: 

449 try: 

450 if self.node.kind.name == "MACRO_DEFINITION": 

451 return self.node.extent.start.offset - 8 

452 

453 return self.node.extent.start.offset 

454 

455 except Exception: 

456 return 0 

457 

458 def __derive_length(self) -> int: 

459 try: 

460 if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]: 

461 end_offset = self.node.extent.end.offset + 1 

462 elif self.node.kind.name == "MACRO_DEFINITION": 

463 end_offset = self.node.extent.end.offset 

464 else: 

465 end_offset = self.node.extent.end.offset 

466 return end_offset - self.__derive_start_offset() 

467 except Exception: 

468 return 0 

469 

470 def __derive_kind(self) -> str: 

471 try: 

472 if self.node.kind.name == "MACRO_DEFINITION": 

473 return str(self.node.kind.name) 

474 if self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]: 

475 if self.node.displayname.startswith("$$") and " " not in self.node.displayname: 

476 return "MatchAll" 

477 if self.node.displayname.startswith("$") and " " not in self.node.displayname: 

478 return "MatchOne" 

479 return str(self.node.kind.name) 

480 except Exception: 

481 return EMPTY_STR 

482 

483 @staticmethod 

484 def remove_wrapper(cursor) -> Cursor: 

485 """AI: Unwrap a cursor through single-child unexposed wrapper nodes to reach the real node.""" 

486 try: 

487 if ClangASTNode._is_wrapped(cursor): 

488 return ClangASTNode.remove_wrapper(list(cursor.children)[0]) 

489 except Exception: 

490 pass 

491 return cursor 

492 

493 @staticmethod 

494 def _is_reference(node) -> bool | None: 

495 # refactor this 

496 try: 

497 print(type(node)) 

498 print(vars(node)) 

499 print(dir(node)) 

500 print(node.__dict__) 

501 node.__dict__["id"] 

502 return True 

503 except Exception: 

504 return False 

505 

506 @staticmethod 

507 @cache 

508 def __is_property(key, value) -> bool: 

509 return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"]) 

510 

511 @staticmethod 

512 def _is_wrapped(cursor) -> bool: 

513 return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1 

514 

515 @property 

516 def is_implicit(self): 

517 """AI: Return whether this node is implicitly generated (part of the translation unit).""" 

518 return self.is_part_of_translation_unit() 

519 

520 

521# def get_ancestor(self, types ): 

522# return get_ancestor(self, types) 

523 

524 

525SYSTEM_MACROS = { 

526 "linux", 

527 "unix", 

528 "_LP64", 

529 "_WIN32", 

530 "_WIN64", 

531 "_ISO_VOLATILE", 

532 "_INTEGRAL_MAX_BITS", 

533 "_CRT_USE_BUILTIN_OFFSETOF", 

534} 

535 

536 

537def is_system_macro(n): 

538 """AI: Return True if n is a predefined/builtin compiler or system macro that should be filtered from output.""" 

539 return n.kind.name == "MACRO_DEFINITION" and ( 

540 n.displayname.startswith("__") 

541 or n.displayname.startswith("_MS") 

542 or n.displayname.startswith("_M_") 

543 or n.displayname in SYSTEM_MACROS 

544 ) 

545 

546 

547class ReferenceHelper: 

548 """AI: Static helpers that populate reference caches between clang AST nodes.""" 

549 

550 @staticmethod 

551 def create_references(ast_node: ClangASTNode) -> None: 

552 """AI: Populate the reference and referenced-by caches for ast_node's translation unit.""" 

553 assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}" 

554 references = [] 

555 node_id: str = ast_node.node.hash 

556 ast_node.translation_unit._references[node_id] = references 

557 ref_fields = ["referenced"] # , 'type.get_declaration()'] 

558 for field in ref_fields: 

559 try: 

560 element = attrgetter(field)(ast_node.node) 

561 if element.kind.name == "NO_DECL_FOUND": 

562 continue 

563 ref_id = element.hash 

564 ref_kind = field.split(".")[0] 

565 properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"} 

566 if node_id == ref_id: 

567 return 

568 reference = Clangastreference(ref_id, ref_kind, properties) 

569 referenced_by = Clangastreference( 

570 node_id, 

571 ref_kind, 

572 {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"}, 

573 ) 

574 try: 

575 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by) 

576 except Exception: 

577 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by] 

578 references.append(reference) 

579 except Exception: 

580 pass