Coverage for src/renaissance/integrations/clang/clang_ast

1import sys

2from collections.abc import Sequence

3from functools import cache

4from pathlib import Path

5from typing import Any, override

7import clang.native

8from clang.cindex import Config, CursorKind, Index, TypeKind

9from clang.cindex import TranslationUnit as ClangCindexTranslationUnit

11from renaissance.integrations.clang.cpp_utils import matches_kind

12from renaissance.integrations.types import (

13 KIND_MAP,

14 BinaryOperation,

15 CompoundStatement,

16 Declaration,

17 DeclarationExpression,

18 Definition,

19 Literal,

20 MacroDef,

21 MatchAll,

22 MatchOne,

23 Statement,

24 TranslationUnit,

25 UnaryOperation,

26 UnknownType,

27)

28from renaissance.syntax_tree import ASTFinder, ASTNode, ASTReference

29from renaissance.utils.ast_utils import match_children, match_props

31EMPTY_DICT = {}

32EMPTY_STR = ""

33EMPTY_LIST = []

35STMT_PARENTS = [CompoundStatement, TranslationUnit]

36IRRELEVANT_PROPS = {"comment"}

37IRRELEVANT_NODES = {"comment"}

38PRINT_ALL_NODES = False

41class Clangastreference:

42 def __init__(self, node_id: str, ref_kind: str, properties: dict[str, Any]) -> None:

43 self.node_id = node_id

44 self.ref_kind = ref_kind

45 self.properties = properties

48class ClangTranslationUnit:

49 cache = []

51 def __init__(self, clang_atu: ClangCindexTranslationUnit, file_name: str):

52 self.clang_atu = clang_atu

53 self.file_name = file_name

54 self.references_initialized = False

55 # print_node_kind(clang_atu.cursor)

56 self.macro_expansions = ClangTranslationUnit._collect_expansions(clang_atu)

57 # references are used as a cache to store the references of a node

58 # they are stored as id for lazy creation

59 self._references: dict[str, list[Clangastreference]] = {}

60 self._referenced_by: dict[str, list[Clangastreference]] = {}

61 self._nodes: dict[str, ClangASTNode] = {}

63 def lazy_create_references(self, node: ClangASTNode) -> None:

64 if self.references_initialized:

65 return

66 node.root.process(ReferenceHelper.create_references)

67 self.references_initialized = True

69 @staticmethod

70 def _collect_expansions(

71 translation_unit: ClangCindexTranslationUnit,

72 ) -> set[tuple[str, int, int]]:

73 result: set[tuple[str, int, int]] = set()

74 for child in translation_unit.cursor.get_children():

75 if child.kind.name == "MACRO_INSTANTIATION":

76 result.add(

77 (

78 child.extent.start.file,

79 child.extent.start.offset,

80 child.extent.end.offset,

81 ),

82 )

83 return result

86class ClangASTNode(ASTNode):

87 @staticmethod

88 def set_library_path() -> None:

89 try:

90 Config.set_library_path(Path(clang.native.__file__).parent)

91 except Exception as e:

92 print(e)

94 set_library_path()

95 index = Index.create()

96 parse_args = [

97 "-fparse-all-comments",

98 "-ferror-limit=0",

99 "-Xclang",

100 "-detailed-preprocessing-record",

101 "-fsyntax-only",

102 ]

103

104 def __init__(

105 self,

106 node,

107 translation_unit: ClangTranslationUnit,

108 parent=None,

109 start_offset: int | None = None,

110 length: int | None = None,

111 insert_kind: str | None = None,

112 ):

113 super().__init__(self if parent is None else parent.root)

114 self.node = node

115 self._children = None

116 self._parent = parent

117 self.translation_unit = translation_unit

118 self.inserted = insert_kind is not None

119 self.show_props = False

120 self._filename = self._get_containing_filename()

121 self._name = self._derive_name()

122 # if the node has not been added to the translation unit, add it

123 # a node might already be added if it is split into multiple nodes

124 # an example is for base types like int, char, etc. which are split into multiple nodes

125 if self.node.hash not in self.translation_unit._nodes:

126 self.translation_unit._nodes[node.hash] = self

127 self._offset = start_offset if start_offset is not None else self.__derive_start_offset()

128 self._length = length if length is not None else self.__derive_length()

129 self._kind = insert_kind if insert_kind is not None else self.__derive_kind()

130 self.ast_type = KIND_MAP.get(self._kind, UnknownType)

131 self.indent = ""

132 # TODO: TextUtils.get_indent(self.content, self._offset)

133 # an fake child is introduced to handle the case where the type of a declaration is not found

134 # for example in the case of a base type.

135 # without the fake child pattern matching on types will be difficult

136 self.__inserted_children = []

137 # NOTE: clang.cindex.{TypeKind,CursorKind} assign their named members via

138 # runtime attribute assignment after the class body (e.g. `TypeKind.INVALID =

139 # TypeKind(0)`), so pyright cannot see them as declared class attributes from

140 # this module, hence the `pyright: ignore[reportAttributeAccessIssue]` below.

141 if (

142 insert_kind is None

143 and not self.node.location.is_in_system_header

144 and self.node.kind.is_declaration()

145 and self.node.type.kind != TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]

146 ):

147 loc_offset: int = self.node.location.offset

148 length = len(self.node.spelling.encode(sys.getdefaultencoding()))

149 insert_child = ClangASTNode(self.node, self.translation_unit, self, loc_offset, length, "DECL_LOC")

150 insert_child._children = []

151 self.__inserted_children.append(insert_child)

152 if self.node.type.get_declaration().kind is CursorKind.NO_DECL_FOUND: # pyright: ignore[reportAttributeAccessIssue]

153 my_type = (

154 self.node.type

155 if self.node.result_type.kind == TypeKind.INVALID # pyright: ignore[reportAttributeAccessIssue]

156 else self.node.result_type

157 )

158 length_ref = len(my_type.spelling.encode(sys.getdefaultencoding()))

159 insert_child = ClangASTNode(

160 self.node,

161 self.translation_unit,

162 self,

163 self._offset,

164 length_ref,

165 CursorKind.TYPE_REF.name, # pyright: ignore[reportAttributeAccessIssue]

166 )

167 insert_child._children = []

168 self.__inserted_children.append(insert_child)

169

170 self._children = []

171 for n in self.__inserted_children:

172 self._children.append(n)

173 for n in self.node.get_children():

174 if not is_system_macro(n) and n.kind.name != "MACRO_INSTANTIATION":

175 self._children.append(ClangASTNode(ClangASTNode.remove_wrapper(n), self.translation_unit, self))

176

177 self._properties = self._derive_properties()

178 if self.ast_type == DeclarationExpression:

179 self._properties["name"] = self._name

180

181 def __eq__(self, other):

182 return (

183 other

184 and isinstance(other, type(self))

185 and self.ast_type == other.ast_type

186 and match_props(self.properties, other.properties, IRRELEVANT_PROPS)

187 and match_children(self.children, other.children, IRRELEVANT_NODES)

188 )

189

190 def __hash__(self):

191 return hash((self.ast_type, frozenset(self.properties.items())))

192

193 @override

194 @staticmethod

195 def load(file_path: Path, extra_args: Sequence[str], working_dir: Path) -> ClangASTNode:

196 args = [*extra_args, *ClangASTNode.parse_args]

197 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(working_dir / file_path, args=args[3:])

198 ClangASTNode.check_diagnostics(translation_unit, file_path.name)

199 root_node = ClangASTNode(

200 translation_unit.cursor,

201 ClangTranslationUnit(translation_unit, file_name=str(file_path)),

202 None,

203 )

204 return root_node

205

206 @override

207 @staticmethod

208 def load_from_text(

209 text: str,

210 file_name: str,

211 extra_args: Sequence[str] = None,

212 working_dir: Path = None,

213 ) -> ClangASTNode:

214 # Convert file_content to bytes

215 file_content_bytes = text.encode(sys.getfilesystemencoding())

216 # add to cache to avoid reading the file again

217 ASTNode.cache[file_name] = file_content_bytes

218 args = [*ClangASTNode.parse_args, *extra_args] if extra_args is not None else [*ClangASTNode.parse_args]

219 translation_unit: ClangCindexTranslationUnit = ClangASTNode.index.parse(file_name, unsaved_files=[(file_name, text)], args=args)

220 ClangASTNode.check_diagnostics(translation_unit, file_name)

221 try:

222 root_node = ClangASTNode(

223 translation_unit.cursor,

224 ClangTranslationUnit(translation_unit, file_name=str(file_name)),

225 None,

226 )

227 except Exception as e:

228 print(e)

229 raise e

230 ClangASTNode.check_diagnostics(translation_unit, file_name)

231 return root_node

232

233 @staticmethod

234 def check_diagnostics(translation_unit: ClangCindexTranslationUnit, file_name: str) -> None:

235 has_error = False

236 errors = ""

237 for d in translation_unit.diagnostics:

238 if d.severity >= 3:

239 has_error = True

240 errors += f"{d.severity}: {d.spelling} at {d.location}\n"

241 print(f"{d.severity}: {d.spelling} at {d.location}")

242 if has_error:

243 raise Exception(f"Error parsing: {file_name} \n+ errors: {errors}")

244

245 def _derive_name(self) -> str:

246 try:

247 # NOTE: see the clang.cindex enum note near __init__ above.

248 if self.node.type.kind == TypeKind.RECORD: # pyright: ignore[reportAttributeAccessIssue]

249 return self.node.type.spelling

250 except Exception as e:

251 print(e)

252 try:

253 return self.node.spelling

254 except Exception as e:

255 print(e)

256 return EMPTY_STR

257

258 def _get_containing_filename(self) -> str:

259 if self is self.root:

260 return self.translation_unit.clang_atu.spelling

261 try:

262 return self.node.location.file.name

263 except Exception:

264 return EMPTY_STR

265

266 @override

267 @property

268 def extended_end_offset(self) -> int:

269 try:

270 end_offset = self._offset + self._length

271 if (

272 (not self._is_statement_or_declaration())

273 and (self.parent and self.parent.ast_type in STMT_PARENTS)

274 and self.ast_type not in [MacroDef]

275 ):

276 content = self.root.binary_file_content()

277 while end_offset < len(content) and content[end_offset - 1] not in b";":

278 end_offset += 1

279 return end_offset

280 except Exception:

281 return 0

282

283 def _is_statement_or_declaration(self):

284 print(f"{self.ast_type} is statement: {self.kind}")

285 return isinstance(self.ast_type(), (Statement, Declaration, Definition))

286

287 @override

288 def matches_kind(self, node: ASTNode) -> bool:

289 return matches_kind(self.ast_type, node.ast_type)

290

291 def _derive_properties(self) -> dict[str, int | str]:

292 result = {}

293 offsets = (self.filename, self.offset, self.end_offset)

294 if offsets in self.translation_unit.macro_expansions:

295 result["macro_expansion"] = self.text

296

297 if self.ast_type == BinaryOperation:

298 # TODO remove below code after clang release that supports the getOpCode() statement

299 children = self.children

300 start_offset = children[0].offset + children[0].length

301 end_offset = children[1].offset

302 operator = self.content(start_offset, end_offset)

303 result["operator"] = operator.strip()

304 # next statement works in C++ but not in Python (yet) will be released later

305 # result['operator'] = self.node.getOpCode()

306 elif self.ast_type == UnaryOperation:

307 # TODO remove below code after clang release that supports the getOpCode() statement

308 child = self.children[0]

309 # list all attributes of self.node excluding the once starting with _

310

311 if child.offset > self.offset:

312 start_offset = self.offset

313 end_offset = child.offset

314 prefix_operator = True

315 else:

316 start_offset = child.offset + child.length

317 end_offset = self.offset + self.length

318 prefix_operator = False

319

320 operator = self.content(start_offset, end_offset)

321 result["operator"] = operator.strip()

322 result["prefixOperator"] = prefix_operator

323 # next statement works in C++ but not in Python (yet) will be released later

324 # result['operator'] = self.node.getOpCode()

325 elif isinstance(self.ast_type(), Literal) or self.ast_type == DeclarationExpression:

326 self._add_tokens(result, "LITERAL")

327

328 is_all = {

329 attr[len("is_") :]: True

330 for attr in dir(self.node)

331 if attr.startswith("is_") and callable(getattr(self.node, attr)) and getattr(self.node, attr)()

332 }

333 result.update(is_all)

334 return result

335

336 @override

337 @property

338 def is_statement(self) -> bool:

339 """Pretty good definition."""

340 return self.parent is not None and self.parent.ast_type in STMT_PARENTS

341

342 @override

343 @property

344 def referenced_by(self) -> Sequence[ASTReference]:

345 self.translation_unit.lazy_create_references(self)

346 node_id = self.node.hash

347 ref_by = self.translation_unit._referenced_by.get(node_id, EMPTY_LIST)

348 # if both the function declaration and function definition are available

349 # the references are stored in the function definition,

350 # but we want them to also show up in the declaration

351 if len(ref_by) == 0:

352 definition = self._get_function_definition()

353 if definition:

354 ref_by = self.translation_unit._referenced_by.get(definition.node.hash, EMPTY_LIST)

355 return list(

356 ASTReference(

357 self.translation_unit._nodes[ref.node_id],

358 ref.ref_kind,

359 ref.properties,

360 )

361 for ref in ref_by

362 )

363

364 def _get_function_definition(self):

365 # NOTE: see the clang.cindex enum note near __init__ above.

366 if self.node.type.kind == TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]

367 signature = self.node.displayname

368 semantic_parent = self.node.semantic_parent.hash

369

370 def has_body(node):

371 return any(

372 c.kind == CursorKind.COMPOUND_STMT # pyright: ignore[reportAttributeAccessIssue]

373 for c in node.node.get_children()

374 )

375

376 def is_match(node):

377 if node._kind != self._kind:

378 return False

379 if node.node.type.kind != TypeKind.FUNCTIONPROTO: # pyright: ignore[reportAttributeAccessIssue]

380 return False

381 if node.node.semantic_parent.hash != semantic_parent:

382 return False

383 if node.node.displayname != signature:

384 return False

385 return has_body(node)

386

387 if has_body(self):

388 return None

389 body = ASTFinder.find_all(self.root, is_match).find_first().or_else(None) # type: ignore

390 if isinstance(body, ClangASTNode):

391 return body

392 return None

393

394 @override

395 @property

396 def references(self) -> Sequence[ASTReference]:

397 self.translation_unit.lazy_create_references(self)

398 return list(

399 ASTReference(

400 self.translation_unit._nodes[ref.node_id],

401 ref.ref_kind,

402 ref.properties,

403 )

404 for ref in self.translation_unit._references.get(self.node.hash, EMPTY_LIST)

405 )

406

407 def _add_tokens(self, result: dict[str, str], *token_kind):

408 for token in self.node.get_tokens():

409 # find all attr of token that are of type str or int

410 kind = str(token.kind).split(".")[-1]

411 if kind in token_kind:

412 result[kind] = token.spelling

413

414 def __derive_start_offset(self) -> int:

415 try:

416 if self.node.kind.name == "MACRO_DEFINITION":

417 return self.node.extent.start.offset - 8

418

419 return self.node.extent.start.offset

420

421 except Exception:

422 return 0

423

424 def __derive_length(self) -> int:

425 try:

426 if self.node.kind.name in ["VAR_DECL", "STRUCT_DECL"]:

427 end_offset = self.node.extent.end.offset + 1

428 elif self.node.kind.name in ["MACRO_DEFINITION"]:

429 end_offset = self.node.extent.end.offset

430 else:

431 end_offset = self.node.extent.end.offset

432 return end_offset - self.__derive_start_offset()

433 except Exception:

434 return 0

435

436 def __derive_kind(self) -> str:

437 try:

438 if self.node.kind.name == "MACRO_DEFINITION":

439 return str(self.node.kind.name)

440 if self.node.kind.name in ["UNEXPOSED_EXPR", "VAR_DECL", "DECL_REF_EXPR"]:

441 if self.node.displayname.startswith("$$") and " " not in self.node.displayname:

442 return MatchAll.__name__

443 if self.node.displayname.startswith("$") and " " not in self.node.displayname:

444 return MatchOne.__name__

445 return str(self.node.kind.name)

446 except Exception:

447 return EMPTY_STR

448

449 @staticmethod

450 def remove_wrapper(cursor):

451 try:

452 if ClangASTNode._is_wrapped(cursor):

453 return ClangASTNode.remove_wrapper(list(cursor.children)[0])

454 except Exception:

455 pass

456 return cursor

457

458 @staticmethod

459 def _is_reference(node):

460 # refactor this

461 try:

462 print(type(node))

463 print(vars(node))

464 print(dir(node))

465 print(node.__dict__)

466 node.__dict__["id"]

467 return True

468 except Exception:

469 return False

470

471 @staticmethod

472 @cache

473 def __is_property(key, value):

474 return callable(value) and any(key.startswith(tag) for tag in ["is_", "get"])

475

476 @staticmethod

477 def _is_wrapped(cursor):

478 return cursor.kind.is_unexposed() and len(list(cursor.children)) == 1

479

480 @property

481 def is_implicit(self):

482 return self.is_part_of_translation_unit()

483

484

485# def get_ancestor(self, types ):

486# return get_ancestor(self, types)

487

488

489SYSTEM_MACROS = {

490 "linux",

491 "unix",

492 "_LP64",

493 "_WIN32",

494 "_WIN64",

495 "_ISO_VOLATILE",

496 "_INTEGRAL_MAX_BITS",

497}

498

499

500def is_system_macro(n):

501 return n.kind.name == "MACRO_DEFINITION" and (

502 n.displayname.startswith("__")

503 or n.displayname.startswith("_MS")

504 or n.displayname.startswith("_M_")

505 or n.displayname in SYSTEM_MACROS

506 )

507

508

509class ReferenceHelper:

510 @staticmethod

511 def create_references(ast_node: ClangASTNode) -> None:

512 assert isinstance(ast_node, ClangASTNode), f"Expected ClangASTNode but got {type(ast_node)}"

513 references = []

514 node_id: str = ast_node.node.hash

515 ast_node.translation_unit._references[node_id] = references

516 ref_fields = ["referenced"] # , 'type.get_declaration()']

517 for field in ref_fields:

518 try:

519 element = eval("ast_node.node." + field)

520 if element.kind.name == "NO_DECL_FOUND":

521 continue

522 ref_id = element.hash

523 ref_kind = field.split(".")[0]

524 properties = {k: p for k, p in element.__dict__.items() if not k.startswith("_") and k != "hash"}

525 if node_id == ref_id:

526 return

527 reference = Clangastreference(ref_id, ref_kind, properties)

528 referenced_by = Clangastreference(

529 node_id,

530 ref_kind,

531 {k: p for k, p in ast_node.node.__dict__.items() if k != "hash"},

532 )

533 try:

534 ast_node.translation_unit._referenced_by[ref_id].append(referenced_by)

535 except Exception:

536 ast_node.translation_unit._referenced_by[ref_id] = [referenced_by]

537 references.append(reference)

538 except Exception:

539 pass

Coverage for src / renaissance / integrations / clang / clang_ast_node.py: 83%

332 statements