From b5625337f7657b98bd20a9e94315561f275ee348 Mon Sep 17 00:00:00 2001 From: Nicolas Dessart Date: Tue, 23 Mar 2021 17:45:54 +0100 Subject: [PATCH] [DEV] add advanced_macro flag Change-Id: I5027bc6a22b916c056cdc04be4a8731fc59b132c --- ctypeslib/clang2py.py | 10 ++++- ctypeslib/codegen/clangparser.py | 30 ++++++++++++--- ctypeslib/codegen/codegenerator.py | 18 ++++++--- ctypeslib/codegen/cursorhandler.py | 24 +++++++++--- ctypeslib/codegen/typedesc.py | 3 +- ctypeslib/codegen/util.py | 62 ++++++++++++++++++------------ test/test_macro_advanced.py | 6 ++- test/util.py | 3 +- 8 files changed, 110 insertions(+), 46 deletions(-) diff --git a/ctypeslib/clang2py.py b/ctypeslib/clang2py.py index f024fd6..c53429e 100755 --- a/ctypeslib/clang2py.py +++ b/ctypeslib/clang2py.py @@ -64,6 +64,11 @@ def windows_dlls(option, opt, value, parser): parser = argparse.ArgumentParser(prog='clang2py', description='Version %s. Generate python code from C headers' % (version)) + parser.add_argument("--advanced-macro", + dest="advanced_macro", + action="store_true", + help="enable function-like macro generation for those which are 'translapilable' to Python", + default=False) parser.add_argument("-c", "--comments", dest="generate_comments", action="store_true", @@ -305,8 +310,9 @@ def windows_dlls(option, opt, value, parser): searched_dlls=dlls, preloaded_dlls=options.preload, types=options.kind, - flags=clang_opts) - finally: + flags=clang_opts, + advanced_macro=options.advanced_macro) + except: if output_file is not None: output_file.close() os.remove(options.output) diff --git a/ctypeslib/codegen/clangparser.py b/ctypeslib/codegen/clangparser.py index d7f96b5..ffd8438 100644 --- a/ctypeslib/codegen/clangparser.py +++ b/ctypeslib/codegen/clangparser.py @@ -4,14 +4,15 @@ import logging import os -from ctypeslib.codegen.cindex import Index, TranslationUnit +from ctypeslib.codegen.cindex import Index, TranslationUnit, TargetInfo from ctypeslib.codegen.cindex import TypeKind from ctypeslib.codegen.hash import hash_combine +from ctypeslib.codegen import cache from ctypeslib.codegen import cursorhandler +from ctypeslib.codegen import preprocess from ctypeslib.codegen import typedesc from ctypeslib.codegen import typehandler -from ctypeslib.codegen import cache from ctypeslib.codegen import util from ctypeslib.codegen.handler import DuplicateDefinitionException from ctypeslib.codegen.handler import InvalidDefinitionError @@ -79,15 +80,22 @@ def __init__(self, flags): self.typekind_handler = typehandler.TypeHandler(self) self.__filter_location = None self.__processed_location = set() + self._advanced_macro = False + self.interpreter_namespace = {} def init_parsing_options(self): """Set the Translation Unit to skip functions bodies per default.""" self.tu_options = TranslationUnit.PARSE_SKIP_FUNCTION_BODIES - def activate_macros_parsing(self): + def activate_macros_parsing(self, advanced_macro=False): """Activates the detailled code parsing options in the Translation Unit.""" self.tu_options |= TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD + self._advanced_macro = advanced_macro + + @property + def advanced_macro(self): + return self._advanced_macro def activate_comment_parsing(self): """Activates the comment parsing options in the Translation Unit.""" @@ -103,12 +111,12 @@ def filter_location(self, src_files): @cache.cached_pure_method() def _do_parse(self, filename): if os.path.abspath(filename) in self.__processed_location: - return + return None index = Index.create() tu = index.parse(filename, self.flags, options=self.tu_options) if not tu: log.warning("unable to load input") - return + return None return tu def parse(self, filename): @@ -124,6 +132,9 @@ def parse(self, filename): . for each TYPEREF ?? """ self.tu = self._do_parse(filename) + if self.tu is None: + return + self.ti = TargetInfo.from_translation_unit(self.tu) self._parse_tu_diagnostics(self.tu, filename) root = self.tu.cursor for node in root.get_children(): @@ -331,6 +342,12 @@ def get_ctypes_name(self, typekind): def get_ctypes_size(self, typekind): return self.ctypes_sizes[typekind] + def get_pointer_width(self): + return self.ti.pointer_width + + def get_platform_triple(self): + return self.ti.triple + def parse_cursor(self, cursor): """Forward parsing calls to dedicated CursorKind Handlder""" return self.cursorkind_handler.parse_cursor(cursor) @@ -425,3 +442,6 @@ def get_result(self): log.debug("parsed items order: %s", result) return tuple(result) + + def interprete(self, expr): + preprocess.exec_processed_macro(expr, self.interpreter_namespace) diff --git a/ctypeslib/codegen/codegenerator.py b/ctypeslib/codegen/codegenerator.py index 1055773..9871f81 100644 --- a/ctypeslib/codegen/codegenerator.py +++ b/ctypeslib/codegen/codegenerator.py @@ -157,6 +157,8 @@ def enable_macro_processing(self): If a structure type is used, declare our ctypes.Structure extension type """ self.enable_macro_processing = lambda: True + if not self.parser.advanced_macro: + return import pkgutil shared = pkgutil.get_data('ctypeslib', 'codegen/preprocess.py').decode() print(shared, file=self.imports) @@ -276,11 +278,14 @@ def Macro(self, macro): macro_body = " ".join(str(_) for _ in macro.body) ret.write("stream", f"# {macro.name} = {macro_body} # macro") elif macro_args: - macro_func = process_macro_function(macro.name, macro.args, macro.body) - if macro_func is not None: - ret.write("stream", f"\n# macro function {macro.name}{macro_func}") + if self.parser.advanced_macro: + macro_func = process_macro_function(macro.name, macro.args, macro.body) + if macro_func is not None: + ret.write("stream", f"\n# macro function {macro.name}{macro_func}") + else: + ret.write("stream", f"\n# invalid macro function {macro.name}{macro.body}") else: - ret.write("stream", f"\n# invalid macro function {macro.name}{macro.body}") + ret.write("stream", f"\n# macro function {macro.name}") elif isinstance(macro_body, bool): ret.write("stream", f"{macro.name} = {macro_body} # macro") @@ -1152,14 +1157,15 @@ def generate_code(srcfiles, generate_docstrings=False, generate_locations=False, filter_location=False, - flags=None + flags=None, + advanced_macro=False ): # expressions is a sequence of compiled regular expressions, # symbols is a sequence of names parser = clangparser.Clang_Parser(flags or []) # if macros are not needed, use a faster TranslationUnit if typedesc.Macro in types: - parser.activate_macros_parsing() + parser.activate_macros_parsing(advanced_macro) if generate_comments is True: parser.activate_comment_parsing() diff --git a/ctypeslib/codegen/cursorhandler.py b/ctypeslib/codegen/cursorhandler.py index 8173840..b435992 100644 --- a/ctypeslib/codegen/cursorhandler.py +++ b/ctypeslib/codegen/cursorhandler.py @@ -11,12 +11,15 @@ from ctypeslib.codegen.handler import DuplicateDefinitionException from ctypeslib.codegen.handler import InvalidDefinitionError from ctypeslib.codegen.cache import cached_pure_method +from ctypeslib.codegen.preprocess import ( + is_identifier, + process_macro_function, + remove_outermost_parentheses, +) from ctypeslib.codegen.util import ( contains_invalid_code, expand_macro_function, - is_identifier, log_entity, - remove_outermost_parentheses, ) @@ -661,7 +664,11 @@ def _literal_handling(self, cursor): tokens.consume() token = tokens.current call_args = self._macro_args_handling(tokens, call_args=True) - value = expand_macro_function(macro, call_args) + expansion_limit = None + if self.parser.advanced_macro: + expansion_limit = 1 + value = expand_macro_function( + macro, call_args, namespace=self.parser.interpreter_namespace, limit=expansion_limit) token = tokens.current else: value = macro.body @@ -1251,7 +1258,6 @@ def MACRO_DEFINITION(self, cursor): elif len(tokens) == 3 and tokens[1] == '-': value = ''.join(tokens[1:]) elif tokens[1] == '(': - # TODO, differentiate between function-like macro and expression in () # function macro or an expression. tokens = remove_outermost_parentheses(tokens[1:]) if tokens and tokens[0] == "(": @@ -1268,7 +1274,8 @@ def MACRO_DEFINITION(self, cursor): value = str_tokens elif all(map(lambda a: a == "," or is_identifier(str(a)), tokens)): - # no-op function + # TODO FIX differentiation between function-like macro and expression in () + # no-op function ? args = list(map( lambda a: str(a).strip(), tokens )) @@ -1305,7 +1312,12 @@ def MACRO_DEFINITION(self, cursor): if name in ['NULL', '__thread'] or value in ['__null', '__thread']: value = None log.debug('MACRO: #define %s%s %s', name, args or '', value) - obj = typedesc.Macro(name, args, value) + func = None + if args: + func = process_macro_function(name, args, value) + if func is not None: + self.parser.interprete(func) + obj = typedesc.Macro(name, args, value, func) self.set_location(obj, cursor) # set the comment in the obj obj.comment = comment diff --git a/ctypeslib/codegen/typedesc.py b/ctypeslib/codegen/typedesc.py index 4ea4bec..f57eadc 100644 --- a/ctypeslib/codegen/typedesc.py +++ b/ctypeslib/codegen/typedesc.py @@ -87,13 +87,14 @@ class Macro(T): """a C preprocessor definition with arguments""" - def __init__(self, name, args, body): + def __init__(self, name, args, body, func): """all arguments are strings, args is the literal argument list *with* the parens around it: Example: Macro("CD_INDRIVE", "(status)", "((int)status > 0)")""" self.name = name self.args = args self.body = body + self.func = func class File(T): diff --git a/ctypeslib/codegen/util.py b/ctypeslib/codegen/util.py index 88973b3..fb7f5a1 100644 --- a/ctypeslib/codegen/util.py +++ b/ctypeslib/codegen/util.py @@ -8,15 +8,16 @@ from collections.abc import Iterable import logging +import sys import textwrap from ctypeslib.codegen import typedesc -from ctypeslib.codegen.preprocess import * +from ctypeslib.codegen.preprocess import eval_processed_macro -log = logging.getLogger('utils') +log = logging.getLogger("utils") -def get_tu(source, lang='c', all_warnings=False, flags=None): +def get_tu(source, lang="c", all_warnings=False, flags=None): """Obtain a translation unit from source and language. By default, the translation unit is created from source file "t." @@ -28,17 +29,17 @@ def get_tu(source, lang='c', all_warnings=False, flags=None): all_warnings is a convenience argument to enable all compiler warnings. """ args = list(flags or []) - name = 'memory_input.c' - if lang == 'cpp': - name = 'memory_input.cpp' - args.append('-std=c++11') - elif lang == 'objc': - name = 'memory_input.m' - elif lang != 'c': - raise Exception('Unknown language: %s' % lang) + name = "memory_input.c" + if lang == "cpp": + name = "memory_input.cpp" + args.append("-std=c++11") + elif lang == "objc": + name = "memory_input.m" + elif lang != "c": + raise Exception("Unknown language: %s" % lang) if all_warnings: - args += ['-Wall', '-Wextra'] + args += ["-Wall", "-Wextra"] return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)]) @@ -67,7 +68,6 @@ def get_cursor(source, spelling): result = get_cursor(cursor, spelling) if result is not None: return result - return None @@ -105,6 +105,7 @@ def new_decorator(f): g.__doc__ = f.__doc__ g.__dict__.update(f.__dict__) return g + new_decorator.__name__ = dec.__name__ new_decorator.__doc__ = dec.__doc__ new_decorator.__dict__.update(dec.__dict__) @@ -116,18 +117,18 @@ def log_entity(func): def fn(*args, **kwargs): cursor = next(arg for arg in args if isinstance(arg, (Type, Cursor))) name = args[0].get_unique_name(cursor) - if name == '': + if name == "": parent = cursor.semantic_parent if parent: - name = 'child of %s' % parent.displayname + name = "child of %s" % parent.displayname log.debug("%s: displayname:'%s'", func.__name__, name) # print 'calling {}'.format(func.__name__) return func(*args, **kwargs) + return fn class ADict(dict): - def __getattr__(self, name): try: return self[name] @@ -135,11 +136,24 @@ def __getattr__(self, name): raise AttributeError(name) -def expand_macro_function(macro, args): +def expand_macro_function(macro, args, namespace=None, limit=None, max_recursion=None): args = ", ".join(args) code = f"{macro.name}({args})" + if max_recursion is None: + max_recursion = sys.getrecursionlimit() + max_eval = limit or max_recursion try: - return eval_processed_macro(code) + prev = eval_processed_macro(code, namespace=namespace) + for i in range(1, max_eval + 1): + if limit is not None and limit == i: + return prev + value = eval_processed_macro(str(prev), namespace=namespace) + if prev == value: + return value + prev = value + raise RecursionError( + f"maximum recursion depth exceeded in {macro.name} expansion" + ) except (SyntaxError, NameError): return typedesc.InvalidGeneratedMacro(code) @@ -185,10 +199,10 @@ def body_is_all_string_tokens(macro_body): __all__ = [ - 'get_cursor', - 'get_cursors', - 'get_tu', - 'from_c_float_literal', - 'remove_outermost_parentheses', - 'replace_builtins', + "get_cursor", + "get_cursors", + "get_tu", + "from_c_float_literal", + "remove_outermost_parentheses", + "replace_builtins", ] diff --git a/test/test_macro_advanced.py b/test/test_macro_advanced.py index 70441a6..85c6445 100644 --- a/test/test_macro_advanced.py +++ b/test/test_macro_advanced.py @@ -16,8 +16,12 @@ class Macro(ClangTest): # @unittest.skip('') def setUp(self): - # we need to generate macro. Which is very long for some reasons. + # We need to generate macro (including function-like macro) + # This used to take a long time to process but some performance + # improvements have been implemented and I am not sure if it's + # still the case for common workloads. (See: codegen.cache). self.full_parsing_options = True + self.advanced_macro = True def test_bitwise(self): self.convert(textwrap.dedent(""" diff --git a/test/util.py b/test/util.py index 825b9f1..1e4fc53 100644 --- a/test/util.py +++ b/test/util.py @@ -27,6 +27,7 @@ class ClangTest(unittest.TestCase): namespace = None text_output = None full_parsing_options = False + advanced_macro = False def _gen(self, ofi, fname, flags=None, dlls=None): """Take a file input and generate the code. @@ -36,7 +37,7 @@ def _gen(self, ofi, fname, flags=None, dlls=None): # leave the new parser accessible for tests self.parser = clangparser.Clang_Parser(flags) if self.full_parsing_options: - self.parser.activate_macros_parsing() + self.parser.activate_macros_parsing(self.advanced_macro) self.parser.activate_comment_parsing() with open(fname): pass