[DEV] add advanced_macro flag

Change-Id: I5027bc6a22b916c056cdc04be4a8731fc59b132c

ctypeslib/clang2py.py

@@ -64,6 +64,11 @@ def windows_dlls(option, opt, value, parser):
 
     parser = argparse.ArgumentParser(prog='clang2py',
                                      description='Version %s. Generate python code from C headers' % (version))
+    parser.add_argument("--advanced-macro",
+                        dest="advanced_macro",
+                        action="store_true",
+                        help="enable function-like macro generation for those which are 'translapilable' to Python",
+                        default=False)
     parser.add_argument("-c", "--comments",
                         dest="generate_comments",
                         action="store_true",
@@ -305,8 +310,9 @@ def windows_dlls(option, opt, value, parser):
                       searched_dlls=dlls,
                       preloaded_dlls=options.preload,
                       types=options.kind,
-                      flags=clang_opts)
-    finally:
+                      flags=clang_opts,
+                      advanced_macro=options.advanced_macro)
+    except:
         if output_file is not None:
             output_file.close()
             os.remove(options.output)

ctypeslib/codegen/clangparser.py

@@ -4,14 +4,15 @@
 import logging
 import os
 
-from ctypeslib.codegen.cindex import Index, TranslationUnit
+from ctypeslib.codegen.cindex import Index, TranslationUnit, TargetInfo
 from ctypeslib.codegen.cindex import TypeKind
 from ctypeslib.codegen.hash import hash_combine
 
+from ctypeslib.codegen import cache
 from ctypeslib.codegen import cursorhandler
+from ctypeslib.codegen import preprocess
 from ctypeslib.codegen import typedesc
 from ctypeslib.codegen import typehandler
-from ctypeslib.codegen import cache
 from ctypeslib.codegen import util
 from ctypeslib.codegen.handler import DuplicateDefinitionException
 from ctypeslib.codegen.handler import InvalidDefinitionError
@@ -79,15 +80,22 @@ def __init__(self, flags):
         self.typekind_handler = typehandler.TypeHandler(self)
         self.__filter_location = None
         self.__processed_location = set()
+        self._advanced_macro = False
+        self.interpreter_namespace = {}
 
     def init_parsing_options(self):
         """Set the Translation Unit to skip functions bodies per default."""
         self.tu_options = TranslationUnit.PARSE_SKIP_FUNCTION_BODIES
 
-    def activate_macros_parsing(self):
+    def activate_macros_parsing(self, advanced_macro=False):
         """Activates the detailled code parsing options in the Translation
         Unit."""
         self.tu_options |= TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD
+        self._advanced_macro = advanced_macro
+
+    @property
+    def advanced_macro(self):
+        return self._advanced_macro
 
     def activate_comment_parsing(self):
         """Activates the comment parsing options in the Translation Unit."""
@@ -103,12 +111,12 @@ def filter_location(self, src_files):
     @cache.cached_pure_method()
     def _do_parse(self, filename):
         if os.path.abspath(filename) in self.__processed_location:
-            return
+            return None
         index = Index.create()
         tu = index.parse(filename, self.flags, options=self.tu_options)
         if not tu:
             log.warning("unable to load input")
-            return
+            return None
         return tu
 
     def parse(self, filename):
@@ -124,6 +132,9 @@ def parse(self, filename):
         . for each TYPEREF ??
         """
         self.tu = self._do_parse(filename)
+        if self.tu is None:
+            return
+        self.ti = TargetInfo.from_translation_unit(self.tu)
         self._parse_tu_diagnostics(self.tu, filename)
         root = self.tu.cursor
         for node in root.get_children():
@@ -331,6 +342,12 @@ def get_ctypes_name(self, typekind):
     def get_ctypes_size(self, typekind):
         return self.ctypes_sizes[typekind]
 
+    def get_pointer_width(self):
+        return self.ti.pointer_width
+
+    def get_platform_triple(self):
+        return self.ti.triple
+
     def parse_cursor(self, cursor):
         """Forward parsing calls to dedicated CursorKind Handlder"""
         return self.cursorkind_handler.parse_cursor(cursor)
@@ -425,3 +442,6 @@ def get_result(self):
 
         log.debug("parsed items order: %s", result)
         return tuple(result)
+
+    def interprete(self, expr):
+        preprocess.exec_processed_macro(expr, self.interpreter_namespace)

ctypeslib/codegen/codegenerator.py

@@ -157,6 +157,8 @@ def enable_macro_processing(self):
         If a structure type is used, declare our ctypes.Structure extension type
         """
         self.enable_macro_processing = lambda: True
+        if not self.parser.advanced_macro:
+            return
         import pkgutil
         shared = pkgutil.get_data('ctypeslib', 'codegen/preprocess.py').decode()
         print(shared, file=self.imports)
@@ -276,11 +278,14 @@ def Macro(self, macro):
                 macro_body = " ".join(str(_) for _ in macro.body)
                 ret.write("stream", f"# {macro.name} = {macro_body} # macro")
         elif macro_args:
-            macro_func = process_macro_function(macro.name, macro.args, macro.body)
-            if macro_func is not None:
-                ret.write("stream", f"\n# macro function {macro.name}{macro_func}")
+            if self.parser.advanced_macro:
+                macro_func = process_macro_function(macro.name, macro.args, macro.body)
+                if macro_func is not None:
+                    ret.write("stream", f"\n# macro function {macro.name}{macro_func}")
+                else:
+                    ret.write("stream", f"\n# invalid macro function {macro.name}{macro.body}")
             else:
-                ret.write("stream", f"\n# invalid macro function {macro.name}{macro.body}")
+                ret.write("stream", f"\n# macro function {macro.name}")
 
         elif isinstance(macro_body, bool):
             ret.write("stream", f"{macro.name} = {macro_body} # macro")
@@ -1152,14 +1157,15 @@ def generate_code(srcfiles,
                   generate_docstrings=False,
                   generate_locations=False,
                   filter_location=False,
-                  flags=None
+                  flags=None,
+                  advanced_macro=False
                   ):
     # expressions is a sequence of compiled regular expressions,
     # symbols is a sequence of names
     parser = clangparser.Clang_Parser(flags or [])
     # if macros are not needed, use a faster TranslationUnit
     if typedesc.Macro in types:
-        parser.activate_macros_parsing()
+        parser.activate_macros_parsing(advanced_macro)
     if generate_comments is True:
         parser.activate_comment_parsing()
 

ctypeslib/codegen/cursorhandler.py

@@ -11,12 +11,15 @@
 from ctypeslib.codegen.handler import DuplicateDefinitionException
 from ctypeslib.codegen.handler import InvalidDefinitionError
 from ctypeslib.codegen.cache import cached_pure_method
+from ctypeslib.codegen.preprocess import (
+    is_identifier,
+    process_macro_function,
+    remove_outermost_parentheses,
+)
 from ctypeslib.codegen.util import (
     contains_invalid_code,
     expand_macro_function,
-    is_identifier,
     log_entity,
-    remove_outermost_parentheses,
 )
 
 
@@ -661,7 +664,11 @@ def _literal_handling(self, cursor):
                                     tokens.consume()
                                     token = tokens.current
                                     call_args = self._macro_args_handling(tokens, call_args=True)
-                                    value = expand_macro_function(macro, call_args)
+                                    expansion_limit = None
+                                    if self.parser.advanced_macro:
+                                        expansion_limit = 1
+                                    value = expand_macro_function(
+                                        macro, call_args, namespace=self.parser.interpreter_namespace, limit=expansion_limit)
                                     token = tokens.current
                                 else:
                                     value = macro.body
@@ -1251,7 +1258,6 @@ def MACRO_DEFINITION(self, cursor):
             elif len(tokens) == 3 and tokens[1] == '-':
                 value = ''.join(tokens[1:])
             elif tokens[1] == '(':
-                # TODO, differentiate between function-like macro and expression in ()
                 # function macro or an expression.
                 tokens = remove_outermost_parentheses(tokens[1:])
                 if tokens and tokens[0] == "(":
@@ -1268,7 +1274,8 @@ def MACRO_DEFINITION(self, cursor):
                         value = str_tokens
 
                 elif all(map(lambda a: a == "," or is_identifier(str(a)), tokens)):
-                    # no-op function
+                    # TODO FIX differentiation between function-like macro and expression in ()
+                    # no-op function ?
                     args = list(map(
                         lambda a: str(a).strip(), tokens
                     ))
@@ -1305,7 +1312,12 @@ def MACRO_DEFINITION(self, cursor):
         if name in ['NULL', '__thread'] or value in ['__null', '__thread']:
             value = None
         log.debug('MACRO: #define %s%s %s', name, args or '', value)
-        obj = typedesc.Macro(name, args, value)
+        func = None
+        if args:
+            func = process_macro_function(name, args, value)
+            if func is not None:
+                self.parser.interprete(func)
+        obj = typedesc.Macro(name, args, value, func)
         self.set_location(obj, cursor)
         # set the comment in the obj
         obj.comment = comment

ctypeslib/codegen/typedesc.py

@@ -87,13 +87,14 @@ class Macro(T):
 
     """a C preprocessor definition with arguments"""
 
-    def __init__(self, name, args, body):
+    def __init__(self, name, args, body, func):
         """all arguments are strings, args is the literal argument list
         *with* the parens around it:
         Example: Macro("CD_INDRIVE", "(status)", "((int)status > 0)")"""
         self.name = name
         self.args = args
         self.body = body
+        self.func = func
 
 
 class File(T):

ctypeslib/codegen/util.py

@@ -8,15 +8,16 @@
 from collections.abc import Iterable
 
 import logging
+import sys
 import textwrap
 
 from ctypeslib.codegen import typedesc
-from ctypeslib.codegen.preprocess import *
+from ctypeslib.codegen.preprocess import eval_processed_macro
 
-log = logging.getLogger('utils')
+log = logging.getLogger("utils")
 
 
-def get_tu(source, lang='c', all_warnings=False, flags=None):
+def get_tu(source, lang="c", all_warnings=False, flags=None):
     """Obtain a translation unit from source and language.
 
     By default, the translation unit is created from source file "t.<ext>"
@@ -28,17 +29,17 @@ def get_tu(source, lang='c', all_warnings=False, flags=None):
     all_warnings is a convenience argument to enable all compiler warnings.
     """
     args = list(flags or [])
-    name = 'memory_input.c'
-    if lang == 'cpp':
-        name = 'memory_input.cpp'
-        args.append('-std=c++11')
-    elif lang == 'objc':
-        name = 'memory_input.m'
-    elif lang != 'c':
-        raise Exception('Unknown language: %s' % lang)
+    name = "memory_input.c"
+    if lang == "cpp":
+        name = "memory_input.cpp"
+        args.append("-std=c++11")
+    elif lang == "objc":
+        name = "memory_input.m"
+    elif lang != "c":
+        raise Exception("Unknown language: %s" % lang)
 
     if all_warnings:
-        args += ['-Wall', '-Wextra']
+        args += ["-Wall", "-Wextra"]
 
     return TranslationUnit.from_source(name, args, unsaved_files=[(name, source)])
 
@@ -67,7 +68,6 @@ def get_cursor(source, spelling):
         result = get_cursor(cursor, spelling)
         if result is not None:
             return result
-
     return None
 
 
@@ -105,6 +105,7 @@ def new_decorator(f):
         g.__doc__ = f.__doc__
         g.__dict__.update(f.__dict__)
         return g
+
     new_decorator.__name__ = dec.__name__
     new_decorator.__doc__ = dec.__doc__
     new_decorator.__dict__.update(dec.__dict__)
@@ -116,30 +117,43 @@ def log_entity(func):
     def fn(*args, **kwargs):
         cursor = next(arg for arg in args if isinstance(arg, (Type, Cursor)))
         name = args[0].get_unique_name(cursor)
-        if name == '':
+        if name == "":
             parent = cursor.semantic_parent
             if parent:
-                name = 'child of %s' % parent.displayname
+                name = "child of %s" % parent.displayname
         log.debug("%s: displayname:'%s'", func.__name__, name)
         # print 'calling {}'.format(func.__name__)
         return func(*args, **kwargs)
+
     return fn
 
 
 class ADict(dict):
-
     def __getattr__(self, name):
         try:
             return self[name]
         except KeyError:
             raise AttributeError(name)
 
 
-def expand_macro_function(macro, args):
+def expand_macro_function(macro, args, namespace=None, limit=None, max_recursion=None):
     args = ", ".join(args)
     code = f"{macro.name}({args})"
+    if max_recursion is None:
+        max_recursion = sys.getrecursionlimit()
+    max_eval = limit or max_recursion
     try:
-        return eval_processed_macro(code)
+        prev = eval_processed_macro(code, namespace=namespace)
+        for i in range(1, max_eval + 1):
+            if limit is not None and limit == i:
+                return prev
+            value = eval_processed_macro(str(prev), namespace=namespace)
+            if prev == value:
+                return value
+            prev = value
+        raise RecursionError(
+            f"maximum recursion depth exceeded in {macro.name} expansion"
+        )
     except (SyntaxError, NameError):
         return typedesc.InvalidGeneratedMacro(code)
 
@@ -185,10 +199,10 @@ def body_is_all_string_tokens(macro_body):
 
 
 __all__ = [
-    'get_cursor',
-    'get_cursors',
-    'get_tu',
-    'from_c_float_literal',
-    'remove_outermost_parentheses',
-    'replace_builtins',
+    "get_cursor",
+    "get_cursors",
+    "get_tu",
+    "from_c_float_literal",
+    "remove_outermost_parentheses",
+    "replace_builtins",
 ]

test/test_macro_advanced.py

@@ -16,8 +16,12 @@ class Macro(ClangTest):
     # @unittest.skip('')
 
     def setUp(self):
-        # we need to generate macro. Which is very long for some reasons.
+        # We need to generate macro (including function-like macro)
+        # This used to take a long time to process but some performance
+        # improvements have been implemented and I am not sure if it's
+        # still the case for common workloads. (See: codegen.cache).
         self.full_parsing_options = True
+        self.advanced_macro = True
 
     def test_bitwise(self):
         self.convert(textwrap.dedent("""

test/util.py

@@ -27,6 +27,7 @@ class ClangTest(unittest.TestCase):
     namespace = None
     text_output = None
     full_parsing_options = False
+    advanced_macro = False
 
     def _gen(self, ofi, fname, flags=None, dlls=None):
         """Take a file input and generate the code.
@@ -36,7 +37,7 @@ def _gen(self, ofi, fname, flags=None, dlls=None):
         # leave the new parser accessible for tests
         self.parser = clangparser.Clang_Parser(flags)
         if self.full_parsing_options:
-            self.parser.activate_macros_parsing()
+            self.parser.activate_macros_parsing(self.advanced_macro)
             self.parser.activate_comment_parsing()
         with open(fname):
             pass