Improve support for macros

atom.mk

@@ -14,6 +14,10 @@ LOCAL_COPY_FILES := \
 	ctypeslib/codegen/__init__.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
 	ctypeslib/codegen/typedesc.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
 	ctypeslib/codegen/typehandler.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
+	ctypeslib/codegen/preprocess.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
+	ctypeslib/codegen/cache.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
+	ctypeslib/codegen/cindex.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
+	ctypeslib/codegen/hash.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
 	ctypeslib/codegen/util.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
 	ctypeslib/data/fundamental_type_name.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
 	ctypeslib/data/headers.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
@@ -41,6 +45,10 @@ define pybinding-macro
 
 
 PRIVATE_SO_FILES = $(shell echo "$4" | sed "s#:# #g")
+PRIVATE_PYBINDING_BUNDLE = $$(shell echo $5)
+PRIVATE_C_INCLUDES := $$(call module-get-listed-export,host.libclang,C_INCLUDES)
+PRIVATE_C_INCLUDES := $$(PRIVATE_C_INCLUDES:%=-I%)
+$(call local-get-build-dir)/$1.py: PRIVATE_C_INCLUDES := $$(PRIVATE_C_INCLUDES)
 $(call local-get-build-dir)/$1.py: PRIVATE_SO_FILES := $$(PRIVATE_SO_FILES)
 $(call local-get-build-dir)/$1.py: PRIVATE_SRC_FILES = \
 	$$(foreach header, $$(shell echo "$3" | sed "s#:# #g"), \
@@ -52,6 +60,7 @@ $(call local-get-build-dir)/$1.py: PRIVATE_OBJECT_FLAGS := $$(foreach lib, $$(sh
 $(call local-get-build-dir)/$1.py: $(shell echo "$4" | sed "s#:# #g")
 	@echo "$$(PRIVATE_MODULE): Generating $1 python binding"
 	@echo "Private object flags: $$(PRIVATE_OBJECT_FLAGS)"
+	@echo "Private includes: $$(PRIVATE_C_INCLUDES)"
 	@echo "Private so files: $$(PRIVATE_SO_FILES)"
 
 	$(Q) PYTHONPATH=$(HOST_OUT_STAGING)/usr/lib/python/site-packages \
@@ -68,16 +77,33 @@ $(call local-get-build-dir)/$1.py: $(shell echo "$4" | sed "s#:# #g")
 			$$$$(sed -n -e 's/TARGET_GLOBAL_C_INCLUDES :=//p' $$(PRIVATE_OBJECT_FLAGS) | tr ' ' '\n' | sed -E 's/^(.+)/-I\1/') \
 			$$$$(sed -n -e 's/PRIVATE_GLOBAL_CFLAGS :=//p' $$(PRIVATE_OBJECT_FLAGS)) \
 			$$$$(sed -n -e 's/PRIVATE_CFLAGS :=//p' $$(PRIVATE_OBJECT_FLAGS)) \
+			$$(PRIVATE_C_INCLUDES) \
+			-D__PYBINDING_MACRO__=1 \
 			-fno-unsigned-char \
 		"
 
 LOCAL_CLEAN_FILES += $(call local-get-build-dir)/$1.py
-LOCAL_COPY_FILES += $(call local-get-build-dir)/$1.py:usr/lib/python/site-packages/
-LOCAL_DEPENDS_HOST_MODULES += host.pybinding
+
+LOCAL_COPY_FILES += $(call local-get-build-dir)/$1.py:usr/lib/python/site-packages/$1/__init__.py
+
+LOCAL_DEPENDS_HOST_MODULES += host.pybinding host.libclang
 LOCAL_DEPENDS_MODULES := python
 LOCAL_LIBRARIES += $(shell echo "$2" | sed "s#:# #g")
 
 endef
 
 # Register the macro in alchemy
 $(call local-register-custom-macro,pybinding-macro)
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE := tst-ctypeslib-basic-types
+
+LOCAL_C_INCLUDES := $(LOCAL_PATH)/tests $(LOCAL_PATH)/src
+LOCAL_CFLAGS := -Wall -Wextra -Werror -std=c99 -pedantic -fpic
+
+LOCAL_SRC_FILES := test/data/test-basic-types.c
+
+LOCAL_LIBRARIES := host.pybinding
+
+include $(BUILD_LIBRARY)

ctypeslib/__init__.py

@@ -22,13 +22,14 @@
 # configure python-clang to use the local clang library
 try:
     from ctypes.util import find_library
-    from clang import cindex
+    from ctypeslib.codegen import cindex
     # debug for python-haystack travis-ci
     v1 = ["clang-%d" % _ for _ in range(14, 6, -1)]
     v2 = ["clang-%f" % _ for _ in range(6, 3, -1)]
     v_list = v1 + v2 + ["clang-3.9", "clang-3.8", "clang-3.7"]
     for version in ["libclang", "clang"] + v_list:
         if find_library(version) is not None:
+            from ctypeslib.codegen import cindex
             cindex.Config.set_library_file(find_library(version))
             break
     else:
@@ -40,6 +41,7 @@
                     cindex.Config.set_library_file(f)
 
     def clang_version():
+        from ctypeslib.codegen import cindex
         return cindex.Config.library_file
 except ImportError as e:
     print(e)

ctypeslib/clang2py.py

@@ -64,6 +64,11 @@ def windows_dlls(option, opt, value, parser):
 
     parser = argparse.ArgumentParser(prog='clang2py',
                                      description='Version %s. Generate python code from C headers' % (version))
+    parser.add_argument("--advanced-macro",
+                        dest="advanced_macro",
+                        action="store_true",
+                        help="enable function-like macro generation for those which are 'translapilable' to Python",
+                        default=False)
     parser.add_argument("-c", "--comments",
                         dest="generate_comments",
                         action="store_true",
@@ -305,8 +310,9 @@ def windows_dlls(option, opt, value, parser):
                       searched_dlls=dlls,
                       preloaded_dlls=options.preload,
                       types=options.kind,
-                      flags=clang_opts)
-    finally:
+                      flags=clang_opts,
+                      advanced_macro=options.advanced_macro)
+    except:
         if output_file is not None:
             output_file.close()
             os.remove(options.output)

ctypeslib/codegen/cache.py

@@ -0,0 +1,85 @@
+import functools
+import itertools
+import types
+
+from ctypeslib.codegen.hash import hashable_dict
+
+
+_Tee = itertools.tee([], 1)[0].__class__
+
+
+disable_cache = False
+
+
+def _get_function_fullname(function):
+    return f"{function.__module__}.{function.__qualname__}"
+
+
+_cache_functions = {
+    "ctypeslib.codegen.cindex.Config.lib",
+    "ctypeslib.codegen.cindex.Cursor.get_tokens",
+    "ctypeslib.codegen.cindex.SourceLocation.__contains__",
+    "ctypeslib.codegen.cindex.Token.cursor",
+    # # The following aren't worth caching
+    # "ctypeslib.codegen.cindex.Cursor.kind",
+    # "ctypeslib.codegen.cindex.Token.kind",
+    # "ctypeslib.codegen.cindex.TokenGroup.get_tokens",
+    # "ctypeslib.codegen.cindex.TranslationUnit.from_source",
+    # "ctypeslib.codegen.cursorhandler.CursorHandler.MACRO_DEFINITION",
+}
+
+
+def cached(cache_key=None):
+    def decorator(function):
+        global disable_cache, _cache_functions
+        if disable_cache or not _get_function_fullname(function) in _cache_functions:
+            return function
+
+        cache = {}
+        args_names = function.__code__.co_varnames[: function.__code__.co_argcount]
+
+        def wrapper(*args, **kwds):
+            wargs = dict(zip(args_names, args))
+            wargs.update(kwds)
+            wargs = hashable_dict(wargs)
+            if cache_key is None:
+                key = hash(wargs)
+            elif isinstance(cache_key, (int, slice)):
+                key = tuple(wargs.values())[cache_key]
+            else:
+                key = cache_key(wargs)
+            try:
+                return cache[key]
+            except KeyError:
+                value = function(*args, **kwds)
+                if isinstance(value, types.GeneratorType):
+                    # flatten the generator
+                    value = tuple(value)
+                cache[key] = value
+                return value
+
+        return functools.update_wrapper(wrapper, function)
+
+    return decorator
+
+
+def cached_pure_method():
+    return cached(cache_key=slice(1, None))
+
+
+def cached_property():
+    def decorator(function):
+        return property(cached()(function))
+    return decorator
+
+
+def cached_classmethod():
+    def decorator(function):
+        return classmethod(cached()(function))
+    return decorator
+
+
+def cached_staticmethod():
+    def decorator(function):
+        return staticmethod(cached()(function))
+    return decorator