Add plugin system

We have a powerful system for defining custom type, object, symbol, and
debug info finders, but those currently require manual setup by users.
The next step is a plugin system so that these (and more) can be set up
automatically.

Plugins are simply Python modules that define hook functions (currently
there's only one hook). Plugins are registered as package entry points
(shout out to Stephen Brennan for the suggestion) and can be configured
further via environment variables. They are still called when using
libdrgn directly (assuming libdrgn was compiled with Python support).

Signed-off-by: Omar Sandoval <[email protected]>

_drgn_util/plugins.py

@@ -0,0 +1,108 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# SPDX-License-Identifier: LGPL-2.1-or-later
+
+import fnmatch
+import importlib  # noqa: F401
+import logging
+import os
+import runpy
+import sys
+from types import SimpleNamespace
+from typing import List, Tuple
+
+logger = logging.getLogger("drgn.plugins")
+
+_plugins = None
+
+
+def _load_plugins() -> List[Tuple[str, object]]:
+    plugins: List[Tuple[str, object]] = []
+    enabled_entry_points = {}
+
+    env = os.getenv("DRGN_PLUGINS")
+    if env:
+        for item in env.split(","):
+            if not item:
+                # Ignore empty items for convenience.
+                continue
+            name, sep, value = item.partition("=")
+            if sep:
+                try:
+                    if value.startswith("/") or value.startswith("."):
+                        plugin: object = SimpleNamespace(**runpy.run_path(value))
+                    else:
+                        plugin = importlib.import_module(value)
+                except Exception:
+                    logger.warning("failed to load %r:", value, exc_info=True)
+                else:
+                    plugins.append((name, plugin))
+                    logger.debug("loaded %r", item)
+            else:
+                enabled_entry_points[name] = False
+
+    env = os.getenv("DRGN_DISABLE_PLUGINS")
+    # If all plugins are disabled, avoid the entry point machinery entirely.
+    if env != "*" or enabled_entry_points:
+        group = "drgn.plugins"
+
+        if sys.version_info >= (3, 10):
+            import importlib.metadata  # novermin
+
+            entry_points = importlib.metadata.entry_points(group=group)  # novermin
+        elif sys.version_info >= (3, 8):
+            import importlib.metadata  # novermin
+
+            entry_points = importlib.metadata.entry_points()[group]  # novermin
+        else:
+            import pkg_resources
+
+            entry_points = pkg_resources.iter_entry_points(group)
+
+        disable_plugins = env.split(",") if env else []
+        for entry_point in entry_points:
+            if entry_point.name in enabled_entry_points:
+                enabled_entry_points[entry_point.name] = True
+            elif any(
+                fnmatch.fnmatch(entry_point.name, disable)
+                for disable in disable_plugins
+            ):
+                continue
+            try:
+                plugin = entry_point.load()
+            except Exception:
+                logger.warning("failed to load %r:", entry_point.value, exc_info=True)
+            else:
+                plugins.append((entry_point.name, plugin))
+                logger.debug("loaded %r", entry_point.name)
+
+        missing_entry_points = [
+            key for key, value in enabled_entry_points.items() if not value
+        ]
+        if missing_entry_points:
+            missing_entry_points.sort()
+            logger.warning(
+                "not found: %s",
+                ", ".join([repr(name) for name in missing_entry_points]),
+            )
+
+    plugins.sort(
+        key=lambda plugin: (getattr(plugin[1], "drgn_priority", 50), plugin[0])
+    )
+    return plugins
+
+
+def call_plugins(hook_name: str, *args: object) -> None:
+    global _plugins
+    if _plugins is None:
+        _plugins = _load_plugins()
+
+    for name, plugin in _plugins:
+        try:
+            hook = getattr(plugin, hook_name)
+        except AttributeError:
+            continue
+
+        try:
+            hook(*args)
+        except Exception:
+            logger.warning("%r %s failed:", name, hook_name, exc_info=True)

docs/advanced_usage.rst

@@ -199,11 +199,95 @@ program "memory":
 :meth:`drgn.Program.register_object_finder()` are the equivalent methods for
 plugging in types and objects.
 
+.. _writing-plugins:
+
+Writing Plugins
+---------------
+
+In order for drgn to load a plugin automatically, it must be registered as an
+`entry point <https://packaging.python.org/specifications/entry-points/>`_ for
+the ``drgn.plugins`` group. Here is a minimal example. First:
+
+.. code-block:: console
+
+    $ mkdir drgn_plugin_example
+    $ cd drgn_plugin_example
+
+Then, create ``pyproject.toml`` with the following contents:
+
+.. code-block:: toml
+    :caption: pyproject.toml
+    :emphasize-lines: 5-6
+
+    [project]
+    name = 'drgn_plugin_example'
+    version = '0.0.1'
+
+    [project.entry-points.'drgn.plugins']
+    example = 'drgn_plugin_example'
+
+See the `Python Packaging User Guide
+<https://packaging.python.org/guides/writing-pyproject-toml/>`_ for a complete
+description of ``pyproject.toml``. We are most interested in the last two
+lines, which define the entry point. In ``example = 'drgn_plugin_example'``,
+``example`` is the plugin name, and ``drgn_plugin_example`` is the plugin
+module.
+
+Create ``drgn_plugin_example.py`` with the following contents:
+
+.. code-block:: python3
+    :caption: drgn_plugin_example.py
+
+    import drgn
+
+    # Optional; the default is 50.
+    drgn_priority = 100
+
+    def example_debug_info_finder(modules: list[drgn.Module]) -> None:
+        if isinstance(module, drgn.MainModule):
+            module.try_file("/my/vmlinux")
+
+    def drgn_prog_set(prog: drgn.Program) -> None:
+        if prog.flags & drgn.ProgramFlags.IS_LINUX_KERNEL:
+            prog.register_debug_info_finder(
+                "example", example_debug_info_finder, enable_index=-1
+            )
+
+This is a typical usage of the :func:`drgn_prog_set()` hook to register
+finders. See :ref:`plugins` for more details.
+
+After creating the above files, the plugin can be installed with
+``pip install .``.
+
 Environment Variables
 ---------------------
 
 Some of drgn's behavior can be modified through environment variables:
 
+.. envvar:: DRGN_DISABLE_PLUGINS
+
+    Comma-separated list of plugins to disable. Each item is a glob pattern
+    matching plugin entry point names.
+
+.. envvar:: DRGN_PLUGINS
+
+    Comma-separated list of plugins to enable. Each item is either a plugin
+    entry point name, a file path, or a module name. Empty items are ignored.
+
+    An item not containing ``=`` is interpreted as a plugin entry point name.
+    This takes precedence over :envvar:`DRGN_DISABLE_PLUGINS`.
+
+    An item containing ``=`` is interpreted as an extra plugin to load manually
+    instead of via an entry point. The string before ``=`` is the plugin name.
+    The string after ``=`` is the value. If the value starts with ``.`` or
+    ``/``, it is the file path of a Python module. Otherwise, it is a module
+    name.
+
+    So, ``DRGN_DISABLE_PLUGINS=* DRGN_PLUGINS=foo,bar=/hello/world.py,baz=my.module``
+    results in three plugins being loaded: the entry point ``foo``, the file
+    ``/hello/world.py`` as ``bar``, and the module ``my.module`` as ``baz``.
+    All other plugins are disabled.
+
 .. envvar:: DRGN_MAX_DEBUG_INFO_ERRORS
 
     The maximum number of warnings about missing debugging information to log

docs/api_reference.rst

@@ -218,6 +218,40 @@ CLI
 
 .. drgndoc:: cli
 
+.. _plugins:
+
+Plugins
+-------
+
+drgn can be extended with plugins. A drgn plugin is a Python module defining
+one or more hook functions that are called at specific times.
+
+drgn loads plugins before calling a hook for the first time. By default, it
+loads installed modules registered as :ref:`entry points <writing-plugins>` for
+the ``drgn.plugins`` group. The :envvar:`DRGN_DISABLE_PLUGINS` and
+:envvar:`DRGN_PLUGINS` environment variables can be used to configure what
+plugins are loaded.
+
+The following hooks are currently defined:
+
+.. py:currentmodule:: None
+
+.. function:: drgn_prog_set(prog: drgn.Program) -> None
+
+    Called after the program target has been set (e.g., one of
+    :meth:`drgn.Program.set_core_dump()`, :meth:`drgn.Program.set_kernel()`, or
+    :meth:`drgn.Program.set_pid()` has been called).
+
+Plugins can also define the following settings:
+
+.. data:: drgn_priority
+    :type: int
+
+    Defines the order that plugins are called in. Plugins with lower values are
+    called earlier (e.g., a plugin with ``drgn_priority = 1`` is called before
+    one with ``drgn_priority = 2``). Plugins with equal values are called in an
+    unspecified order. The default if not defined is 50.
+
 Logging
 -------
 

libdrgn/Makefile.am

@@ -166,6 +166,7 @@ libdrgnimpl_la_SOURCES += python/constants.c \
 		      python/module_section_addresses.c \
 		      python/object.c \
 		      python/platform.c \
+		      python/plugins.c \
 		      python/program.c \
 		      python/stack_trace.c \
 		      python/symbol.c \

libdrgn/plugins.h

@@ -0,0 +1,17 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#ifndef DRGN_PLUGINS_H
+#define DRGN_PLUGINS_H
+
+#include <stdbool.h>
+
+struct drgn_program;
+
+#if ENABLE_PYTHON
+void drgn_call_plugins_prog(const char *name, struct drgn_program *prog);
+#else
+static inline void drgn_call_plugins_prog(const char *name, struct drgn_program *prog) {}
+#endif
+
+#endif /* DRGN_PLUGINS_H */

libdrgn/program.c

@@ -31,6 +31,7 @@
 #include "memory_reader.h"
 #include "minmax.h"
 #include "object.h"
+#include "plugins.h"
 #include "program.h"
 #include "serialize.h"
 #include "symbol.h"
@@ -666,6 +667,7 @@ drgn_program_set_core_dump_fd_internal(struct drgn_program *prog, int fd,
 			goto out_segments;
 	}
 
+	drgn_call_plugins_prog("drgn_prog_set", prog);
 	return NULL;
 
 out_segments:
@@ -769,6 +771,8 @@ drgn_program_set_pid(struct drgn_program *prog, pid_t pid)
 
 	prog->pid = pid;
 	prog->flags |= DRGN_PROGRAM_IS_LIVE | DRGN_PROGRAM_IS_LOCAL;
+
+	drgn_call_plugins_prog("drgn_prog_set", prog);
 	return NULL;
 
 out_segments:

libdrgn/python/plugins.c

@@ -0,0 +1,32 @@
+// Copyright (c) Meta Platforms, Inc. and affiliates.
+// SPDX-License-Identifier: LGPL-2.1-or-later
+
+#include "drgnpy.h"
+#include "../plugins.h"
+
+void drgn_call_plugins_prog(const char *name, struct drgn_program *prog)
+{
+	PyGILState_guard();
+
+	static PyObject *call_plugins;
+	if (!call_plugins) {
+		_cleanup_pydecref_ PyObject *_drgn_util_plugins_module =
+			PyImport_ImportModule("_drgn_util.plugins");
+		if (!_drgn_util_plugins_module) {
+			PyErr_WriteUnraisable(NULL);
+			return;
+		}
+		call_plugins = PyObject_GetAttrString(_drgn_util_plugins_module,
+						      "call_plugins");
+		if (!call_plugins) {
+			PyErr_WriteUnraisable(NULL);
+			return;
+		}
+	}
+
+	Program *prog_obj = container_of(prog, Program, prog);
+	_cleanup_pydecref_ PyObject *res =
+		PyObject_CallFunction(call_plugins, "sO", name, prog_obj);
+	if (!res)
+		PyErr_WriteUnraisable(call_plugins);
+}