From 3817fc75a90e9ee7eaa27276b9beeb34e4454cda Mon Sep 17 00:00:00 2001 From: Andrej Orsula Date: Sat, 9 Mar 2024 00:17:29 +0100 Subject: [PATCH] Development towards 0.4.0 (#17) * Build docs.rs documentation with all features enabled Signed-off-by: Andrej Orsula * Make Config fields public Signed-off-by: Andrej Orsula * Bump to 0.4.0 Signed-off-by: Andrej Orsula * Reenable non-proc macro doc tests Signed-off-by: Andrej Orsula * Rearrange instructions Signed-off-by: Andrej Orsula * Fix instructions for `macros` feature Signed-off-by: Andrej Orsula * Add "py" to forbidden function/type names Signed-off-by: Andrej Orsula * Disable some debug asserts due to the uncertainty of Python code Signed-off-by: Andrej Orsula * Improve error handling in `Codegen::generate()` Signed-off-by: Andrej Orsula * Make kwargs optional Signed-off-by: Andrej Orsula * Reenable support for generating bindings to builtin functions Signed-off-by: Andrej Orsula * Improve docstring processing Signed-off-by: Andrej Orsula * Make Codegen API more ergonomic Signed-off-by: Andrej Orsula * Re-export `pyo3` directly from `pyo3_bindgen` Signed-off-by: Andrej Orsula * Fix loading of libpython symbols in `import_python!` procedural macro Signed-off-by: Andrej Orsula * Add "macros" to default features Signed-off-by: Andrej Orsula * Update documentation Signed-off-by: Andrej Orsula * Add examples Signed-off-by: Andrej Orsula * Disable `pygal` example Signed-off-by: Andrej Orsula * Slightly simplify README example Signed-off-by: Andrej Orsula * Update information about `pyo3_bindgen_macros` Signed-off-by: Andrej Orsula * Update documentation Signed-off-by: Andrej Orsula * Fix TOML formatting Signed-off-by: Andrej Orsula --------- Signed-off-by: Andrej Orsula --- Cargo.lock | 19 +++- Cargo.toml | 17 ++- README.md | 106 +++++++++++------- examples/Cargo.toml | 31 +++++ examples/build.rs | 8 ++ examples/math.rs | 26 +++++ examples/os_sys.rs | 34 ++++++ examples/pygal.rs | 33 ++++++ examples/random.rs | 22 ++++ pyo3_bindgen/Cargo.toml | 6 +- pyo3_bindgen/src/lib.rs | 97 ++++++++++------ pyo3_bindgen_engine/src/codegen.rs | 37 +++--- pyo3_bindgen_engine/src/config.rs | 35 +++--- pyo3_bindgen_engine/src/lib.rs | 3 - pyo3_bindgen_engine/src/syntax/class.rs | 9 +- .../src/syntax/common/attribute_variant.rs | 5 +- pyo3_bindgen_engine/src/syntax/common/path.rs | 2 +- pyo3_bindgen_engine/src/syntax/function.rs | 25 ++--- pyo3_bindgen_engine/src/syntax/module.rs | 9 +- pyo3_bindgen_engine/src/syntax/property.rs | 28 ++--- pyo3_bindgen_engine/src/typing/from_py.rs | 28 ++--- pyo3_bindgen_engine/src/typing/into_rs.rs | 22 ++++ pyo3_bindgen_engine/src/utils/error.rs | 4 + pyo3_bindgen_engine/src/utils/mod.rs | 1 + pyo3_bindgen_engine/src/utils/text.rs | 52 +++++++++ pyo3_bindgen_engine/tests/bindgen.rs | 8 +- pyo3_bindgen_macros/Cargo.toml | 9 +- pyo3_bindgen_macros/build.rs | 4 + pyo3_bindgen_macros/src/lib.rs | 16 ++- pyo3_bindgen_macros/src/utils.rs | 35 ++++++ 30 files changed, 542 insertions(+), 189 deletions(-) create mode 100644 examples/Cargo.toml create mode 100644 examples/build.rs create mode 100644 examples/math.rs create mode 100644 examples/os_sys.rs create mode 100644 examples/pygal.rs create mode 100644 examples/random.rs create mode 100644 pyo3_bindgen_engine/src/utils/text.rs create mode 100644 pyo3_bindgen_macros/build.rs create mode 100644 pyo3_bindgen_macros/src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index de2aa6a..1ccabd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -279,6 +279,14 @@ version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +[[package]] +name = "examples" +version = "0.4.0" +dependencies = [ + "pyo3", + "pyo3_bindgen", +] + [[package]] name = "float-cmp" version = "0.9.0" @@ -567,15 +575,16 @@ dependencies = [ [[package]] name = "pyo3_bindgen" -version = "0.3.1" +version = "0.4.0" dependencies = [ + "pyo3", "pyo3_bindgen_engine", "pyo3_bindgen_macros", ] [[package]] name = "pyo3_bindgen_cli" -version = "0.3.1" +version = "0.4.0" dependencies = [ "assert_cmd", "clap", @@ -587,7 +596,7 @@ dependencies = [ [[package]] name = "pyo3_bindgen_engine" -version = "0.3.1" +version = "0.4.0" dependencies = [ "criterion", "indoc", @@ -605,9 +614,11 @@ dependencies = [ [[package]] name = "pyo3_bindgen_macros" -version = "0.3.1" +version = "0.4.0" dependencies = [ + "libc", "pyo3", + "pyo3-build-config", "pyo3_bindgen_engine", "syn", ] diff --git a/Cargo.toml b/Cargo.toml index caf2e89..8b2d55f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,7 @@ [workspace] members = [ + # Examples of usage + "examples", # Public API "pyo3_bindgen", # CLI tool @@ -9,6 +11,12 @@ members = [ # Procedural macros "pyo3_bindgen_macros", ] +default-members = [ + "pyo3_bindgen", + "pyo3_bindgen_cli", + "pyo3_bindgen_engine", + "pyo3_bindgen_macros", +] resolver = "2" [workspace.package] @@ -20,18 +28,19 @@ license = "MIT OR Apache-2.0" readme = "README.md" repository = "https://github.com/AndrejOrsula/pyo3_bindgen" rust-version = "1.74" -version = "0.3.1" +version = "0.4.0" [workspace.dependencies] -pyo3_bindgen = { path = "pyo3_bindgen", version = "0.3.1" } -pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.3.1" } -pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.3.1" } +pyo3_bindgen = { path = "pyo3_bindgen", version = "0.4.0" } +pyo3_bindgen_engine = { path = "pyo3_bindgen_engine", version = "0.4.0" } +pyo3_bindgen_macros = { path = "pyo3_bindgen_macros", version = "0.4.0" } assert_cmd = { version = "2" } clap = { version = "4.5", features = ["derive"] } criterion = { version = "0.5" } indoc = { version = "2" } itertools = { version = "0.12" } +libc = { version = "0.2" } predicates = { version = "3" } prettyplease = { version = "0.2" } proc-macro2 = { version = "1" } diff --git a/README.md b/README.md index 4ed0a9b..1326929 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ An example of a generated Rust function signature and its intended usage is show ```py   def answer_to(question: str) -> int: - """Returns answer to question.""" + """Returns answer to a question.""" return 42 @@ -44,9 +44,9 @@ if __name__ == "__main__": ```rs -/// Returns answer to question. +/// Returns answer to a question. pub fn answer_to<'py>( - py: ::pyo3::marker::Python<'py>, + py: ::pyo3::Python<'py>, question: &str, ) -> ::pyo3::PyResult { ... // Calls function via `pyo3` @@ -81,91 +81,115 @@ The workspace contains these packages: - **[pyo3_bindgen](pyo3_bindgen):** Public API for generation of bindings (in `build.rs` or via procedural macros) - **[pyo3_bindgen_cli](pyo3_bindgen_cli):** CLI tool for generation of bindings via `pyo3_bindgen` executable - **[pyo3_bindgen_engine](pyo3_bindgen_engine):** The underlying engine for generation of bindings -- **[pyo3_bindgen_macros](pyo3_bindgen_macros):** \[Experimental\] Procedural macros for in-place generation +- **[pyo3_bindgen_macros](pyo3_bindgen_macros):** Procedural macros for in-place generation ## Instructions -Add `pyo3` as a dependency and `pyo3_bindgen` as a build dependency to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest (`auto-initialize` feature of `pyo3` is optional and shown here for your convenience). +### Option 1: Build script + +First, add `pyo3_bindgen` as a **build dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest. To actually use the generated bindings, you will also need to add `pyo3` as a regular dependency (or use the re-exported `pyo3_bindgen::pyo3` module). ```toml +[build-dependencies] +pyo3_bindgen = { version = "0.4" } + [dependencies] pyo3 = { version = "0.20", features = ["auto-initialize"] } - -[build-dependencies] -pyo3_bindgen = { version = "0.3" } ``` -### Option 1: Build script +Then, create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the selected Python modules. In this example, the bindings are simultaneously generated for the "os", "posixpath", and "sys" Python modules. At the end of the generation process, the Rust bindings are written to `${OUT_DIR}/bindings.rs`. -Create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the `py_module` Python module. +> With this approach, you can also customize the generation process via [`pyo3_bindgen::Config`](https://docs.rs/pyo3_bindgen/latest/pyo3_bindgen/struct.Config.html) that can be passed to the constructor, e.g. `Codegen::new(Config::builder().include_private(true).build())`. ```rs -// build.rs -use pyo3_bindgen::{Codegen, Config}; +//! build.rs +use pyo3_bindgen::Codegen; fn main() -> Result<(), Box> { - // Generate Rust bindings to Python modules - Codegen::new(Config::default())? - .module_name("py_module")? - .build(std::path::Path::new(&std::env::var("OUT_DIR")?).join("bindings.rs"))?; + Codegen::default() + .module_names(["os", "posixpath", "sys"])? + .build(format!("{}/bindings.rs", std::env::var("OUT_DIR")?))?; Ok(()) } ``` -Afterwards, include the generated bindings anywhere in your crate. +Afterwards, you can include the generated Rust code via the `include!` macro anywhere in your crate and use the generated bindings as regular Rust modules. However, the bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held. ```rs +//! src/main.rs include!(concat!(env!("OUT_DIR"), "/bindings.rs")); -pub use py_module::*; + +fn main() -> pyo3::PyResult<()> { + pyo3::Python::with_gil(|py| { + // Get the path to the Python executable via "sys" Python module + let python_exe_path = sys::executable(py)?; + // Get the current working directory via "os" Python module + let current_dir = os::getcwd(py)?; + // Get the relative path to the Python executable via "posixpath" Python module + let relpath_to_python_exe = posixpath::relpath(py, python_exe_path, current_dir)?; + + println!("Relative path to Python executable: '{relpath_to_python_exe}'"); + Ok(()) + }) +} ``` -### Option 2: CLI tool +### Option 2: Procedural macros (experimental) -Install the `pyo3_bindgen` executable with `cargo`. +As an alternative to build scripts, you can use procedural macros to generate the bindings in-place. First, add `pyo3_bindgen_macros` as a **regular dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest. -```bash -cargo install --locked pyo3_bindgen_cli +```toml +[dependencies] +pyo3_bindgen = { version = "0.4" } ``` -Afterwards, run the `pyo3_bindgen` executable while passing the name of the target Python module. +Subsequently, the `import_python!` macro can be used to generate Rust bindings for the selected Python modules anywhere in your crate. As demonstrated in the example below, Rust bindings are generated for the "math" Python module and can directly be used in the same scope. Similar to the previous approach, the generated bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held. -```bash -# Pass `--help` to show the usage and available options -pyo3_bindgen -m py_module -o bindings.rs -``` +> As opposed to using build scripts, this approach does not offer the same level of customization via `pyo3_bindgen::Config`. Furthermore, the procedural macro is quite experimental and might not work in all cases. -### Option 3 \[Experimental\]: Procedural macros +```rs +use pyo3_bindgen::import_python; +import_python!("math"); + +// Which Pi do you prefer? +// a) 🐍 Pi from Python "math" module +// b) 🦀 Pi from Rust standard library +// c) 🥧 Pi from your favourite bakery +pyo3::Python::with_gil(|py| { + let python_pi = math::pi(py).unwrap(); + let rust_pi = std::f64::consts::PI; + assert_eq!(python_pi, rust_pi); +}) +``` -> **Note:** This feature is experimental and will probably fail in many cases. It is recommended to use build scripts instead. +### Option 3: CLI tool -Enable the `macros` feature of `pyo3_bindgen`. +For a quick start and testing purposes, you can use the `pyo3_bindgen` executable to generate and inspect bindings for the selected Python modules. The executable is available as a standalone package and can be installed via `cargo`. -```toml -[build-dependencies] -pyo3_bindgen = { version = "0.3", features = ["macros"] } +```bash +cargo install --locked pyo3_bindgen_cli ``` -Then, you can call the `import_python!` macro anywhere in your crate. +Afterwards, run the `pyo3_bindgen` executable to generate Rust bindings for the selected Python modules. The generated bindings are printed to STDOUT by default, but they can also be written to a file via the `-o` option (see `pyo3_bindgen --help` for more options). -```rs -pyo3_bindgen::import_python!("py_module"); -pub use py_module::*; +```bash +pyo3_bindgen -m os sys numpy -o bindings.rs ``` ## Status This project is in early development, and as such, the API of the generated bindings is not yet stable. -- Not all Python types are mapped to their Rust equivalents yet. For this reason, some additional typecasting might be currently required when using the generated bindings (e.g. `let typed_value: py_module::MyClass = get_value()?.extract()?;`). -- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Here are some preliminary results for version `0.3.0` with the default configuration (measured: parsing IO & codegen | not measured: compilation of the generated bindings, which takes much longer): +- The binding generation is primarily designed to be used inside build scripts or via procedural macros. Therefore, the performance of the codegen process is [benchmarked](./pyo3_bindgen_engine/benches/bindgen.rs) to understand the potential impact on build times. Here are some preliminary results for version `0.3` with the default configuration (measured: parsing IO & codegen | not measured: compilation of the generated bindings, which takes much longer): - `sys`: 1.24 ms (0.66k total LoC) - `os`: 8.38 ms (3.88k total LoC) - `numpy`: 1.02 s (294k total LoC) - `torch`: 7.05 s (1.08M total LoC) - The generation of bindings should never panic as long as the target Python module can be successfully imported. If it does, please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as a bug. - The generated bindings should always be compilable and usable in Rust. If you encounter any issues, consider manually fixing the problematic parts of the bindings and please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as a bug. -- However, the generated bindings are based on the introspection of the target Python module. Therefore, the correctness of the generated bindings is directly dependent on the quality of the type annotations and docstrings in the target Python module. Ideally, the generated bindings should be considered unsafe and serve as a starting point for safe and idiomatic Rust APIs. -- Although implemented, the procedural macro does not work in many cases because PyO3 fails to import the target Python module when used from within a `proc_macro` crate. Therefore, it is recommended to use build scripts instead for now. +- However, the generated bindings are based on the introspection of the target Python module. Therefore, the completeness and correctness of the generated bindings are directly dependent on the quality of the module structure, type annotations and docstrings in the target Python module. Ideally, the generated bindings should be considered unsafe and serve as a starting point for safe and idiomatic Rust APIs. If you find that something in the generated bindings is incorrect or missing, please [report](https://github.com/AndrejOrsula/pyo3_bindgen/issues/new) this as well. +- Not all Python types are mapped to their Rust equivalents yet. For this reason, some additional type-casting might be required when using the generated bindings (e.g. `let typed_value: MyType = any_value.extract()?;`). +- Although implemented, the procedural macro might not work in many cases. Therefore, it is recommended that the build scripts be used wherever possible. ## License diff --git a/examples/Cargo.toml b/examples/Cargo.toml new file mode 100644 index 0000000..8aeb35a --- /dev/null +++ b/examples/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "examples" +authors.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +version.workspace = true +publish = false + +[dependencies] +pyo3 = { workspace = true, features = ["auto-initialize"] } +pyo3_bindgen = { workspace = true } + +[build-dependencies] +pyo3_bindgen = { workspace = true } + +[[example]] +name = "math" +path = "math.rs" + +[[example]] +name = "os_sys" +path = "os_sys.rs" + +# [[example]] +# name = "pygal" +# path = "pygal.rs" + +[[example]] +name = "random" +path = "random.rs" diff --git a/examples/build.rs b/examples/build.rs new file mode 100644 index 0000000..d7cee3f --- /dev/null +++ b/examples/build.rs @@ -0,0 +1,8 @@ +use pyo3_bindgen::Codegen; + +fn main() -> Result<(), Box> { + Codegen::default() + .module_names(["os", "posixpath", "sys"])? + .build(format!("{}/bindings.rs", std::env::var("OUT_DIR")?))?; + Ok(()) +} diff --git a/examples/math.rs b/examples/math.rs new file mode 100644 index 0000000..fbf9d21 --- /dev/null +++ b/examples/math.rs @@ -0,0 +1,26 @@ +//! Example demonstrating the use of the `import_python!` macro for the "math" module. +//! +//! Python equivalent: +//! +//! ```py +//! import math +//! +//! python_pi = math.pi +//! assert python_pi == 3.141592653589793 +//! print(f"Python Pi: {python_pi}") +//! ``` + +pyo3_bindgen::import_python!("math"); + +fn main() { + // Which Pi do you prefer? + // a) 🐍 Pi from Python "math" module + // b) 🦀 Pi from Rust standard library + // c) 🥧 Pi from your favorite bakery + pyo3::Python::with_gil(|py| { + let python_pi = math::pi(py).unwrap(); + let rust_pi = std::f64::consts::PI; + assert_eq!(python_pi, rust_pi); + println!("Python Pi: {}", python_pi); + }) +} diff --git a/examples/os_sys.rs b/examples/os_sys.rs new file mode 100644 index 0000000..1242c7b --- /dev/null +++ b/examples/os_sys.rs @@ -0,0 +1,34 @@ +//! Example demonstrating the use of the `pyo3_bindgen` crate via build script for +//! the "os", "posixpath", and "sys" Python modules. +//! +//! See `build.rs` for more details about the generation. +//! +//! Python equivalent: +//! +//! ```py +//! import os +//! import posixpath +//! import sys +//! +//! python_exe_path = sys.executable +//! current_dir = os.getcwd() +//! relpath_to_python_exe = posixpath.relpath(python_exe_path, current_dir) +//! +//! print(f"Relative path to Python executable: '{relpath_to_python_exe}'") +//! ``` + +include!(concat!(env!("OUT_DIR"), "/bindings.rs")); + +fn main() -> pyo3::PyResult<()> { + pyo3::Python::with_gil(|py| { + // Get the path to the Python executable via "sys" Python module + let python_exe_path = sys::executable(py)?; + // Get the current working directory via "os" Python module + let current_dir = os::getcwd(py)?; + // Get the relative path to the Python executable via "posixpath" Python module + let relpath_to_python_exe = posixpath::relpath(py, python_exe_path, current_dir)?; + + println!("Relative path to Python executable: '{relpath_to_python_exe}'"); + Ok(()) + }) +} diff --git a/examples/pygal.rs b/examples/pygal.rs new file mode 100644 index 0000000..10ef64a --- /dev/null +++ b/examples/pygal.rs @@ -0,0 +1,33 @@ +//! Example demonstrating the use of the `import_python!` macro for the "pygal" module. +//! +//! Python equivalent: +//! +//! ```py +//! import pygal +//! +//! bar_chart = pygal.Bar(style=pygal.style.DarkStyle) +//! bar_chart.add("Fibonacci", [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55]) +//! bar_chart.add("Padovan", [1, 1, 1, 2, 2, 3, 4, 5, 7, 9, 12]) +//! svg = bar_chart.render(false) +//! print(svg) +//! ``` + +pyo3_bindgen::import_python!("pygal"); + +fn main() -> pyo3::PyResult<()> { + pyo3::Python::with_gil(|py| { + let bar_chart = pygal::Bar::new( + py, + (), + Some(pyo3::types::IntoPyDict::into_py_dict( + [("style", pygal::style::DarkStyle::new(py, None)?)], + py, + )), + )?; + bar_chart.add(py, "Fibonacci", [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55], None)?; + bar_chart.add(py, "Padovan", [1, 1, 1, 2, 2, 3, 4, 5, 7, 9, 12], None)?; + let svg = bar_chart.render(py, false, None)?; + println!("{svg}"); + Ok(()) + }) +} diff --git a/examples/random.rs b/examples/random.rs new file mode 100644 index 0000000..4903e2c --- /dev/null +++ b/examples/random.rs @@ -0,0 +1,22 @@ +//! Example demonstrating the use of the `import_python!` macro for the "random" module. +//! +//! Python equivalent: +//! +//! ```py +//! import random +//! +//! rand_f64 = random.random() +//! assert 0.0 <= rand_f64 <= 1.0 +//! print(f"Random f64: {rand_f64}") +//! ``` + +pyo3_bindgen::import_python!("random"); + +fn main() -> pyo3::PyResult<()> { + pyo3::Python::with_gil(|py| { + let rand_f64: f64 = random::random(py)?.extract()?; + assert!((0.0..=1.0).contains(&rand_f64)); + println!("Random f64: {}", rand_f64); + Ok(()) + }) +} diff --git a/pyo3_bindgen/Cargo.toml b/pyo3_bindgen/Cargo.toml index 7d286af..06cf217 100644 --- a/pyo3_bindgen/Cargo.toml +++ b/pyo3_bindgen/Cargo.toml @@ -12,14 +12,18 @@ rust-version.workspace = true version.workspace = true [dependencies] +pyo3 = { workspace = true } pyo3_bindgen_engine = { workspace = true } pyo3_bindgen_macros = { workspace = true, optional = true } [features] -default = [] +default = ["macros"] macros = ["pyo3_bindgen_macros"] [lib] name = "pyo3_bindgen" path = "src/lib.rs" crate-type = ["rlib"] + +[package.metadata.docs.rs] +all-features = true diff --git a/pyo3_bindgen/src/lib.rs b/pyo3_bindgen/src/lib.rs index 44fc354..973d693 100644 --- a/pyo3_bindgen/src/lib.rs +++ b/pyo3_bindgen/src/lib.rs @@ -10,75 +10,104 @@ //! //! ## Instructions //! -//! Add `pyo3` as a dependency and `pyo3_bindgen` as a build dependency to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest (`auto-initialize` feature of `pyo3` is optional and shown here for your convenience). +//! ### Option 1: Build script +//! +//! First, add `pyo3_bindgen` as a **build dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest. To actually use the generated bindings, you will also need to add `pyo3` as a regular dependency (or use the re-exported `pyo3_bindgen::pyo3` module). //! //! ```toml +//! [build-dependencies] +//! pyo3_bindgen = { version = "0.4" } +//! //! [dependencies] //! pyo3 = { version = "0.20", features = ["auto-initialize"] } -//! -//! [build-dependencies] -//! pyo3_bindgen = { version = "0.3" } //! ``` //! -//! ### Option 1: Build script +//! Then, create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the selected Python modules. In this example, the bindings are simultaneously generated for the "os", "posixpath", and "sys" Python modules. At the end of the generation process, the Rust bindings are written to `${OUT_DIR}/bindings.rs`. //! -//! Create a [`build.rs`](https://doc.rust-lang.org/cargo/reference/build-scripts.html) script in the root of your crate that generates bindings to the `py_module` Python module. +//! > With this approach, you can also customize the generation process via [`pyo3_bindgen::Config`](https://docs.rs/pyo3_bindgen/latest/pyo3_bindgen/struct.Config.html) that can be passed to `Codegen::new` constructor, e.g. `Codegen::new(Config::builder().include_private(true).build())`. //! //! ```no_run -//! // build.rs -//! use pyo3_bindgen::{Codegen, Config}; +//! //! build.rs +//! use pyo3_bindgen::Codegen; //! //! fn main() -> Result<(), Box> { -//! // Generate Rust bindings to Python modules -//! Codegen::new(Config::default())? -//! .module_name("py_module")? -//! .build(std::path::Path::new(&std::env::var("OUT_DIR")?).join("bindings.rs"))?; +//! Codegen::default() +//! .module_names(["os", "posixpath", "sys"])? +//! .build(format!("{}/bindings.rs", std::env::var("OUT_DIR")?))?; //! Ok(()) //! } //! ``` //! -//! Afterwards, include the generated bindings anywhere in your crate. +//! Afterwards, you can include the generated Rust code via the `include!` macro anywhere in your crate and use the generated bindings as regular Rust modules. However, the bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held. //! //! ```ignore +//! //! src/main.rs //! include!(concat!(env!("OUT_DIR"), "/bindings.rs")); -//! pub use py_module::*; +//! +//! fn main() -> pyo3::PyResult<()> { +//! # pyo3::prepare_freethreaded_python(); +//! pyo3::Python::with_gil(|py| { +//! // Get the path to the Python executable via "sys" Python module +//! let python_exe_path = sys::executable(py)?; +//! // Get the current working directory via "os" Python module +//! let current_dir = os::getcwd(py)?; +//! // Get the relative path to the Python executable via "posixpath" Python module +//! let relpath_to_python_exe = posixpath::relpath(py, python_exe_path, current_dir)?; +//! +//! println!("Relative path to Python executable: '{relpath_to_python_exe}'"); +//! Ok(()) +//! }) +//! } //! ``` //! -//! ### Option 2: CLI tool +//! ### Option 2: Procedural macros (experimental) //! -//! Install the `pyo3_bindgen` executable with `cargo`. +//! As an alternative to build scripts, you can use procedural macros to generate the bindings in-place. First, add `pyo3_bindgen_macros` as a **regular dependency** to your [`Cargo.toml`](https://doc.rust-lang.org/cargo/reference/manifest.html) manifest. //! -//! ```bash -//! cargo install --locked pyo3_bindgen_cli +//! ```toml +//! [dependencies] +//! pyo3_bindgen = { version = "0.4" } //! ``` //! -//! Afterwards, run the `pyo3_bindgen` executable while passing the name of the target Python module. +//! Subsequently, the `import_python!` macro can be used to generate Rust bindings for the selected Python modules anywhere in your crate. As demonstrated in the example below, Rust bindings are generated for the "math" Python module and can directly be used in the same scope. Similar to the previous approach, the generated bindings must be used within the `pyo3::Python::with_gil` closure to ensure that Python [GIL](https://wiki.python.org/moin/GlobalInterpreterLock) is held. //! -//! ```bash -//! # Pass `--help` to show the usage and available options -//! pyo3_bindgen -m py_module -o bindings.rs -//! ``` +//! > As opposed to using build scripts, this approach does not offer the same level of customization via `pyo3_bindgen::Config`. Furthermore, the procedural macro is quite experimental and might not work in all cases. //! -//! ### Option 3 \[Experimental\]: Procedural macros +//! ``` +//! use pyo3_bindgen::import_python; +//! import_python!("math"); +//! +//! // Which Pi do you prefer? +//! // a) 🐍 Pi from Python "math" module +//! // b) 🦀 Pi from Rust standard library +//! // c) 🥧 Pi from your favourite bakery +//! # pyo3::prepare_freethreaded_python(); +//! pyo3::Python::with_gil(|py| { +//! let python_pi = math::pi(py).unwrap(); +//! let rust_pi = std::f64::consts::PI; +//! assert_eq!(python_pi, rust_pi); +//! }) +//! ``` //! -//! > **Note:** This feature is experimental and will probably fail in many cases. It is recommended to use build scripts instead. +//! ### Option 3: CLI tool //! -//! Enable the `macros` feature of `pyo3_bindgen`. +//! For a quick start and testing purposes, you can use the `pyo3_bindgen` executable to generate and inspect bindings for the selected Python modules. The executable is available as a standalone package and can be installed via `cargo`. //! -//! ```toml -//! [build-dependencies] -//! pyo3_bindgen = { version = "0.3", features = ["macros"] } +//! ```bash +//! cargo install --locked pyo3_bindgen_cli //! ``` //! -//! Then, you can call the `import_python!` macro anywhere in your crate. +//! Afterwards, run the `pyo3_bindgen` executable to generate Rust bindings for the selected Python modules. The generated bindings are printed to STDOUT by default, but they can also be written to a file via the `-o` option (see `pyo3_bindgen --help` for more options). //! -//! ```ignore -//! pyo3_bindgen::import_python!("py_module"); -//! pub use py_module::*; +//! ```bash +//! pyo3_bindgen -m os sys numpy -o bindings.rs //! ``` +// Public re-export of PyO3 for convenience +pub use pyo3; + // Public API re-exports from engine -pub use pyo3_bindgen_engine::{pyo3, Codegen, Config, PyBindgenError, PyBindgenResult}; +pub use pyo3_bindgen_engine::{Codegen, Config, PyBindgenError, PyBindgenResult}; // Public API re-exports from macros #[cfg(feature = "macros")] diff --git a/pyo3_bindgen_engine/src/codegen.rs b/pyo3_bindgen_engine/src/codegen.rs index 7b641c5..a7757a2 100644 --- a/pyo3_bindgen_engine/src/codegen.rs +++ b/pyo3_bindgen_engine/src/codegen.rs @@ -1,6 +1,6 @@ use crate::{ syntax::{Ident, Import, Module, Path}, - Config, Result, + Config, PyBindgenError, Result, }; use itertools::Itertools; use rustc_hash::FxHashSet as HashSet; @@ -14,10 +14,10 @@ use rustc_hash::FxHashSet as HashSet; /// default configuration, all submodules, classes, functions, and parameters /// will be recursively parsed and included in the generated bindings. /// -/// ```no_run +/// ``` /// # use pyo3_bindgen_engine::{Codegen, Config}; /// fn main() -> Result<(), Box> { -/// Codegen::new(Config::default())? +/// Codegen::new(Config::default()) /// .module_name("os")? /// .module_name("sys")? /// .generate()?; @@ -31,10 +31,10 @@ use rustc_hash::FxHashSet as HashSet; /// respective submodules, classes, functions, and parameters. No direct attributes /// or submodules of the `html` top-level module will be included. /// -/// ```no_run -/// # use pyo3_bindgen_engine::{Codegen, Config}; +/// ``` +/// # use pyo3_bindgen_engine::Codegen; /// fn main() -> Result<(), Box> { -/// Codegen::new(Config::default())? +/// Codegen::default() /// .module_names(&["html.entities", "html.parser"])? /// .generate()?; /// Ok(()) @@ -48,11 +48,12 @@ pub struct Codegen { impl Codegen { /// Create a new `Codegen` engine with the given configuration. - pub fn new(cfg: Config) -> Result { - Ok(Self { + #[must_use] + pub fn new(cfg: Config) -> Self { + Self { cfg, ..Default::default() - }) + } } /// Add a Python module to the list of modules for which to generate bindings. @@ -95,7 +96,11 @@ impl Codegen { } /// Add multiple Python modules to the list of modules for which to generate bindings. - pub fn modules(mut self, modules: &[&pyo3::types::PyModule]) -> Result { + pub fn modules<'py>( + mut self, + modules: impl AsRef<[&'py pyo3::types::PyModule]>, + ) -> Result { + let modules = modules.as_ref(); self.modules.reserve(modules.len()); for module in modules { self = self.module(module)?; @@ -104,7 +109,8 @@ impl Codegen { } /// Add multiple Python modules by their names to the list of modules for which to generate bindings. - pub fn module_names(mut self, module_names: &[&str]) -> Result { + pub fn module_names<'a>(mut self, module_names: impl AsRef<[&'a str]>) -> Result { + let module_names = module_names.as_ref(); self.modules.reserve(module_names.len()); for module_name in module_names { self = self.module_name(module_name)?; @@ -114,10 +120,11 @@ impl Codegen { /// Generate the Rust FFI bindings for all modules added to the engine. pub fn generate(mut self) -> Result { - assert!( - !self.modules.is_empty(), - "There are no modules for which to generate bindings" - ); + if self.modules.is_empty() { + return Err(PyBindgenError::CodegenError( + "There are no modules for which to generate bindings".to_string(), + )); + } // Parse external modules (if enabled) if self.cfg.generate_dependencies { diff --git a/pyo3_bindgen_engine/src/config.rs b/pyo3_bindgen_engine/src/config.rs index 3bc7d1f..cd24b9c 100644 --- a/pyo3_bindgen_engine/src/config.rs +++ b/pyo3_bindgen_engine/src/config.rs @@ -1,15 +1,16 @@ use crate::syntax::{Ident, Path}; /// Array of forbidden attribute names that are reserved for internal use by derived traits -pub const FORBIDDEN_FUNCTION_NAMES: [&str; 4] = ["get_type", "obj", "repr", "str"]; +pub const FORBIDDEN_FUNCTION_NAMES: [&str; 5] = ["get_type", "obj", "py", "repr", "str"]; /// Array of forbidden type names -pub const FORBIDDEN_TYPE_NAMES: [&str; 6] = [ +pub const FORBIDDEN_TYPE_NAMES: [&str; 7] = [ "_collections._tuplegetter", "AsyncState", "getset_descriptor", "member_descriptor", "method_descriptor", "property", + "py", ]; /// Default array of blocklisted attribute names @@ -20,51 +21,51 @@ const DEFAULT_BLOCKLIST_ATTRIBUTE_NAMES: [&str; 4] = ["builtins", "testing", "te pub struct Config { /// Flag that determines whether to recursively generate code for all submodules of the target modules. #[builder(default = true)] - pub(crate) traverse_submodules: bool, + pub traverse_submodules: bool, /// Flag that determines whether to generate code for prelude modules (Python `__all__` attribute). #[builder(default = true)] - pub(crate) generate_preludes: bool, + pub generate_preludes: bool, /// Flag that determines whether to generate code for imports. #[builder(default = true)] - pub(crate) generate_imports: bool, + pub generate_imports: bool, /// Flag that determines whether to generate code for classes. #[builder(default = true)] - pub(crate) generate_classes: bool, + pub generate_classes: bool, /// Flag that determines whether to generate code for type variables. #[builder(default = true)] - pub(crate) generate_type_vars: bool, + pub generate_type_vars: bool, /// Flag that determines whether to generate code for functions. #[builder(default = true)] - pub(crate) generate_functions: bool, + pub generate_functions: bool, /// Flag that determines whether to generate code for properties. #[builder(default = true)] - pub(crate) generate_properties: bool, + pub generate_properties: bool, /// Flag that determines whether to documentation for the generate code. /// The documentation is based on Python docstrings. #[builder(default = true)] - pub(crate) generate_docs: bool, + pub generate_docs: bool, /// List of blocklisted attribute names that are skipped during the code generation. #[builder(default = DEFAULT_BLOCKLIST_ATTRIBUTE_NAMES.iter().map(|&s| s.to_string()).collect())] - pub(crate) blocklist_names: Vec, + pub blocklist_names: Vec, /// Flag that determines whether private attributes are considered while parsing the Python code. #[builder(default = false)] - pub(crate) include_private: bool, + pub include_private: bool, /// Flag that determines whether to generate code for all dependencies of the target modules. /// The list of dependent modules is derived from the imports of the target modules. /// /// Warning: This feature is not fully supported yet. #[builder(default = false)] - pub(crate) generate_dependencies: bool, + pub generate_dependencies: bool, /// Flag that suppresses the generation of Python STDOUT while parsing the Python code. #[builder(default = true)] - pub(crate) suppress_python_stdout: bool, + pub suppress_python_stdout: bool, /// Flag that suppresses the generation of Python STDERR while parsing the Python code. #[builder(default = true)] - pub(crate) suppress_python_stderr: bool, + pub suppress_python_stderr: bool, } impl Default for Config { @@ -78,7 +79,7 @@ impl Config { &self, attr_name: &Ident, attr_module: &Path, - attr_type: &pyo3::types::PyType, + _attr_type: &pyo3::types::PyType, ) -> bool { if // Skip always forbidden attribute names @@ -91,8 +92,6 @@ impl Config { self.blocklist_names.iter().any(|blocklist_match| { attr_name.as_py() == blocklist_match }) || - // Skip builtin functions - attr_type.is_subclass_of::().unwrap_or(false) || // Skip `__future__` attributes attr_module.iter().any(|segment| segment.as_py() == "__future__") || // Skip `typing` attributes diff --git a/pyo3_bindgen_engine/src/lib.rs b/pyo3_bindgen_engine/src/lib.rs index 4ccbcb4..4f7f3ce 100644 --- a/pyo3_bindgen_engine/src/lib.rs +++ b/pyo3_bindgen_engine/src/lib.rs @@ -14,6 +14,3 @@ use utils::result::Result; pub use codegen::Codegen; pub use config::Config; pub use utils::{error::PyBindgenError, result::PyBindgenResult}; - -// Public re-export of PyO3 for convenience -pub use pyo3; diff --git a/pyo3_bindgen_engine/src/syntax/class.rs b/pyo3_bindgen_engine/src/syntax/class.rs index 95ff3fe..015f3cf 100644 --- a/pyo3_bindgen_engine/src/syntax/class.rs +++ b/pyo3_bindgen_engine/src/syntax/class.rs @@ -145,13 +145,8 @@ impl Class { // Documentation if cfg.generate_docs { - if let Some(docstring) = &self.docstring { - // Trim the docstring and add a leading whitespace (looks better in the generated code) - let mut docstring = docstring.trim().trim_end_matches('/').to_owned(); - docstring.insert(0, ' '); - // Replace all double quotes with single quotes - docstring = docstring.replace('"', "'"); - + if let Some(mut docstring) = self.docstring.clone() { + crate::utils::text::format_docstring(&mut docstring); output.extend(quote::quote! { #[doc = #docstring] }); diff --git a/pyo3_bindgen_engine/src/syntax/common/attribute_variant.rs b/pyo3_bindgen_engine/src/syntax/common/attribute_variant.rs index 70771fa..ccdd722 100644 --- a/pyo3_bindgen_engine/src/syntax/common/attribute_variant.rs +++ b/pyo3_bindgen_engine/src/syntax/common/attribute_variant.rs @@ -41,6 +41,9 @@ impl AttributeVariant { let is_class = attr_type .is_subclass_of::() .unwrap_or(false); + let is_builtin_function = attr_type + .is_subclass_of::() + .unwrap_or(false); let is_function = inspect .call_method1(pyo3::intern!(py, "isfunction"), (attr,))? .is_true()?; @@ -64,7 +67,7 @@ impl AttributeVariant { AttributeVariant::Module } else if is_class { AttributeVariant::Class - } else if is_function { + } else if is_builtin_function || is_function { AttributeVariant::Function } else if is_method { AttributeVariant::Method diff --git a/pyo3_bindgen_engine/src/syntax/common/path.rs b/pyo3_bindgen_engine/src/syntax/common/path.rs index 175df8a..957bb4a 100644 --- a/pyo3_bindgen_engine/src/syntax/common/path.rs +++ b/pyo3_bindgen_engine/src/syntax/common/path.rs @@ -194,7 +194,7 @@ impl Path { } } - pub fn import_quote(&self, py: pyo3::marker::Python) -> proc_macro2::TokenStream { + pub fn import_quote(&self, py: pyo3::Python) -> proc_macro2::TokenStream { // Find the last package and import it via py.import, then get the rest of the path via getattr() let mut package_path = self.root().unwrap_or_else(|| unreachable!()); for i in (1..self.len()).rev() { diff --git a/pyo3_bindgen_engine/src/syntax/function.rs b/pyo3_bindgen_engine/src/syntax/function.rs index 93ef5d2..73f2ba4 100644 --- a/pyo3_bindgen_engine/src/syntax/function.rs +++ b/pyo3_bindgen_engine/src/syntax/function.rs @@ -69,10 +69,10 @@ impl Function { ); let annotation = match kind { ParameterKind::VarPositional => Type::PyTuple(vec![Type::Unknown]), - ParameterKind::VarKeyword => Type::PyDict { + ParameterKind::VarKeyword => Type::Optional(Box::new(Type::PyDict { key_type: Box::new(Type::Unknown), value_type: Box::new(Type::Unknown), - }, + })), _ => { let annotation = param.getattr(pyo3::intern!(py, "annotation"))?; if annotation.is(param.getattr(pyo3::intern!(py, "empty"))?) { @@ -215,10 +215,10 @@ impl Function { Parameter { name: Ident::from_rs("kwargs"), kind: ParameterKind::VarKeyword, - annotation: Type::PyDict { + annotation: Type::Optional(Box::new(Type::PyDict { key_type: Box::new(Type::Unknown), value_type: Box::new(Type::Unknown), - }, + })), default: None, }, ]; @@ -273,10 +273,10 @@ impl Function { Parameter { name: Ident::from_rs("kwargs"), kind: ParameterKind::VarKeyword, - annotation: Type::PyDict { + annotation: Type::Optional(Box::new(Type::PyDict { key_type: Box::new(Type::Unknown), value_type: Box::new(Type::Unknown), - }, + })), default: None, }, ]; @@ -303,10 +303,10 @@ impl Function { Parameter { name: Ident::from_rs("kwargs"), kind: ParameterKind::VarKeyword, - annotation: Type::PyDict { + annotation: Type::Optional(Box::new(Type::PyDict { key_type: Box::new(Type::Unknown), value_type: Box::new(Type::Unknown), - }, + })), default: None, }, ], @@ -326,13 +326,8 @@ impl Function { // Documentation if cfg.generate_docs { - if let Some(docstring) = &self.docstring { - // Trim the docstring and add a leading whitespace (looks better in the generated code) - let mut docstring = docstring.trim().trim_end_matches('/').to_owned(); - docstring.insert(0, ' '); - // Replace all double quotes with single quotes - docstring = docstring.replace('"', "'"); - + if let Some(mut docstring) = self.docstring.clone() { + crate::utils::text::format_docstring(&mut docstring); output.extend(quote::quote! { #[doc = #docstring] }); diff --git a/pyo3_bindgen_engine/src/syntax/module.rs b/pyo3_bindgen_engine/src/syntax/module.rs index 038cbb7..629a8e7 100644 --- a/pyo3_bindgen_engine/src/syntax/module.rs +++ b/pyo3_bindgen_engine/src/syntax/module.rs @@ -333,13 +333,8 @@ impl Module { // Documentation if cfg.generate_docs { - if let Some(docstring) = &self.docstring { - // Trim the docstring and add a leading whitespace (looks better in the generated code) - let mut docstring = docstring.trim().trim_end_matches('/').to_owned(); - docstring.insert(0, ' '); - // Replace all double quotes with single quotes - docstring = docstring.replace('"', "'"); - + if let Some(mut docstring) = self.docstring.clone() { + crate::utils::text::format_docstring(&mut docstring); output.extend(quote::quote! { #[doc = #docstring] }); diff --git a/pyo3_bindgen_engine/src/syntax/property.rs b/pyo3_bindgen_engine/src/syntax/property.rs index c564504..419cb12 100644 --- a/pyo3_bindgen_engine/src/syntax/property.rs +++ b/pyo3_bindgen_engine/src/syntax/property.rs @@ -173,13 +173,8 @@ impl Property { // Documentation if cfg.generate_docs { - if let Some(docstring) = &self.docstring { - // Trim the docstring and add a leading whitespace (looks better in the generated code) - let mut docstring = docstring.trim().trim_end_matches('/').to_owned(); - docstring.insert(0, ' '); - // Replace all double quotes with single quotes - docstring = docstring.replace('"', "'"); - + if let Some(mut docstring) = self.docstring.clone() { + crate::utils::text::format_docstring(&mut docstring); output.extend(quote::quote! { #[doc = #docstring] }); @@ -255,23 +250,20 @@ impl Property { pub fn generate_setter( &self, - _cfg: &Config, + cfg: &Config, scoped_function_idents: &[&Ident], local_types: &HashMap, ) -> Result { let mut output = proc_macro2::TokenStream::new(); // Documentation - if let Some(docstring) = &self.setter_docstring { - // Trim the docstring and add a leading whitespace (looks better in the generated code) - let mut docstring = docstring.trim().trim_end_matches('/').to_owned(); - docstring.insert(0, ' '); - // Replace all double quotes with single quotes - docstring = docstring.replace('"', "'"); - - output.extend(quote::quote! { - #[doc = #docstring] - }); + if cfg.generate_docs { + if let Some(mut docstring) = self.setter_docstring.clone() { + crate::utils::text::format_docstring(&mut docstring); + output.extend(quote::quote! { + #[doc = #docstring] + }); + } } // Function diff --git a/pyo3_bindgen_engine/src/typing/from_py.rs b/pyo3_bindgen_engine/src/typing/from_py.rs index 357ae4a..5519ffe 100644 --- a/pyo3_bindgen_engine/src/typing/from_py.rs +++ b/pyo3_bindgen_engine/src/typing/from_py.rs @@ -140,31 +140,31 @@ impl Type { } } Self::Optional(..) => { - debug_assert_eq!(inner_types.len(), 1); + // debug_assert_eq!(inner_types.len(), 1); Self::Optional(Box::new(inner_types[0].clone())) } Self::PyDict { .. } => { - debug_assert_eq!(inner_types.len(), 2); + // debug_assert_eq!(inner_types.len(), 2); Self::PyDict { key_type: Box::new(inner_types[0].clone()), value_type: Box::new(inner_types[1].clone()), } } Self::PyFrozenSet(..) => { - debug_assert_eq!(inner_types.len(), 1); + // debug_assert_eq!(inner_types.len(), 1); Self::PyFrozenSet(Box::new(inner_types[0].clone())) } Self::PyList(..) => { - debug_assert_eq!(inner_types.len(), 1); + // debug_assert_eq!(inner_types.len(), 1); Self::PyList(Box::new(inner_types[0].clone())) } Self::PySet(..) => { - debug_assert_eq!(inner_types.len(), 1); + // debug_assert_eq!(inner_types.len(), 1); Self::PySet(Box::new(inner_types[0].clone())) } Self::PyTuple(..) => Self::PyTuple(inner_types), Self::PyFunction { .. } => { - debug_assert!(!inner_types.is_empty()); + // debug_assert!(!inner_types.is_empty()); Self::PyFunction { param_types: match inner_types.len() { 1 => Vec::default(), @@ -179,7 +179,7 @@ impl Type { } } Self::PyType => { - debug_assert_eq!(inner_types.len(), 1); + // debug_assert_eq!(inner_types.len(), 1); inner_types[0].clone() } _ => { @@ -252,7 +252,7 @@ impl std::str::FromStr for Type { .map(|x| x.trim().to_owned()) .collect_vec(); repair_complex_sequence(&mut inner_types, ','); - debug_assert_eq!(inner_types.len(), 2); + // debug_assert_eq!(inner_types.len(), 2); let inner_types = inner_types .iter() .map(|x| Self::from_str(x)) @@ -386,7 +386,7 @@ impl std::str::FromStr for Type { .map(|x| x.trim().to_owned()) .collect_vec(); repair_complex_sequence(&mut inner_types, ','); - debug_assert!(!inner_types.is_empty()); + // debug_assert!(!inner_types.is_empty()); let inner_types = inner_types .iter() .map(|x| Self::from_str(x)) @@ -469,11 +469,11 @@ impl std::str::FromStr for Type { // TODO: Refactor `repair_complex_sequence()` into something more sensible /// Repairs complex wrapped sequences. fn repair_complex_sequence(sequence: &mut Vec, separator: char) { - debug_assert!(!sequence.is_empty()); - debug_assert!({ - let merged_sequence = sequence.iter().join(""); - merged_sequence.matches('[').count() == merged_sequence.matches(']').count() - }); + // debug_assert!(!sequence.is_empty()); + // debug_assert!({ + // let merged_sequence = sequence.iter().join(""); + // merged_sequence.matches('[').count() == merged_sequence.matches(']').count() + // }); let mut traversed_all_elements = false; let mut start_index = 0; diff --git a/pyo3_bindgen_engine/src/typing/into_rs.rs b/pyo3_bindgen_engine/src/typing/into_rs.rs index 93d0e35..bef02ab 100644 --- a/pyo3_bindgen_engine/src/typing/into_rs.rs +++ b/pyo3_bindgen_engine/src/typing/into_rs.rs @@ -70,6 +70,28 @@ impl Type { let #ident = #ident.as_ref(py); } } + Self::Optional(inner_type) => match inner_type.as_ref() { + Self::PyDict { + key_type, + value_type, + } if !key_type.is_hashable() + || value_type + .clone() + .into_rs(local_types) + .owned + .to_string() + .contains("PyAny") => + { + quote! { + let #ident = if let Some(#ident) = #ident { + ::pyo3::types::IntoPyDict::into_py_dict(#ident, py) + } else { + ::pyo3::types::PyDict::new(py) + }; + } + } + _ => proc_macro2::TokenStream::new(), + }, _ => proc_macro2::TokenStream::new(), } } diff --git a/pyo3_bindgen_engine/src/utils/error.rs b/pyo3_bindgen_engine/src/utils/error.rs index f1f445e..d9b4d84 100644 --- a/pyo3_bindgen_engine/src/utils/error.rs +++ b/pyo3_bindgen_engine/src/utils/error.rs @@ -9,6 +9,10 @@ pub enum PyBindgenError { PyDowncastError, #[error(transparent)] SynError(#[from] syn::Error), + #[error("Failed to parse Python code: {0}")] + ParseError(String), + #[error("Failed to generate Rust code: {0}")] + CodegenError(String), } impl From> for PyBindgenError { diff --git a/pyo3_bindgen_engine/src/utils/mod.rs b/pyo3_bindgen_engine/src/utils/mod.rs index 09821bb..2620963 100644 --- a/pyo3_bindgen_engine/src/utils/mod.rs +++ b/pyo3_bindgen_engine/src/utils/mod.rs @@ -3,3 +3,4 @@ pub mod error; pub(crate) mod io; pub mod result; +pub(crate) mod text; diff --git a/pyo3_bindgen_engine/src/utils/text.rs b/pyo3_bindgen_engine/src/utils/text.rs new file mode 100644 index 0000000..e381431 --- /dev/null +++ b/pyo3_bindgen_engine/src/utils/text.rs @@ -0,0 +1,52 @@ +/// Sanitize and format the given docstring. +pub fn format_docstring(docstring: &mut String) { + // Remove leading and trailing whitespace for each line + *docstring = docstring + .lines() + .map(str::trim) + .collect::>() + .join("\n"); + + // Remove trailing slashes + while docstring.ends_with('/') { + docstring.pop(); + docstring.truncate(docstring.trim_end().len()); + } + + // Remove duplicate whitespace characters (except line breaks) + conditioned_dedup(docstring, |c| c.is_whitespace() && c != '\n'); + + // Remove duplicate backticks to avoid potential doctests + conditioned_dedup(docstring, |c| c == '`'); + + // If the docstring has multiple lines, make sure it is properly formatted + if docstring.contains('\n') { + // Make sure the first line is not empty + while docstring.starts_with('\n') { + docstring.remove(0); + } + // Make sure it ends with a single newline + if docstring.ends_with('\n') { + while docstring.ends_with("\n\n") { + docstring.pop(); + } + } else { + docstring.push('\n'); + } + } + // Pad the docstring with a leading whitespace (looks better in the generated code) + docstring.insert(0, ' '); +} + +/// Remove duplicate characters from the input string that satisfy the given predicate. +fn conditioned_dedup(input: &mut String, mut predicate: impl FnMut(char) -> bool) { + let mut previous = None; + input.retain(|c| { + if predicate(c) { + Some(c) != std::mem::replace(&mut previous, Some(c)) + } else { + previous = None; + true + } + }); +} diff --git a/pyo3_bindgen_engine/tests/bindgen.rs b/pyo3_bindgen_engine/tests/bindgen.rs index d65e682..4eb6c15 100644 --- a/pyo3_bindgen_engine/tests/bindgen.rs +++ b/pyo3_bindgen_engine/tests/bindgen.rs @@ -181,9 +181,13 @@ test_bindgen! { &'py self, py: ::pyo3::marker::Python<'py>, p_my_arg1: &::std::collections::HashMap<::std::string::String, i64>, - p_kwargs: impl ::pyo3::types::IntoPyDict, + p_kwargs: ::std::option::Option<&'py ::pyo3::types::PyDict>, ) -> ::pyo3::PyResult<&'py ::pyo3::types::PyAny> { - let p_kwargs = ::pyo3::types::IntoPyDict::into_py_dict(p_kwargs, py); + let p_kwargs = if let Some(p_kwargs) = p_kwargs { + ::pyo3::types::IntoPyDict::into_py_dict(p_kwargs, py) + } else { + ::pyo3::types::PyDict::new(py) + }; ::pyo3::FromPyObject::extract(self.0.call_method( ::pyo3::intern!(py, "my_method"), ::pyo3::types::PyTuple::new(py, [::pyo3::ToPyObject::to_object(&p_my_arg1, py)]), diff --git a/pyo3_bindgen_macros/Cargo.toml b/pyo3_bindgen_macros/Cargo.toml index 904f1fa..a8567e2 100644 --- a/pyo3_bindgen_macros/Cargo.toml +++ b/pyo3_bindgen_macros/Cargo.toml @@ -15,8 +15,15 @@ version.workspace = true pyo3_bindgen_engine = { workspace = true } syn = { workspace = true } -[dev-dependencies] +[target.'cfg(unix)'.dependencies] +libc = { workspace = true } pyo3 = { workspace = true } +[target.'cfg(not(unix))'.dev-dependencies] +pyo3 = { workspace = true } + +[build-dependencies] +pyo3-build-config = { workspace = true } + [lib] proc-macro = true diff --git a/pyo3_bindgen_macros/build.rs b/pyo3_bindgen_macros/build.rs new file mode 100644 index 0000000..16dc4f4 --- /dev/null +++ b/pyo3_bindgen_macros/build.rs @@ -0,0 +1,4 @@ +fn main() { + // Expose #[cfg] flags of pyo3 + pyo3_build_config::use_pyo3_cfgs(); +} diff --git a/pyo3_bindgen_macros/src/lib.rs b/pyo3_bindgen_macros/src/lib.rs index caa40d4..58e018a 100644 --- a/pyo3_bindgen_macros/src/lib.rs +++ b/pyo3_bindgen_macros/src/lib.rs @@ -1,6 +1,7 @@ //! Procedural macros for automatic generation of Rust FFI bindings to Python modules. mod parser; +mod utils; /// Procedural macro for generating Rust bindings to Python modules in-place. /// @@ -12,7 +13,7 @@ mod parser; /// /// Here is a simple example of how to use the macro to generate bindings for the `sys` module. /// -/// ```ignore +/// ``` /// # use pyo3_bindgen_macros::import_python; /// import_python!("sys"); /// pub use sys::*; @@ -20,7 +21,7 @@ mod parser; /// /// For consistency, the top-level package is always included in the generated bindings. /// -/// ```ignore +/// ``` /// # use pyo3_bindgen_macros::import_python; /// import_python!("html.parser"); /// pub use html::parser::*; @@ -28,15 +29,24 @@ mod parser; /// /// Furthermore, the actual name of the package is always used regardless of how it is aliased. /// -/// ```ignore +/// ``` /// # use pyo3_bindgen_macros::import_python; /// import_python!("os.path"); /// pub use posixpath::*; /// ``` #[proc_macro] pub fn import_python(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + // Parse the macro arguments let parser::Args { module_name } = syn::parse_macro_input!(input as parser::Args); + // On Unix systems, ensure that the symbols of the libpython shared library are loaded globally + #[cfg(unix)] + utils::try_load_libpython_symbols().unwrap_or_else(|err| { + eprintln!( + "Failed to load libpython symbols, code generation might not work as expected:\n{err}" + ); + }); + // Generate the bindings pyo3_bindgen_engine::Codegen::default() .module_name(&module_name) diff --git a/pyo3_bindgen_macros/src/utils.rs b/pyo3_bindgen_macros/src/utils.rs new file mode 100644 index 0000000..16da82e --- /dev/null +++ b/pyo3_bindgen_macros/src/utils.rs @@ -0,0 +1,35 @@ +/// Ensure that the symbols of the libpython shared library are loaded globally. +/// +/// # Explanation +/// +/// On Unix, rustc loads proc-macro crates with `RTLD_LOCAL`, which (at least +/// on Linux) means all their dependencies (in this case: libpython) don't +/// get their symbols made available globally either. This means that loading +/// Python modules might fail, as these modules refer to symbols of libpython. +/// This function tries to (re)load the right version of libpython, but this +/// time with `RTLD_GLOBAL` enabled. +/// +/// # Disclaimer +/// +/// This function is adapted from the [inline-python](https://crates.io/crates/inline-python) crate +/// ([source code](https://github.com/fusion-engineering/inline-python/blob/24b04b59c0e7f059bbf319e7227054023b3fba55/macros/src/run.rs#L6-L25)). +#[cfg(unix)] +pub fn try_load_libpython_symbols() -> pyo3::PyResult<()> { + #[cfg(not(PyPy))] + pyo3::prepare_freethreaded_python(); + pyo3::Python::with_gil(|py| { + let fn_get_config_var = py + .import(pyo3::intern!(py, "sysconfig"))? + .getattr(pyo3::intern!(py, "get_config_var"))?; + let libpython_dir = fn_get_config_var.call1(("LIBDIR",))?.to_string(); + let libpython_so_name = fn_get_config_var.call1(("INSTSONAME",))?.to_string(); + let libpython_so_path = std::path::Path::new(&libpython_dir).join(libpython_so_name); + unsafe { + libc::dlopen( + std::ffi::CString::new(libpython_so_path.to_string_lossy().as_ref())?.as_ptr(), + libc::RTLD_GLOBAL | libc::RTLD_NOW, + ); + } + Ok(()) + }) +}