diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5042ac9cee3..f4b13f80dd7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -656,14 +656,17 @@ jobs: # ubuntu x86_64 -> windows x86_64 - os: "ubuntu-latest" target: "x86_64-pc-windows-gnu" - flags: "-i python3.12 --features abi3 --features generate-import-lib" - manylinux: off + flags: "-i python3.12 --features generate-import-lib" # macos x86_64 -> aarch64 - os: "macos-13" # last x86_64 macos runners target: "aarch64-apple-darwin" # macos aarch64 -> x86_64 - os: "macos-latest" target: "x86_64-apple-darwin" + # windows x86_64 -> aarch64 + - os: "windows-latest" + target: "aarch64-pc-windows-msvc" + flags: "-i python3.12 --features generate-import-lib" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -677,11 +680,18 @@ jobs: - name: Setup cross-compiler if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} run: sudo apt-get install -y mingw-w64 llvm - - uses: PyO3/maturin-action@v1 + - name: Compile version-specific library + uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} manylinux: ${{ matrix.manylinux }} args: --release -m examples/maturin-starter/Cargo.toml ${{ matrix.flags }} + - name: Compile abi3 library + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + manylinux: ${{ matrix.manylinux }} + args: --release -m examples/maturin-starter/Cargo.toml --features abi3 ${{ matrix.flags }} test-cross-compilation-windows: needs: [fmt] diff --git a/build.rs b/build.rs index 5d638291f3b..68a658bf285 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,9 @@ use std::env; use pyo3_build_config::pyo3_build_script_impl::{cargo_env_var, errors::Result}; -use pyo3_build_config::{bail, print_feature_cfgs, InterpreterConfig}; +use pyo3_build_config::{ + add_python_framework_link_args, bail, print_feature_cfgs, InterpreterConfig, +}; fn ensure_auto_initialize_ok(interpreter_config: &InterpreterConfig) -> Result<()> { if cargo_env_var("CARGO_FEATURE_AUTO_INITIALIZE").is_some() && !interpreter_config.shared { @@ -42,6 +44,9 @@ fn configure_pyo3() -> Result<()> { // Emit cfgs like `invalid_from_utf8_lint` print_feature_cfgs(); + // Make `cargo test` etc work on macOS with Xcode bundled Python + add_python_framework_link_args(); + Ok(()) } diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 1a806304d22..d3474fedaf7 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -144,7 +144,17 @@ rustflags = [ ] ``` -Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. These can be resolved with another addition to `.cargo/config.toml`: +Using the MacOS system python3 (`/usr/bin/python3`, as opposed to python installed via homebrew, pyenv, nix, etc.) may result in runtime errors such as `Library not loaded: @rpath/Python3.framework/Versions/3.8/Python3`. + +The easiest way to set the correct linker arguments is to add a `build.rs` with the following content: + +```rust,ignore +fn main() { + pyo3_build_config::add_python_framework_link_args(); +} +``` + +Alternatively it can be resolved with another addition to `.cargo/config.toml`: ```toml [build] @@ -153,16 +163,6 @@ rustflags = [ ] ``` -Alternatively, one can include in `build.rs`: - -```rust -fn main() { - println!( - "cargo:rustc-link-arg=-Wl,-rpath,/Library/Developer/CommandLineTools/Library/Frameworks" - ); -} -``` - For more discussion on and workarounds for MacOS linking problems [see this issue](https://github.com/PyO3/pyo3/issues/1800#issuecomment-906786649). Finally, don't forget that on MacOS the `extension-module` feature will cause `cargo test` to fail without the `--no-default-features` flag (see [the FAQ](https://pyo3.rs/main/faq.html#i-cant-run-cargo-test-or-i-cant-build-in-a-cargo-workspace-im-having-linker-issues-like-symbol-not-found-or-undefined-reference-to-_pyexc_systemerror)). diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md index 573fd7b2609..8ee9a2e100e 100644 --- a/guide/src/free-threading.md +++ b/guide/src/free-threading.md @@ -156,20 +156,40 @@ freethreaded build, holding a `'py` lifetime means only that the thread is currently attached to the Python interpreter -- other threads can be simultaneously interacting with the interpreter. -The main reason for obtaining a `'py` lifetime is to interact with Python +You still need to obtain a `'py` lifetime is to interact with Python objects or call into the CPython C API. If you are not yet attached to the Python runtime, you can register a thread using the [`Python::with_gil`] function. Threads created via the Python [`threading`] module do not not need to -do this, but all other OS threads that interact with the Python runtime must -explicitly attach using `with_gil` and obtain a `'py` liftime. - -Since there is no GIL in the free-threaded build, releasing the GIL for -long-running tasks is no longer necessary to ensure other threads run, but you -should still detach from the interpreter runtime using [`Python::allow_threads`] -when doing long-running tasks that do not require the CPython runtime. The -garbage collector can only run if all threads are detached from the runtime (in -a stop-the-world state), so detaching from the runtime allows freeing unused -memory. +do this, and pyo3 will handle setting up the [`Python<'py>`] token when CPython +calls into your extension. + +### Global synchronization events can cause hangs and deadlocks + +The free-threaded build triggers global synchronization events in the following +situations: + +* During garbage collection in order to get a globally consistent view of + reference counts and references between objects +* In Python 3.13, when the first background thread is started in + order to mark certain objects as immortal +* When either `sys.settrace` or `sys.setprofile` are called in order to + instrument running code objects and threads +* Before `os.fork()` is called. + +This is a non-exhaustive list and there may be other situations in future Python +versions that can trigger global synchronization events. + +This means that you should detach from the interpreter runtime using +[`Python::allow_threads`] in exactly the same situations as you should detach +from the runtime in the GIL-enabled build: when doing long-running tasks that do +not require the CPython runtime or when doing any task that needs to re-attach +to the runtime (see the [guide +section](parallelism.md#sharing-python-objects-between-rust-threads) that +covers this). In the former case, you would observe a hang on threads that are +waiting on the long-running task to complete, and in the latter case you would +see a deadlock while a thread tries to attach after the runtime triggers a +global synchronization event, but the spawning thread prevents the +synchronization event from completing. ### Exceptions and panics for multithreaded access of mutable `pyclass` instances diff --git a/guide/src/parallelism.md b/guide/src/parallelism.md index a288b14be19..64ff1c8c9c0 100644 --- a/guide/src/parallelism.md +++ b/guide/src/parallelism.md @@ -1,6 +1,6 @@ # Parallelism -CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. +CPython has the infamous [Global Interpreter Lock](https://docs.python.org/3/glossary.html#term-global-interpreter-lock) (GIL), which prevents several threads from executing Python bytecode in parallel. This makes threading in Python a bad fit for [CPU-bound](https://en.wikipedia.org/wiki/CPU-bound) tasks and often forces developers to accept the overhead of multiprocessing. There is an experimental "free-threaded" version of CPython 3.13 that does not have a GIL, see the PyO3 docs on [free-threaded Python](./free-threading.md) for more information about that. In PyO3 parallelism can be easily achieved in Rust-only code. Let's take a look at our [word-count](https://github.com/PyO3/pyo3/blob/main/examples/word-count/src/lib.rs) example, where we have a `search` function that utilizes the [rayon](https://github.com/rayon-rs/rayon) crate to count words in parallel. ```rust,no_run @@ -117,4 +117,61 @@ test_word_count_python_sequential 27.3985 (15.82) 45.452 You can see that the Python threaded version is not much slower than the Rust sequential version, which means compared to an execution on a single CPU core the speed has doubled. +## Sharing Python objects between Rust threads + +In the example above we made a Python interface to a low-level rust function, +and then leveraged the python `threading` module to run the low-level function +in parallel. It is also possible to spawn threads in Rust that acquire the GIL +and operate on Python objects. However, care must be taken to avoid writing code +that deadlocks with the GIL in these cases. + +* Note: This example is meant to illustrate how to drop and re-acquire the GIL + to avoid creating deadlocks. Unless the spawned threads subsequently + release the GIL or you are using the free-threaded build of CPython, you + will not see any speedups due to multi-threaded parallelism using `rayon` + to parallelize code that acquires and holds the GIL for the entire + execution of the spawned thread. + +In the example below, we share a `Vec` of User ID objects defined using the +`pyclass` macro and spawn threads to process the collection of data into a `Vec` +of booleans based on a predicate using a rayon parallel iterator: + +```rust,no_run +use pyo3::prelude::*; + +// These traits let us use int_par_iter and map +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; + +#[pyclass] +struct UserID { + id: i64, +} + +let allowed_ids: Vec = Python::with_gil(|outer_py| { + let instances: Vec> = (0..10).map(|x| Py::new(outer_py, UserID { id: x }).unwrap()).collect(); + outer_py.allow_threads(|| { + instances.par_iter().map(|instance| { + Python::with_gil(|inner_py| { + instance.borrow(inner_py).id > 5 + }) + }).collect() + }) +}); +assert!(allowed_ids.into_iter().filter(|b| *b).count() == 4); +``` + +It's important to note that there is an `outer_py` GIL lifetime token as well as +an `inner_py` token. Sharing GIL lifetime tokens between threads is not allowed +and threads must individually acquire the GIL to access data wrapped by a python +object. + +It's also important to see that this example uses [`Python::allow_threads`] to +wrap the code that spawns OS threads via `rayon`. If this example didn't use +`allow_threads`, a rayon worker thread would block on acquiring the GIL while a +thread that owns the GIL spins forever waiting for the result of the rayon +thread. Calling `allow_threads` allows the GIL to be released in the thread +collecting the results from the worker threads. You should always call +`allow_threads` in situations that spawn worker threads, but especially so in +cases where worker threads need to acquire the GIL, to prevent deadlocks. + [`Python::allow_threads`]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.allow_threads diff --git a/guide/src/performance.md b/guide/src/performance.md index fb2288dd566..5a57585c4a0 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -97,6 +97,19 @@ impl PartialEq for FooBound<'_> { } ``` +## Calling Python callables (`__call__`) +CPython support multiple calling protocols: [`tp_call`] and [`vectorcall`]. [`vectorcall`] is a more efficient protocol unlocking faster calls. +PyO3 will try to dispatch Python `call`s using the [`vectorcall`] calling convention to archive maximum performance if possible and falling back to [`tp_call`] otherwise. +This is implemented using the (internal) `PyCallArgs` trait. It defines how Rust types can be used as Python `call` arguments. This trait is currently implemented for +- Rust tuples, where each member implements `IntoPyObject`, +- `Bound<'_, PyTuple>` +- `Py` +Rust tuples may make use of [`vectorcall`] where as `Bound<'_, PyTuple>` and `Py` can only use [`tp_call`]. For maximum performance prefer using Rust tuples as arguments. + + +[`tp_call`]: https://docs.python.org/3/c-api/call.html#the-tp-call-protocol +[`vectorcall`]: https://docs.python.org/3/c-api/call.html#the-vectorcall-protocol + ## Disable the global reference pool PyO3 uses global mutable state to keep track of deferred reference count updates implied by `impl Drop for Py` being called without the GIL being held. The necessary synchronization to obtain and apply these reference count updates when PyO3-based code next acquires the GIL is somewhat expensive and can become a significant part of the cost of crossing the Python-Rust boundary. diff --git a/newsfragments/4768.added.md b/newsfragments/4768.added.md new file mode 100644 index 00000000000..1ce9c6f5b92 --- /dev/null +++ b/newsfragments/4768.added.md @@ -0,0 +1 @@ +Added `PyCallArgs` trait for arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance. \ No newline at end of file diff --git a/newsfragments/4768.changed.md b/newsfragments/4768.changed.md new file mode 100644 index 00000000000..6b09fd0e093 --- /dev/null +++ b/newsfragments/4768.changed.md @@ -0,0 +1 @@ +`PyAnyMethods::call` an friends now require `PyCallArgs` for their positional arguments. \ No newline at end of file diff --git a/newsfragments/4788.fixed.md b/newsfragments/4788.fixed.md new file mode 100644 index 00000000000..804cd60fd3d --- /dev/null +++ b/newsfragments/4788.fixed.md @@ -0,0 +1,4 @@ +* Fixed thread-unsafe access of dict internals in BoundDictIterator on the + free-threaded build. +* Avoided creating unnecessary critical sections in BoundDictIterator + implementation on the free-threaded build. diff --git a/newsfragments/4789.added.md b/newsfragments/4789.added.md new file mode 100644 index 00000000000..fab564a8962 --- /dev/null +++ b/newsfragments/4789.added.md @@ -0,0 +1,3 @@ +* Added `PyList::locked_for_each`, which is equivalent to `PyList::for_each` on + the GIL-enabled build and uses a critical section to lock the list on the + free-threaded build, similar to `PyDict::locked_for_each`. diff --git a/newsfragments/4789.changed.md b/newsfragments/4789.changed.md new file mode 100644 index 00000000000..d20419e8f23 --- /dev/null +++ b/newsfragments/4789.changed.md @@ -0,0 +1,2 @@ +* Operations that process a PyList via an iterator now use a critical section + on the free-threaded build to amortize synchronization cost and prevent race conditions. diff --git a/newsfragments/4800.fixed.md b/newsfragments/4800.fixed.md new file mode 100644 index 00000000000..615e622a963 --- /dev/null +++ b/newsfragments/4800.fixed.md @@ -0,0 +1 @@ +fix: cross-compilation compatibility checks for Windows diff --git a/newsfragments/4802.fixed.md b/newsfragments/4802.fixed.md new file mode 100644 index 00000000000..55d79c71734 --- /dev/null +++ b/newsfragments/4802.fixed.md @@ -0,0 +1 @@ +Fixed missing struct fields on GraalPy when subclassing builtin classes diff --git a/newsfragments/4808.fixed.md b/newsfragments/4808.fixed.md new file mode 100644 index 00000000000..2e7c3a8a23c --- /dev/null +++ b/newsfragments/4808.fixed.md @@ -0,0 +1 @@ +Fix generating import lib for python3.13t when `abi3` feature is enabled. diff --git a/newsfragments/4814.fixed.md b/newsfragments/4814.fixed.md new file mode 100644 index 00000000000..6634efc2b9f --- /dev/null +++ b/newsfragments/4814.fixed.md @@ -0,0 +1 @@ +`derive(FromPyObject)` support raw identifiers like `r#box`. \ No newline at end of file diff --git a/newsfragments/4822.changed.md b/newsfragments/4822.changed.md new file mode 100644 index 00000000000..a06613292c4 --- /dev/null +++ b/newsfragments/4822.changed.md @@ -0,0 +1 @@ +Bumped `target-lexicon` dependency to 0.13 diff --git a/newsfragments/4832.fixed.md b/newsfragments/4832.fixed.md new file mode 100644 index 00000000000..13df6deae57 --- /dev/null +++ b/newsfragments/4832.fixed.md @@ -0,0 +1 @@ +`#[pyclass]` complex enums support more than 12 variant fields. \ No newline at end of file diff --git a/newsfragments/4833.added.md b/newsfragments/4833.added.md new file mode 100644 index 00000000000..4e1e0005305 --- /dev/null +++ b/newsfragments/4833.added.md @@ -0,0 +1 @@ +Add `pyo3_build_config::add_python_framework_link_args` build script API to set rpath when using macOS system Python. diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 1e951b29bff..2378d30757f 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -13,12 +13,12 @@ rust-version = "1.63" [dependencies] once_cell = "1" -python3-dll-a = { version = "0.2.11", optional = true } -target-lexicon = "0.12.14" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [build-dependencies] -python3-dll-a = { version = "0.2.11", optional = true } -target-lexicon = "0.12.14" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [features] default = [] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 43702eebef9..05bb58ac7aa 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -22,7 +22,7 @@ use std::{ pub use target_lexicon::Triple; -use target_lexicon::{Environment, OperatingSystem}; +use target_lexicon::{Architecture, Environment, OperatingSystem}; use crate::{ bail, ensure, @@ -167,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -245,6 +247,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -253,6 +256,7 @@ print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -289,6 +293,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) }; let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -359,6 +364,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -396,6 +402,9 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { Some(value) => value == "1", @@ -424,6 +433,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -497,9 +507,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let mut lib_dir = None; let mut executable = None; let mut pointer_width = None; - let mut build_flags = None; + let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -528,6 +539,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -535,10 +547,12 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let version = version.ok_or("missing value for version")?; let implementation = implementation.unwrap_or(PythonImplementation::CPython); let abi3 = abi3.unwrap_or(false); + let build_flags = build_flags.unwrap_or_default(); + let gil_disabled = build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED); // Fixup lib_name if it's not set let lib_name = lib_name.or_else(|| { if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::()) { - default_lib_name_for_target(version, implementation, abi3, &target) + default_lib_name_for_target(version, implementation, abi3, gil_disabled, &target) } else { None } @@ -553,9 +567,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) lib_dir, executable, pointer_width, - build_flags: build_flags.unwrap_or_default(), + build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } @@ -565,7 +580,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) // Auto generate python3.dll import libraries for Windows targets. if self.lib_dir.is_none() { let target = target_triple_from_env(); - let py_version = if self.implementation == PythonImplementation::CPython && self.abi3 { + let py_version = if self.implementation == PythonImplementation::CPython + && self.abi3 + && !self.is_free_threaded() + { None } else { Some(self.version) @@ -645,6 +663,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { writeln!(writer, "extra_build_script_line={}", line) @@ -893,6 +912,9 @@ pub struct CrossCompileConfig { /// The compile target triple (e.g. aarch64-unknown-linux-gnu) target: Triple, + + /// Python ABI flags, used to detect free-threaded Python builds. + abiflags: Option, } impl CrossCompileConfig { @@ -907,7 +929,7 @@ impl CrossCompileConfig { ) -> Result> { if env_vars.any() || Self::is_cross_compiling_from_to(host, target) { let lib_dir = env_vars.lib_dir_path()?; - let version = env_vars.parse_version()?; + let (version, abiflags) = env_vars.parse_version()?; let implementation = env_vars.parse_implementation()?; let target = target.clone(); @@ -916,6 +938,7 @@ impl CrossCompileConfig { version, implementation, target, + abiflags, })) } else { Ok(None) @@ -935,11 +958,13 @@ impl CrossCompileConfig { // Not cross-compiling to compile for 32-bit Python from windows 64-bit compatible |= target.operating_system == OperatingSystem::Windows - && host.operating_system == OperatingSystem::Windows; + && host.operating_system == OperatingSystem::Windows + && matches!(target.architecture, Architecture::X86_32(_)) + && host.architecture == Architecture::X86_64; // Not cross-compiling to compile for x86-64 Python from macOS arm64 and vice versa - compatible |= target.operating_system == OperatingSystem::Darwin - && host.operating_system == OperatingSystem::Darwin; + compatible |= matches!(target.operating_system, OperatingSystem::Darwin(_)) + && matches!(host.operating_system, OperatingSystem::Darwin(_)); !compatible } @@ -989,22 +1014,25 @@ impl CrossCompileEnvVars { } /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value - /// into `PythonVersion`. - fn parse_version(&self) -> Result> { - let version = self - .pyo3_cross_python_version - .as_ref() - .map(|os_string| { + /// into `PythonVersion` and ABI flags. + fn parse_version(&self) -> Result<(Option, Option)> { + match self.pyo3_cross_python_version.as_ref() { + Some(os_string) => { let utf8_str = os_string .to_str() .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid a UTF-8 string")?; - utf8_str + let (utf8_str, abiflags) = if let Some(version) = utf8_str.strip_suffix('t') { + (version, Some("t".to_string())) + } else { + (utf8_str, None) + }; + let version = utf8_str .parse() - .context("failed to parse PYO3_CROSS_PYTHON_VERSION") - }) - .transpose()?; - - Ok(version) + .context("failed to parse PYO3_CROSS_PYTHON_VERSION")?; + Ok((Some(version), abiflags)) + } + None => Ok((None, None)), + } } /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value @@ -1530,16 +1558,27 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result Result Option { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows(version, implementation, abi3, false, false, false).unwrap()) + Some( + default_lib_name_windows(version, implementation, abi3, false, false, gil_disabled) + .unwrap(), + ) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None, false).unwrap()) + Some(default_lib_name_unix(version, implementation, None, gil_disabled).unwrap()) } else { None } @@ -1906,7 +1951,14 @@ pub fn make_interpreter_config() -> Result { // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - let py_version = if interpreter_config.abi3 { + let gil_disabled = interpreter_config + .build_flags + .0 + .contains(&BuildFlag::Py_GIL_DISABLED); + let py_version = if interpreter_config.implementation == PythonImplementation::CPython + && interpreter_config.abi3 + && !gil_disabled + { None } else { Some(interpreter_config.version) @@ -1975,6 +2027,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2003,6 +2056,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2024,6 +2078,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -2050,6 +2105,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2072,6 +2128,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2174,6 +2231,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2203,6 +2261,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2229,6 +2288,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2252,6 +2312,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2275,6 +2336,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2309,6 +2371,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2343,6 +2406,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2377,6 +2441,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2413,6 +2478,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2706,7 +2772,7 @@ mod tests { assert_eq!( env_vars.parse_version().unwrap(), - Some(PythonVersion { major: 3, minor: 9 }) + (Some(PythonVersion { major: 3, minor: 9 }), None), ); let env_vars = CrossCompileEnvVars { @@ -2716,7 +2782,25 @@ mod tests { pyo3_cross_python_implementation: None, }; - assert_eq!(env_vars.parse_version().unwrap(), None); + assert_eq!(env_vars.parse_version().unwrap(), (None, None)); + + let env_vars = CrossCompileEnvVars { + pyo3_cross: None, + pyo3_cross_lib_dir: None, + pyo3_cross_python_version: Some("3.13t".into()), + pyo3_cross_python_implementation: None, + }; + + assert_eq!( + env_vars.parse_version().unwrap(), + ( + Some(PythonVersion { + major: 3, + minor: 13 + }), + Some("t".into()) + ), + ); let env_vars = CrossCompileEnvVars { pyo3_cross: None, @@ -2742,6 +2826,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2764,6 +2849,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2799,6 +2885,11 @@ mod tests { version: Some(interpreter_config.version), implementation: Some(interpreter_config.implementation), target: triple!("x86_64-unknown-linux-gnu"), + abiflags: if interpreter_config.is_free_threaded() { + Some("t".into()) + } else { + None + }, }; let sysconfigdata_path = match find_sysconfigdata(&cross) { @@ -2823,6 +2914,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2898,6 +2990,16 @@ mod tests { .is_none()); } + #[test] + fn test_is_cross_compiling_from_to() { + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("aarch64-pc-windows-msvc") + ) + .unwrap() + .is_some()); + } + #[test] fn test_run_python_script() { // as above, this should be okay in CI where Python is presumed installed @@ -2937,6 +3039,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), @@ -2976,6 +3079,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3023,6 +3127,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3056,6 +3161,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -3074,6 +3180,7 @@ mod tests { version: None, implementation: None, target: triple!("x86_64-unknown-linux-gnu"), + abiflags: None, }) .unwrap_err(); diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 554200040e4..9070f6d7401 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -65,7 +65,7 @@ pub fn add_extension_module_link_args() { } fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Write) { - if triple.operating_system == OperatingSystem::Darwin { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) { writeln!(writer, "cargo:rustc-cdylib-link-arg=-undefined").unwrap(); writeln!(writer, "cargo:rustc-cdylib-link-arg=dynamic_lookup").unwrap(); } else if triple == &Triple::from_str("wasm32-unknown-emscripten").unwrap() { @@ -74,6 +74,44 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap(); + _add_python_framework_link_args( + &interpreter_config, + &impl_::target_triple_from_env(), + impl_::is_linking_libpython(), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!( + writer, + "cargo:rustc-link-arg=-Wl,-rpath,{}", + framework_prefix + ) + .unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. @@ -160,6 +198,10 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 83 { println!("cargo:rustc-cfg=io_error_more"); } + + if rustc_minor_version >= 85 { + println!("cargo:rustc-cfg=fn_ptr_eq"); + } } /// Registers `pyo3`s config names as reachable cfg expressions @@ -185,6 +227,7 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(c_str_lit)"); println!("cargo:rustc-check-cfg=cfg(rustc_has_once_lock)"); println!("cargo:rustc-check-cfg=cfg(io_error_more)"); + println!("cargo:rustc-check-cfg=cfg(fn_ptr_eq)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) @@ -301,4 +344,49 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } } diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index 1899545011a..ce6c9b94735 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -17,7 +17,6 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_ extern "C" { #[cfg(all( not(PyPy), - not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] diff --git a/pyo3-ffi/src/cpython/abstract_.rs b/pyo3-ffi/src/cpython/abstract_.rs index 83295e58f61..477ad02b747 100644 --- a/pyo3-ffi/src/cpython/abstract_.rs +++ b/pyo3-ffi/src/cpython/abstract_.rs @@ -1,5 +1,7 @@ use crate::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +#[cfg(not(all(Py_3_11, GraalPy)))] +use std::os::raw::c_char; +use std::os::raw::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; diff --git a/pyo3-ffi/src/cpython/complexobject.rs b/pyo3-ffi/src/cpython/complexobject.rs index 255f9c27034..4cc86db5667 100644 --- a/pyo3-ffi/src/cpython/complexobject.rs +++ b/pyo3-ffi/src/cpython/complexobject.rs @@ -19,7 +19,6 @@ pub struct Py_complex { #[repr(C)] pub struct PyComplexObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub cval: Py_complex, } diff --git a/pyo3-ffi/src/cpython/floatobject.rs b/pyo3-ffi/src/cpython/floatobject.rs index 8c7ee88543d..e7caa441c5d 100644 --- a/pyo3-ffi/src/cpython/floatobject.rs +++ b/pyo3-ffi/src/cpython/floatobject.rs @@ -6,7 +6,6 @@ use std::os::raw::c_double; #[repr(C)] pub struct PyFloatObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub ob_fval: c_double, } diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index 17348b2f7bd..4be310a8c88 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -2,7 +2,7 @@ use crate::object::*; use crate::PyFrameObject; #[cfg(not(any(PyPy, GraalPy)))] use crate::_PyErr_StackItem; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(GraalPy)))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index 963ddfbea87..694e6bc4290 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -2,7 +2,7 @@ use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] #[repr(C)] pub struct PyListObject { pub ob_base: PyVarObject, @@ -10,7 +10,7 @@ pub struct PyListObject { pub allocated: Py_ssize_t, } -#[cfg(any(PyPy, GraalPy))] +#[cfg(PyPy)] pub struct PyListObject { pub ob_base: PyObject, } diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 35ddf25a2de..75eef11aae3 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -211,8 +211,6 @@ pub type printfunc = #[derive(Debug)] pub struct PyTypeObject { pub ob_base: object::PyVarObject, - #[cfg(GraalPy)] - pub ob_size: Py_ssize_t, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, pub tp_itemsize: Py_ssize_t, diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 3e0270ddc8f..98a19abeb81 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -1,3 +1,4 @@ +#[cfg(not(all(Py_3_11, GraalPy)))] use libc::size_t; use std::os::raw::c_int; diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index ca08b44a95c..c6e10e5f07b 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -6,19 +6,19 @@ use crate::Py_ssize_t; #[derive(Debug)] pub struct PyBaseExceptionObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, } @@ -134,19 +134,19 @@ pub struct PyOSErrorObject { #[derive(Debug)] pub struct PyStopIterationObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, pub value: *mut PyObject, diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 1d988d2bef0..9616d4372cc 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -5,7 +5,6 @@ use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, - #[cfg(not(GraalPy))] pub ob_item: [*mut PyObject; 1], } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 1414b4ceb38..fae626b8d25 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,4 +1,4 @@ -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] use crate::Py_hash_t; use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t}; use libc::wchar_t; @@ -250,9 +250,8 @@ impl From for u32 { #[repr(C)] pub struct PyASCIIObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub length: Py_ssize_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hash: Py_hash_t, /// A bit field with various properties. /// @@ -265,9 +264,8 @@ pub struct PyASCIIObject { /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; - #[cfg(not(GraalPy))] pub state: u32, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } @@ -379,11 +377,9 @@ impl PyASCIIObject { #[repr(C)] pub struct PyCompactUnicodeObject { pub _base: PyASCIIObject, - #[cfg(not(GraalPy))] pub utf8_length: Py_ssize_t, - #[cfg(not(GraalPy))] pub utf8: *mut c_char, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr_length: Py_ssize_t, } @@ -398,7 +394,6 @@ pub union PyUnicodeObjectData { #[repr(C)] pub struct PyUnicodeObject { pub _base: PyCompactUnicodeObject, - #[cfg(not(GraalPy))] pub data: PyUnicodeObjectData, } diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 76d12151afc..e529e0fce50 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -9,13 +9,12 @@ use crate::PyCapsule_Import; #[cfg(GraalPy)] use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -#[cfg(not(GraalPy))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr; use std::sync::Once; use std::{cell::UnsafeCell, ffi::CStr}; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; @@ -27,13 +26,10 @@ const _PyDateTime_DATETIME_DATASIZE: usize = 10; /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub days: c_int, - #[cfg(not(GraalPy))] pub seconds: c_int, - #[cfg(not(GraalPy))] pub microseconds: c_int, } @@ -56,19 +52,17 @@ pub struct _PyDateTime_BaseTime { /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -77,11 +71,11 @@ pub struct PyDateTime_Time { /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } @@ -101,19 +95,17 @@ pub struct _PyDateTime_BaseDateTime { /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseDateTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index fc3484be102..3f086ac1e92 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -129,6 +129,9 @@ pub struct PyVarObject { pub ob_base: PyObject, #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, + // On GraalPy the field is physically there, but not always populated. We hide it to prevent accidental misuse + #[cfg(GraalPy)] + pub _ob_size_graalpy: Py_ssize_t, } // skipped private _PyVarObject_CAST diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index f42f9730f1b..4f14e04a695 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,9 @@ -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::{c_char, c_void}; +use std::os::raw::c_char; +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +use std::os::raw::c_void; use std::os::raw::{c_int, c_ulong}; @@ -10,7 +12,7 @@ extern "C" { // skipped non-limited _Py_HashPointer // skipped non-limited _Py_HashPointerRaw - #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] + #[cfg(not(any(Py_LIMITED_API, PyPy)))] pub fn _Py_HashBytes(src: *const c_void, len: Py_ssize_t) -> Py_hash_t; } diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 14c8755e9be..565c54da1f3 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -3,6 +3,7 @@ use crate::utils::Ctx; use proc_macro2::TokenStream; use quote::{format_ident, quote}; use syn::{ + ext::IdentExt, parenthesized, parse::{Parse, ParseStream}, parse_quote, @@ -323,11 +324,11 @@ impl<'a> Container<'a> { fn build_struct(&self, struct_fields: &[NamedStructField<'_>], ctx: &Ctx) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; let self_ty = &self.path; - let struct_name = &self.name(); - let mut fields: Punctuated = Punctuated::new(); + let struct_name = self.name(); + let mut fields: Punctuated = Punctuated::new(); for field in struct_fields { - let ident = &field.ident; - let field_name = ident.to_string(); + let ident = field.ident; + let field_name = ident.unraw().to_string(); let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { FieldGetter::GetAttr(Some(name)) => { quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 914cd19c1a0..d9fb6652017 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -19,8 +19,9 @@ use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyfunction::ConstructorAttribute; use crate::pyimpl::{gen_py_const, get_cfg_attributes, PyClassMethodsType}; use crate::pymethod::{ - impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, MethodAndSlotDef, PropertyType, - SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, __RICHCMP__, __STR__, + impl_py_class_attribute, impl_py_getter_def, impl_py_setter_def, MethodAndMethodDef, + MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, + __RICHCMP__, __STR__, }; use crate::pyversions::is_abi3_before; use crate::utils::{self, apply_renaming_rule, Ctx, LitCStr, PythonDoc}; @@ -1185,34 +1186,30 @@ fn impl_complex_enum_variant_cls( } fn impl_complex_enum_variant_match_args( - ctx: &Ctx, + ctx @ Ctx { pyo3_path, .. }: &Ctx, variant_cls_type: &syn::Type, field_names: &mut Vec, -) -> (MethodAndMethodDef, syn::ImplItemConst) { +) -> syn::Result<(MethodAndMethodDef, syn::ImplItemFn)> { let ident = format_ident!("__match_args__"); - let match_args_const_impl: syn::ImplItemConst = { - let args_tp = field_names.iter().map(|_| { - quote! { &'static str } - }); + let mut match_args_impl: syn::ImplItemFn = { parse_quote! { - #[allow(non_upper_case_globals)] - const #ident: ( #(#args_tp,)* ) = ( - #(stringify!(#field_names),)* - ); + #[classattr] + fn #ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'_, #pyo3_path::types::PyTuple>> { + #pyo3_path::types::PyTuple::new::<&str, _>(py, [ + #(stringify!(#field_names),)* + ]) + } } }; - let spec = ConstSpec { - rust_ident: ident, - attributes: ConstAttributes { - is_class_attr: true, - name: None, - }, - }; - - let variant_match_args = gen_py_const(variant_cls_type, &spec, ctx); + let spec = FnSpec::parse( + &mut match_args_impl.sig, + &mut match_args_impl.attrs, + Default::default(), + )?; + let variant_match_args = impl_py_class_attribute(variant_cls_type, &spec, ctx)?; - (variant_match_args, match_args_const_impl) + Ok((variant_match_args, match_args_impl)) } fn impl_complex_enum_struct_variant_cls( @@ -1260,7 +1257,7 @@ fn impl_complex_enum_struct_variant_cls( } let (variant_match_args, match_args_const_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; field_getters.push(variant_match_args); @@ -1268,6 +1265,7 @@ fn impl_complex_enum_struct_variant_cls( #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { + #[allow(clippy::too_many_arguments)] fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#fields_with_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident { #(#field_names,)* }; <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) @@ -1434,7 +1432,7 @@ fn impl_complex_enum_tuple_variant_cls( slots.push(variant_getitem); let (variant_match_args, match_args_method_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names); + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; field_getters.push(variant_match_args); @@ -1442,6 +1440,7 @@ fn impl_complex_enum_tuple_variant_cls( #[doc(hidden)] #[allow(non_snake_case)] impl #variant_cls { + #[allow(clippy::too_many_arguments)] fn __pymethod_constructor__(py: #pyo3_path::Python<'_>, #(#field_names : #field_types,)*) -> #pyo3_path::PyClassInitializer<#variant_cls> { let base_value = #enum_name::#variant_ident ( #(#field_names,)* ); <#pyo3_path::PyClassInitializer<#enum_name> as ::std::convert::From<#enum_name>>::from(base_value).add_subclass(#variant_cls) diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index 3d2975e4885..c21f6d4556e 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -526,7 +526,7 @@ fn impl_clear_slot(cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx) -> syn::Result }) } -fn impl_py_class_attribute( +pub(crate) fn impl_py_class_attribute( cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx, diff --git a/pyo3-macros-backend/src/quotes.rs b/pyo3-macros-backend/src/quotes.rs index 47b82605bd1..d961b4c0acd 100644 --- a/pyo3-macros-backend/src/quotes.rs +++ b/pyo3-macros-backend/src/quotes.rs @@ -17,6 +17,7 @@ pub(crate) fn ok_wrap(obj: TokenStream, ctx: &Ctx) -> TokenStream { let pyo3_path = pyo3_path.to_tokens_spanned(*output_span); quote_spanned! { *output_span => { let obj = #obj; + #[allow(clippy::useless_conversion)] #pyo3_path::impl_::wrap::converter(&obj).wrap(obj).map_err(::core::convert::Into::<#pyo3_path::PyErr>::into) }} } diff --git a/src/call.rs b/src/call.rs new file mode 100644 index 00000000000..1da1b67530b --- /dev/null +++ b/src/call.rs @@ -0,0 +1,150 @@ +//! Defines how Python calls are dispatched, see [`PyCallArgs`].for more information. + +use crate::ffi_ptr_ext::FfiPtrExt as _; +use crate::types::{PyAnyMethods as _, PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, IntoPyObjectExt as _, Py, PyAny, PyResult}; + +pub(crate) mod private { + use super::*; + + pub trait Sealed {} + + impl Sealed for () {} + impl Sealed for Bound<'_, PyTuple> {} + impl Sealed for Py {} + + pub struct Token; +} + +/// This trait marks types that can be used as arguments to Python function +/// calls. +/// +/// This trait is currently implemented for Rust tuple (up to a size of 12), +/// [`Bound<'py, PyTuple>`] and [`Py`]. Custom types that are +/// convertable to `PyTuple` via `IntoPyObject` need to do so before passing it +/// to `call`. +/// +/// This trait is not intended to used by downstream crates directly. As such it +/// has no publicly available methods and cannot be implemented ouside of +/// `pyo3`. The corresponding public API is available through [`call`] +/// ([`call0`], [`call1`] and friends) on [`PyAnyMethods`]. +/// +/// # What is `PyCallArgs` used for? +/// `PyCallArgs` is used internally in `pyo3` to dispatch the Python calls in +/// the most optimal way for the current build configuration. Certain types, +/// such as Rust tuples, do allow the usage of a faster calling convention of +/// the Python interpreter (if available). More types that may take advantage +/// from this may be added in the future. +/// +/// [`call0`]: crate::types::PyAnyMethods::call0 +/// [`call1`]: crate::types::PyAnyMethods::call1 +/// [`call`]: crate::types::PyAnyMethods::call +/// [`PyAnyMethods`]: crate::types::PyAnyMethods +#[cfg_attr( + diagnostic_namespace, + diagnostic::on_unimplemented( + message = "`{Self}` cannot used as a Python `call` argument", + note = "`PyCallArgs` is implemented for Rust tuples, `Bound<'py, PyTuple>` and `Py`", + note = "if your type is convertable to `PyTuple` via `IntoPyObject`, call `.into_pyobject(py)` manually", + note = "if you meant to pass the type as a single argument, wrap it in a 1-tuple, `(,)`" + ) +)] +pub trait PyCallArgs<'py>: Sized + private::Sealed { + #[doc(hidden)] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult>; + + #[doc(hidden)] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, PyString>, + _: private::Token, + ) -> PyResult> { + object + .getattr(method_name) + .and_then(|method| method.call1(self)) + } +} + +impl<'py> PyCallArgs<'py> for () { + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, PyDict>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call(function, kwargs, token) + } + + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: private::Token, + ) -> PyResult> { + let args = self.into_pyobject_or_pyerr(function.py())?; + args.call_positional(function, token) + } +} + +impl<'py> PyCallArgs<'py> for Bound<'py, PyTuple> { + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, '_, PyDict>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr()) + .assume_owned_or_err(function.py()) + } + } + + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut()) + .assume_owned_or_err(function.py()) + } + } +} + +impl<'py> PyCallArgs<'py> for Py { + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, '_, PyDict>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), kwargs.as_ptr()) + .assume_owned_or_err(function.py()) + } + } + + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: private::Token, + ) -> PyResult> { + unsafe { + ffi::PyObject_Call(function.as_ptr(), self.as_ptr(), std::ptr::null_mut()) + .assume_owned_or_err(function.py()) + } + } +} diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index ac5c6e3e3f0..59bb2b4bd5a 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -43,18 +43,27 @@ pub fn weaklist_offset() -> ffi::Py_ssize_t { PyClassObject::::weaklist_offset() } +mod sealed { + pub trait Sealed {} + + impl Sealed for super::PyClassDummySlot {} + impl Sealed for super::PyClassDictSlot {} + impl Sealed for super::PyClassWeakRefSlot {} + impl Sealed for super::ThreadCheckerImpl {} + impl Sealed for super::SendablePyClass {} +} + /// Represents the `__dict__` field for `#[pyclass]`. -pub trait PyClassDict { +pub trait PyClassDict: sealed::Sealed { /// Initial form of a [PyObject](crate::ffi::PyObject) `__dict__` reference. const INIT: Self; /// Empties the dictionary of its key-value pairs. #[inline] fn clear_dict(&mut self, _py: Python<'_>) {} - private_decl! {} } /// Represents the `__weakref__` field for `#[pyclass]`. -pub trait PyClassWeakRef { +pub trait PyClassWeakRef: sealed::Sealed { /// Initializes a `weakref` instance. const INIT: Self; /// Clears the weak references to the given object. @@ -64,19 +73,16 @@ pub trait PyClassWeakRef { /// - The GIL must be held. #[inline] unsafe fn clear_weakrefs(&mut self, _obj: *mut ffi::PyObject, _py: Python<'_>) {} - private_decl! {} } /// Zero-sized dummy field. pub struct PyClassDummySlot; impl PyClassDict for PyClassDummySlot { - private_impl! {} const INIT: Self = PyClassDummySlot; } impl PyClassWeakRef for PyClassDummySlot { - private_impl! {} const INIT: Self = PyClassDummySlot; } @@ -88,7 +94,6 @@ impl PyClassWeakRef for PyClassDummySlot { pub struct PyClassDictSlot(*mut ffi::PyObject); impl PyClassDict for PyClassDictSlot { - private_impl! {} const INIT: Self = Self(std::ptr::null_mut()); #[inline] fn clear_dict(&mut self, _py: Python<'_>) { @@ -106,7 +111,6 @@ impl PyClassDict for PyClassDictSlot { pub struct PyClassWeakRefSlot(*mut ffi::PyObject); impl PyClassWeakRef for PyClassWeakRefSlot { - private_impl! {} const INIT: Self = Self(std::ptr::null_mut()); #[inline] unsafe fn clear_weakrefs(&mut self, obj: *mut ffi::PyObject, _py: Python<'_>) { @@ -1034,12 +1038,11 @@ impl PyClassNewTextSignature for &'_ PyClassImplCollector { // Thread checkers #[doc(hidden)] -pub trait PyClassThreadChecker: Sized { +pub trait PyClassThreadChecker: Sized + sealed::Sealed { fn ensure(&self); fn check(&self) -> bool; fn can_drop(&self, py: Python<'_>) -> bool; fn new() -> Self; - private_decl! {} } /// Default thread checker for `#[pyclass]`. @@ -1062,7 +1065,6 @@ impl PyClassThreadChecker for SendablePyClass { fn new() -> Self { SendablePyClass(PhantomData) } - private_impl! {} } /// Thread checker for `#[pyclass(unsendable)]` types. @@ -1111,7 +1113,6 @@ impl PyClassThreadChecker for ThreadCheckerImpl { fn new() -> Self { ThreadCheckerImpl(thread::current().id()) } - private_impl! {} } /// Trait denoting that this class is suitable to be used as a base type for PyClass. diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 58d0c93c240..e6cbaec86f5 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -21,6 +21,7 @@ use std::panic::{catch_unwind, AssertUnwindSafe}; use std::ptr::null_mut; use super::trampoline; +use crate::internal_tricks::{clear_eq, traverse_eq}; /// Python 3.8 and up - __ipow__ has modulo argument correctly populated. #[cfg(Py_3_8)] @@ -364,7 +365,7 @@ unsafe fn call_super_traverse( // First find the current type by the current_traverse function loop { traverse = get_slot(ty, TP_TRAVERSE); - if traverse == Some(current_traverse) { + if traverse_eq(traverse, current_traverse) { break; } ty = get_slot(ty, TP_BASE); @@ -375,7 +376,7 @@ unsafe fn call_super_traverse( } // Get first base which has a different traverse function - while traverse == Some(current_traverse) { + while traverse_eq(traverse, current_traverse) { ty = get_slot(ty, TP_BASE); if ty.is_null() { break; @@ -429,7 +430,7 @@ unsafe fn call_super_clear( // First find the current type by the current_clear function loop { clear = ty.get_slot(TP_CLEAR); - if clear == Some(current_clear) { + if clear_eq(clear, current_clear) { break; } let base = ty.get_slot(TP_BASE); @@ -441,7 +442,7 @@ unsafe fn call_super_clear( } // Get first base which has a different clear function - while clear == Some(current_clear) { + while clear_eq(clear, current_clear) { let base = ty.get_slot(TP_BASE); if base.is_null() { break; diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 97b13aff2a8..1c011253254 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -1,21 +1,4 @@ -use crate::ffi::{Py_ssize_t, PY_SSIZE_T_MAX}; -pub struct PrivateMarker; - -macro_rules! private_decl { - () => { - /// This trait is private to implement; this method exists to make it - /// impossible to implement outside the crate. - fn __private__(&self) -> crate::internal_tricks::PrivateMarker; - }; -} - -macro_rules! private_impl { - () => { - fn __private__(&self) -> crate::internal_tricks::PrivateMarker { - crate::internal_tricks::PrivateMarker - } - }; -} +use crate::ffi::{self, Py_ssize_t, PY_SSIZE_T_MAX}; macro_rules! pyo3_exception { ($doc: expr, $name: ident, $base: ty) => { @@ -47,3 +30,31 @@ pub(crate) const fn ptr_from_ref(t: &T) -> *const T { pub(crate) fn ptr_from_mut(t: &mut T) -> *mut T { t as *mut T } + +// TODO: use ptr::fn_addr_eq on MSRV 1.85 +pub(crate) fn clear_eq(f: Option, g: ffi::inquiry) -> bool { + #[cfg(fn_ptr_eq)] + { + let Some(f) = f else { return false }; + std::ptr::fn_addr_eq(f, g) + } + + #[cfg(not(fn_ptr_eq))] + { + f == Some(g) + } +} + +// TODO: use ptr::fn_addr_eq on MSRV 1.85 +pub(crate) fn traverse_eq(f: Option, g: ffi::traverseproc) -> bool { + #[cfg(fn_ptr_eq)] + { + let Some(f) = f else { return false }; + std::ptr::fn_addr_eq(f, g) + } + + #[cfg(not(fn_ptr_eq))] + { + f == Some(g) + } +} diff --git a/src/lib.rs b/src/lib.rs index 46a2fd53d32..e5146c81c00 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -428,6 +428,7 @@ mod internal_tricks; mod internal; pub mod buffer; +pub mod call; pub mod conversion; mod conversions; #[cfg(feature = "experimental-async")] diff --git a/src/tests/hygiene/pyclass.rs b/src/tests/hygiene/pyclass.rs index 4270da34be3..17c3ce41e11 100644 --- a/src/tests/hygiene/pyclass.rs +++ b/src/tests/hygiene/pyclass.rs @@ -92,6 +92,44 @@ pub enum TupleEnumEqOrd { Variant2(u32), } +#[crate::pyclass(crate = "crate")] +pub enum ComplexEnumManyVariantFields { + ManyStructFields { + field_1: u16, + field_2: u32, + field_3: u32, + field_4: i32, + field_5: u32, + field_6: u32, + field_7: u8, + field_8: u32, + field_9: i32, + field_10: u32, + field_11: u32, + field_12: u32, + field_13: u32, + field_14: i16, + field_15: u32, + }, + ManyTupleFields( + u16, + u32, + u32, + i32, + u32, + u32, + u8, + u32, + i32, + u32, + u32, + u32, + u32, + i16, + u32, + ), +} + #[crate::pyclass(str = "{x}, {y}, {z}")] #[pyo3(crate = "crate")] pub struct PointFmt { diff --git a/src/types/any.rs b/src/types/any.rs index d060c187631..1ebc5d40a0b 100644 --- a/src/types/any.rs +++ b/src/types/any.rs @@ -1,3 +1,4 @@ +use crate::call::PyCallArgs; use crate::class::basic::CompareOp; use crate::conversion::{AsPyPointer, FromPyObjectBound, IntoPyObject}; use crate::err::{DowncastError, DowncastIntoError, PyErr, PyResult}; @@ -10,7 +11,7 @@ use crate::py_result_ext::PyResultExt; use crate::type_object::{PyTypeCheck, PyTypeInfo}; #[cfg(not(any(PyPy, GraalPy)))] use crate::types::PySuper; -use crate::types::{PyDict, PyIterator, PyList, PyString, PyTuple, PyType}; +use crate::types::{PyDict, PyIterator, PyList, PyString, PyType}; use crate::{err, ffi, Borrowed, BoundObject, IntoPyObjectExt, Python}; use std::cell::UnsafeCell; use std::cmp::Ordering; @@ -436,7 +437,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// ``` fn call(&self, args: A, kwargs: Option<&Bound<'py, PyDict>>) -> PyResult> where - A: IntoPyObject<'py, Target = PyTuple>; + A: PyCallArgs<'py>; /// Calls the object without arguments. /// @@ -491,7 +492,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { /// ``` fn call1(&self, args: A) -> PyResult> where - A: IntoPyObject<'py, Target = PyTuple>; + A: PyCallArgs<'py>; /// Calls a method on the object. /// @@ -538,7 +539,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { ) -> PyResult> where N: IntoPyObject<'py, Target = PyString>, - A: IntoPyObject<'py, Target = PyTuple>; + A: PyCallArgs<'py>; /// Calls a method on the object without arguments. /// @@ -614,7 +615,7 @@ pub trait PyAnyMethods<'py>: crate::sealed::Sealed { fn call_method1(&self, name: N, args: A) -> PyResult> where N: IntoPyObject<'py, Target = PyString>, - A: IntoPyObject<'py, Target = PyTuple>; + A: PyCallArgs<'py>; /// Returns whether the object is considered to be true. /// @@ -1209,25 +1210,17 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { fn call(&self, args: A, kwargs: Option<&Bound<'py, PyDict>>) -> PyResult> where - A: IntoPyObject<'py, Target = PyTuple>, + A: PyCallArgs<'py>, { - fn inner<'py>( - any: &Bound<'py, PyAny>, - args: Borrowed<'_, 'py, PyTuple>, - kwargs: Option<&Bound<'py, PyDict>>, - ) -> PyResult> { - unsafe { - ffi::PyObject_Call( - any.as_ptr(), - args.as_ptr(), - kwargs.map_or(std::ptr::null_mut(), |dict| dict.as_ptr()), - ) - .assume_owned_or_err(any.py()) - } + if let Some(kwargs) = kwargs { + args.call( + self.as_borrowed(), + kwargs.as_borrowed(), + crate::call::private::Token, + ) + } else { + args.call_positional(self.as_borrowed(), crate::call::private::Token) } - - let py = self.py(); - inner(self, args.into_pyobject_or_pyerr(py)?.as_borrowed(), kwargs) } #[inline] @@ -1237,9 +1230,9 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { fn call1(&self, args: A) -> PyResult> where - A: IntoPyObject<'py, Target = PyTuple>, + A: PyCallArgs<'py>, { - self.call(args, None) + args.call_positional(self.as_borrowed(), crate::call::private::Token) } #[inline] @@ -1251,10 +1244,14 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { ) -> PyResult> where N: IntoPyObject<'py, Target = PyString>, - A: IntoPyObject<'py, Target = PyTuple>, + A: PyCallArgs<'py>, { - self.getattr(name) - .and_then(|method| method.call(args, kwargs)) + if kwargs.is_none() { + self.call_method1(name, args) + } else { + self.getattr(name) + .and_then(|method| method.call(args, kwargs)) + } } #[inline] @@ -1273,9 +1270,14 @@ impl<'py> PyAnyMethods<'py> for Bound<'py, PyAny> { fn call_method1(&self, name: N, args: A) -> PyResult> where N: IntoPyObject<'py, Target = PyString>, - A: IntoPyObject<'py, Target = PyTuple>, + A: PyCallArgs<'py>, { - self.call_method(name, args, None) + let name = name.into_pyobject_or_pyerr(self.py())?; + args.call_method_positional( + self.as_borrowed(), + name.as_borrowed(), + crate::call::private::Token, + ) } fn is_truthy(&self) -> PyResult { diff --git a/src/types/dict.rs b/src/types/dict.rs index b3c8e37962b..0d2e6ff335f 100644 --- a/src/types/dict.rs +++ b/src/types/dict.rs @@ -181,7 +181,8 @@ pub trait PyDictMethods<'py>: crate::sealed::Sealed { /// Iterates over the contents of this dictionary while holding a critical section on the dict. /// This is useful when the GIL is disabled and the dictionary is shared between threads. /// It is not guaranteed that the dictionary will not be modified during iteration when the - /// closure calls arbitrary Python code that releases the current critical section. + /// closure calls arbitrary Python code that releases the critical section held by the + /// iterator. Otherwise, the dictionary will not be modified during iteration. /// /// This method is a small performance optimization over `.iter().try_for_each()` when the /// nightly feature is not enabled because we cannot implement an optimised version of @@ -396,19 +397,26 @@ impl<'a, 'py> Borrowed<'a, 'py, PyDict> { /// Iterates over the contents of this dictionary without incrementing reference counts. /// /// # Safety - /// It must be known that this dictionary will not be modified during iteration. + /// It must be known that this dictionary will not be modified during iteration, + /// for example, when parsing arguments in a keyword arguments dictionary. pub(crate) unsafe fn iter_borrowed(self) -> BorrowedDictIter<'a, 'py> { BorrowedDictIter::new(self) } } fn dict_len(dict: &Bound<'_, PyDict>) -> Py_ssize_t { - #[cfg(any(not(Py_3_8), PyPy, GraalPy, Py_LIMITED_API))] + #[cfg(any(not(Py_3_8), PyPy, GraalPy, Py_LIMITED_API, Py_GIL_DISABLED))] unsafe { ffi::PyDict_Size(dict.as_ptr()) } - #[cfg(all(Py_3_8, not(PyPy), not(GraalPy), not(Py_LIMITED_API)))] + #[cfg(all( + Py_3_8, + not(PyPy), + not(GraalPy), + not(Py_LIMITED_API), + not(Py_GIL_DISABLED) + ))] unsafe { (*dict.as_ptr().cast::()).ma_used } @@ -429,8 +437,11 @@ enum DictIterImpl { } impl DictIterImpl { + #[deny(unsafe_op_in_unsafe_fn)] #[inline] - fn next<'py>( + /// Safety: the dict should be locked with a critical section on the free-threaded build + /// and otherwise not shared between threads in code that releases the GIL. + unsafe fn next_unchecked<'py>( &mut self, dict: &Bound<'py, PyDict>, ) -> Option<(Bound<'py, PyAny>, Bound<'py, PyAny>)> { @@ -440,7 +451,7 @@ impl DictIterImpl { remaining, ppos, .. - } => crate::sync::with_critical_section(dict, || { + } => { let ma_used = dict_len(dict); // These checks are similar to what CPython does. @@ -470,20 +481,20 @@ impl DictIterImpl { let mut key: *mut ffi::PyObject = std::ptr::null_mut(); let mut value: *mut ffi::PyObject = std::ptr::null_mut(); - if unsafe { ffi::PyDict_Next(dict.as_ptr(), ppos, &mut key, &mut value) } != 0 { + if unsafe { ffi::PyDict_Next(dict.as_ptr(), ppos, &mut key, &mut value) != 0 } { *remaining -= 1; let py = dict.py(); // Safety: // - PyDict_Next returns borrowed values // - we have already checked that `PyDict_Next` succeeded, so we can assume these to be non-null Some(( - unsafe { key.assume_borrowed_unchecked(py) }.to_owned(), - unsafe { value.assume_borrowed_unchecked(py) }.to_owned(), + unsafe { key.assume_borrowed_unchecked(py).to_owned() }, + unsafe { value.assume_borrowed_unchecked(py).to_owned() }, )) } else { None } - }), + } } } @@ -504,7 +515,17 @@ impl<'py> Iterator for BoundDictIterator<'py> { #[inline] fn next(&mut self) -> Option { - self.inner.next(&self.dict) + #[cfg(Py_GIL_DISABLED)] + { + self.inner + .with_critical_section(&self.dict, |inner| unsafe { + inner.next_unchecked(&self.dict) + }) + } + #[cfg(not(Py_GIL_DISABLED))] + { + unsafe { self.inner.next_unchecked(&self.dict) } + } } #[inline] @@ -522,7 +543,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut accum = init; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { accum = f(accum, x); } accum @@ -539,7 +560,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut accum = init; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { accum = f(accum, x)? } R::from_output(accum) @@ -554,7 +575,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if !f(x) { return false; } @@ -571,7 +592,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if f(x) { return true; } @@ -588,7 +609,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { P: FnMut(&Self::Item) -> bool, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if predicate(&x) { return Some(x); } @@ -605,7 +626,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { F: FnMut(Self::Item) -> Option, { self.inner.with_critical_section(&self.dict, |inner| { - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if let found @ Some(_) = f(x) { return found; } @@ -623,7 +644,7 @@ impl<'py> Iterator for BoundDictIterator<'py> { { self.inner.with_critical_section(&self.dict, |inner| { let mut acc = 0; - while let Some(x) = inner.next(&self.dict) { + while let Some(x) = unsafe { inner.next_unchecked(&self.dict) } { if predicate(x) { return Some(acc); } diff --git a/src/types/list.rs b/src/types/list.rs index 3db72d07fed..76da36d00b9 100644 --- a/src/types/list.rs +++ b/src/types/list.rs @@ -18,7 +18,7 @@ use crate::types::sequence::PySequenceMethods; /// [`Py`][crate::Py] or [`Bound<'py, PyList>`][Bound]. /// /// For APIs available on `list` objects, see the [`PyListMethods`] trait which is implemented for -/// [`Bound<'py, PyDict>`][Bound]. +/// [`Bound<'py, PyList>`][Bound]. #[repr(transparent)] pub struct PyList(PyAny); @@ -179,7 +179,9 @@ pub trait PyListMethods<'py>: crate::sealed::Sealed { /// # Safety /// /// Caller must verify that the index is within the bounds of the list. - #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] + /// On the free-threaded build, caller must verify they have exclusive access to the list + /// via a lock or by holding the innermost critical section on the list. + #[cfg(not(Py_LIMITED_API))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny>; /// Takes the slice `self[low:high]` and returns it as a new list. @@ -239,6 +241,17 @@ pub trait PyListMethods<'py>: crate::sealed::Sealed { /// Returns an iterator over this list's items. fn iter(&self) -> BoundListIterator<'py>; + /// Iterates over the contents of this list while holding a critical section on the list. + /// This is useful when the GIL is disabled and the list is shared between threads. + /// It is not guaranteed that the list will not be modified during iteration when the + /// closure calls arbitrary Python code that releases the critical section held by the + /// iterator. Otherwise, the list will not be modified during iteration. + /// + /// This is equivalent to for_each if the GIL is enabled. + fn locked_for_each(&self, closure: F) -> PyResult<()> + where + F: Fn(Bound<'py, PyAny>) -> PyResult<()>; + /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. fn sort(&self) -> PyResult<()>; @@ -302,7 +315,7 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { /// # Safety /// /// Caller must verify that the index is within the bounds of the list. - #[cfg(not(any(Py_LIMITED_API, Py_GIL_DISABLED)))] + #[cfg(not(Py_LIMITED_API))] unsafe fn get_item_unchecked(&self, index: usize) -> Bound<'py, PyAny> { // PyList_GET_ITEM return borrowed ptr; must make owned for safety (see #890). ffi::PyList_GET_ITEM(self.as_ptr(), index as Py_ssize_t) @@ -440,6 +453,14 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { BoundListIterator::new(self.clone()) } + /// Iterates over a list while holding a critical section, calling a closure on each item + fn locked_for_each(&self, closure: F) -> PyResult<()> + where + F: Fn(Bound<'py, PyAny>) -> PyResult<()>, + { + crate::sync::with_critical_section(self, || self.iter().try_for_each(closure)) + } + /// Sorts the list in-place. Equivalent to the Python expression `l.sort()`. fn sort(&self) -> PyResult<()> { err::error_on_minusone(self.py(), unsafe { ffi::PyList_Sort(self.as_ptr()) }) @@ -462,29 +483,123 @@ impl<'py> PyListMethods<'py> for Bound<'py, PyList> { } } +// New types for type checking when using BoundListIterator associated methods, like +// BoundListIterator::next_unchecked. +struct Index(usize); +struct Length(usize); + /// Used by `PyList::iter()`. pub struct BoundListIterator<'py> { list: Bound<'py, PyList>, - index: usize, - length: usize, + index: Index, + length: Length, } impl<'py> BoundListIterator<'py> { fn new(list: Bound<'py, PyList>) -> Self { - let length: usize = list.len(); - BoundListIterator { + Self { + index: Index(0), + length: Length(list.len()), list, - index: 0, - length, } } - unsafe fn get_item(&self, index: usize) -> Bound<'py, PyAny> { - #[cfg(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED))] - let item = self.list.get_item(index).expect("list.get failed"); - #[cfg(not(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED)))] - let item = self.list.get_item_unchecked(index); - item + /// # Safety + /// + /// On the free-threaded build, caller must verify they have exclusive + /// access to the list by holding a lock or by holding the innermost + /// critical section on the list. + #[inline] + #[cfg(not(Py_LIMITED_API))] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn next_unchecked( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let length = length.0.min(list.len()); + let my_index = index.0; + + if index.0 < length { + let item = unsafe { list.get_item_unchecked(my_index) }; + index.0 += 1; + Some(item) + } else { + None + } + } + + #[cfg(Py_LIMITED_API)] + fn next( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let length = length.0.min(list.len()); + let my_index = index.0; + + if index.0 < length { + let item = list.get_item(my_index).expect("get-item failed"); + index.0 += 1; + Some(item) + } else { + None + } + } + + /// # Safety + /// + /// On the free-threaded build, caller must verify they have exclusive + /// access to the list by holding a lock or by holding the innermost + /// critical section on the list. + #[inline] + #[cfg(not(Py_LIMITED_API))] + #[deny(unsafe_op_in_unsafe_fn)] + unsafe fn next_back_unchecked( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let current_length = length.0.min(list.len()); + + if index.0 < current_length { + let item = unsafe { list.get_item_unchecked(current_length - 1) }; + length.0 = current_length - 1; + Some(item) + } else { + None + } + } + + #[inline] + #[cfg(Py_LIMITED_API)] + fn next_back( + index: &mut Index, + length: &mut Length, + list: &Bound<'py, PyList>, + ) -> Option> { + let current_length = (length.0).min(list.len()); + + if index.0 < current_length { + let item = list.get_item(current_length - 1).expect("get-item failed"); + length.0 = current_length - 1; + Some(item) + } else { + None + } + } + + #[cfg(not(Py_LIMITED_API))] + fn with_critical_section( + &mut self, + f: impl FnOnce(&mut Index, &mut Length, &Bound<'py, PyList>) -> R, + ) -> R { + let Self { + index, + length, + list, + } = self; + crate::sync::with_critical_section(list, || f(index, length, list)) } } @@ -493,13 +608,20 @@ impl<'py> Iterator for BoundListIterator<'py> { #[inline] fn next(&mut self) -> Option { - let length = self.length.min(self.list.len()); - if self.index < length { - let item = unsafe { self.get_item(self.index) }; - self.index += 1; - Some(item) - } else { - None + #[cfg(not(Py_LIMITED_API))] + { + self.with_critical_section(|index, length, list| unsafe { + Self::next_unchecked(index, length, list) + }) + } + #[cfg(Py_LIMITED_API)] + { + let Self { + index, + length, + list, + } = self; + Self::next(index, length, list) } } @@ -510,52 +632,183 @@ impl<'py> Iterator for BoundListIterator<'py> { } #[inline] - fn nth(&mut self, n: usize) -> Option { - let length = self.length.min(self.list.len()); - let target_index = self.index + n; - if self.index + n < length { - let item = unsafe { self.get_item(target_index) }; - self.index = target_index + 1; - Some(item) - } else { - self.index = self.list.len(); + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn fold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + accum = f(accum, x); + } + accum + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, feature = "nightly"))] + fn try_fold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: std::ops::Try, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + accum = f(accum, x)? + } + R::from_output(accum) + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn all(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if !f(x) { + return false; + } + } + true + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn any(&mut self, mut f: F) -> bool + where + Self: Sized, + F: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if f(x) { + return true; + } + } + false + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn find

(&mut self, mut predicate: P) -> Option + where + Self: Sized, + P: FnMut(&Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if predicate(&x) { + return Some(x); + } + } None - } + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn find_map(&mut self, mut f: F) -> Option + where + Self: Sized, + F: FnMut(Self::Item) -> Option, + { + self.with_critical_section(|index, length, list| { + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if let found @ Some(_) = f(x) { + return found; + } + } + None + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn position

(&mut self, mut predicate: P) -> Option + where + Self: Sized, + P: FnMut(Self::Item) -> bool, + { + self.with_critical_section(|index, length, list| { + let mut acc = 0; + while let Some(x) = unsafe { Self::next_unchecked(index, length, list) } { + if predicate(x) { + return Some(acc); + } + acc += 1; + } + None + }) } } impl DoubleEndedIterator for BoundListIterator<'_> { #[inline] fn next_back(&mut self) -> Option { - let length = self.length.min(self.list.len()); - - if self.index < length { - let item = unsafe { self.get_item(length - 1) }; - self.length = length - 1; - Some(item) - } else { - None + #[cfg(not(Py_LIMITED_API))] + { + self.with_critical_section(|index, length, list| unsafe { + Self::next_back_unchecked(index, length, list) + }) + } + #[cfg(Py_LIMITED_API)] + { + let Self { + index, + length, + list, + } = self; + Self::next_back(index, length, list) } } #[inline] - fn nth_back(&mut self, n: usize) -> Option { - let length = self.length.min(self.list.len()); - if self.index + n < length { - let target_index = length - n - 1; - let item = unsafe { self.get_item(target_index) }; - self.length = target_index; - Some(item) - } else { - self.length = length; - None - } + #[cfg(all(Py_GIL_DISABLED, not(feature = "nightly")))] + fn rfold(mut self, init: B, mut f: F) -> B + where + Self: Sized, + F: FnMut(B, Self::Item) -> B, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_back_unchecked(index, length, list) } { + accum = f(accum, x); + } + accum + }) + } + + #[inline] + #[cfg(all(Py_GIL_DISABLED, feature = "nightly"))] + fn try_rfold(&mut self, init: B, mut f: F) -> R + where + Self: Sized, + F: FnMut(B, Self::Item) -> R, + R: std::ops::Try, + { + self.with_critical_section(|index, length, list| { + let mut accum = init; + while let Some(x) = unsafe { Self::next_back_unchecked(index, length, list) } { + accum = f(accum, x)? + } + R::from_output(accum) + }) } } impl ExactSizeIterator for BoundListIterator<'_> { fn len(&self) -> usize { - self.length.saturating_sub(self.index) + self.length.0.saturating_sub(self.index.0) } } @@ -585,7 +838,7 @@ mod tests { use crate::types::list::PyListMethods; use crate::types::sequence::PySequenceMethods; use crate::types::{PyList, PyTuple}; - use crate::{ffi, IntoPyObject, Python}; + use crate::{ffi, IntoPyObject, PyResult, Python}; #[test] fn test_new() { @@ -748,130 +1001,166 @@ mod tests { } #[test] - fn test_iter_nth() { + fn test_iter_rev() { Python::with_gil(|py| { - let v = vec![6, 7, 8, 9, 10]; - let ob = (&v).into_pyobject(py).unwrap(); + let v = vec![2, 3, 5, 7]; + let ob = v.into_pyobject(py).unwrap(); let list = ob.downcast::().unwrap(); - let mut iter = list.iter(); - iter.next(); - assert_eq!(iter.nth(1).unwrap().extract::().unwrap(), 8); - assert_eq!(iter.nth(1).unwrap().extract::().unwrap(), 10); - assert!(iter.nth(1).is_none()); + let mut iter = list.iter().rev(); - let v: Vec = vec![]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); + assert_eq!(iter.size_hint(), (4, Some(4))); - let mut iter = list.iter(); - iter.next(); - assert!(iter.nth(1).is_none()); + assert_eq!(iter.next().unwrap().extract::().unwrap(), 7); + assert_eq!(iter.size_hint(), (3, Some(3))); - let v = vec![1, 2, 3]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); + assert_eq!(iter.next().unwrap().extract::().unwrap(), 5); + assert_eq!(iter.size_hint(), (2, Some(2))); - let mut iter = list.iter(); - assert!(iter.nth(10).is_none()); + assert_eq!(iter.next().unwrap().extract::().unwrap(), 3); + assert_eq!(iter.size_hint(), (1, Some(1))); - let v = vec![6, 7, 8, 9, 10]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); - let mut iter = list.iter(); - assert_eq!(iter.next().unwrap().extract::().unwrap(), 6); - assert_eq!(iter.nth(2).unwrap().extract::().unwrap(), 9); - assert_eq!(iter.next().unwrap().extract::().unwrap(), 10); + assert_eq!(iter.next().unwrap().extract::().unwrap(), 2); + assert_eq!(iter.size_hint(), (0, Some(0))); - let mut iter = list.iter(); - iter.nth_back(1); - assert_eq!(iter.nth(2).unwrap().extract::().unwrap(), 8); + assert!(iter.next().is_none()); assert!(iter.next().is_none()); }); } #[test] - fn test_iter_nth_back() { + fn test_iter_all() { Python::with_gil(|py| { - let v = vec![1, 2, 3, 4, 5]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); - - let mut iter = list.iter(); - assert_eq!(iter.nth_back(0).unwrap().extract::().unwrap(), 5); - assert_eq!(iter.nth_back(1).unwrap().extract::().unwrap(), 3); - assert!(iter.nth_back(2).is_none()); - - let v: Vec = vec![]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); - - let mut iter = list.iter(); - assert!(iter.nth_back(0).is_none()); - assert!(iter.nth_back(1).is_none()); - - let v = vec![1, 2, 3]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); - - let mut iter = list.iter(); - assert!(iter.nth_back(5).is_none()); + let list = PyList::new(py, [true, true, true]).unwrap(); + assert!(list.iter().all(|x| x.extract::().unwrap())); - let v = vec![1, 2, 3, 4, 5]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); + let list = PyList::new(py, [true, false, true]).unwrap(); + assert!(!list.iter().all(|x| x.extract::().unwrap())); + }); + } - let mut iter = list.iter(); - iter.next_back(); // Consume the last element - assert_eq!(iter.nth_back(1).unwrap().extract::().unwrap(), 3); - assert_eq!(iter.next_back().unwrap().extract::().unwrap(), 2); - assert_eq!(iter.nth_back(0).unwrap().extract::().unwrap(), 1); + #[test] + fn test_iter_any() { + Python::with_gil(|py| { + let list = PyList::new(py, [true, true, true]).unwrap(); + assert!(list.iter().any(|x| x.extract::().unwrap())); - let v = vec![1, 2, 3, 4, 5]; - let ob = (&v).into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); + let list = PyList::new(py, [true, false, true]).unwrap(); + assert!(list.iter().any(|x| x.extract::().unwrap())); - let mut iter = list.iter(); - assert_eq!(iter.nth_back(1).unwrap().extract::().unwrap(), 4); - assert_eq!(iter.nth_back(2).unwrap().extract::().unwrap(), 1); - - let mut iter2 = list.iter(); - iter2.next_back(); - assert_eq!(iter2.nth_back(1).unwrap().extract::().unwrap(), 3); - assert_eq!(iter2.next_back().unwrap().extract::().unwrap(), 2); - - let mut iter3 = list.iter(); - iter3.nth(1); - assert_eq!(iter3.nth_back(2).unwrap().extract::().unwrap(), 3); - assert!(iter3.nth_back(0).is_none()); + let list = PyList::new(py, [false, false, false]).unwrap(); + assert!(!list.iter().any(|x| x.extract::().unwrap())); }); } #[test] - fn test_iter_rev() { - Python::with_gil(|py| { - let v = vec![2, 3, 5, 7]; - let ob = v.into_pyobject(py).unwrap(); - let list = ob.downcast::().unwrap(); - - let mut iter = list.iter().rev(); + fn test_iter_find() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, ["hello", "world"]).unwrap(); + assert_eq!( + Some("world".to_string()), + list.iter() + .find(|v| v.extract::().unwrap() == "world") + .map(|v| v.extract::().unwrap()) + ); + assert_eq!( + None, + list.iter() + .find(|v| v.extract::().unwrap() == "foobar") + .map(|v| v.extract::().unwrap()) + ); + }); + } - assert_eq!(iter.size_hint(), (4, Some(4))); + #[test] + fn test_iter_position() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, ["hello", "world"]).unwrap(); + assert_eq!( + Some(1), + list.iter() + .position(|v| v.extract::().unwrap() == "world") + ); + assert_eq!( + None, + list.iter() + .position(|v| v.extract::().unwrap() == "foobar") + ); + }); + } - assert_eq!(iter.next().unwrap().extract::().unwrap(), 7); - assert_eq!(iter.size_hint(), (3, Some(3))); + #[test] + fn test_iter_fold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .fold(0, |acc, v| acc + v.extract::().unwrap()); + assert_eq!(sum, 6); + }); + } - assert_eq!(iter.next().unwrap().extract::().unwrap(), 5); - assert_eq!(iter.size_hint(), (2, Some(2))); + #[test] + fn test_iter_fold_out_of_bounds() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list.iter().fold(0, |_, _| { + // clear the list to create a pathological fold operation + // that mutates the list as it processes it + for _ in 0..3 { + list.del_item(0).unwrap(); + } + -5 + }); + assert_eq!(sum, -5); + assert!(list.len() == 0); + }); + } - assert_eq!(iter.next().unwrap().extract::().unwrap(), 3); - assert_eq!(iter.size_hint(), (1, Some(1))); + #[test] + fn test_iter_rfold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .rfold(0, |acc, v| acc + v.extract::().unwrap()); + assert_eq!(sum, 6); + }); + } - assert_eq!(iter.next().unwrap().extract::().unwrap(), 2); - assert_eq!(iter.size_hint(), (0, Some(0))); + #[test] + fn test_iter_try_fold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .try_fold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .unwrap(); + assert_eq!(sum, 6); + + let list = PyList::new(py, ["foo", "bar"]).unwrap(); + assert!(list + .iter() + .try_fold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .is_err()); + }); + } - assert!(iter.next().is_none()); - assert!(iter.next().is_none()); + #[test] + fn test_iter_try_rfold() { + Python::with_gil(|py: Python<'_>| { + let list = PyList::new(py, [1, 2, 3]).unwrap(); + let sum = list + .iter() + .try_rfold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .unwrap(); + assert_eq!(sum, 6); + + let list = PyList::new(py, ["foo", "bar"]).unwrap(); + assert!(list + .iter() + .try_rfold(0, |acc, v| PyResult::Ok(acc + v.extract::()?)) + .is_err()); }); } @@ -1004,7 +1293,7 @@ mod tests { }); } - #[cfg(not(any(Py_LIMITED_API, PyPy, Py_GIL_DISABLED)))] + #[cfg(not(Py_LIMITED_API))] #[test] fn test_list_get_item_unchecked_sanity() { Python::with_gil(|py| { diff --git a/src/types/mod.rs b/src/types/mod.rs index d84f099e773..39ab3fd501e 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -23,7 +23,7 @@ pub use self::float::{PyFloat, PyFloatMethods}; pub use self::frame::PyFrame; pub use self::frozenset::{PyFrozenSet, PyFrozenSetBuilder, PyFrozenSetMethods}; pub use self::function::PyCFunction; -#[cfg(all(not(Py_LIMITED_API), not(all(PyPy, not(Py_3_8))), not(GraalPy)))] +#[cfg(all(not(Py_LIMITED_API), not(all(PyPy, not(Py_3_8)))))] pub use self::function::PyFunction; pub use self::iterator::PyIterator; pub use self::list::{PyList, PyListMethods}; diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 216a376d833..a5de938140a 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -566,6 +566,255 @@ macro_rules! tuple_conversion ({$length:expr,$(($refN:ident, $n:tt, $T:ident)),+ } } + impl<'py, $($T),+> crate::call::private::Sealed for ($($T,)+) where $($T: IntoPyObject<'py>,)+ {} + impl<'py, $($T),+> crate::call::PyCallArgs<'py> for ($($T,)+) + where + $($T: IntoPyObject<'py>,)+ + { + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, '_, crate::types::PyDict>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = function.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + // Prepend one null argument for `PY_VECTORCALL_ARGUMENTS_OFFSET`. + let mut args = [std::ptr::null_mut(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_VectorcallDict( + function.as_ptr(), + args.as_mut_ptr().add(1), + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + kwargs.as_ptr(), + ) + .assume_owned_or_err(py) + } + } + + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = function.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + if $length == 1 { + unsafe { + ffi::PyObject_CallOneArg( + function.as_ptr(), + args_bound[0].as_ptr() + ) + .assume_owned_or_err(py) + } + } else { + // Prepend one null argument for `PY_VECTORCALL_ARGUMENTS_OFFSET`. + let mut args = [std::ptr::null_mut(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_Vectorcall( + function.as_ptr(), + args.as_mut_ptr().add(1), + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + std::ptr::null_mut(), + ) + .assume_owned_or_err(py) + } + } + } + + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, crate::types::PyString>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = object.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + if $length == 1 { + unsafe { + ffi::PyObject_CallMethodOneArg( + object.as_ptr(), + method_name.as_ptr(), + args_bound[0].as_ptr(), + ) + .assume_owned_or_err(py) + } + } else { + let mut args = [object.as_ptr(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_VectorcallMethod( + method_name.as_ptr(), + args.as_mut_ptr(), + // +1 for the receiver. + 1 + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + std::ptr::null_mut(), + ) + .assume_owned_or_err(py) + } + } + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, crate::types::PyDict>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(function.py())?.call(function, kwargs, token) + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(function.py())?.call_positional(function, token) + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, crate::types::PyString>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(object.py())?.call_method_positional(object, method_name, token) + } + } + + impl<'a, 'py, $($T),+> crate::call::private::Sealed for &'a ($($T,)+) where $(&'a $T: IntoPyObject<'py>,)+ $($T: 'a,)+ /*MSRV */ {} + impl<'a, 'py, $($T),+> crate::call::PyCallArgs<'py> for &'a ($($T,)+) + where + $(&'a $T: IntoPyObject<'py>,)+ + $($T: 'a,)+ // MSRV + { + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, '_, crate::types::PyDict>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = function.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + // Prepend one null argument for `PY_VECTORCALL_ARGUMENTS_OFFSET`. + let mut args = [std::ptr::null_mut(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_VectorcallDict( + function.as_ptr(), + args.as_mut_ptr().add(1), + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + kwargs.as_ptr(), + ) + .assume_owned_or_err(py) + } + } + + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = function.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + if $length == 1 { + unsafe { + ffi::PyObject_CallOneArg( + function.as_ptr(), + args_bound[0].as_ptr() + ) + .assume_owned_or_err(py) + } + } else { + // Prepend one null argument for `PY_VECTORCALL_ARGUMENTS_OFFSET`. + let mut args = [std::ptr::null_mut(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_Vectorcall( + function.as_ptr(), + args.as_mut_ptr().add(1), + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + std::ptr::null_mut(), + ) + .assume_owned_or_err(py) + } + } + } + + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API))))] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, crate::types::PyString>, + _: crate::call::private::Token, + ) -> PyResult> { + let py = object.py(); + // We need this to drop the arguments correctly. + let args_bound = [$(self.$n.into_bound_py_any(py)?,)*]; + if $length == 1 { + unsafe { + ffi::PyObject_CallMethodOneArg( + object.as_ptr(), + method_name.as_ptr(), + args_bound[0].as_ptr(), + ) + .assume_owned_or_err(py) + } + } else { + let mut args = [object.as_ptr(), $(args_bound[$n].as_ptr()),*]; + unsafe { + ffi::PyObject_VectorcallMethod( + method_name.as_ptr(), + args.as_mut_ptr(), + // +1 for the receiver. + 1 + $length + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + std::ptr::null_mut(), + ) + .assume_owned_or_err(py) + } + } + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call( + self, + function: Borrowed<'_, 'py, PyAny>, + kwargs: Borrowed<'_, 'py, crate::types::PyDict>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(function.py())?.call(function, kwargs, token) + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call_positional( + self, + function: Borrowed<'_, 'py, PyAny>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(function.py())?.call_positional(function, token) + } + + #[cfg(not(all(Py_3_9, not(any(PyPy, GraalPy, Py_LIMITED_API)))))] + fn call_method_positional( + self, + object: Borrowed<'_, 'py, PyAny>, + method_name: Borrowed<'_, 'py, crate::types::PyString>, + token: crate::call::private::Token, + ) -> PyResult> { + self.into_pyobject_or_pyerr(object.py())?.call_method_positional(object, method_name, token) + } + } + #[allow(deprecated)] impl <$($T: IntoPy),+> IntoPy> for ($($T,)+) { fn into_py(self, py: Python<'_>) -> Py { diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 05d9ccd6d2e..10b692a604c 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -20,6 +20,7 @@ fn test_compile_errors() { t.compile_fail("tests/ui/invalid_pymethod_enum.rs"); t.compile_fail("tests/ui/invalid_pymethod_names.rs"); t.compile_fail("tests/ui/invalid_pymodule_args.rs"); + t.compile_fail("tests/ui/invalid_pycallargs.rs"); t.compile_fail("tests/ui/reject_generics.rs"); t.compile_fail("tests/ui/invalid_closure.rs"); t.compile_fail("tests/ui/pyclass_send.rs"); diff --git a/tests/test_frompyobject.rs b/tests/test_frompyobject.rs index 344a47acf72..2192caf1f7c 100644 --- a/tests/test_frompyobject.rs +++ b/tests/test_frompyobject.rs @@ -648,3 +648,41 @@ fn test_transparent_from_py_with() { assert_eq!(result, expected); }); } + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +pub struct WithKeywordAttr { + r#box: usize, +} + +#[pyclass] +pub struct WithKeywordAttrC { + #[pyo3(get)] + r#box: usize, +} + +#[test] +fn test_with_keyword_attr() { + Python::with_gil(|py| { + let cls = WithKeywordAttrC { r#box: 3 }.into_pyobject(py).unwrap(); + let result = cls.extract::().unwrap(); + let expected = WithKeywordAttr { r#box: 3 }; + assert_eq!(result, expected); + }); +} + +#[derive(Debug, FromPyObject, PartialEq, Eq)] +pub struct WithKeywordItem { + #[pyo3(item)] + r#box: usize, +} + +#[test] +fn test_with_keyword_item() { + Python::with_gil(|py| { + let dict = PyDict::new(py); + dict.set_item("box", 3).unwrap(); + let result = dict.extract::().unwrap(); + let expected = WithKeywordItem { r#box: 3 }; + assert_eq!(result, expected); + }); +} diff --git a/tests/ui/invalid_pycallargs.rs b/tests/ui/invalid_pycallargs.rs new file mode 100644 index 00000000000..b77dbb20dcb --- /dev/null +++ b/tests/ui/invalid_pycallargs.rs @@ -0,0 +1,8 @@ +use pyo3::prelude::*; + +fn main() { + Python::with_gil(|py| { + let any = py.None().into_bound(py); + any.call1("foo"); + }) +} diff --git a/tests/ui/invalid_pycallargs.stderr b/tests/ui/invalid_pycallargs.stderr new file mode 100644 index 00000000000..93c0bc19b7f --- /dev/null +++ b/tests/ui/invalid_pycallargs.stderr @@ -0,0 +1,29 @@ +error[E0277]: `&str` cannot used as a Python `call` argument + --> tests/ui/invalid_pycallargs.rs:6:19 + | +6 | any.call1("foo"); + | ----- ^^^^^ the trait `PyCallArgs<'_>` is not implemented for `&str` + | | + | required by a bound introduced by this call + | + = note: `PyCallArgs` is implemented for Rust tuples, `Bound<'py, PyTuple>` and `Py` + = note: if your type is convertable to `PyTuple` via `IntoPyObject`, call `.into_pyobject(py)` manually + = note: if you meant to pass the type as a single argument, wrap it in a 1-tuple, `(,)` + = help: the following other types implement trait `PyCallArgs<'py>`: + &'a (T0, T1) + &'a (T0, T1, T2) + &'a (T0, T1, T2, T3) + &'a (T0, T1, T2, T3, T4) + &'a (T0, T1, T2, T3, T4, T5) + &'a (T0, T1, T2, T3, T4, T5, T6) + &'a (T0, T1, T2, T3, T4, T5, T6, T7) + &'a (T0, T1, T2, T3, T4, T5, T6, T7, T8) + and $N others +note: required by a bound in `call1` + --> src/types/any.rs + | + | fn call1(&self, args: A) -> PyResult> + | ----- required by a bound in this associated function + | where + | A: PyCallArgs<'py>; + | ^^^^^^^^^^^^^^^ required by this bound in `PyAnyMethods::call1`