From fde4c629e6dd564395107e3ce1ea1365b43d8989 Mon Sep 17 00:00:00 2001 From: ijl Date: Tue, 7 Jan 2025 15:37:00 +0000 Subject: [PATCH] build maintenance --- .github/workflows/artifact.yaml | 12 +- .github/workflows/debug.yaml | 8 +- .github/workflows/stale.yaml | 3 +- Cargo.lock | 76 ++- Cargo.toml | 8 +- README.md | 4 +- build.rs | 2 +- include/pyo3/pyo3-build-config/Cargo.toml | 10 +- include/pyo3/pyo3-build-config/src/impl_.rs | 544 +++++++++++++++--- .../pyo3/pyo3-build-config/src/import_lib.rs | 2 + include/pyo3/pyo3-build-config/src/lib.rs | 101 +++- include/pyo3/pyo3-ffi/Cargo.toml | 6 +- include/pyo3/pyo3-ffi/src/abstract_.rs | 1 - include/pyo3/pyo3-ffi/src/compat/py_3_13.rs | 21 + .../pyo3/pyo3-ffi/src/cpython/abstract_.rs | 4 +- .../pyo3-ffi/src/cpython/complexobject.rs | 1 - .../pyo3-ffi/src/cpython/critical_section.rs | 18 + .../pyo3/pyo3-ffi/src/cpython/dictobject.rs | 9 + .../pyo3/pyo3-ffi/src/cpython/floatobject.rs | 1 - .../pyo3/pyo3-ffi/src/cpython/genobject.rs | 2 +- .../pyo3/pyo3-ffi/src/cpython/listobject.rs | 4 +- include/pyo3/pyo3-ffi/src/cpython/object.rs | 2 - include/pyo3/pyo3-ffi/src/cpython/objimpl.rs | 1 + include/pyo3/pyo3-ffi/src/cpython/pyerrors.rs | 28 +- .../pyo3/pyo3-ffi/src/cpython/tupleobject.rs | 1 - .../pyo3-ffi/src/cpython/unicodeobject.rs | 13 +- include/pyo3/pyo3-ffi/src/datetime.rs | 93 +-- include/pyo3/pyo3-ffi/src/lib.rs | 166 ++++-- include/pyo3/pyo3-ffi/src/listobject.rs | 4 + include/pyo3/pyo3-ffi/src/methodobject.rs | 6 +- include/pyo3/pyo3-ffi/src/moduleobject.rs | 14 +- include/pyo3/pyo3-ffi/src/object.rs | 3 + include/pyo3/pyo3-ffi/src/pyhash.rs | 8 +- pyproject.toml | 6 +- requirements.txt | 4 +- script/install-fedora | 9 +- src/deserialize/backend/yyjson.rs | 14 +- src/deserialize/pyobject.rs | 2 +- src/lib.rs | 10 - src/util.rs | 6 + test/requirements.txt | 2 - test/test_append_newline.py | 6 +- test/test_error.py | 3 +- test/test_fixture.py | 4 +- test/test_fragment.py | 3 +- test/test_indent.py | 3 +- test/test_issue331.py | 6 +- test/test_jsonchecker.py | 3 +- test/test_parsing.py | 3 +- test/test_roundtrip.py | 3 +- test/test_sort_keys.py | 3 +- test/test_transform.py | 3 +- test/test_type.py | 17 - test/util.py | 14 +- 54 files changed, 966 insertions(+), 334 deletions(-) diff --git a/.github/workflows/artifact.yaml b/.github/workflows/artifact.yaml index 9fbdc201..6d06c0b0 100644 --- a/.github/workflows/artifact.yaml +++ b/.github/workflows/artifact.yaml @@ -3,7 +3,7 @@ on: push env: CARGO_UNSTABLE_SPARSE_REGISTRY: "true" PIP_DISABLE_PIP_VERSION_CHECK: "1" - RUST_TOOLCHAIN: "nightly-2024-11-22" + RUST_TOOLCHAIN: "nightly-2025-01-07" UNSAFE_PYO3_BUILD_FREE_THREADED: "1" UNSAFE_PYO3_SKIP_VERSION_CHECK: "1" UV_LINK_MODE: "copy" @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - - run: python3 -m pip install --user --upgrade pip "maturin==1.7.8" wheel + - run: python3 -m pip install --user --upgrade pip "maturin>=1,<2" wheel - name: Vendor dependencies run: | @@ -287,7 +287,6 @@ jobs: LDFLAGS: "-Wl,--as-needed" RUSTFLAGS: "-C lto=fat -Z mir-opt-level=4 -Z threads=2 -D warnings -C target-feature=-crt-static" with: - maturin-version: 1.7.8 rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" rustup-components: rust-src target: "${{ matrix.platform.target }}" @@ -392,7 +391,6 @@ jobs: LDFLAGS: "-Wl,--as-needed" RUSTFLAGS: "${{ matrix.target.rustflags }}" with: - maturin-version: 1.7.8 target: "${{ matrix.target.target }}" rust-toolchain: "${{ env.RUST_TOOLCHAIN }}" rustup-components: rust-src @@ -450,7 +448,7 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python python${{ matrix.python.version }} - uv pip install --upgrade "maturin==1.7.8" -r test/requirements.txt -r integration/requirements.txt + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt mkdir .cargo cp ci/config.toml .cargo/config.toml @@ -524,7 +522,7 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh uv venv --python python${{ matrix.python.version }} - uv pip install --upgrade "maturin==1.7.8" -r test/requirements.txt -r integration/requirements.txt + uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt mkdir .cargo cp ci/config.toml .cargo/config.toml @@ -599,7 +597,7 @@ jobs: run: | cargo fetch --target "${{ matrix.platform.target }}" & - python.exe -m pip install --upgrade pip "maturin==1.7.8" wheel + python.exe -m pip install --upgrade pip "maturin>=1,<2" wheel python.exe -m pip install -r test\requirements.txt -r integration\requirements.txt mkdir .cargo diff --git a/.github/workflows/debug.yaml b/.github/workflows/debug.yaml index 1171a3a7..0111dd40 100644 --- a/.github/workflows/debug.yaml +++ b/.github/workflows/debug.yaml @@ -8,9 +8,9 @@ jobs: fail-fast: false matrix: profile: [ - { rust: "1.72", features: "" }, - { rust: "1.72", features: "--features=yyjson" }, - { rust: "nightly-2024-11-22", features: "--features=avx512,yyjson,unstable-simd" }, + { rust: "1.82", features: "" }, + { rust: "1.82", features: "--features=yyjson" }, + { rust: "nightly-2025-01-07", features: "--features=avx512,yyjson,unstable-simd" }, ] python: [ { version: '3.13' }, @@ -30,7 +30,7 @@ jobs: with: python-version: '${{ matrix.python.version }}' - - run: python -m pip install --user --upgrade pip "maturin==1.7.8" wheel + - run: python -m pip install --user --upgrade pip "maturin>=1,<2" wheel - uses: actions/checkout@v4 diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 3dc0091d..881fed82 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -8,9 +8,10 @@ permissions: jobs: stale: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/stale@v9 with: days-before-stale: 7 days-before-close: 1 + exempt-assignees: ijl diff --git a/Cargo.lock b/Cargo.lock index 2a5f20e5..83e3e362 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "compact_str" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6050c3a16ddab2e412160b31f2c871015704239bca62f72f6e5f0be631d3f644" +checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" dependencies = [ "castaway", "cfg-if", @@ -86,9 +86,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.13" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "540654e97a3f4470a492cd30ff187bc95d89557a903a2bbf112e2fae98104ef2" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" [[package]] name = "itoap" @@ -98,15 +98,20 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "jiff" -version = "0.1.14" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d9d414fc817d3e3d62b2598616733f76c4cc74fbac96069674739b881295c8" +checksum = "ed0ce60560149333a8e41ca7dc78799c47c5fd435e2bc18faf6a054382eec037" +dependencies = [ + "portable-atomic", + "portable-atomic-util", + "serde", +] [[package]] name = "libc" -version = "0.2.164" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433bfe06b8c75da9b2e3fbea6e5329ff87748f0b144ef75306e674c3f6f7c13f" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "memchr" @@ -147,6 +152,21 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "portable-atomic" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "280dc24453071f1b63954171985a0b0d30058d287960968b9b2aca264c8d4ee6" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -158,7 +178,7 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.23.0-dev" +version = "0.23.3" dependencies = [ "once_cell", "target-lexicon", @@ -166,7 +186,7 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.23.0-dev" +version = "0.23.3" dependencies = [ "libc", "pyo3-build-config", @@ -174,18 +194,18 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -195,18 +215,18 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.217" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" dependencies = [ "proc-macro2", "quote", @@ -215,9 +235,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.135" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" dependencies = [ "itoa", "memchr", @@ -251,9 +271,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "syn" -version = "2.0.89" +version = "2.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d46482f1c1c87acd84dea20c1bf5ebff4c757009ed6bf19cfd36fb10e92c4e" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" dependencies = [ "proc-macro2", "quote", @@ -262,9 +282,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "dc12939a1c9b9d391e0b7135f72fd30508b73450753e28341fed159317582a77" [[package]] name = "unicode-ident" @@ -274,9 +294,9 @@ checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" [[package]] name = "unwinding" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2c6cb20f236dae10c69b0b45d82ef50af8b7e45c10e429e7901d26b49b4dbf3" +checksum = "51f06a05848f650946acef3bf525fe96612226b61f74ae23ffa4e98bfbb8ab3c" dependencies = [ "gimli", ] @@ -295,6 +315,6 @@ checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] name = "xxhash-rust" -version = "0.8.12" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" diff --git a/Cargo.toml b/Cargo.toml index b90a9519..d12a2d7f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["ijl "] description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" edition = "2021" resolver = "2" -rust-version = "1.72" +rust-version = "1.82" license = "Apache-2.0 OR MIT" repository = "https://github.com/ijl/orjson" homepage = "https://github.com/ijl/orjson" @@ -14,8 +14,8 @@ keywords = ["fast", "json", "dataclass", "dataclasses", "datetime", "rfc", "8259 include = [ "Cargo.toml", "CHANGELOG.md", - "data", - "include", + "include/pyo3", + "include/yyjson", "LICENSE-APACHE", "LICENSE-MIT", "pyproject.toml", @@ -71,7 +71,7 @@ uuid = { version = "1", default-features = false } xxhash-rust = { version = "^0.8", default-features = false, features = ["xxh3"] } [build-dependencies] -cc = { version = "1" } +cc = { version = "=1.2.1" } pyo3-build-config = { path = "include/pyo3/pyo3-build-config" } version_check = { version = "0.9" } diff --git a/README.md b/README.md index 75b1d4f2..19da4e9b 100644 --- a/README.md +++ b/README.md @@ -1078,7 +1078,7 @@ It benefits from also having a C build environment to compile a faster deserialization backend. See this project's `manylinux_2_28` builds for an example using clang and LTO. -The project's own CI tests against `nightly-2024-11-22` and stable 1.72. It +The project's own CI tests against `nightly-2025-01-07` and stable 1.72. It is prudent to pin the nightly version because that channel can introduce breaking changes. There is a significant performance benefit to using nightly. @@ -1101,5 +1101,5 @@ tests should be run as part of the build. It can be run with ## License -orjson was written by ijl <>, copyright 2018 - 2024, available +orjson was written by ijl <>, copyright 2018 - 2025, available to you under either the Apache 2 license or MIT license at your choice. diff --git a/build.rs b/build.rs index b3fc40c9..0979eabf 100644 --- a/build.rs +++ b/build.rs @@ -52,7 +52,7 @@ fn main() { } #[cfg(any(target_arch = "x86_64", target_arch = "aarch64"))] - if let Some(64) = python_config.pointer_width { + if matches!(python_config.pointer_width, Some(64)) { println!("cargo:rustc-cfg=feature=\"inline_int\""); } diff --git a/include/pyo3/pyo3-build-config/Cargo.toml b/include/pyo3/pyo3-build-config/Cargo.toml index a5549df3..0facef81 100644 --- a/include/pyo3/pyo3-build-config/Cargo.toml +++ b/include/pyo3/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.23.0-dev" +version = "0.23.3" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -12,12 +12,12 @@ edition = "2021" [dependencies] once_cell = "1" -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12.14" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [build-dependencies] -python3-dll-a = { version = "0.2.6", optional = true } -target-lexicon = "0.12.14" +python3-dll-a = { version = "0.2.12", optional = true } +target-lexicon = "0.13" [features] default = [] diff --git a/include/pyo3/pyo3-build-config/src/impl_.rs b/include/pyo3/pyo3-build-config/src/impl_.rs index ec652591..0f90d573 100644 --- a/include/pyo3/pyo3-build-config/src/impl_.rs +++ b/include/pyo3/pyo3-build-config/src/impl_.rs @@ -6,6 +6,8 @@ #[path = "import_lib.rs"] mod import_lib; +#[cfg(test)] +use std::cell::RefCell; use std::{ collections::{HashMap, HashSet}, env, @@ -15,13 +17,12 @@ use std::{ io::{BufRead, BufReader, Read, Write}, path::{Path, PathBuf}, process::{Command, Stdio}, - str, - str::FromStr, + str::{self, FromStr}, }; pub use target_lexicon::Triple; -use target_lexicon::{Environment, OperatingSystem}; +use target_lexicon::{Architecture, Environment, OperatingSystem}; use crate::{ bail, ensure, @@ -41,6 +42,11 @@ const MINIMUM_SUPPORTED_VERSION_GRAALPY: PythonVersion = PythonVersion { /// Maximum Python version that can be used as minimum required Python version with abi3. pub(crate) const ABI3_MAX_MINOR: u8 = 12; +#[cfg(test)] +thread_local! { + static READ_ENV_VARS: RefCell> = const { RefCell::new(Vec::new()) }; +} + /// Gets an environment variable owned by cargo. /// /// Environment variables set by cargo are expected to be valid UTF8. @@ -54,6 +60,12 @@ pub fn env_var(var: &str) -> Option { if cfg!(feature = "resolve-config") { println!("cargo:rerun-if-env-changed={}", var); } + #[cfg(test)] + { + READ_ENV_VARS.with(|env_vars| { + env_vars.borrow_mut().push(var.to_owned()); + }); + } env::var_os(var) } @@ -155,6 +167,8 @@ pub struct InterpreterConfig { /// /// Serialized to multiple `extra_build_script_line` values. pub extra_build_script_lines: Vec, + /// macOS Python3.framework requires special rpath handling + pub python_framework_prefix: Option, } impl InterpreterConfig { @@ -168,6 +182,7 @@ impl InterpreterConfig { for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { out.push(format!("cargo:rustc-cfg=Py_3_{}", i)); } + println!("cargo::rustc-check-cfg=cfg(Py_3_14)"); match self.implementation { PythonImplementation::CPython => {} @@ -176,7 +191,7 @@ impl InterpreterConfig { } // If Py_GIL_DISABLED is set, do not build with limited API support - if self.abi3 && !self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) { + if self.abi3 && !self.is_free_threaded() { out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned()); } @@ -233,6 +248,7 @@ WINDOWS = platform.system() == "Windows" # macOS framework packages use shared linking FRAMEWORK = bool(get_config_var("PYTHONFRAMEWORK")) +FRAMEWORK_PREFIX = get_config_var("PYTHONFRAMEWORKPREFIX") # unix-style shared library enabled SHARED = bool(get_config_var("Py_ENABLE_SHARED")) @@ -241,6 +257,7 @@ print("implementation", platform.python_implementation()) print("version_major", sys.version_info[0]) print("version_minor", sys.version_info[1]) print("shared", PYPY or GRAALPY or ANACONDA or WINDOWS or FRAMEWORK or SHARED) +print("python_framework_prefix", FRAMEWORK_PREFIX) print_if_set("ld_version", get_config_var("LDVERSION")) print_if_set("libdir", get_config_var("LIBDIR")) print_if_set("base_prefix", base_prefix) @@ -248,6 +265,7 @@ print("executable", sys.executable) print("calcsize_pointer", struct.calcsize("P")) print("mingw", get_platform().startswith("mingw")) print("ext_suffix", get_config_var("EXT_SUFFIX")) +print("gil_disabled", get_config_var("Py_GIL_DISABLED")) "#; let output = run_python_script(interpreter.as_ref(), SCRIPT)?; let map: HashMap = parse_script_output(&output); @@ -276,6 +294,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) }; let shared = map["shared"].as_str() == "True"; + let python_framework_prefix = map.get("python_framework_prefix").cloned(); let version = PythonVersion { major: map["version_major"] @@ -290,6 +309,13 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let implementation = map["implementation"].parse()?; + let gil_disabled = match map["gil_disabled"].as_str() { + "1" => true, + "0" => false, + "None" => false, + _ => panic!("Unknown Py_GIL_DISABLED value"), + }; + let lib_name = if cfg!(windows) { default_lib_name_windows( version, @@ -300,13 +326,15 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // on Windows from sysconfig - e.g. ext_suffix may be // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), - ) + gil_disabled, + )? } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), - ) + gil_disabled, + )? }; let lib_dir = if cfg!(windows) { @@ -337,6 +365,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags: BuildFlags::from_interpreter(interpreter)?, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -374,12 +403,20 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) Some(s) => !s.is_empty(), _ => false, }; + let python_framework_prefix = sysconfigdata + .get_value("PYTHONFRAMEWORKPREFIX") + .map(str::to_string); let lib_dir = get_key!(sysconfigdata, "LIBDIR").ok().map(str::to_string); + let gil_disabled = match sysconfigdata.get_value("Py_GIL_DISABLED") { + Some(value) => value == "1", + None => false, + }; let lib_name = Some(default_lib_name_unix( version, implementation, sysconfigdata.get_value("LDVERSION"), - )); + gil_disabled, + )?); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) .ok(); @@ -397,6 +434,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix, }) } @@ -405,7 +443,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) /// The `abi3` features, if set, may apply an `abi3` constraint to the Python version. #[allow(dead_code)] // only used in build.rs pub(super) fn from_pyo3_config_file_env() -> Option> { - cargo_env_var("PYO3_CONFIG_FILE").map(|path| { + env_var("PYO3_CONFIG_FILE").map(|path| { let path = Path::new(&path); println!("cargo:rerun-if-changed={}", path.display()); // Absolute path is necessary because this build script is run with a cwd different to the @@ -470,9 +508,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let mut lib_dir = None; let mut executable = None; let mut pointer_width = None; - let mut build_flags = None; + let mut build_flags: Option = None; let mut suppress_build_script_link_lines = None; let mut extra_build_script_lines = vec![]; + let mut python_framework_prefix = None; for (i, line) in lines.enumerate() { let line = line.context("failed to read line from config")?; @@ -501,6 +540,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) "extra_build_script_line" => { extra_build_script_lines.push(value.to_string()); } + "python_framework_prefix" => parse_value!(python_framework_prefix, value), unknown => warn!("unknown config key `{}`", unknown), } } @@ -508,10 +548,12 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) let version = version.ok_or("missing value for version")?; let implementation = implementation.unwrap_or(PythonImplementation::CPython); let abi3 = abi3.unwrap_or(false); + let build_flags = build_flags.unwrap_or_default(); + let gil_disabled = build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED); // Fixup lib_name if it's not set let lib_name = lib_name.or_else(|| { if let Ok(Ok(target)) = env::var("TARGET").map(|target| target.parse::()) { - default_lib_name_for_target(version, implementation, abi3, &target) + default_lib_name_for_target(version, implementation, abi3, gil_disabled, &target) } else { None } @@ -526,9 +568,10 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) lib_dir, executable, pointer_width, - build_flags: build_flags.unwrap_or_default(), + build_flags, suppress_build_script_link_lines: suppress_build_script_link_lines.unwrap_or(false), extra_build_script_lines, + python_framework_prefix, }) } @@ -538,9 +581,25 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) // Auto generate python3.dll import libraries for Windows targets. if self.lib_dir.is_none() { let target = target_triple_from_env(); - let py_version = if self.abi3 { None } else { Some(self.version) }; - self.lib_dir = - import_lib::generate_import_lib(&target, self.implementation, py_version)?; + let py_version = if self.implementation == PythonImplementation::CPython + && self.abi3 + && !self.is_free_threaded() + { + None + } else { + Some(self.version) + }; + let abiflags = if self.is_free_threaded() { + Some("t") + } else { + None + }; + self.lib_dir = import_lib::generate_import_lib( + &target, + self.implementation, + py_version, + abiflags, + )?; } Ok(()) } @@ -605,6 +664,7 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) write_option_line!(executable)?; write_option_line!(pointer_width)?; write_line!(build_flags)?; + write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { writeln!(writer, "extra_build_script_line={}", line) @@ -645,10 +705,18 @@ print("ext_suffix", get_config_var("EXT_SUFFIX")) ) } - /// Lowers the configured version to the abi3 version, if set. + pub fn is_free_threaded(&self) -> bool { + self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) + } + + /// Updates configured ABI to build for to the requested abi3 version + /// This is a no-op for platforms where abi3 is not supported fn fixup_for_abi3_version(&mut self, abi3_version: Option) -> Result<()> { - // PyPy doesn't support abi3; don't adjust the version - if self.implementation.is_pypy() || self.implementation.is_graalpy() { + // PyPy, GraalPy, and the free-threaded build don't support abi3; don't adjust the version + if self.implementation.is_pypy() + || self.implementation.is_graalpy() + || self.is_free_threaded() + { return Ok(()); } @@ -676,6 +744,14 @@ pub struct PythonVersion { } impl PythonVersion { + pub const PY313: Self = PythonVersion { + major: 3, + minor: 13, + }; + const PY310: Self = PythonVersion { + major: 3, + minor: 10, + }; const PY37: Self = PythonVersion { major: 3, minor: 7 }; } @@ -837,6 +913,9 @@ pub struct CrossCompileConfig { /// The compile target triple (e.g. aarch64-unknown-linux-gnu) target: Triple, + + /// Python ABI flags, used to detect free-threaded Python builds. + abiflags: Option, } impl CrossCompileConfig { @@ -851,7 +930,7 @@ impl CrossCompileConfig { ) -> Result> { if env_vars.any() || Self::is_cross_compiling_from_to(host, target) { let lib_dir = env_vars.lib_dir_path()?; - let version = env_vars.parse_version()?; + let (version, abiflags) = env_vars.parse_version()?; let implementation = env_vars.parse_implementation()?; let target = target.clone(); @@ -860,6 +939,7 @@ impl CrossCompileConfig { version, implementation, target, + abiflags, })) } else { Ok(None) @@ -879,11 +959,13 @@ impl CrossCompileConfig { // Not cross-compiling to compile for 32-bit Python from windows 64-bit compatible |= target.operating_system == OperatingSystem::Windows - && host.operating_system == OperatingSystem::Windows; + && host.operating_system == OperatingSystem::Windows + && matches!(target.architecture, Architecture::X86_32(_)) + && host.architecture == Architecture::X86_64; // Not cross-compiling to compile for x86-64 Python from macOS arm64 and vice versa - compatible |= target.operating_system == OperatingSystem::Darwin - && host.operating_system == OperatingSystem::Darwin; + compatible |= matches!(target.operating_system, OperatingSystem::Darwin(_)) + && matches!(host.operating_system, OperatingSystem::Darwin(_)); !compatible } @@ -933,22 +1015,25 @@ impl CrossCompileEnvVars { } /// Parses `PYO3_CROSS_PYTHON_VERSION` environment variable value - /// into `PythonVersion`. - fn parse_version(&self) -> Result> { - let version = self - .pyo3_cross_python_version - .as_ref() - .map(|os_string| { + /// into `PythonVersion` and ABI flags. + fn parse_version(&self) -> Result<(Option, Option)> { + match self.pyo3_cross_python_version.as_ref() { + Some(os_string) => { let utf8_str = os_string .to_str() .ok_or("PYO3_CROSS_PYTHON_VERSION is not valid a UTF-8 string")?; - utf8_str + let (utf8_str, abiflags) = if let Some(version) = utf8_str.strip_suffix('t') { + (version, Some("t".to_string())) + } else { + (utf8_str, None) + }; + let version = utf8_str .parse() - .context("failed to parse PYO3_CROSS_PYTHON_VERSION") - }) - .transpose()?; - - Ok(version) + .context("failed to parse PYO3_CROSS_PYTHON_VERSION")?; + Ok((Some(version), abiflags)) + } + None => Ok((None, None)), + } } /// Parses `PYO3_CROSS_PYTHON_IMPLEMENTATION` environment variable value @@ -1091,11 +1176,7 @@ impl BuildFlags { Self( BuildFlags::ALL .iter() - .filter(|flag| { - config_map - .get_value(flag.to_string()) - .map_or(false, |value| value == "1") - }) + .filter(|flag| config_map.get_value(flag.to_string()) == Some("1")) .cloned() .collect(), ) @@ -1106,10 +1187,15 @@ impl BuildFlags { /// the interpreter and printing variables of interest from /// sysconfig.get_config_vars. fn from_interpreter(interpreter: impl AsRef) -> Result { - // sysconfig is missing all the flags on windows, so we can't actually - // query the interpreter directly for its build flags. + // sysconfig is missing all the flags on windows for Python 3.12 and + // older, so we can't actually query the interpreter directly for its + // build flags on those versions. if cfg!(windows) { - return Ok(Self::new()); + let script = String::from("import sys;print(sys.version_info < (3, 13))"); + let stdout = run_python_script(interpreter.as_ref(), &script)?; + if stdout.trim_end() == "True" { + return Ok(Self::new()); + } } let mut script = String::from("import sysconfig\n"); @@ -1473,22 +1559,34 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result Result InterpreterConfig { +fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result { // FIXME: PyPy & GraalPy do not support the Stable ABI. let implementation = PythonImplementation::CPython; let abi3 = true; @@ -1528,12 +1627,13 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf abi3, false, false, - )) + false, + )?) } else { None }; - InterpreterConfig { + Ok(InterpreterConfig { implementation, version, shared: true, @@ -1545,7 +1645,8 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], - } + python_framework_prefix: None, + }) } /// Detects the cross compilation target interpreter configuration from all @@ -1585,28 +1686,24 @@ fn load_cross_compile_config( Ok(config) } -// Link against python3.lib for the stable ABI on Windows. -// See https://www.python.org/dev/peps/pep-0384/#linkage -// -// This contains only the limited ABI symbols. +// These contains only the limited ABI symbols. const WINDOWS_ABI3_LIB_NAME: &str = "python3"; +const WINDOWS_ABI3_DEBUG_LIB_NAME: &str = "python3_d"; fn default_lib_name_for_target( version: PythonVersion, implementation: PythonImplementation, abi3: bool, + gil_disabled: bool, target: &Triple, ) -> Option { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows( - version, - implementation, - abi3, - false, - false, - )) + Some( + default_lib_name_windows(version, implementation, abi3, false, false, gil_disabled) + .unwrap(), + ) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None)) + Some(default_lib_name_unix(version, implementation, None, gil_disabled).unwrap()) } else { None } @@ -1618,18 +1715,36 @@ fn default_lib_name_windows( abi3: bool, mingw: bool, debug: bool, -) -> String { - if debug { + gil_disabled: bool, +) -> Result { + if debug && version < PythonVersion::PY310 { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - format!("python{}{}_d", version.major, version.minor) - } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { - WINDOWS_ABI3_LIB_NAME.to_owned() + Ok(format!("python{}{}_d", version.major, version.minor)) + } else if abi3 && !(gil_disabled || implementation.is_pypy() || implementation.is_graalpy()) { + if debug { + Ok(WINDOWS_ABI3_DEBUG_LIB_NAME.to_owned()) + } else { + Ok(WINDOWS_ABI3_LIB_NAME.to_owned()) + } } else if mingw { + ensure!( + !gil_disabled, + "MinGW free-threaded builds are not currently tested or supported" + ); // https://packages.msys2.org/base/mingw-w64-python - format!("python{}.{}", version.major, version.minor) + Ok(format!("python{}.{}", version.major, version.minor)) + } else if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + if debug { + Ok(format!("python{}{}t_d", version.major, version.minor)) + } else { + Ok(format!("python{}{}t", version.major, version.minor)) + } + } else if debug { + Ok(format!("python{}{}_d", version.major, version.minor)) } else { - format!("python{}{}", version.major, version.minor) + Ok(format!("python{}{}", version.major, version.minor)) } } @@ -1637,26 +1752,32 @@ fn default_lib_name_unix( version: PythonVersion, implementation: PythonImplementation, ld_version: Option<&str>, -) -> String { + gil_disabled: bool, +) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => format!("python{}", ld_version), + Some(ld_version) => Ok(format!("python{}", ld_version)), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone - format!("python{}.{}", version.major, version.minor) + if gil_disabled { + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + Ok(format!("python{}.{}t", version.major, version.minor)) + } else { + Ok(format!("python{}.{}", version.major, version.minor)) + } } else { // Work around https://bugs.python.org/issue36707 - format!("python{}.{}m", version.major, version.minor) + Ok(format!("python{}.{}m", version.major, version.minor)) } } }, PythonImplementation::PyPy => match ld_version { - Some(ld_version) => format!("pypy{}-c", ld_version), - None => format!("pypy{}.{}-c", version.major, version.minor), + Some(ld_version) => Ok(format!("pypy{}-c", ld_version)), + None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), }, - PythonImplementation::GraalPy => "python-native".to_string(), + PythonImplementation::GraalPy => Ok("python-native".to_string()), } } @@ -1826,12 +1947,19 @@ pub fn make_interpreter_config() -> Result { ); }; - let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap()); + let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap())?; // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] { - let py_version = if interpreter_config.abi3 { + let gil_disabled = interpreter_config + .build_flags + .0 + .contains(&BuildFlag::Py_GIL_DISABLED); + let py_version = if interpreter_config.implementation == PythonImplementation::CPython + && interpreter_config.abi3 + && !gil_disabled + { None } else { Some(interpreter_config.version) @@ -1840,6 +1968,7 @@ pub fn make_interpreter_config() -> Result { &host, interpreter_config.implementation, py_version, + None, )?; } @@ -1899,6 +2028,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1927,6 +2057,7 @@ mod tests { }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1948,6 +2079,7 @@ mod tests { version: MINIMUM_SUPPORTED_VERSION, suppress_build_script_link_lines: true, extra_build_script_lines: vec!["cargo:test1".to_string(), "cargo:test2".to_string()], + python_framework_prefix: None, }; let mut buf: Vec = Vec::new(); config.to_writer(&mut buf).unwrap(); @@ -1974,6 +2106,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -1996,6 +2129,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2098,6 +2232,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2127,6 +2262,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); @@ -2153,6 +2289,7 @@ mod tests { version: PythonVersion::PY37, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2163,7 +2300,7 @@ mod tests { let min_version = "3.7".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 7 }, @@ -2176,6 +2313,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2186,7 +2324,7 @@ mod tests { let min_version = "3.9".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 9 }, @@ -2199,6 +2337,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2233,6 +2372,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2267,6 +2407,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2301,6 +2442,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2337,6 +2479,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ); } @@ -2351,9 +2494,20 @@ mod tests { false, false, false, - ), + false, + ) + .unwrap(), "python39", ); + assert!(super::default_lib_name_windows( + PythonVersion { major: 3, minor: 9 }, + CPython, + false, + false, + false, + true, + ) + .is_err()); assert_eq!( super::default_lib_name_windows( PythonVersion { major: 3, minor: 9 }, @@ -2361,7 +2515,9 @@ mod tests { true, false, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( @@ -2371,7 +2527,9 @@ mod tests { false, true, false, - ), + false, + ) + .unwrap(), "python3.9", ); assert_eq!( @@ -2381,7 +2539,9 @@ mod tests { true, true, false, - ), + false, + ) + .unwrap(), "python3", ); assert_eq!( @@ -2391,7 +2551,9 @@ mod tests { true, false, false, - ), + false, + ) + .unwrap(), "python39", ); assert_eq!( @@ -2401,10 +2563,12 @@ mod tests { false, false, true, - ), + false, + ) + .unwrap(), "python39_d", ); - // abi3 debug builds on windows use version-specific lib + // abi3 debug builds on windows use version-specific lib on 3.9 and older // to workaround https://github.com/python/cpython/issues/101614 assert_eq!( super::default_lib_name_windows( @@ -2413,9 +2577,97 @@ mod tests { true, false, true, - ), + false, + ) + .unwrap(), "python39_d", ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 10 + }, + CPython, + true, + false, + true, + false, + ) + .unwrap(), + "python3_d", + ); + // Python versions older than 3.13 don't support gil_disabled + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + false, + false, + true, + ) + .is_err()); + // mingw and free-threading are incompatible (until someone adds support) + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + true, + false, + true, + ) + .is_err()); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + true, // abi3 true should not affect the free-threaded lib name + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + true, + true, + ) + .unwrap(), + "python313t_d", + ); } #[test] @@ -2423,16 +2675,34 @@ mod tests { use PythonImplementation::*; // Defaults to python3.7m for CPython 3.7 assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 7 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 7 }, + CPython, + None, + false + ) + .unwrap(), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 8 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 8 }, + CPython, + None, + false + ) + .unwrap(), "python3.8", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, CPython, None), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + CPython, + None, + false + ) + .unwrap(), "python3.9", ); // Can use ldversion to override for CPython @@ -2440,21 +2710,56 @@ mod tests { super::default_lib_name_unix( PythonVersion { major: 3, minor: 9 }, CPython, - Some("3.7md") - ), + Some("3.7md"), + false + ) + .unwrap(), "python3.7md", ); // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None), + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false) + .unwrap(), "pypy3.9-c", ); assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, Some("3.9d")), + super::default_lib_name_unix( + PythonVersion { major: 3, minor: 9 }, + PyPy, + Some("3.9d"), + false + ) + .unwrap(), "pypy3.9d-c", ); + + // free-threading adds a t suffix + assert_eq!( + super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + None, + true + ) + .unwrap(), + "python3.13t", + ); + // 3.12 and older are incompatible with gil_disabled + assert!(super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + None, + true, + ) + .is_err()); } #[test] @@ -2468,7 +2773,7 @@ mod tests { assert_eq!( env_vars.parse_version().unwrap(), - Some(PythonVersion { major: 3, minor: 9 }) + (Some(PythonVersion { major: 3, minor: 9 }), None), ); let env_vars = CrossCompileEnvVars { @@ -2478,7 +2783,25 @@ mod tests { pyo3_cross_python_implementation: None, }; - assert_eq!(env_vars.parse_version().unwrap(), None); + assert_eq!(env_vars.parse_version().unwrap(), (None, None)); + + let env_vars = CrossCompileEnvVars { + pyo3_cross: None, + pyo3_cross_lib_dir: None, + pyo3_cross_python_version: Some("3.13t".into()), + pyo3_cross_python_implementation: None, + }; + + assert_eq!( + env_vars.parse_version().unwrap(), + ( + Some(PythonVersion { + major: 3, + minor: 13 + }), + Some("t".into()) + ), + ); let env_vars = CrossCompileEnvVars { pyo3_cross: None, @@ -2504,6 +2827,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; config @@ -2526,6 +2850,7 @@ mod tests { version: PythonVersion { major: 3, minor: 7 }, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert!(config @@ -2561,6 +2886,11 @@ mod tests { version: Some(interpreter_config.version), implementation: Some(interpreter_config.implementation), target: triple!("x86_64-unknown-linux-gnu"), + abiflags: if interpreter_config.is_free_threaded() { + Some("t".into()) + } else { + None + }, }; let sysconfigdata_path = match find_sysconfigdata(&cross) { @@ -2585,6 +2915,7 @@ mod tests { version: interpreter_config.version, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, } ) } @@ -2660,6 +2991,16 @@ mod tests { .is_none()); } + #[test] + fn test_is_cross_compiling_from_to() { + assert!(cross_compiling_from_to( + &triple!("x86_64-pc-windows-msvc"), + &triple!("aarch64-pc-windows-msvc") + ) + .unwrap() + .is_some()); + } + #[test] fn test_run_python_script() { // as above, this should be okay in CI where Python is presumed installed @@ -2699,6 +3040,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( interpreter_config.build_script_outputs(), @@ -2738,6 +3080,7 @@ mod tests { build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -2785,6 +3128,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -2818,6 +3162,7 @@ mod tests { build_flags, suppress_build_script_link_lines: false, extra_build_script_lines: vec![], + python_framework_prefix: None, }; assert_eq!( @@ -2836,6 +3181,7 @@ mod tests { version: None, implementation: None, target: triple!("x86_64-unknown-linux-gnu"), + abiflags: None, }) .unwrap_err(); @@ -2848,4 +3194,12 @@ mod tests { " )); } + + #[test] + fn test_from_pyo3_config_file_env_rebuild() { + READ_ENV_VARS.with(|vars| vars.borrow_mut().clear()); + let _ = InterpreterConfig::from_pyo3_config_file_env(); + // it's possible that other env vars were also read, hence just checking for contains + READ_ENV_VARS.with(|vars| assert!(vars.borrow().contains(&"PYO3_CONFIG_FILE".to_string()))); + } } diff --git a/include/pyo3/pyo3-build-config/src/import_lib.rs b/include/pyo3/pyo3-build-config/src/import_lib.rs index 0925a861..ee934441 100644 --- a/include/pyo3/pyo3-build-config/src/import_lib.rs +++ b/include/pyo3/pyo3-build-config/src/import_lib.rs @@ -19,6 +19,7 @@ pub(super) fn generate_import_lib( target: &Triple, py_impl: PythonImplementation, py_version: Option, + abiflags: Option<&str>, ) -> Result> { if target.operating_system != OperatingSystem::Windows { return Ok(None); @@ -50,6 +51,7 @@ pub(super) fn generate_import_lib( ImportLibraryGenerator::new(&arch, &env) .version(py_version.map(|v| (v.major, v.minor))) .implementation(implementation) + .abiflags(abiflags) .generate(&out_lib_dir) .context("failed to generate python3.dll import library")?; diff --git a/include/pyo3/pyo3-build-config/src/lib.rs b/include/pyo3/pyo3-build-config/src/lib.rs index 6e295c99..9070f6d7 100644 --- a/include/pyo3/pyo3-build-config/src/lib.rs +++ b/include/pyo3/pyo3-build-config/src/lib.rs @@ -65,7 +65,7 @@ pub fn add_extension_module_link_args() { } fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Write) { - if triple.operating_system == OperatingSystem::Darwin { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) { writeln!(writer, "cargo:rustc-cdylib-link-arg=-undefined").unwrap(); writeln!(writer, "cargo:rustc-cdylib-link-arg=dynamic_lookup").unwrap(); } else if triple == &Triple::from_str("wasm32-unknown-emscripten").unwrap() { @@ -74,6 +74,44 @@ fn _add_extension_module_link_args(triple: &Triple, mut writer: impl std::io::Wr } } +/// Adds linker arguments suitable for linking against the Python framework on macOS. +/// +/// This should be called from a build script. +/// +/// The following link flags are added: +/// - macOS: `-Wl,-rpath,` +/// +/// All other platforms currently are no-ops. +#[cfg(feature = "resolve-config")] +pub fn add_python_framework_link_args() { + let interpreter_config = pyo3_build_script_impl::resolve_interpreter_config().unwrap(); + _add_python_framework_link_args( + &interpreter_config, + &impl_::target_triple_from_env(), + impl_::is_linking_libpython(), + std::io::stdout(), + ) +} + +#[cfg(feature = "resolve-config")] +fn _add_python_framework_link_args( + interpreter_config: &InterpreterConfig, + triple: &Triple, + link_libpython: bool, + mut writer: impl std::io::Write, +) { + if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { + if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { + writeln!( + writer, + "cargo:rustc-link-arg=-Wl,-rpath,{}", + framework_prefix + ) + .unwrap(); + } + } +} + /// Loads the configuration determined from the build environment. /// /// Because this will never change in a given compilation run, this is cached in a `once_cell`. @@ -138,6 +176,10 @@ fn resolve_cross_compile_config_path() -> Option { pub fn print_feature_cfgs() { let rustc_minor_version = rustc_minor_version().unwrap_or(0); + if rustc_minor_version >= 70 { + println!("cargo:rustc-cfg=rustc_has_once_lock"); + } + // invalid_from_utf8 lint was added in Rust 1.74 if rustc_minor_version >= 74 { println!("cargo:rustc-cfg=invalid_from_utf8_lint"); @@ -152,6 +194,14 @@ pub fn print_feature_cfgs() { if rustc_minor_version >= 79 { println!("cargo:rustc-cfg=diagnostic_namespace"); } + + if rustc_minor_version >= 83 { + println!("cargo:rustc-cfg=io_error_more"); + } + + if rustc_minor_version >= 85 { + println!("cargo:rustc-cfg=fn_ptr_eq"); + } } /// Registers `pyo3`s config names as reachable cfg expressions @@ -175,13 +225,15 @@ pub fn print_expected_cfgs() { println!("cargo:rustc-check-cfg=cfg(pyo3_leak_on_drop_without_reference_pool)"); println!("cargo:rustc-check-cfg=cfg(diagnostic_namespace)"); println!("cargo:rustc-check-cfg=cfg(c_str_lit)"); + println!("cargo:rustc-check-cfg=cfg(rustc_has_once_lock)"); + println!("cargo:rustc-check-cfg=cfg(io_error_more)"); + println!("cargo:rustc-check-cfg=cfg(fn_ptr_eq)"); // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=impl_::ABI3_MAX_MINOR + 1 { println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); } - println!("cargo::rustc-check-cfg=cfg(Py_3_14)"); } /// Private exports used in PyO3's build.rs @@ -292,4 +344,49 @@ mod tests { cargo:rustc-cdylib-link-arg=-sWASM_BIGINT\n" ); } + + #[cfg(feature = "resolve-config")] + #[test] + fn python_framework_link_args() { + let mut buf = Vec::new(); + + let interpreter_config = InterpreterConfig { + implementation: PythonImplementation::CPython, + version: PythonVersion { + major: 3, + minor: 13, + }, + shared: true, + abi3: false, + lib_name: None, + lib_dir: None, + executable: None, + pointer_width: None, + build_flags: BuildFlags::default(), + suppress_build_script_link_lines: false, + extra_build_script_lines: vec![], + python_framework_prefix: Some( + "/Applications/Xcode.app/Contents/Developer/Library/Frameworks".to_string(), + ), + }; + // Does nothing on non-mac + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-pc-windows-msvc").unwrap(), + true, + &mut buf, + ); + assert_eq!(buf, Vec::new()); + + _add_python_framework_link_args( + &interpreter_config, + &Triple::from_str("x86_64-apple-darwin").unwrap(), + true, + &mut buf, + ); + assert_eq!( + std::str::from_utf8(&buf).unwrap(), + "cargo:rustc-link-arg=-Wl,-rpath,/Applications/Xcode.app/Contents/Developer/Library/Frameworks\n" + ); + } } diff --git a/include/pyo3/pyo3-ffi/Cargo.toml b/include/pyo3/pyo3-ffi/Cargo.toml index e569ca1d..14a4f958 100644 --- a/include/pyo3/pyo3-ffi/Cargo.toml +++ b/include/pyo3/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.23.0-dev" +version = "0.23.3" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -12,7 +12,7 @@ edition = "2021" links = "python" [dependencies] -libc = "0.2.62" +libc = "0.2" [features] @@ -41,4 +41,4 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.23.0-dev", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", features = ["resolve-config"] } diff --git a/include/pyo3/pyo3-ffi/src/abstract_.rs b/include/pyo3/pyo3-ffi/src/abstract_.rs index 18995450..ce6c9b94 100644 --- a/include/pyo3/pyo3-ffi/src/abstract_.rs +++ b/include/pyo3/pyo3-ffi/src/abstract_.rs @@ -17,7 +17,6 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_ extern "C" { #[cfg(all( not(PyPy), - not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] diff --git a/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs b/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs index 9f44ced6..59289cb7 100644 --- a/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs +++ b/include/pyo3/pyo3-ffi/src/compat/py_3_13.rs @@ -83,3 +83,24 @@ compat_function!( 1 } ); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Extend( + list: *mut crate::PyObject, + iterable: *mut crate::PyObject, + ) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, crate::PY_SSIZE_T_MAX, crate::PY_SSIZE_T_MAX, iterable) + } +); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyList_Clear(list: *mut crate::PyObject) -> std::os::raw::c_int { + crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) + } +); diff --git a/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs b/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs index 83295e58..477ad02b 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/abstract_.rs @@ -1,5 +1,7 @@ use crate::{PyObject, Py_ssize_t}; -use std::os::raw::{c_char, c_int}; +#[cfg(not(all(Py_3_11, GraalPy)))] +use std::os::raw::c_char; +use std::os::raw::c_int; #[cfg(not(Py_3_11))] use crate::Py_buffer; diff --git a/include/pyo3/pyo3-ffi/src/cpython/complexobject.rs b/include/pyo3/pyo3-ffi/src/cpython/complexobject.rs index 255f9c27..4cc86db5 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/complexobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/complexobject.rs @@ -19,7 +19,6 @@ pub struct Py_complex { #[repr(C)] pub struct PyComplexObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub cval: Py_complex, } diff --git a/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs b/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs index 97b2f5e0..7b91f793 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/critical_section.rs @@ -9,6 +9,24 @@ pub struct PyCriticalSection { _cs_mutex: *mut PyMutex, } +#[cfg(Py_GIL_DISABLED)] +impl Default for PyCriticalSection { + fn default() -> Self { + PyCriticalSection { + _cs_prev: 0, + _cs_mutex: core::ptr::null_mut(), + } + } +} + +#[cfg(Py_GIL_DISABLED)] +impl PyCriticalSection { + #[inline] + pub fn is_locked(&self) -> bool { + !self._cs_mutex.is_null() + } +} + #[repr(C)] #[cfg(Py_GIL_DISABLED)] pub struct PyCriticalSection2 { diff --git a/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs b/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs index 79dcbfdb..f67a7725 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/dictobject.rs @@ -36,6 +36,15 @@ extern "C" { item: *mut PyObject, hash: crate::Py_hash_t, ) -> c_int; + + #[cfg(Py_3_13)] + pub fn _PyDict_SetItem_KnownHash_LockHeld( + mp: *mut PyDictObject, + name: *mut PyObject, + value: *mut PyObject, + hash: crate::Py_hash_t, + ) -> c_int; + // skipped _PyDict_DelItem_KnownHash // skipped _PyDict_DelItemIf // skipped _PyDict_NewKeysForClass diff --git a/include/pyo3/pyo3-ffi/src/cpython/floatobject.rs b/include/pyo3/pyo3-ffi/src/cpython/floatobject.rs index 8c7ee885..e7caa441 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/floatobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/floatobject.rs @@ -6,7 +6,6 @@ use std::os::raw::c_double; #[repr(C)] pub struct PyFloatObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub ob_fval: c_double, } diff --git a/include/pyo3/pyo3-ffi/src/cpython/genobject.rs b/include/pyo3/pyo3-ffi/src/cpython/genobject.rs index 17348b2f..4be310a8 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/genobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/genobject.rs @@ -2,7 +2,7 @@ use crate::object::*; use crate::PyFrameObject; #[cfg(not(any(PyPy, GraalPy)))] use crate::_PyErr_StackItem; -#[cfg(Py_3_11)] +#[cfg(all(Py_3_11, not(GraalPy)))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; diff --git a/include/pyo3/pyo3-ffi/src/cpython/listobject.rs b/include/pyo3/pyo3-ffi/src/cpython/listobject.rs index 963ddfbe..694e6bc4 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/listobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/listobject.rs @@ -2,7 +2,7 @@ use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] #[repr(C)] pub struct PyListObject { pub ob_base: PyVarObject, @@ -10,7 +10,7 @@ pub struct PyListObject { pub allocated: Py_ssize_t, } -#[cfg(any(PyPy, GraalPy))] +#[cfg(PyPy)] pub struct PyListObject { pub ob_base: PyObject, } diff --git a/include/pyo3/pyo3-ffi/src/cpython/object.rs b/include/pyo3/pyo3-ffi/src/cpython/object.rs index 35ddf25a..75eef11a 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/object.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/object.rs @@ -211,8 +211,6 @@ pub type printfunc = #[derive(Debug)] pub struct PyTypeObject { pub ob_base: object::PyVarObject, - #[cfg(GraalPy)] - pub ob_size: Py_ssize_t, pub tp_name: *const c_char, pub tp_basicsize: Py_ssize_t, pub tp_itemsize: Py_ssize_t, diff --git a/include/pyo3/pyo3-ffi/src/cpython/objimpl.rs b/include/pyo3/pyo3-ffi/src/cpython/objimpl.rs index 3e0270dd..98a19abe 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/objimpl.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/objimpl.rs @@ -1,3 +1,4 @@ +#[cfg(not(all(Py_3_11, GraalPy)))] use libc::size_t; use std::os::raw::c_int; diff --git a/include/pyo3/pyo3-ffi/src/cpython/pyerrors.rs b/include/pyo3/pyo3-ffi/src/cpython/pyerrors.rs index ca08b44a..c6e10e5f 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/pyerrors.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/pyerrors.rs @@ -6,19 +6,19 @@ use crate::Py_ssize_t; #[derive(Debug)] pub struct PyBaseExceptionObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, } @@ -134,19 +134,19 @@ pub struct PyOSErrorObject { #[derive(Debug)] pub struct PyStopIterationObject { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub dict: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub args: *mut PyObject, - #[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] + #[cfg(all(Py_3_11, not(PyPy)))] pub notes: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub traceback: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub context: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub cause: *mut PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub suppress_context: char, pub value: *mut PyObject, diff --git a/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs b/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs index 1d988d2b..9616d437 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/tupleobject.rs @@ -5,7 +5,6 @@ use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, - #[cfg(not(GraalPy))] pub ob_item: [*mut PyObject; 1], } diff --git a/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs b/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs index 1414b4ce..fae626b8 100644 --- a/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/include/pyo3/pyo3-ffi/src/cpython/unicodeobject.rs @@ -1,4 +1,4 @@ -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(PyPy))] use crate::Py_hash_t; use crate::{PyObject, Py_UCS1, Py_UCS2, Py_UCS4, Py_ssize_t}; use libc::wchar_t; @@ -250,9 +250,8 @@ impl From for u32 { #[repr(C)] pub struct PyASCIIObject { pub ob_base: PyObject, - #[cfg(not(GraalPy))] pub length: Py_ssize_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hash: Py_hash_t, /// A bit field with various properties. /// @@ -265,9 +264,8 @@ pub struct PyASCIIObject { /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; - #[cfg(not(GraalPy))] pub state: u32, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } @@ -379,11 +377,9 @@ impl PyASCIIObject { #[repr(C)] pub struct PyCompactUnicodeObject { pub _base: PyASCIIObject, - #[cfg(not(GraalPy))] pub utf8_length: Py_ssize_t, - #[cfg(not(GraalPy))] pub utf8: *mut c_char, - #[cfg(not(any(Py_3_12, GraalPy)))] + #[cfg(not(Py_3_12))] pub wstr_length: Py_ssize_t, } @@ -398,7 +394,6 @@ pub union PyUnicodeObjectData { #[repr(C)] pub struct PyUnicodeObject { pub _base: PyCompactUnicodeObject, - #[cfg(not(GraalPy))] pub data: PyUnicodeObjectData, } diff --git a/include/pyo3/pyo3-ffi/src/datetime.rs b/include/pyo3/pyo3-ffi/src/datetime.rs index 7283b6d4..e529e0fc 100644 --- a/include/pyo3/pyo3-ffi/src/datetime.rs +++ b/include/pyo3/pyo3-ffi/src/datetime.rs @@ -4,17 +4,17 @@ //! and covers the various date and time related objects in the Python `datetime` //! standard library module. +#[cfg(not(PyPy))] +use crate::PyCapsule_Import; #[cfg(GraalPy)] use crate::{PyLong_AsLong, PyLong_Check, PyObject_GetAttrString, Py_DecRef}; use crate::{PyObject, PyObject_TypeCheck, PyTypeObject, Py_TYPE}; -use std::cell::UnsafeCell; -#[cfg(not(GraalPy))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr; +use std::sync::Once; +use std::{cell::UnsafeCell, ffi::CStr}; #[cfg(not(PyPy))] -use {crate::PyCapsule_Import, std::ffi::CString}; -#[cfg(not(any(PyPy, GraalPy)))] use {crate::Py_hash_t, std::os::raw::c_uchar}; // Type struct wrappers const _PyDateTime_DATE_DATASIZE: usize = 4; @@ -26,13 +26,10 @@ const _PyDateTime_DATETIME_DATASIZE: usize = 10; /// Structure representing a `datetime.timedelta`. pub struct PyDateTime_Delta { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub days: c_int, - #[cfg(not(GraalPy))] pub seconds: c_int, - #[cfg(not(GraalPy))] pub microseconds: c_int, } @@ -55,19 +52,17 @@ pub struct _PyDateTime_BaseTime { /// Structure representing a `datetime.time`. pub struct PyDateTime_Time { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_TIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -76,11 +71,11 @@ pub struct PyDateTime_Time { /// Structure representing a `datetime.date` pub struct PyDateTime_Date { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATE_DATASIZE], } @@ -100,19 +95,17 @@ pub struct _PyDateTime_BaseDateTime { /// Structure representing a `datetime.datetime`. pub struct PyDateTime_DateTime { pub ob_base: PyObject, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub hashcode: Py_hash_t, - #[cfg(not(GraalPy))] pub hastzinfo: c_char, - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub data: [c_uchar; _PyDateTime_DATETIME_DATASIZE], - #[cfg(not(any(PyPy, GraalPy)))] + #[cfg(not(PyPy))] pub fold: c_uchar, /// # Safety /// /// Care should be taken when reading this field. If the time does not have a /// tzinfo then CPython may allocate as a `_PyDateTime_BaseDateTime` without this field. - #[cfg(not(GraalPy))] pub tzinfo: *mut PyObject, } @@ -593,6 +586,8 @@ pub struct PyDateTime_CAPI { // Python already shares this object between threads, so it's no more evil for us to do it too! unsafe impl Sync for PyDateTime_CAPI {} +pub const PyDateTime_CAPSULE_NAME: &CStr = c_str!("datetime.datetime_CAPI"); + /// Returns a pointer to a `PyDateTime_CAPI` instance /// /// # Note @@ -600,33 +595,38 @@ unsafe impl Sync for PyDateTime_CAPI {} /// `PyDateTime_IMPORT` is called #[inline] pub unsafe fn PyDateTimeAPI() -> *mut PyDateTime_CAPI { - *PyDateTimeAPI_impl.0.get() -} - -#[inline] -pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { - (*PyDateTimeAPI()).TimeZone_UTC + *PyDateTimeAPI_impl.ptr.get() } /// Populates the `PyDateTimeAPI` object pub unsafe fn PyDateTime_IMPORT() { - // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use - // `PyCapsule_Import` will behave unexpectedly in pypy. - #[cfg(PyPy)] - let py_datetime_c_api = PyDateTime_Import(); - - #[cfg(not(PyPy))] - let py_datetime_c_api = { - // PyDateTime_CAPSULE_NAME is a macro in C - let PyDateTime_CAPSULE_NAME = CString::new("datetime.datetime_CAPI").unwrap(); - - PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI - }; + if !PyDateTimeAPI_impl.once.is_completed() { + // PyPy expects the C-API to be initialized via PyDateTime_Import, so trying to use + // `PyCapsule_Import` will behave unexpectedly in pypy. + #[cfg(PyPy)] + let py_datetime_c_api = PyDateTime_Import(); + + #[cfg(not(PyPy))] + let py_datetime_c_api = + PyCapsule_Import(PyDateTime_CAPSULE_NAME.as_ptr(), 1) as *mut PyDateTime_CAPI; + + if py_datetime_c_api.is_null() { + return; + } - *PyDateTimeAPI_impl.0.get() = py_datetime_c_api; + // Protect against race conditions when the datetime API is concurrently + // initialized in multiple threads. UnsafeCell.get() cannot panic so this + // won't panic either. + PyDateTimeAPI_impl.once.call_once(|| { + *PyDateTimeAPI_impl.ptr.get() = py_datetime_c_api; + }); + } } -// skipped non-limited PyDateTime_TimeZone_UTC +#[inline] +pub unsafe fn PyDateTime_TimeZone_UTC() -> *mut PyObject { + (*PyDateTimeAPI()).TimeZone_UTC +} /// Type Check macros /// @@ -739,8 +739,13 @@ extern "C" { // Rust specific implementation details -struct PyDateTimeAPISingleton(UnsafeCell<*mut PyDateTime_CAPI>); +struct PyDateTimeAPISingleton { + once: Once, + ptr: UnsafeCell<*mut PyDateTime_CAPI>, +} unsafe impl Sync for PyDateTimeAPISingleton {} -static PyDateTimeAPI_impl: PyDateTimeAPISingleton = - PyDateTimeAPISingleton(UnsafeCell::new(ptr::null_mut())); +static PyDateTimeAPI_impl: PyDateTimeAPISingleton = PyDateTimeAPISingleton { + once: Once::new(), + ptr: UnsafeCell::new(ptr::null_mut()), +}; diff --git a/include/pyo3/pyo3-ffi/src/lib.rs b/include/pyo3/pyo3-ffi/src/lib.rs index c6157401..7bdba117 100644 --- a/include/pyo3/pyo3-ffi/src/lib.rs +++ b/include/pyo3/pyo3-ffi/src/lib.rs @@ -43,10 +43,39 @@ //! PyO3 uses `rustc`'s `--cfg` flags to enable or disable code used for different Python versions. //! If you want to do this for your own crate, you can do so with the [`pyo3-build-config`] crate. //! -//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`: Marks code that is only enabled when -//! compiling for a given minimum Python version. +//! - `Py_3_7`, `Py_3_8`, `Py_3_9`, `Py_3_10`, `Py_3_11`, `Py_3_12`, `Py_3_13`: Marks code that is +//! only enabled when compiling for a given minimum Python version. //! - `Py_LIMITED_API`: Marks code enabled when the `abi3` feature flag is enabled. +//! - `Py_GIL_DISABLED`: Marks code that runs only in the free-threaded build of CPython. //! - `PyPy` - Marks code enabled when compiling for PyPy. +//! - `GraalPy` - Marks code enabled when compiling for GraalPy. +//! +//! Additionally, you can query for the values `Py_DEBUG`, `Py_REF_DEBUG`, +//! `Py_TRACE_REFS`, and `COUNT_ALLOCS` from `py_sys_config` to query for the +//! corresponding C build-time defines. For example, to conditionally define +//! debug code using `Py_DEBUG`, you could do: +//! +//! ```rust,ignore +//! #[cfg(py_sys_config = "Py_DEBUG")] +//! println!("only runs if python was compiled with Py_DEBUG") +//! ``` +//! +//! To use these attributes, add [`pyo3-build-config`] as a build dependency in +//! your `Cargo.toml`: +//! +//! ```toml +//! [build-dependencies] +#![doc = concat!("pyo3-build-config =\"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! And then either create a new `build.rs` file in the project root or modify +//! the existing `build.rs` file to call `use_pyo3_cfgs()`: +//! +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs(); +//! } +//! ``` //! //! # Minimum supported Rust and Python versions //! @@ -79,11 +108,29 @@ //! [dependencies.pyo3-ffi] #![doc = concat!("version = \"", env!("CARGO_PKG_VERSION"), "\"")] //! features = ["extension-module"] +//! +//! [build-dependencies] +//! # This is only necessary if you need to configure your build based on +//! # the Python version or the compile-time configuration for the interpreter. +#![doc = concat!("pyo3_build_config = \"", env!("CARGO_PKG_VERSION"), "\"")] +//! ``` +//! +//! If you need to use conditional compilation based on Python version or how +//! Python was compiled, you need to add `pyo3-build-config` as a +//! `build-dependency` in your `Cargo.toml` as in the example above and either +//! create a new `build.rs` file or modify an existing one so that +//! `pyo3_build_config::use_pyo3_cfgs()` gets called at build time: +//! +//! **`build.rs`** +//! ```rust,ignore +//! fn main() { +//! pyo3_build_config::use_pyo3_cfgs() +//! } //! ``` //! //! **`src/lib.rs`** //! ```rust -//! use std::os::raw::c_char; +//! use std::os::raw::{c_char, c_long}; //! use std::ptr; //! //! use pyo3_ffi::*; @@ -93,14 +140,14 @@ //! m_name: c_str!("string_sum").as_ptr(), //! m_doc: c_str!("A Python module written in Rust.").as_ptr(), //! m_size: 0, -//! m_methods: unsafe { METHODS.as_mut_ptr().cast() }, +//! m_methods: unsafe { METHODS as *const [PyMethodDef] as *mut PyMethodDef }, //! m_slots: std::ptr::null_mut(), //! m_traverse: None, //! m_clear: None, //! m_free: None, //! }; //! -//! static mut METHODS: [PyMethodDef; 2] = [ +//! static mut METHODS: &[PyMethodDef] = &[ //! PyMethodDef { //! ml_name: c_str!("sum_as_string").as_ptr(), //! ml_meth: PyMethodDefPointer { @@ -110,58 +157,99 @@ //! ml_doc: c_str!("returns the sum of two integers as a string").as_ptr(), //! }, //! // A zeroed PyMethodDef to mark the end of the array. -//! PyMethodDef::zeroed() +//! PyMethodDef::zeroed(), //! ]; //! //! // The module initialization function, which must be named `PyInit_`. //! #[allow(non_snake_case)] //! #[no_mangle] //! pub unsafe extern "C" fn PyInit_string_sum() -> *mut PyObject { -//! PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)) +//! let module = PyModule_Create(ptr::addr_of_mut!(MODULE_DEF)); +//! if module.is_null() { +//! return module; +//! } +//! #[cfg(Py_GIL_DISABLED)] +//! { +//! if PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED) < 0 { +//! Py_DECREF(module); +//! return std::ptr::null_mut(); +//! } +//! } +//! module //! } //! -//! pub unsafe extern "C" fn sum_as_string( -//! _self: *mut PyObject, -//! args: *mut *mut PyObject, -//! nargs: Py_ssize_t, -//! ) -> *mut PyObject { -//! if nargs != 2 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! c_str!("sum_as_string() expected 2 positional arguments").as_ptr(), +//! /// A helper to parse function arguments +//! /// If we used PyO3's proc macros they'd handle all of this boilerplate for us :) +//! unsafe fn parse_arg_as_i32(obj: *mut PyObject, n_arg: usize) -> Option { +//! if PyLong_Check(obj) == 0 { +//! let msg = format!( +//! "sum_as_string expected an int for positional argument {}\0", +//! n_arg //! ); -//! return std::ptr::null_mut(); +//! PyErr_SetString(PyExc_TypeError, msg.as_ptr().cast::()); +//! return None; //! } //! -//! let arg1 = *args; -//! if PyLong_Check(arg1) == 0 { -//! PyErr_SetString( -//! PyExc_TypeError, -//! c_str!("sum_as_string() expected an int for positional argument 1").as_ptr(), -//! ); -//! return std::ptr::null_mut(); +//! // Let's keep the behaviour consistent on platforms where `c_long` is bigger than 32 bits. +//! // In particular, it is an i32 on Windows but i64 on most Linux systems +//! let mut overflow = 0; +//! let i_long: c_long = PyLong_AsLongAndOverflow(obj, &mut overflow); +//! +//! #[allow(irrefutable_let_patterns)] // some platforms have c_long equal to i32 +//! if overflow != 0 { +//! raise_overflowerror(obj); +//! None +//! } else if let Ok(i) = i_long.try_into() { +//! Some(i) +//! } else { +//! raise_overflowerror(obj); +//! None //! } +//! } //! -//! let arg1 = PyLong_AsLong(arg1); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); +//! unsafe fn raise_overflowerror(obj: *mut PyObject) { +//! let obj_repr = PyObject_Str(obj); +//! if !obj_repr.is_null() { +//! let mut size = 0; +//! let p = PyUnicode_AsUTF8AndSize(obj_repr, &mut size); +//! if !p.is_null() { +//! let s = std::str::from_utf8_unchecked(std::slice::from_raw_parts( +//! p.cast::(), +//! size as usize, +//! )); +//! let msg = format!("cannot fit {} in 32 bits\0", s); +//! +//! PyErr_SetString(PyExc_OverflowError, msg.as_ptr().cast::()); +//! } +//! Py_DECREF(obj_repr); //! } +//! } //! -//! let arg2 = *args.add(1); -//! if PyLong_Check(arg2) == 0 { +//! pub unsafe extern "C" fn sum_as_string( +//! _self: *mut PyObject, +//! args: *mut *mut PyObject, +//! nargs: Py_ssize_t, +//! ) -> *mut PyObject { +//! if nargs != 2 { //! PyErr_SetString( //! PyExc_TypeError, -//! c_str!("sum_as_string() expected an int for positional argument 2").as_ptr(), +//! c_str!("sum_as_string expected 2 positional arguments").as_ptr(), //! ); //! return std::ptr::null_mut(); //! } //! -//! let arg2 = PyLong_AsLong(arg2); -//! if !PyErr_Occurred().is_null() { -//! return ptr::null_mut(); -//! } +//! let (first, second) = (*args, *args.add(1)); +//! +//! let first = match parse_arg_as_i32(first, 1) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; +//! let second = match parse_arg_as_i32(second, 2) { +//! Some(x) => x, +//! None => return std::ptr::null_mut(), +//! }; //! -//! match arg1.checked_add(arg2) { +//! match first.checked_add(second) { //! Some(sum) => { //! let string = sum.to_string(); //! PyUnicode_FromStringAndSize(string.as_ptr().cast::(), string.len() as isize) @@ -201,6 +289,12 @@ //! [manually][manual_builds]. Both offer more flexibility than `maturin` but require further //! configuration. //! +//! This example stores the module definition statically and uses the `PyModule_Create` function +//! in the CPython C API to register the module. This is the "old" style for registering modules +//! and has the limitation that it cannot support subinterpreters. You can also create a module +//! using the new multi-phase initialization API that does support subinterpreters. See the +//! `sequential` project located in the `examples` directory at the root of the `pyo3-ffi` crate +//! for a worked example of how to this using `pyo3-ffi`. //! //! # Using Python from Rust //! @@ -226,7 +320,7 @@ #![doc = concat!("[manual_builds]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/building-and-distribution.html#manual-builds \"Manual builds - Building and Distribution - PyO3 user guide\"")] //! [setuptools-rust]: https://github.com/PyO3/setuptools-rust "Setuptools plugin for Rust extensions" //! [PEP 384]: https://www.python.org/dev/peps/pep-0384 "PEP 384 -- Defining a Stable ABI" -#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features Reference - PyO3 user guide\"")] +#![doc = concat!("[Features chapter of the guide]: https://pyo3.rs/v", env!("CARGO_PKG_VERSION"), "/features.html#features-reference \"Features eference - PyO3 user guide\"")] #![allow( missing_docs, non_camel_case_types, diff --git a/include/pyo3/pyo3-ffi/src/listobject.rs b/include/pyo3/pyo3-ffi/src/listobject.rs index 9d8b7ed6..881a8a87 100644 --- a/include/pyo3/pyo3-ffi/src/listobject.rs +++ b/include/pyo3/pyo3-ffi/src/listobject.rs @@ -50,6 +50,10 @@ extern "C" { arg3: Py_ssize_t, arg4: *mut PyObject, ) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Extend(list: *mut PyObject, iterable: *mut PyObject) -> c_int; + #[cfg(Py_3_13)] + pub fn PyList_Clear(list: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Sort")] pub fn PyList_Sort(arg1: *mut PyObject) -> c_int; #[cfg_attr(PyPy, link_name = "PyPyList_Reverse")] diff --git a/include/pyo3/pyo3-ffi/src/methodobject.rs b/include/pyo3/pyo3-ffi/src/methodobject.rs index bd214409..37e1e206 100644 --- a/include/pyo3/pyo3-ffi/src/methodobject.rs +++ b/include/pyo3/pyo3-ffi/src/methodobject.rs @@ -50,7 +50,7 @@ pub type PyCFunctionFast = unsafe extern "C" fn( ) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] -#[deprecated(note = "renamed to `PyCFunctionFast`")] +#[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFast`"))] pub type _PyCFunctionFast = PyCFunctionFast; pub type PyCFunctionWithKeywords = unsafe extern "C" fn( @@ -68,6 +68,7 @@ pub type PyCFunctionFastWithKeywords = unsafe extern "C" fn( ) -> *mut PyObject; #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] +#[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFastWithKeywords`"))] pub type _PyCFunctionFastWithKeywords = PyCFunctionFastWithKeywords; #[cfg(all(Py_3_9, not(Py_LIMITED_API)))] @@ -152,7 +153,7 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - #[deprecated(note = "renamed to `PyCFunctionFast`")] + #[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFast`"))] pub _PyCFunctionFast: PyCFunctionFast, /// This variant corresponds with [`METH_FASTCALL`]. @@ -161,6 +162,7 @@ pub union PyMethodDefPointer { /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] + #[cfg_attr(Py_3_10, deprecated(note = "renamed to `PyCFunctionFastWithKeywords`"))] pub _PyCFunctionFastWithKeywords: PyCFunctionFastWithKeywords, /// This variant corresponds with [`METH_FASTCALL`] | [`METH_KEYWORDS`]. diff --git a/include/pyo3/pyo3-ffi/src/moduleobject.rs b/include/pyo3/pyo3-ffi/src/moduleobject.rs index ff6458f4..2417664a 100644 --- a/include/pyo3/pyo3-ffi/src/moduleobject.rs +++ b/include/pyo3/pyo3-ffi/src/moduleobject.rs @@ -88,6 +88,10 @@ pub const Py_mod_create: c_int = 1; pub const Py_mod_exec: c_int = 2; #[cfg(Py_3_12)] pub const Py_mod_multiple_interpreters: c_int = 3; +#[cfg(Py_3_13)] +pub const Py_mod_gil: c_int = 4; + +// skipped private _Py_mod_LAST_SLOT #[cfg(Py_3_12)] pub const Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED: *mut c_void = 0 as *mut c_void; @@ -96,7 +100,15 @@ pub const Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED: *mut c_void = 1 as *mut c_void #[cfg(Py_3_12)] pub const Py_MOD_PER_INTERPRETER_GIL_SUPPORTED: *mut c_void = 2 as *mut c_void; -// skipped non-limited _Py_mod_LAST_SLOT +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; +#[cfg(Py_3_13)] +pub const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; + +#[cfg(all(not(Py_LIMITED_API), Py_GIL_DISABLED))] +extern "C" { + pub fn PyUnstable_Module_SetGIL(module: *mut PyObject, gil: *mut c_void) -> c_int; +} #[repr(C)] pub struct PyModuleDef { diff --git a/include/pyo3/pyo3-ffi/src/object.rs b/include/pyo3/pyo3-ffi/src/object.rs index d2fa1930..51083a8e 100644 --- a/include/pyo3/pyo3-ffi/src/object.rs +++ b/include/pyo3/pyo3-ffi/src/object.rs @@ -129,6 +129,9 @@ pub struct PyVarObject { pub ob_base: PyObject, #[cfg(not(GraalPy))] pub ob_size: Py_ssize_t, + // On GraalPy the field is physically there, but not always populated. We hide it to prevent accidental misuse + #[cfg(GraalPy)] + pub _ob_size_graalpy: Py_ssize_t, } // skipped private _PyVarObject_CAST diff --git a/include/pyo3/pyo3-ffi/src/pyhash.rs b/include/pyo3/pyo3-ffi/src/pyhash.rs index 19459903..8b0b6771 100644 --- a/include/pyo3/pyo3-ffi/src/pyhash.rs +++ b/include/pyo3/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,9 @@ -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] +#[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::{c_char, c_void}; +use std::os::raw::c_char; +#[cfg(not(any(Py_LIMITED_API, PyPy)))] +use std::os::raw::c_void; use std::os::raw::{c_int, c_ulong}; @@ -10,7 +12,7 @@ extern "C" { // skipped non-limited _Py_HashPointer // skipped non-limited _Py_HashPointerRaw - #[cfg(not(all(Py_3_14, any(Py_LIMITED_API, PyPy, GraalPy))))] + #[cfg(not(all(Py_3_14, any(Py_LIMITED_API, PyPy))))] pub fn _Py_HashBytes(src: *const c_void, len: Py_ssize_t) -> Py_hash_t; #[cfg(Py_3_14)] diff --git a/pyproject.toml b/pyproject.toml index f1c42170..656ad360 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ Changelog = "https://github.com/ijl/orjson/blob/master/CHANGELOG.md" [build-system] build-backend = "maturin" -requires = ["maturin==1.7.8"] +requires = ["maturin>=1,<2"] [tool.maturin] python-source = "pysrc" @@ -39,7 +39,9 @@ include = [ { format = "sdist", path = ".cargo/*" }, { format = "sdist", path = "build.rs" }, { format = "sdist", path = "Cargo.lock" }, - { format = "sdist", path = "include/**/*" }, + { format = "sdist", path = "include/cargo/**/*" }, + { format = "sdist", path = "include/pyo3/**/*" }, + { format = "sdist", path = "include/yyjson/**/*" }, ] [tool.ruff] diff --git a/requirements.txt b/requirements.txt index a1c591ca..d8c942aa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -r bench/requirements.txt -r integration/requirements.txt -r test/requirements.txt -maturin==1.7.8 +maturin>=1,<2 mypy==1.13.0 -ruff==0.8.0 +ruff==0.8.5 diff --git a/script/install-fedora b/script/install-fedora index 7c1f9589..af8e1a53 100755 --- a/script/install-fedora +++ b/script/install-fedora @@ -2,13 +2,6 @@ set -eou pipefail -# export PYTHON=python3.11 -# export PYTHON_PACKAGE=python3.11 -# export RUST_TOOLCHAIN=nightly-2024-11-22 -# export TARGET=x86_64-unknown-linux-gnu -# export VENV=.venv -# export CARGO_TARGET_DIR=/tmp/orjson - export VENV="${VENV:-.venv}" export CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-target}" @@ -28,4 +21,4 @@ rm -rf "${VENV}" uv venv --python "${PYTHON}" "${VENV}" source "${VENV}/bin/activate" -uv pip install --upgrade "maturin==1.7.8" -r test/requirements.txt -r integration/requirements.txt +uv pip install --upgrade "maturin>=1,<2" -r test/requirements.txt -r integration/requirements.txt diff --git a/src/deserialize/backend/yyjson.rs b/src/deserialize/backend/yyjson.rs index c5538b4b..9840e183 100644 --- a/src/deserialize/backend/yyjson.rs +++ b/src/deserialize/backend/yyjson.rs @@ -85,8 +85,8 @@ pub(crate) fn deserialize( ElementType::Null => parse_none(), ElementType::True => parse_true(), ElementType::False => parse_false(), - ElementType::Array => unreachable!(), - ElementType::Object => unreachable!(), + ElementType::Array => unreachable_unchecked!(), + ElementType::Object => unreachable_unchecked!(), }; unsafe { yyjson_doc_free(doc) }; Ok(pyval) @@ -149,7 +149,7 @@ impl ElementType { TAG_FALSE => Self::False, TAG_ARRAY => Self::Array, TAG_OBJECT => Self::Object, - _ => unreachable!(), + _ => unreachable_unchecked!(), } } } @@ -221,8 +221,8 @@ fn populate_yy_array(list: *mut pyo3_ffi::PyObject, elem: *mut yyjson_val) { ElementType::Null => parse_none(), ElementType::True => parse_true(), ElementType::False => parse_false(), - ElementType::Array => unreachable!(), - ElementType::Object => unreachable!(), + ElementType::Array => unreachable_unchecked!(), + ElementType::Object => unreachable_unchecked!(), }; append_to_list!(dptr, pyval.as_ptr()); } @@ -283,8 +283,8 @@ fn populate_yy_object(dict: *mut pyo3_ffi::PyObject, elem: *mut yyjson_val) { ElementType::Null => parse_none(), ElementType::True => parse_true(), ElementType::False => parse_false(), - ElementType::Array => unreachable!(), - ElementType::Object => unreachable!(), + ElementType::Array => unreachable_unchecked!(), + ElementType::Object => unreachable_unchecked!(), }; add_to_dict!(dict, pykey, pyval.as_ptr()); reverse_pydict_incref!(pykey); diff --git a/src/deserialize/pyobject.rs b/src/deserialize/pyobject.rs index 9ad9b090..e486c916 100644 --- a/src/deserialize/pyobject.rs +++ b/src/deserialize/pyobject.rs @@ -16,7 +16,7 @@ pub fn get_unicode_key(key_str: &str) -> *mut pyo3_ffi::PyObject { unsafe { let entry = KEY_MAP .get_mut() - .unwrap_or_else(|| unreachable!()) + .unwrap_or_else(|| unreachable_unchecked!()) .entry(&hash) .or_insert_with( || hash, diff --git a/src/lib.rs b/src/lib.rs index 64ace9e1..11fd6054 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -154,16 +154,6 @@ pub unsafe extern "C" fn orjson_init_exec(mptr: *mut PyObject) -> c_int { 0 } -#[cfg(Py_3_13)] -#[allow(non_upper_case_globals)] -const Py_mod_gil: c_int = 4; -#[cfg(Py_3_13)] -#[allow(non_upper_case_globals, dead_code, fuzzy_provenance_casts)] -const Py_MOD_GIL_USED: *mut c_void = 0 as *mut c_void; -#[cfg(Py_3_13)] -#[allow(non_upper_case_globals, dead_code, fuzzy_provenance_casts)] -const Py_MOD_GIL_NOT_USED: *mut c_void = 1 as *mut c_void; - #[cfg(not(Py_3_12))] const PYMODULEDEF_LEN: usize = 2; #[cfg(all(Py_3_12, not(Py_3_13)))] diff --git a/src/util.rs b/src/util.rs index 8add0f75..a6344343 100644 --- a/src/util.rs +++ b/src/util.rs @@ -305,3 +305,9 @@ macro_rules! popcnt { core::mem::transmute::($val).count_ones() as usize }; } + +macro_rules! unreachable_unchecked { + () => { + unsafe { core::hint::unreachable_unchecked() } + }; +} diff --git a/test/requirements.txt b/test/requirements.txt index 2c04102f..97bee4ca 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -6,5 +6,3 @@ time-machine < 2.15;sys_platform=="linux" and platform_machine=="x86_64" and pyt psutil;(sys_platform=="linux" or sys_platform == "macos") and platform_machine=="x86_64" and python_version<"3.13" pytest pytz -typing_extensions;python_version<"3.8" -xxhash==1.4.3;sys_platform=="linux" and platform_machine=="x86_64" and python_version<"3.9" # creates non-compact ASCII for test_str_ascii diff --git a/test/test_append_newline.py b/test/test_append_newline.py index e6369026..00a8a181 100644 --- a/test/test_append_newline.py +++ b/test/test_append_newline.py @@ -2,7 +2,7 @@ import orjson -from .util import read_fixture_obj +from .util import needs_data, read_fixture_obj class TestAppendNewline: @@ -12,6 +12,7 @@ def test_dumps_newline(self): """ assert orjson.dumps([], option=orjson.OPT_APPEND_NEWLINE) == b"[]\n" + @needs_data def test_twitter_newline(self): """ loads(),dumps() twitter.json OPT_APPEND_NEWLINE @@ -19,6 +20,7 @@ def test_twitter_newline(self): val = read_fixture_obj("twitter.json.xz") assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_canada(self): """ loads(), dumps() canada.json OPT_APPEND_NEWLINE @@ -26,6 +28,7 @@ def test_canada(self): val = read_fixture_obj("canada.json.xz") assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_citm_catalog_newline(self): """ loads(), dumps() citm_catalog.json OPT_APPEND_NEWLINE @@ -33,6 +36,7 @@ def test_citm_catalog_newline(self): val = read_fixture_obj("citm_catalog.json.xz") assert orjson.loads(orjson.dumps(val, option=orjson.OPT_APPEND_NEWLINE)) == val + @needs_data def test_github_newline(self): """ loads(), dumps() github.json OPT_APPEND_NEWLINE diff --git a/test/test_error.py b/test/test_error.py index 29e4c731..069020be 100644 --- a/test/test_error.py +++ b/test/test_error.py @@ -6,7 +6,7 @@ import orjson -from .util import read_fixture_str +from .util import needs_data, read_fixture_str ASCII_TEST = b"""\ { @@ -81,6 +81,7 @@ def test_four_byte(self): {"pos": 19, "lineno": 4, "colno": 1}, ) + @needs_data def test_tab(self): data = read_fixture_str("fail26.json", "jsonchecker") with pytest.raises(json.decoder.JSONDecodeError) as json_exc_info: diff --git a/test/test_fixture.py b/test/test_fixture.py index 4121c6d0..f98bd3ba 100644 --- a/test/test_fixture.py +++ b/test/test_fixture.py @@ -4,9 +4,10 @@ import orjson -from .util import read_fixture_bytes, read_fixture_str +from .util import needs_data, read_fixture_bytes, read_fixture_str +@needs_data class TestFixture: def test_twitter(self): """ @@ -16,6 +17,7 @@ def test_twitter(self): read = orjson.loads(val) assert orjson.loads(orjson.dumps(read)) == read + @needs_data def test_canada(self): """ loads(), dumps() canada.json diff --git a/test/test_fragment.py b/test/test_fragment.py index 734cb069..0d330a6f 100644 --- a/test/test_fragment.py +++ b/test/test_fragment.py @@ -9,7 +9,7 @@ except ImportError: pandas = None # type: ignore -from .util import read_fixture_bytes +from .util import needs_data, read_fixture_bytes class TestFragment: @@ -99,6 +99,7 @@ def default(value): ) +@needs_data class TestFragmentParsing: def _run_test(self, filename: str): data = read_fixture_bytes(filename, "parsing") diff --git a/test/test_indent.py b/test/test_indent.py index 4179a20f..67f2a87c 100644 --- a/test/test_indent.py +++ b/test/test_indent.py @@ -5,9 +5,10 @@ import orjson -from .util import read_fixture_obj +from .util import needs_data, read_fixture_obj +@needs_data class TestIndentedOutput: def test_equivalent(self): """ diff --git a/test/test_issue331.py b/test/test_issue331.py index 7b16912d..6223a187 100644 --- a/test/test_issue331.py +++ b/test/test_issue331.py @@ -4,7 +4,7 @@ import orjson -from .util import read_fixture_bytes +from .util import needs_data, read_fixture_bytes FIXTURE_ISSUE_335 = { "pfkrpavmb": "maxyjzmvacdwjfiifmzwbztjmnqdsjesykpf", @@ -253,6 +253,7 @@ } +@needs_data def test_issue331_1_pretty(): as_bytes = read_fixture_bytes("issue331_1.json.xz") as_obj = orjson.loads(as_bytes) @@ -260,6 +261,7 @@ def test_issue331_1_pretty(): assert orjson.loads(orjson.dumps(as_obj, option=orjson.OPT_INDENT_2)) == as_obj +@needs_data def test_issue331_1_compact(): as_bytes = read_fixture_bytes("issue331_1.json.xz") as_obj = orjson.loads(as_bytes) @@ -267,6 +269,7 @@ def test_issue331_1_compact(): assert orjson.loads(orjson.dumps(as_obj)) == as_obj +@needs_data def test_issue331_2_pretty(): as_bytes = read_fixture_bytes("issue331_2.json.xz") as_obj = orjson.loads(as_bytes) @@ -274,6 +277,7 @@ def test_issue331_2_pretty(): assert orjson.loads(orjson.dumps(as_obj, option=orjson.OPT_INDENT_2)) == as_obj +@needs_data def test_issue331_2_compact(): as_bytes = read_fixture_bytes("issue331_2.json.xz") as_obj = orjson.loads(as_bytes) diff --git a/test/test_jsonchecker.py b/test/test_jsonchecker.py index 88703e64..36bae0de 100644 --- a/test/test_jsonchecker.py +++ b/test/test_jsonchecker.py @@ -7,11 +7,12 @@ import orjson -from .util import read_fixture_str +from .util import needs_data, read_fixture_str PATTERN_1 = '["JSON Test Pattern pass1",{"object with 1 member":["array with 1 element"]},{},[],-42,true,false,null,{"integer":1234567890,"real":-9876.54321,"e":1.23456789e-13,"E":1.23456789e34,"":2.3456789012e76,"zero":0,"one":1,"space":" ","quote":"\\"","backslash":"\\\\","controls":"\\b\\f\\n\\r\\t","slash":"/ & /","alpha":"abcdefghijklmnopqrstuvwyz","ALPHA":"ABCDEFGHIJKLMNOPQRSTUVWYZ","digit":"0123456789","0123456789":"digit","special":"`1~!@#$%^&*()_+-={\':[,]}|;.?","hex":"ģ䕧覫췯ꯍ\uef4a","true":true,"false":false,"null":null,"array":[],"object":{},"address":"50 St. James Street","url":"http://www.JSON.org/","comment":"// /* */":" "," s p a c e d ":[1,2,3,4,5,6,7],"compact":[1,2,3,4,5,6,7],"jsontext":"{\\"object with 1 member\\":[\\"array with 1 element\\"]}","quotes":"" \\" %22 0x22 034 "","/\\\\\\"쫾몾ꮘﳞ볚\uef4a\\b\\f\\n\\r\\t`1~!@#$%^&*()_+-=[]{}|;:\',./<>?":"A key can be any string"},0.5,98.6,99.44,1066,10.0,1.0,0.1,1.0,2.0,2.0,"rosebud"]'.encode() +@needs_data class TestJsonChecker: def _run_fail_json(self, filename, exc=orjson.JSONDecodeError): data = read_fixture_str(filename, "jsonchecker") diff --git a/test/test_parsing.py b/test/test_parsing.py index 8b25129c..ae435762 100644 --- a/test/test_parsing.py +++ b/test/test_parsing.py @@ -4,9 +4,10 @@ import orjson -from .util import read_fixture_bytes +from .util import needs_data, read_fixture_bytes +@needs_data class TestJSONTestSuiteParsing: def _run_fail_json(self, filename, exc=orjson.JSONDecodeError): data = read_fixture_bytes(filename, "parsing") diff --git a/test/test_roundtrip.py b/test/test_roundtrip.py index 0acc2868..ac308877 100644 --- a/test/test_roundtrip.py +++ b/test/test_roundtrip.py @@ -2,9 +2,10 @@ import orjson -from .util import read_fixture_str +from .util import needs_data, read_fixture_str +@needs_data class TestJsonChecker: def _run_roundtrip_json(self, filename): data = read_fixture_str(filename, "roundtrip") diff --git a/test/test_sort_keys.py b/test/test_sort_keys.py index 2d794350..f666d615 100644 --- a/test/test_sort_keys.py +++ b/test/test_sort_keys.py @@ -2,9 +2,10 @@ import orjson -from .util import read_fixture_obj +from .util import needs_data, read_fixture_obj +@needs_data class TestDictSortKeys: # citm_catalog is already sorted def test_twitter_sorted(self): diff --git a/test/test_transform.py b/test/test_transform.py index caee07b2..9f134f05 100644 --- a/test/test_transform.py +++ b/test/test_transform.py @@ -4,13 +4,14 @@ import orjson -from .util import read_fixture_bytes +from .util import needs_data, read_fixture_bytes def _read_file(filename): return read_fixture_bytes(filename, "transform").strip(b"\n").strip(b"\r") +@needs_data class TestJSONTestSuiteTransform: def _pass_transform(self, filename, reference=None): data = _read_file(filename) diff --git a/test/test_type.py b/test/test_type.py index 81c89307..3a8395a7 100644 --- a/test/test_type.py +++ b/test/test_type.py @@ -4,12 +4,6 @@ import sys import pytest - -try: - import xxhash -except ImportError: - xxhash = None - import orjson @@ -260,17 +254,6 @@ def test_str_surrogates_dumps(self): orjson.JSONEncodeError, orjson.dumps, b"\xed\xa0\xbd\xed\xba\x80" ) # \ud83d\ude80 - @pytest.mark.skipif( - xxhash is None, reason="xxhash install broken on win, python3.9, Azure" - ) - def test_str_ascii(self): - """ - str is ASCII but not compact - """ - digest = xxhash.xxh32_hexdigest("12345") - for _ in range(2): - assert orjson.dumps(digest) == b'"b30d56b4"' - def test_bytes_dumps(self): """ bytes dumps not supported diff --git a/test/util.py b/test/util.py index bc00b934..fdbda41a 100644 --- a/test/util.py +++ b/test/util.py @@ -5,9 +5,11 @@ from pathlib import Path from typing import Any, Dict +import pytest + import orjson -dirname = os.path.join(os.path.dirname(__file__), "../data") +data_dir = os.path.join(os.path.dirname(__file__), "../data") STR_CACHE: Dict[str, str] = {} @@ -16,9 +18,9 @@ def read_fixture_bytes(filename, subdir=None): if subdir is None: - path = Path(dirname, filename) + path = Path(data_dir, filename) else: - path = Path(dirname, subdir, filename) + path = Path(data_dir, subdir, filename) if path.suffix == ".xz": contents = lzma.decompress(path.read_bytes()) else: @@ -36,3 +38,9 @@ def read_fixture_obj(filename): if filename not in OBJ_CACHE: OBJ_CACHE[filename] = orjson.loads(read_fixture_str(filename)) return OBJ_CACHE[filename] + + +needs_data = pytest.mark.skipif( + not Path(data_dir).exists(), + reason="Test depends on ./data dir that contains fixtures", +)