From 07093e9b48b393320eb358b6e30bcef0906aa9b0 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Sat, 23 Nov 2024 16:30:07 -0700 Subject: [PATCH] Free-threaded build config fixes (#4719) * update build config logic and library name generation * fix free-threaded windows clippy with --features=abi3 * use constant * add release note * apply my self-review comments * use ensure and error handling instead of panicking * skip abi3 fixup on free-threaded build * don't support PYO3_USE_ABI3_FORWARD_COMPATIBILITY on free-threaded build * don't panic in pyo3-ffi in abi3 check * document lack of limited API support * add is_free_threaded() method to InterpreterConfig * implement David's code review suggestions * remove unused imports --- guide/src/free-threading.md | 28 +++- newsfragments/4719.fixed.md | 2 + pyo3-build-config/src/impl_.rs | 259 +++++++++++++++++++++++++-------- pyo3-ffi/build.rs | 35 +++-- 4 files changed, 245 insertions(+), 79 deletions(-) create mode 100644 newsfragments/4719.fixed.md diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md index d867a707795..f212cb0b9a9 100644 --- a/guide/src/free-threading.md +++ b/guide/src/free-threading.md @@ -108,6 +108,15 @@ using single-phase initialization and the [`sequential`](https://github.com/PyO3/pyo3/tree/main/pyo3-ffi/examples/sequential) example for modules using multi-phase initialization. +If you would like to use conditional compilation to trigger different code paths +under the free-threaded build, you can use the `Py_GIL_DISABLED` attribute once +you have configured your crate to generate the necessary build configuration +data. See [the guide +section](./building-and-distribution/multiple-python-versions.md) for more +details about supporting multiple different Python versions, including the +free-threaded build. + + ## Special considerations for the free-threaded build The free-threaded interpreter does not have a GIL, and this can make interacting @@ -234,7 +243,24 @@ needed. For now you should explicitly add locking, possibly using conditional compilation or using the critical section API to avoid creating deadlocks with the GIL. -## Thread-safe single initialization +### Cannot build extensions using the limited API + +The free-threaded build uses a completely new ABI and there is not yet an +equivalent to the limited API for the free-threaded ABI. That means if your +crate depends on PyO3 using the `abi3` feature or an an `abi3-pyxx` feature, +PyO3 will print a warning and ignore that setting when building extensions using +the free-threaded interpreter. + +This means that if your package makes use of the ABI forward compatibility +provided by the limited API to uploads only one wheel for each release of your +package, you will need to update and tooling or instructions to also upload a +version-specific free-threaded wheel. + +See [the guide section](./building-and-distribution/multiple-python-versions.md) +for more details about supporting multiple different Python versions, including +the free-threaded build. + +### Thread-safe single initialization Until version 0.23, PyO3 provided only [`GILOnceCell`] to enable deadlock-free single initialization of data in contexts that might execute arbitrary Python diff --git a/newsfragments/4719.fixed.md b/newsfragments/4719.fixed.md new file mode 100644 index 00000000000..08cb1c8268e --- /dev/null +++ b/newsfragments/4719.fixed.md @@ -0,0 +1,2 @@ +* Fixed an issue that prevented building free-threaded extensions for crates + that request a specific minimum limited API version. diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 6d2326429d2..4e5d3c10656 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -176,7 +176,7 @@ impl InterpreterConfig { } // If Py_GIL_DISABLED is set, do not build with limited API support - if self.abi3 && !self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) { + if self.abi3 && !self.is_free_threaded() { out.push("cargo:rustc-cfg=Py_LIMITED_API".to_owned()); } @@ -309,14 +309,14 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) // `_d.cp312-win_amd64.pyd` for 3.12 debug build map["ext_suffix"].starts_with("_d."), gil_disabled, - ) + )? } else { default_lib_name_unix( version, implementation, map.get("ld_version").map(String::as_str), gil_disabled, - ) + )? }; let lib_dir = if cfg!(windows) { @@ -394,7 +394,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) implementation, sysconfigdata.get_value("LDVERSION"), gil_disabled, - )); + )?); let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") .map(|bytes_width: u32| bytes_width * 8) .ok(); @@ -660,10 +660,18 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) ) } - /// Lowers the configured version to the abi3 version, if set. + pub fn is_free_threaded(&self) -> bool { + self.build_flags.0.contains(&BuildFlag::Py_GIL_DISABLED) + } + + /// Updates configured ABI to build for to the requested abi3 version + /// This is a no-op for platforms where abi3 is not supported fn fixup_for_abi3_version(&mut self, abi3_version: Option) -> Result<()> { - // PyPy doesn't support abi3; don't adjust the version - if self.implementation.is_pypy() || self.implementation.is_graalpy() { + // PyPy, GraalPy, and the free-threaded build don't support abi3; don't adjust the version + if self.implementation.is_pypy() + || self.implementation.is_graalpy() + || self.is_free_threaded() + { return Ok(()); } @@ -691,6 +699,14 @@ pub struct PythonVersion { } impl PythonVersion { + pub const PY313: Self = PythonVersion { + major: 3, + minor: 13, + }; + const PY310: Self = PythonVersion { + major: 3, + minor: 10, + }; const PY37: Self = PythonVersion { major: 3, minor: 7 }; } @@ -1536,7 +1552,7 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result InterpreterConfig { +fn default_abi3_config(host: &Triple, version: PythonVersion) -> Result { // FIXME: PyPy & GraalPy do not support the Stable ABI. let implementation = PythonImplementation::CPython; let abi3 = true; @@ -1549,12 +1565,12 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf false, false, false, - )) + )?) } else { None }; - InterpreterConfig { + Ok(InterpreterConfig { implementation, version, shared: true, @@ -1566,7 +1582,7 @@ fn default_abi3_config(host: &Triple, version: PythonVersion) -> InterpreterConf build_flags: BuildFlags::default(), suppress_build_script_link_lines: false, extra_build_script_lines: vec![], - } + }) } /// Detects the cross compilation target interpreter configuration from all @@ -1606,11 +1622,9 @@ fn load_cross_compile_config( Ok(config) } -// Link against python3.lib for the stable ABI on Windows. -// See https://www.python.org/dev/peps/pep-0384/#linkage -// -// This contains only the limited ABI symbols. +// These contains only the limited ABI symbols. const WINDOWS_ABI3_LIB_NAME: &str = "python3"; +const WINDOWS_ABI3_DEBUG_LIB_NAME: &str = "python3_d"; fn default_lib_name_for_target( version: PythonVersion, @@ -1619,16 +1633,9 @@ fn default_lib_name_for_target( target: &Triple, ) -> Option { if target.operating_system == OperatingSystem::Windows { - Some(default_lib_name_windows( - version, - implementation, - abi3, - false, - false, - false, - )) + Some(default_lib_name_windows(version, implementation, abi3, false, false, false).unwrap()) } else if is_linking_libpython_for_target(target) { - Some(default_lib_name_unix(version, implementation, None, false)) + Some(default_lib_name_unix(version, implementation, None, false).unwrap()) } else { None } @@ -1641,27 +1648,35 @@ fn default_lib_name_windows( mingw: bool, debug: bool, gil_disabled: bool, -) -> String { - if debug { +) -> Result { + if debug && version < PythonVersion::PY310 { // CPython bug: linking against python3_d.dll raises error // https://github.com/python/cpython/issues/101614 - if gil_disabled { - format!("python{}{}t_d", version.major, version.minor) + Ok(format!("python{}{}_d", version.major, version.minor)) + } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { + if debug { + Ok(WINDOWS_ABI3_DEBUG_LIB_NAME.to_owned()) } else { - format!("python{}{}_d", version.major, version.minor) + Ok(WINDOWS_ABI3_LIB_NAME.to_owned()) } - } else if abi3 && !(implementation.is_pypy() || implementation.is_graalpy()) { - WINDOWS_ABI3_LIB_NAME.to_owned() } else if mingw { - if gil_disabled { - panic!("MinGW free-threaded builds are not currently tested or supported") - } + ensure!( + !gil_disabled, + "MinGW free-threaded builds are not currently tested or supported" + ); // https://packages.msys2.org/base/mingw-w64-python - format!("python{}.{}", version.major, version.minor) + Ok(format!("python{}.{}", version.major, version.minor)) } else if gil_disabled { - format!("python{}{}t", version.major, version.minor) + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + if debug { + Ok(format!("python{}{}t_d", version.major, version.minor)) + } else { + Ok(format!("python{}{}t", version.major, version.minor)) + } + } else if debug { + Ok(format!("python{}{}_d", version.major, version.minor)) } else { - format!("python{}{}", version.major, version.minor) + Ok(format!("python{}{}", version.major, version.minor)) } } @@ -1670,30 +1685,31 @@ fn default_lib_name_unix( implementation: PythonImplementation, ld_version: Option<&str>, gil_disabled: bool, -) -> String { +) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => format!("python{}", ld_version), + Some(ld_version) => Ok(format!("python{}", ld_version)), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone if gil_disabled { - format!("python{}.{}t", version.major, version.minor) + ensure!(version >= PythonVersion::PY313, "Cannot compile C extensions for the free-threaded build on Python versions earlier than 3.13, found {}.{}", version.major, version.minor); + Ok(format!("python{}.{}t", version.major, version.minor)) } else { - format!("python{}.{}", version.major, version.minor) + Ok(format!("python{}.{}", version.major, version.minor)) } } else { // Work around https://bugs.python.org/issue36707 - format!("python{}.{}m", version.major, version.minor) + Ok(format!("python{}.{}m", version.major, version.minor)) } } }, PythonImplementation::PyPy => match ld_version { - Some(ld_version) => format!("pypy{}-c", ld_version), - None => format!("pypy{}.{}-c", version.major, version.minor), + Some(ld_version) => Ok(format!("pypy{}-c", ld_version)), + None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), }, - PythonImplementation::GraalPy => "python-native".to_string(), + PythonImplementation::GraalPy => Ok("python-native".to_string()), } } @@ -1863,7 +1879,7 @@ pub fn make_interpreter_config() -> Result { ); }; - let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap()); + let mut interpreter_config = default_abi3_config(&host, abi3_version.unwrap())?; // Auto generate python3.dll import libraries for Windows targets. #[cfg(feature = "python3-dll-a")] @@ -2200,7 +2216,7 @@ mod tests { let min_version = "3.7".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 7 }, @@ -2223,7 +2239,7 @@ mod tests { let min_version = "3.9".parse().unwrap(); assert_eq!( - default_abi3_config(&host, min_version), + default_abi3_config(&host, min_version).unwrap(), InterpreterConfig { implementation: PythonImplementation::CPython, version: PythonVersion { major: 3, minor: 9 }, @@ -2389,9 +2405,19 @@ mod tests { false, false, false, - ), + ) + .unwrap(), "python39", ); + assert!(super::default_lib_name_windows( + PythonVersion { major: 3, minor: 9 }, + CPython, + false, + false, + false, + true, + ) + .is_err()); assert_eq!( super::default_lib_name_windows( PythonVersion { major: 3, minor: 9 }, @@ -2400,7 +2426,8 @@ mod tests { false, false, false, - ), + ) + .unwrap(), "python3", ); assert_eq!( @@ -2411,7 +2438,8 @@ mod tests { true, false, false, - ), + ) + .unwrap(), "python3.9", ); assert_eq!( @@ -2422,7 +2450,8 @@ mod tests { true, false, false, - ), + ) + .unwrap(), "python3", ); assert_eq!( @@ -2433,7 +2462,8 @@ mod tests { false, false, false, - ), + ) + .unwrap(), "python39", ); assert_eq!( @@ -2444,10 +2474,11 @@ mod tests { false, true, false, - ), + ) + .unwrap(), "python39_d", ); - // abi3 debug builds on windows use version-specific lib + // abi3 debug builds on windows use version-specific lib on 3.9 and older // to workaround https://github.com/python/cpython/issues/101614 assert_eq!( super::default_lib_name_windows( @@ -2457,9 +2488,81 @@ mod tests { false, true, false, - ), + ) + .unwrap(), "python39_d", ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 10 + }, + CPython, + true, + false, + true, + false, + ) + .unwrap(), + "python3_d", + ); + // Python versions older than 3.13 don't support gil_disabled + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + false, + false, + true, + ) + .is_err()); + // mingw and free-threading are incompatible (until someone adds support) + assert!(super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + false, + true, + false, + true, + ) + .is_err()); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + false, + true, + ) + .unwrap(), + "python313t", + ); + assert_eq!( + super::default_lib_name_windows( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + false, + false, + true, + true, + ) + .unwrap(), + "python313t_d", + ); } #[test] @@ -2472,7 +2575,8 @@ mod tests { CPython, None, false - ), + ) + .unwrap(), "python3.7m", ); // Defaults to pythonX.Y for CPython 3.8+ @@ -2482,7 +2586,8 @@ mod tests { CPython, None, false - ), + ) + .unwrap(), "python3.8", ); assert_eq!( @@ -2491,7 +2596,8 @@ mod tests { CPython, None, false - ), + ) + .unwrap(), "python3.9", ); // Can use ldversion to override for CPython @@ -2501,13 +2607,15 @@ mod tests { CPython, Some("3.7md"), false - ), + ) + .unwrap(), "python3.7md", ); // PyPy 3.9 includes ldversion assert_eq!( - super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false), + super::default_lib_name_unix(PythonVersion { major: 3, minor: 9 }, PyPy, None, false) + .unwrap(), "pypy3.9-c", ); @@ -2517,9 +2625,36 @@ mod tests { PyPy, Some("3.9d"), false - ), + ) + .unwrap(), "pypy3.9d-c", ); + + // free-threading adds a t suffix + assert_eq!( + super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 13 + }, + CPython, + None, + true + ) + .unwrap(), + "python3.13t", + ); + // 3.12 and older are incompatible with gil_disabled + assert!(super::default_lib_name_unix( + PythonVersion { + major: 3, + minor: 12, + }, + CPython, + None, + true, + ) + .is_err()); } #[test] diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 931838b5e5d..ea023de75fa 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -4,7 +4,7 @@ use pyo3_build_config::{ cargo_env_var, env_var, errors::Result, is_linking_libpython, resolve_interpreter_config, InterpreterConfig, PythonVersion, }, - warn, BuildFlag, PythonImplementation, + warn, PythonImplementation, }; /// Minimum Python version PyO3 supports. @@ -56,15 +56,22 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { interpreter_config.version, versions.min, ); - ensure!( - interpreter_config.version <= versions.max || env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), - "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ - = help: please check if an updated version of PyO3 is available. Current version: {}\n\ - = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", - interpreter_config.version, - versions.max, - std::env::var("CARGO_PKG_VERSION").unwrap(), - ); + if interpreter_config.version > versions.max { + ensure!(!interpreter_config.is_free_threaded(), + "The configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}\n\ + = help: The free-threaded build of CPython does not support the limited API so this check cannot be suppressed.", + interpreter_config.version, versions.max, std::env::var("CARGO_PKG_VERSION").unwrap() + ); + ensure!(env_var("PYO3_USE_ABI3_FORWARD_COMPATIBILITY").map_or(false, |os_str| os_str == "1"), + "the configured Python interpreter version ({}) is newer than PyO3's maximum supported version ({})\n\ + = help: please check if an updated version of PyO3 is available. Current version: {}\n\ + = help: set PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 to suppress this check and build anyway using the stable ABI", + interpreter_config.version, + versions.max, + std::env::var("CARGO_PKG_VERSION").unwrap(), + ); + } } PythonImplementation::PyPy => { let versions = SUPPORTED_VERSIONS_PYPY; @@ -107,14 +114,10 @@ fn ensure_python_version(interpreter_config: &InterpreterConfig) -> Result<()> { if interpreter_config.abi3 { match interpreter_config.implementation { PythonImplementation::CPython => { - if interpreter_config - .build_flags - .0 - .contains(&BuildFlag::Py_GIL_DISABLED) - { + if interpreter_config.is_free_threaded() { warn!( "The free-threaded build of CPython does not yet support abi3 so the build artifacts will be version-specific." - ) + ) } } PythonImplementation::PyPy => warn!(