feat: implement CEP-20 for Python ABI3 packages (#1320)

src/build.rs

@@ -146,7 +146,7 @@ pub async fn run_build(
     for test in output.recipe.tests() {
         if let TestType::PackageContents { package_contents } = test {
             package_contents
-                .run_test(&paths_json, &output.build_configuration.target_platform)
+                .run_test(&paths_json, &output)
                 .into_diagnostic()?;
         }
     }

src/env_vars.rs

@@ -255,7 +255,7 @@ pub fn vars(output: &Output, build_state: &str) -> HashMap<String, Option<String
     insert!(vars, "PIP_NO_INDEX", "True");
 
     // For noarch packages, do not write any bytecode
-    if output.build_configuration.target_platform == Platform::NoArch {
+    if output.recipe.build().is_python_version_independent() {
         insert!(vars, "PYTHONDONTWRITEBYTECODE", "1");
     }
 

src/package_test/content_test.rs

@@ -1,7 +1,7 @@
 use std::path::PathBuf;
 
-use crate::package_test::TestError;
 use crate::recipe::parser::PackageContentsTest;
+use crate::{metadata::Output, package_test::TestError};
 use globset::{Glob, GlobBuilder, GlobSet};
 use rattler_conda_types::{package::PathsJson, Arch, Platform};
 
@@ -162,13 +162,14 @@ impl PackageContentsTest {
     pub fn site_packages_as_globs(
         &self,
         target_platform: &Platform,
+        version_independent: bool,
     ) -> Result<Vec<(String, GlobSet)>, globset::Error> {
         let mut result = Vec::new();
 
-        let site_packages_base = if target_platform.is_windows() {
-            "Lib/site-packages"
-        } else if matches!(target_platform, Platform::NoArch) {
+        let site_packages_base = if version_independent {
             "site-packages"
+        } else if target_platform.is_windows() {
+            "Lib/site-packages"
         } else {
             "lib/python*/site-packages"
         };
@@ -214,10 +215,10 @@ impl PackageContentsTest {
     }
 
     /// Run the package content test
-    pub fn run_test(&self, paths: &PathsJson, target_platform: &Platform) -> Result<(), TestError> {
+    pub fn run_test(&self, paths: &PathsJson, output: &Output) -> Result<(), TestError> {
         let span = tracing::info_span!("Package content test");
         let _enter = span.enter();
-
+        let target_platform = output.target_platform();
         let paths = paths
             .paths
             .iter()
@@ -227,7 +228,10 @@ impl PackageContentsTest {
         let include_globs = self.include_as_globs(target_platform)?;
         let bin_globs = self.bin_as_globs(target_platform)?;
         let lib_globs = self.lib_as_globs(target_platform)?;
-        let site_package_globs = self.site_packages_as_globs(target_platform)?;
+        let site_package_globs = self.site_packages_as_globs(
+            target_platform,
+            output.recipe.build().is_python_version_independent(),
+        )?;
         let file_globs = self.files_as_globs()?;
 
         fn match_glob<'a>(glob: &GlobSet, paths: &'a Vec<&PathBuf>) -> Vec<&'a PathBuf> {
@@ -431,7 +435,9 @@ mod tests {
 
         if !tests.site_packages.is_empty() {
             println!("site_package globs: {:?}", tests.site_packages);
-            let globs = tests.site_packages_as_globs(&test_case.platform).unwrap();
+            let globs = tests
+                .site_packages_as_globs(&test_case.platform, false)
+                .unwrap();
             test_glob_matches(&globs, &test_case.paths)?;
             if !test_case.fail_paths.is_empty() {
                 test_glob_matches(&globs, &test_case.fail_paths).unwrap_err();

src/packaging.rs

@@ -277,7 +277,7 @@ pub fn package_conda(
 
     tracing::info!("Creating entry points");
     // create any entry points or link.json for noarch packages
-    if output.recipe.build().noarch().is_python() {
+    if output.recipe.build().is_python_version_independent() {
         let link_json = File::create(info_folder.join("link.json"))?;
         serde_json::to_writer_pretty(link_json, &output.link_json()?)?;
         tmp.add_files(vec![info_folder.join("link.json")]);

src/packaging/file_mapper.rs

@@ -99,7 +99,6 @@ impl Output {
         dest_folder: &Path,
     ) -> Result<Option<PathBuf>, PackagingError> {
         let target_platform = &self.build_configuration.target_platform;
-        let noarch_type = self.recipe.build().noarch();
         let entry_points = &self.recipe.build().python().entry_points;
 
         let path_rel = path.strip_prefix(prefix)?;
@@ -120,7 +119,7 @@ impl Output {
             }
         }
 
-        if noarch_type.is_python() {
+        if self.recipe.build().is_python_version_independent() {
             // we need to remove files in bin/ that are registered as entry points
             if path_rel.starts_with("bin") {
                 if let Some(name) = path_rel.file_name() {

src/packaging/metadata.rs

@@ -18,7 +18,7 @@ use rattler_conda_types::{
         AboutJson, FileMode, IndexJson, LinkJson, NoArchLinks, PackageFile, PathType, PathsEntry,
         PathsJson, PrefixPlaceholder, PythonEntryPoints, RunExportsJson,
     },
-    Platform,
+    NoArchType, Platform,
 };
 use rattler_digest::{compute_bytes_digest, compute_file_digest};
 
@@ -250,7 +250,7 @@ impl Output {
     /// Create the contents of the index.json file for the given output.
     pub fn index_json(&self) -> Result<IndexJson, PackagingError> {
         let recipe = &self.recipe;
-        let target_platform = self.build_configuration.target_platform;
+        let target_platform = self.target_platform();
 
         let arch = target_platform.arch().map(|a| a.to_string());
         let platform = target_platform.only_platform().map(|p| p.to_string());
@@ -283,6 +283,14 @@ impl Output {
                 return Err(PackagingError::InvalidMetadata("Cannot set python_site_packages_path for a package that is not called `python`".to_string()));
             }
         }
+
+        // Support CEP-20 / ABI3 packages
+        let noarch = if self.recipe.build().is_python_version_independent() {
+            NoArchType::python()
+        } else {
+            *self.recipe.build().noarch()
+        };
+
         Ok(IndexJson {
             name: self.name().clone(),
             version: self.version().clone().into(),
@@ -308,7 +316,7 @@ impl Output {
                 .map(|dep| dep.spec().to_string())
                 .dedup()
                 .collect(),
-            noarch: *recipe.build().noarch(),
+            noarch,
             track_features,
             features: None,
             python_site_packages_path: recipe.build().python().site_packages_path.clone(),

src/post_process/python.rs

@@ -158,7 +158,7 @@ pub fn python(temp_files: &TempFiles, output: &Output) -> Result<HashSet<PathBuf
     let version = output.version();
     let mut result = HashSet::new();
 
-    if !output.recipe.build().noarch().is_python() {
+    if !output.recipe.build().is_python_version_independent() {
         result.extend(compile_pyc(
             output,
             &temp_files.files,

src/recipe/parser/build.rs

@@ -276,6 +276,13 @@ impl Build {
     pub const fn post_process(&self) -> &Vec<PostProcess> {
         &self.post_process
     }
+
+    /// The output is python version independent if the package is
+    /// `noarch: python` or the python version independent flag is set
+    /// which can also be true for `abi3` packages.
+    pub(crate) fn is_python_version_independent(&self) -> bool {
+        self.python().version_independent || self.noarch().is_python()
+    }
 }
 
 impl TryConvertNode<Build> for RenderedNode {
@@ -505,6 +512,12 @@ pub struct Python {
     #[serde(default, skip_serializing_if = "std::ops::Not::not")]
     pub use_python_app_entrypoint: bool,
 
+    /// Whether the package is Python version independent.
+    /// This is used for abi3 packages that are not tied to a specific Python version, but
+    /// still contain compiled code (and thus need to end up in the right subdir).
+    #[serde(default, skip_serializing_if = "std::ops::Not::not")]
+    pub version_independent: bool,
+
     /// The relative site-packages path that a Python build _exports_ for other
     /// packages to use. This setting only makes sense for the `python` package
     /// itself. For example, a python 3.13 version could advertise a
@@ -538,7 +551,8 @@ impl TryConvertNode<Python> for RenderedMappingNode {
             entry_points,
             skip_pyc_compilation,
             use_python_app_entrypoint,
-            site_packages_path
+            site_packages_path,
+            version_independent
         );
         Ok(python)
     }

src/recipe/parser/requirements.rs

@@ -522,10 +522,12 @@ impl TryConvertNode<RunExports> for RenderedMappingNode {
 /// Run exports to ignore
 #[derive(Debug, Default, Clone, Serialize, Deserialize)]
 pub struct IgnoreRunExports {
+    /// Run exports to ignore by name of the package that is exported
     #[serde(default, skip_serializing_if = "IndexSet::is_empty")]
-    pub(super) by_name: IndexSet<PackageName>,
+    pub by_name: IndexSet<PackageName>,
+    /// Run exports to ignore by the package that applies them
     #[serde(default, skip_serializing_if = "IndexSet::is_empty")]
-    pub(super) from_package: IndexSet<PackageName>,
+    pub from_package: IndexSet<PackageName>,
 }
 
 impl IgnoreRunExports {

src/recipe/snapshots/rattler_build__recipe__parser__tests__recipe_windows.snap

@@ -126,6 +126,7 @@ Recipe {
             entry_points: [],
             skip_pyc_compilation: [],
             use_python_app_entrypoint: false,
+            version_independent: false,
             site_packages_path: None,
         },
         dynamic_linking: DynamicLinking {

src/recipe/snapshots/rattler_build__recipe__parser__tests__unix_recipe.snap

@@ -126,6 +126,7 @@ Recipe {
             entry_points: [],
             skip_pyc_compilation: [],
             use_python_app_entrypoint: false,
+            version_independent: false,
             site_packages_path: None,
         },
         dynamic_linking: DynamicLinking {

src/variant_config.rs

@@ -440,7 +440,7 @@ impl VariantConfig {
         // Now we need to convert the stage 1 renders to DiscoveredOutputs
         let mut recipes = IndexSet::new();
         for sx in stage_1 {
-            for ((node, recipe), variant) in sx.into_sorted_outputs()? {
+            for ((node, mut recipe), variant) in sx.into_sorted_outputs()? {
                 let target_platform = if recipe.build().noarch().is_none() {
                     selector_config.target_platform
                 } else {
@@ -454,6 +454,14 @@ impl VariantConfig {
                     .expect("Build string has to be resolved")
                     .to_string();
 
+                if recipe.build().python().version_independent {
+                    recipe
+                        .requirements
+                        .ignore_run_exports
+                        .from_package
+                        .insert("python".parse().unwrap());
+                }
+
                 recipes.insert(DiscoveredOutput {
                     name: recipe.package().name.as_normalized().to_string(),
                     version: recipe.package().version.to_string(),

src/variant_render.rs

@@ -352,7 +352,7 @@ pub(crate) fn stage_1_render(
             additional_variables.extend(extra_use_keys);
 
             // If the recipe is `noarch: python` we can remove an empty python key that comes from the dependencies
-            if output.build().noarch().is_python() {
+            if output.build().is_python_version_independent() {
                 additional_variables.remove(&"python".into());
             }
 

test-data/recipes/abi3/recipe.yaml

@@ -0,0 +1,45 @@
+package:
+  name: python-abi3-package-sample
+  version: 0.0.1
+
+source:
+  url: https://github.com/joerick/python-abi3-package-sample/archive/6f74ae7b31e58ef5f8f09b647364854122e61155.tar.gz
+  sha256: e81fd4d4c4f5b7bc9786d9ee990afc659e14a25ce11182b7b69f826407cc1718
+
+build:
+  number: 0
+  python:
+    version_independent: true
+  script: ${{ PYTHON }} -m pip install . -vv
+
+requirements:
+  build:
+    - ${{ compiler('c') }}
+  host:
+    - python-abi3
+    - python
+    - pip
+    - setuptools
+  run:
+    - python
+
+tests:
+  - python:
+      imports:
+        - spam
+  - script:
+      - export SP_DIR=$(python -c "import site; print(site.getsitepackages()[0])")
+      - abi3audit $SP_DIR/spam.abi3.so -s -v --assume-minimum-abi3 ${{ python_min }}
+    requirements:
+      run:
+        - abi3audit
+
+about:
+  homepage: https://github.com/joerick/python-abi3-package-sample
+  summary: 'ABI3 example'
+  license: Apache-2.0
+  license_file: LICENSE
+
+extra:
+  recipe-maintainers:
+    - isuruf

test-data/recipes/abi3/variants.yaml

@@ -0,0 +1,2 @@
+python_min:
+  - "3.8"

test/end-to-end/test_simple.py

@@ -1200,3 +1200,30 @@ def test_cache_select_files(rattler_build: RattlerBuild, recipes: Path, tmp_path
     assert paths["paths"][0]["path_type"] == "softlink"
     assert paths["paths"][1]["_path"] == "lib/libdav1d.so.7.0.0"
     assert paths["paths"][1]["path_type"] == "hardlink"
+
+
+@pytest.mark.skipif(
+    os.name == "nt", reason="recipe does not support execution on windows"
+)
+def test_abi3(rattler_build: RattlerBuild, recipes: Path, tmp_path: Path):
+    rattler_build.build(recipes / "abi3", tmp_path)
+    pkg = get_extracted_package(tmp_path, "python-abi3-package-sample")
+
+    assert (pkg / "info/paths.json").exists()
+    paths = json.loads((pkg / "info/paths.json").read_text())
+    # ensure that all paths start with `site-packages`
+    for p in paths["paths"]:
+        assert p["_path"].startswith("site-packages")
+
+    actual_paths = [p["_path"] for p in paths["paths"]]
+    if os.name == "nt":
+        assert "site-packages\\spam.dll" in actual_paths
+    else:
+        assert "site-packages/spam.abi3.so" in actual_paths
+
+    # load index.json
+    index = json.loads((pkg / "info/index.json").read_text())
+    assert index["name"] == "python-abi3-package-sample"
+    assert index["noarch"] == "python"
+    assert index["subdir"] == host_subdir()
+    assert index["platform"] == host_subdir().split("-")[0]