feat: add --extra-meta

Cargo.toml

@@ -44,26 +44,27 @@ required-features = ["recipe-generation"]
 [dependencies]
 serde = { version = "1.0.204", features = ["derive"] }
 serde_yaml = "0.9.34"
-rattler = { version = "0.27.2", default-features = false, features = [
+rattler = { version = "0.27.4", default-features = false, features = [
     "cli-tools",
     "indicatif",
 ] }
-rattler_conda_types = { version = "0.26.3", default-features = false }
+rattler_conda_types = { version = "0.27.1", default-features = false }
 rattler_digest = { version = "1.0.0", default-features = false, features = ["serde"] }
-rattler_index = { version = "0.19.21", default-features = false }
-rattler_networking = { version = "0.20.10", default-features = false }
-rattler_repodata_gateway = { version = "0.21.3", default-features = false, features = [
+rattler_index = { version = "0.19.23", default-features = false }
+rattler_networking = { version = "0.21.0", default-features = false }
+rattler_repodata_gateway = { version = "0.21.5", default-features = false, features = [
     "gateway",
 ] }
-rattler_shell = { version = "0.21.3", default-features = false, features = [
+rattler_redaction = { version = "0.1.0" }
+rattler_shell = { version = "0.21.5", default-features = false, features = [
     "sysinfo",
 ] }
-rattler_solve = { version = "1.0.0", default-features = false, features = [
+rattler_solve = { version = "1.0.2", default-features = false, features = [
     "resolvo",
     "serde",
 ] }
-rattler_virtual_packages = { version = "1.0.0", default-features = false }
-rattler_package_streaming = { version = "0.21.7", default-features = false }
+rattler_virtual_packages = { version = "1.0.2", default-features = false }
+rattler_package_streaming = { version = "0.22.1", default-features = false }
 anyhow = "1.0.86"
 walkdir = "2.5.0"
 sha2 = "0.10.8"
@@ -169,16 +170,17 @@ pre-build = [
 ]
 
 [patch.crates-io]
-#rattler = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_conda_types = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_digest = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_index = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_networking = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_repodata_gateway = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_shell = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_solve = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_virtual_packages = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
-#rattler_package_streaming = { git = "https://github.com/baszalmstra/rattler", branch = "fix/quote_nushell_var" }
+rattler = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_conda_types = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_digest = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_index = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_networking = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_repodata_gateway = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_redaction = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_shell = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_solve = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_virtual_packages = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
+rattler_package_streaming = { git = "https://github.com/nichmor/rattler", branch = "feat/add-extra-field-on-about-json" }
 clap-markdown = { git = "https://github.com/ruben-arts/clap-markdown", branch = "main" }
 
 

src/lib.rs

@@ -322,6 +322,13 @@ pub async fn get_build_output(
             finalized_sources: None,
             system_tools: SystemTools::new(),
             build_summary: Arc::new(Mutex::new(BuildSummary::default())),
+            extra_meta: Some(
+                args.extra_meta
+                    .clone()
+                    .unwrap_or_default()
+                    .into_iter()
+                    .collect(),
+            ),
         };
 
         if args.render_only && args.with_solve {

src/macos/link.rs

@@ -627,7 +627,7 @@ mod tests {
         let prefix = Path::new(env!("CARGO_MANIFEST_DIR")).join("test-data/binary_files");
         let tmp_dir = tempdir_in(&prefix)?;
         let bin_dir = tmp_dir.path().join("bin");
-        fs::create_dir(&bin_dir)?;
+        fs::create_dir(bin_dir)?;
         let binary_path = tmp_dir.path().join("bin/zlink-relink-relative");
         fs::copy(prefix.join("zlink-macos"), &binary_path)?;
 

src/metadata.rs

@@ -1,6 +1,6 @@
 //! All the metadata that makes up a recipe file
 use std::{
-    collections::BTreeMap,
+    collections::{BTreeMap, HashMap},
     fmt::{self, Display, Formatter},
     io::Write,
     iter,
@@ -21,6 +21,7 @@ use rattler_index::index;
 use rattler_package_streaming::write::CompressionLevel;
 use rattler_solve::{ChannelPriority, SolveStrategy};
 use serde::{Deserialize, Serialize};
+use serde_json::Value;
 use url::Url;
 
 use crate::{
@@ -322,6 +323,11 @@ pub struct Output {
     pub build_summary: Arc<Mutex<BuildSummary>>,
     /// The system tools that are used to build this output
     pub system_tools: SystemTools,
+    /// Some extra metadata that should be recorded additionally in about.json
+    /// Usually it is used during the CI build to record link to the CI job
+    /// that created this artifact
+    #[serde(skip)]
+    pub extra_meta: Option<BTreeMap<String, Value>>,
 }
 
 impl Output {

src/opt.rs

@@ -1,6 +1,6 @@
 //! Command-line options.
 
-use std::{path::PathBuf, str::FromStr};
+use std::{error::Error, path::PathBuf, str::FromStr};
 
 #[cfg(feature = "recipe-generation")]
 use crate::recipe_generator::GenerateRecipeOpts;
@@ -16,6 +16,7 @@ use clap_complete_nushell::Nushell;
 use clap_verbosity_flag::{InfoLevel, Verbosity};
 use rattler_conda_types::{package::ArchiveType, Platform};
 use rattler_package_streaming::write::CompressionLevel;
+use serde_json::{json, Value};
 use url::Url;
 
 /// Application subcommands.
@@ -337,6 +338,10 @@ pub struct BuildOpts {
     /// If set to `all`, skip packages that already exist in any channel.
     #[arg(long, default_missing_value = "local", default_value = "none", num_args = 0..=1, help_heading = "Modifying result")]
     pub skip_existing: SkipExisting,
+
+    /// Extra metadata to include in about.json
+    #[arg(long, value_parser = parse_key_val)]
+    pub extra_meta: Option<Vec<(String, Value)>>,
 }
 
 fn is_dir(dir: &str) -> Result<PathBuf, String> {
@@ -350,6 +355,14 @@ fn is_dir(dir: &str) -> Result<PathBuf, String> {
     }
 }
 
+/// Parse a single key-value pair
+fn parse_key_val(s: &str) -> Result<(String, Value), Box<dyn Error + Send + Sync + 'static>> {
+    let (key, value) = s
+        .split_once("=")
+        .ok_or_else(|| format!("invalid KEY=value: no `=` found in `{}`", s))?;
+    Ok((key.to_string(), json!(value)))
+}
+
 /// Test options.
 #[derive(Parser)]
 pub struct TestOpts {

src/packaging/metadata.rs

@@ -237,6 +237,7 @@ impl Output {
                 .iter()
                 .map(|c| c.to_string())
                 .collect(),
+            extra: self.extra_meta.clone().unwrap_or_default(),
         };
 
         about_json

src/upload/mod.rs

@@ -10,7 +10,8 @@ use std::{
 use tokio_util::io::ReaderStream;
 
 use miette::{Context, IntoDiagnostic};
-use rattler_networking::{Authentication, AuthenticationStorage, Redact};
+use rattler_networking::{Authentication, AuthenticationStorage};
+use rattler_redaction::Redact;
 use reqwest::Method;
 use tracing::info;
 use url::Url;

test/end-to-end/__snapshots__/test_simple/test_extra_meta_is_recorded_into_about_json.json

@@ -0,0 +1,12 @@
+{
+  "channels": [
+    "https://conda.anaconda.org/conda-forge/"
+  ],
+  "extra": {
+    "flow_run_id": "some_id",
+    "sha": "24ee3"
+  },
+  "home": "https://github.com/uiri/toml",
+  "license": "MIT",
+  "summary": "Python lib for TOML."
+}

test/end-to-end/helpers.py

@@ -24,6 +24,7 @@ def build_args(
         variant_config: Optional[Path] = None,
         custom_channels: Optional[list[str]] = None,
         extra_args: list[str] = None,
+        extra_meta: dict[str, Any] = None,
     ):
         if extra_args is None:
             extra_args = []
@@ -32,6 +33,12 @@ def build_args(
             args += ["--variant-config", str(variant_config)]
         args += ["--output-dir", str(output_folder)]
         args += ["--package-format", str("tar.bz2")]
+        if extra_meta:
+            args += [
+                item
+                for k, v in (extra_meta or {}).items()
+                for item in ["--extra-meta", f"{k}={v}"]
+            ]
 
         if custom_channels:
             for c in custom_channels:
@@ -46,13 +53,15 @@ def build(
         variant_config: Optional[Path] = None,
         custom_channels: Optional[list[str]] = None,
         extra_args: list[str] = None,
+        extra_meta: dict[str, Any] = None,
     ):
         args = self.build_args(
             recipe_folder,
             output_folder,
             variant_config=variant_config,
             custom_channels=custom_channels,
             extra_args=extra_args,
+            extra_meta=extra_meta,
         )
         return self(*args)
 

test/end-to-end/test_simple.py

@@ -875,11 +875,16 @@ def test_downstream_test(
         assert "│ Failing test in downstream package" in e.value.output
         assert "│ Downstream test failed" in e.value.output
 
+
 def test_cache_runexports(
     rattler_build: RattlerBuild, recipes: Path, tmp_path: Path, snapshot_json
 ):
     rattler_build.build(recipes / "cache_run_exports/helper.yaml", tmp_path)
-    rattler_build.build(recipes / "cache_run_exports/recipe_test_1.yaml", tmp_path, extra_args=["--experimental"])
+    rattler_build.build(
+        recipes / "cache_run_exports/recipe_test_1.yaml",
+        tmp_path,
+        extra_args=["--experimental"],
+    )
 
     pkg = get_extracted_package(tmp_path, "cache-run-exports")
 
@@ -900,14 +905,37 @@ def test_cache_runexports(
     print(index)
     assert index.get("depends", []) == []
 
-    rattler_build.build(recipes / "cache_run_exports/recipe_test_2.yaml", tmp_path, extra_args=["--experimental"])
+    rattler_build.build(
+        recipes / "cache_run_exports/recipe_test_2.yaml",
+        tmp_path,
+        extra_args=["--experimental"],
+    )
     pkg = get_extracted_package(tmp_path, "cache-ignore-run-exports")
     index = json.loads((pkg / "info/index.json").read_text())
     assert index["name"] == "cache-ignore-run-exports"
     assert index.get("depends", []) == []
 
-    rattler_build.build(recipes / "cache_run_exports/recipe_test_3.yaml", tmp_path, extra_args=["--experimental"])
+    rattler_build.build(
+        recipes / "cache_run_exports/recipe_test_3.yaml",
+        tmp_path,
+        extra_args=["--experimental"],
+    )
     pkg = get_extracted_package(tmp_path, "cache-ignore-run-exports-by-name")
     index = json.loads((pkg / "info/index.json").read_text())
     assert index["name"] == "cache-ignore-run-exports-by-name"
     assert index.get("depends", []) == []
+
+
+def test_extra_meta_is_recorded_into_about_json(
+    rattler_build: RattlerBuild, recipes: Path, tmp_path: Path, snapshot_json
+):
+    rattler_build.build(
+        recipes / "toml",
+        tmp_path,
+        extra_meta={"flow_run_id": "some_id", "sha": "24ee3"},
+    )
+    pkg = get_extracted_package(tmp_path, "toml")
+
+    about_json = json.loads((pkg / "info/about.json").read_text())
+
+    assert snapshot_json == about_json