From 356c02d3c706bd29678ab278f839dfe00d315204 Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Thu, 28 Nov 2024 21:26:55 +0900 Subject: [PATCH 1/2] rustPlatform.importCargoLock: Support 'cargo-vendor --no-merge-sources' feature --- pkgs/build-support/rust/import-cargo-lock.nix | 71 ++++++++++++---- .../rust/replace-workspace-values.py | 40 ++++----- .../rust/test/import-cargo-lock/default.nix | 1 + .../default.nix | 2 + .../git-dependency-rev-with-hash/Cargo.lock | 81 +++++++++++++++++++ .../git-dependency-rev-with-hash/Cargo.toml | 8 ++ .../git-dependency-rev-with-hash/default.nix | 31 +++++++ .../git-dependency-rev-with-hash/src/main.rs | 9 +++ .../want.toml | 2 +- 9 files changed, 210 insertions(+), 35 deletions(-) create mode 100644 pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.lock create mode 100644 pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.toml create mode 100644 pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/default.nix create mode 100644 pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/src/main.rs diff --git a/pkgs/build-support/rust/import-cargo-lock.nix b/pkgs/build-support/rust/import-cargo-lock.nix index b39b56f4ccb5e..00469eca99f62 100644 --- a/pkgs/build-support/rust/import-cargo-lock.nix +++ b/pkgs/build-support/rust/import-cargo-lock.nix @@ -72,10 +72,26 @@ let builtins.map nameGitSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages) ); - nameGitSha = pkg: let gitParts = parseGit pkg.source; in { - name = "${pkg.name}-${pkg.version}"; - value = gitParts.sha; - }; + namesGitShasWithSha = builtins.listToAttrs ( + builtins.map nameGitShaWithSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages) + ); + + nameGitSha = pkg: + let + gitParts = parseGit pkg.source; + in { + name = "${pkg.name}-${pkg.version}"; + value = gitParts.sha; + }; + + nameGitShaWithSha = pkg: + let + gitParts = parseGit pkg.source; + suffix = if builtins.isNull gitParts then "" else "-" + gitParts.sha; + in { + name = "${pkg.name}-${pkg.version}${suffix}"; + value = gitParts.sha; + }; # Convert the attrset provided through the `outputHashes` argument to a # a mapping from git commit SHA -> output hash. @@ -88,7 +104,7 @@ let gitShaOutputHash = lib.mapAttrs' (nameVer: hash: let unusedHash = throw "A hash was specified for ${nameVer}, but there is no corresponding git dependency."; - rev = namesGitShas.${nameVer} or unusedHash; in { + rev = namesGitShasWithSha.${nameVer} or namesGitShas.${nameVer} or unusedHash; in { name = rev; value = hash; }) outputHashes; @@ -122,12 +138,13 @@ let let gitParts = parseGit pkg.source; registryIndexUrl = lib.removePrefix "registry+" pkg.source; + suffix = if builtins.isNull gitParts then "" else "-" + gitParts.sha; in if (lib.hasPrefix "registry+" pkg.source || lib.hasPrefix "sparse+" pkg.source) && builtins.hasAttr registryIndexUrl registries then let crateTarball = fetchCrate pkg registries.${registryIndexUrl}; - in runCommand "${pkg.name}-${pkg.version}" {} '' + in runCommand "${pkg.name}-${pkg.version}${suffix}" {} '' mkdir $out tar xf "${crateTarball}" -C $out --strip-components=1 @@ -137,13 +154,19 @@ let else if gitParts != null then let missingHash = throw '' - No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version}. You can add + No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version} or ${pkg.name}-${pkg.version}${suffix}. You can add a hash through the `outputHashes` argument of `importCargoLock`: outputHashes = { "${pkg.name}-${pkg.version}" = ""; }; + or + + outputHashes = { + "${pkg.name}-${pkg.version}${suffix}" = ""; + }; + If you use `buildRustPackage`, you can add this attribute to the `cargoLock` attribute set. ''; @@ -163,7 +186,7 @@ let } else missingHash; - in runCommand "${pkg.name}-${pkg.version}" {} '' + in runCommand "${pkg.name}-${pkg.version}${suffix}" {} '' tree=${tree} # If the target package is in a workspace, or if it's the top-level @@ -187,7 +210,7 @@ let done if [[ -z $crateCargoTOML ]]; then - >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree" + >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}${suffix}' in the tree in: $tree" exit 1 fi fi @@ -215,12 +238,16 @@ let ''} ''} - # Set up configuration for the vendor directory. + # Set up configuration for the vendor directory with package name and version. cat > $out/.cargo-config <&2 echo "The key of 'directory' is empty in $crate/.cargo-config." + exit 1 + fi if [[ -z ''${keysSeen[$key]} ]]; then keysSeen[$key]=1 cat "$crate/.cargo-config" >> $out/.cargo/config.toml fi + else + ln -s "$crate" $out/registry/$(basename "$crate" | cut -c 34-) fi done ''; diff --git a/pkgs/build-support/rust/replace-workspace-values.py b/pkgs/build-support/rust/replace-workspace-values.py index 003023ff2560a..fa9ba28336e40 100644 --- a/pkgs/build-support/rust/replace-workspace-values.py +++ b/pkgs/build-support/rust/replace-workspace-values.py @@ -31,34 +31,36 @@ def replace_key( local_dep = table[key] del local_dep["workspace"] - workspace_dep = workspace_manifest[section][key] - if section == "dependencies": - if isinstance(workspace_dep, str): - workspace_dep = {"version": workspace_dep} + if key in workspace_manifest[section]: + workspace_dep = workspace_manifest[section][key] + if isinstance(workspace_dep, str): + workspace_dep = {"version": workspace_dep} - final: dict[str, Any] = workspace_dep.copy() + final: dict[str, Any] = workspace_dep.copy() - merged_features = local_dep.pop("features", []) + workspace_dep.get("features", []) - if merged_features: - final["features"] = merged_features + merged_features = local_dep.pop("features", []) + workspace_dep.get("features", []) + if merged_features: + final["features"] = merged_features - local_default_features = local_dep.pop("default-features", None) - workspace_default_features = workspace_dep.get("default-features") + local_default_features = local_dep.pop("default-features", None) + workspace_default_features = workspace_dep.get("default-features") - if not workspace_default_features and local_default_features: - final["default-features"] = True + if not workspace_default_features and local_default_features: + final["default-features"] = True - optional = local_dep.pop("optional", False) - if optional: - final["optional"] = True + optional = local_dep.pop("optional", False) + if optional: + final["optional"] = True - if local_dep: - raise Exception(f"Unhandled keys in inherited dependency {key}: {local_dep}") + if local_dep: + raise Exception(f"Unhandled keys in inherited dependency {key}: {local_dep}") - table[key] = final + table[key] = final elif section == "package": - table[key] = workspace_dep + if key in workspace_manifest[section]: + workspace_dep = workspace_manifest[section][key] + table[key] = workspace_dep return True diff --git a/pkgs/build-support/rust/test/import-cargo-lock/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/default.nix index 26e6487989c42..356aaa26a2c43 100644 --- a/pkgs/build-support/rust/test/import-cargo-lock/default.nix +++ b/pkgs/build-support/rust/test/import-cargo-lock/default.nix @@ -7,6 +7,7 @@ basicDynamic = callPackage ./basic-dynamic { }; gitDependency = callPackage ./git-dependency { }; gitDependencyRev = callPackage ./git-dependency-rev { }; + gitDependencyRevWithHash = callPackage ./git-dependency-rev-with-hash { }; gitDependencyRevNonWorkspaceNestedCrate = callPackage ./git-dependency-rev-non-workspace-nested-crate { }; gitDependencyTag = callPackage ./git-dependency-tag { }; gitDependencyBranch = callPackage ./git-dependency-branch { }; diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-non-workspace-nested-crate/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-non-workspace-nested-crate/default.nix index 5b1dcc1aef616..f28753948b540 100644 --- a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-non-workspace-nested-crate/default.nix +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-non-workspace-nested-crate/default.nix @@ -2,6 +2,7 @@ rustPlatform, pkg-config, openssl, + zlib, lib, darwin, stdenv, @@ -29,6 +30,7 @@ rustPlatform.buildRustPackage { buildInputs = [ openssl + zlib ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ darwin.apple_sdk.frameworks.Security diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.lock b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.lock new file mode 100644 index 0000000000000..a9735ae8d6833 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.lock @@ -0,0 +1,81 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "git-dependency-rev-with-hash" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "libc" +version = "0.2.94" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" + +[[package]] +name = "ppv-lite86" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" + +[[package]] +name = "rand" +version = "0.8.3" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.1" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.3.0" +source = "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7" +dependencies = [ + "rand_core", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.toml b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.toml new file mode 100644 index 0000000000000..a51a262265644 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "git-dependency-rev-with-hash" +version = "0.1.0" +authors = ["Daniël de Kok "] +edition = "2018" + +[dependencies] +rand = { git = "https://github.com/rust-random/rand.git", rev = "0.8.3" } diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/default.nix b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/default.nix new file mode 100644 index 0000000000000..5229df709bfd1 --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/default.nix @@ -0,0 +1,31 @@ +{ lib, rustPlatform }: +let + fs = lib.fileset; +in +rustPlatform.buildRustPackage { + pname = "git-dependency-rev-with-hash"; + version = "0.1.0"; + + src = fs.toSource { + root = ./.; + fileset = fs.unions [ + ./Cargo.toml + ./Cargo.lock + ./src + ]; + }; + + cargoLock = { + lockFile = ./Cargo.lock; + outputHashes = { + "rand-0.8.3-6ecbe2626b2cc6110a25c97b1702b347574febc7" = + "0l3p174bpwia61vcvxz5mw65a13ri3wy94z04xrnyy5lzciykz4f"; + }; + }; + + doInstallCheck = true; + + installCheckPhase = '' + $out/bin/git-dependency-rev-with-hash + ''; +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/src/main.rs b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/src/main.rs new file mode 100644 index 0000000000000..50b4ed799e43e --- /dev/null +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/src/main.rs @@ -0,0 +1,9 @@ +use rand::Rng; + +fn main() { + let mut rng = rand::thread_rng(); + + // Always draw zero :). + let roll: u8 = rng.gen_range(0..1); + assert_eq!(roll, 0); +} diff --git a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-workspace-inheritance/want.toml b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-workspace-inheritance/want.toml index ec1331455bece..cd1cb594a7698 100644 --- a/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-workspace-inheritance/want.toml +++ b/pkgs/build-support/rust/test/import-cargo-lock/git-dependency-workspace-inheritance/want.toml @@ -12,8 +12,8 @@ keywords = [ bar = "1.0.0" [dependencies.foo] +version = "1.0.0" features = [ "cat", "meow", ] -version = "1.0.0" From 60fe186313677e8dbf04f917a2b4c3d984be4efd Mon Sep 17 00:00:00 2001 From: Junji Hashimoto Date: Sat, 7 Dec 2024 14:22:58 +0900 Subject: [PATCH 2/2] rustPlatform.fetchCargoVendor: Support 'cargo-vendor --no-merge-sources' feature --- pkgs/build-support/rust/fetch-cargo-vendor-util.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/rust/fetch-cargo-vendor-util.py b/pkgs/build-support/rust/fetch-cargo-vendor-util.py index bb70a559fe966..c570e3684c75f 100644 --- a/pkgs/build-support/rust/fetch-cargo-vendor-util.py +++ b/pkgs/build-support/rust/fetch-cargo-vendor-util.py @@ -224,6 +224,7 @@ def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> N def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: lockfile_path = vendor_staging_dir / "Cargo.lock" out_dir.mkdir(exist_ok=True) + (out_dir / "registry").mkdir(exist_ok=True) shutil.copy(lockfile_path, out_dir / "Cargo.lock") cargo_lock_toml = load_toml(lockfile_path) @@ -231,7 +232,7 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: config_lines = [ '[source.vendored-sources]', - 'directory = "@vendor@"', + 'directory = "@vendor@/registry"', '[source.crates-io]', 'replace-with = "vendored-sources"', ] @@ -246,7 +247,6 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: source: str = pkg["source"] dir_name = f"{pkg["name"]}-{pkg["version"]}" - crate_out_dir = out_dir / dir_name if source.startswith("git+"): @@ -254,6 +254,8 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: git_sha_rev = source_info["git_sha_rev"] git_tree = vendor_staging_dir / "git" / git_sha_rev + (out_dir / f"git-{git_sha_rev}").mkdir(exist_ok=True) + crate_out_dir = out_dir / f"git-{git_sha_rev}" / dir_name copy_and_patch_git_crate_subtree(git_tree, pkg["name"], crate_out_dir) @@ -272,10 +274,13 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: config_lines.append(f'git = "{source_info["url"]}"') if source_info["type"] is not None: config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"') - config_lines.append('replace-with = "vendored-sources"') + config_lines.append(f'replace-with = "vendored-sources-git-{git_sha_rev}"') + config_lines.append(f'[source.vendored-sources-git-{git_sha_rev}]') + config_lines.append(f'directory = "@vendor@/git-{git_sha_rev}"') elif source.startswith("registry+"): + crate_out_dir = out_dir / "registry" / dir_name filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz" tarball_path = vendor_staging_dir / "tarballs" / filename