rustPlatform.importCargoLock: allow duplicated packages with git.

pkgs/build-support/rust/fetch-cargo-vendor-util.py

@@ -224,14 +224,15 @@ def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> N
 def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
     lockfile_path = vendor_staging_dir / "Cargo.lock"
     out_dir.mkdir(exist_ok=True)
+    (out_dir / "registry").mkdir(exist_ok=True)
     shutil.copy(lockfile_path, out_dir / "Cargo.lock")
 
     cargo_lock_toml = load_toml(lockfile_path)
     lockfile_version = get_lockfile_version(cargo_lock_toml)
 
     config_lines = [
         '[source.vendored-sources]',
-        'directory = "@vendor@"',
+        'directory = "@vendor@/registry"',
         '[source.crates-io]',
         'replace-with = "vendored-sources"',
     ]
@@ -246,14 +247,15 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
         source: str = pkg["source"]
 
         dir_name = f"{pkg["name"]}-{pkg["version"]}"
-        crate_out_dir = out_dir / dir_name
 
         if source.startswith("git+"):
 
             source_info = parse_git_source(pkg["source"], lockfile_version)
 
             git_sha_rev = source_info["git_sha_rev"]
             git_tree = vendor_staging_dir / "git" / git_sha_rev
+            (out_dir / f"git-{git_sha_rev}").mkdir(exist_ok=True)
+            crate_out_dir = out_dir / f"git-{git_sha_rev}" / dir_name
 
             copy_and_patch_git_crate_subtree(git_tree, pkg["name"], crate_out_dir)
 
@@ -272,10 +274,13 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
             config_lines.append(f'git = "{source_info["url"]}"')
             if source_info["type"] is not None:
                 config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"')
-            config_lines.append('replace-with = "vendored-sources"')
+            config_lines.append(f'replace-with = "vendored-sources-git-{git_sha_rev}"')
+            config_lines.append(f'[source.vendored-sources-git-{git_sha_rev}]')
+            config_lines.append(f'directory = "@vendor@/git-{git_sha_rev}"')
 
         elif source.startswith("registry+"):
 
+            crate_out_dir = out_dir / "registry" / dir_name
             filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz"
             tarball_path = vendor_staging_dir / "tarballs" / filename
 

pkgs/build-support/rust/import-cargo-lock.nix

@@ -72,10 +72,26 @@ let
     builtins.map nameGitSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages)
   );
 
-  nameGitSha = pkg: let gitParts = parseGit pkg.source; in {
-    name = "${pkg.name}-${pkg.version}";
-    value = gitParts.sha;
-  };
+  namesGitShasWithSha = builtins.listToAttrs (
+    builtins.map nameGitShaWithSha (builtins.filter (pkg: lib.hasPrefix "git+" pkg.source) depPackages)
+  );
+
+  nameGitSha = pkg:
+    let
+      gitParts = parseGit pkg.source;
+    in {
+      name = "${pkg.name}-${pkg.version}";
+      value = gitParts.sha;
+    };
+
+  nameGitShaWithSha = pkg:
+    let
+      gitParts = parseGit pkg.source;
+      suffix = if builtins.isNull gitParts then "" else "-" + gitParts.sha;
+    in {
+      name = "${pkg.name}-${pkg.version}${suffix}";
+      value = gitParts.sha;
+    };
 
   # Convert the attrset provided through the `outputHashes` argument to a
   # a mapping from git commit SHA -> output hash.
@@ -88,7 +104,7 @@ let
   gitShaOutputHash = lib.mapAttrs' (nameVer: hash:
     let
       unusedHash = throw "A hash was specified for ${nameVer}, but there is no corresponding git dependency.";
-      rev = namesGitShas.${nameVer} or unusedHash; in {
+      rev = namesGitShasWithSha.${nameVer} or namesGitShas.${nameVer} or unusedHash; in {
       name = rev;
       value = hash;
     }) outputHashes;
@@ -122,12 +138,13 @@ let
     let
       gitParts = parseGit pkg.source;
       registryIndexUrl = lib.removePrefix "registry+" pkg.source;
+      suffix = if builtins.isNull gitParts then "" else "-" + gitParts.sha;
     in
       if (lib.hasPrefix "registry+" pkg.source || lib.hasPrefix "sparse+" pkg.source)
         && builtins.hasAttr registryIndexUrl registries then
       let
         crateTarball = fetchCrate pkg registries.${registryIndexUrl};
-      in runCommand "${pkg.name}-${pkg.version}" {} ''
+      in runCommand "${pkg.name}-${pkg.version}${suffix}" {} ''
         mkdir $out
         tar xf "${crateTarball}" -C $out --strip-components=1
 
@@ -137,13 +154,19 @@ let
       else if gitParts != null then
       let
         missingHash = throw ''
-          No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version}. You can add
+          No hash was found while vendoring the git dependency ${pkg.name}-${pkg.version} or ${pkg.name}-${pkg.version}${suffix}. You can add
           a hash through the `outputHashes` argument of `importCargoLock`:
 
           outputHashes = {
             "${pkg.name}-${pkg.version}" = "<hash>";
           };
 
+          or
+
+          outputHashes = {
+            "${pkg.name}-${pkg.version}${suffix}" = "<hash>";
+          };
+
           If you use `buildRustPackage`, you can add this attribute to the `cargoLock`
           attribute set.
         '';
@@ -163,7 +186,7 @@ let
             }
           else
             missingHash;
-      in runCommand "${pkg.name}-${pkg.version}" {} ''
+      in runCommand "${pkg.name}-${pkg.version}${suffix}" {} ''
         tree=${tree}
 
         # If the target package is in a workspace, or if it's the top-level
@@ -187,7 +210,7 @@ let
           done
 
           if [[ -z $crateCargoTOML ]]; then
-            >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}' in the tree in: $tree"
+            >&2 echo "Cannot find path for crate '${pkg.name}-${pkg.version}${suffix}' in the tree in: $tree"
             exit 1
           fi
         fi
@@ -215,12 +238,16 @@ let
           ''}
         ''}
 
-        # Set up configuration for the vendor directory.
+        # Set up configuration for the vendor directory with package name and version.
         cat > $out/.cargo-config <<EOF
-        [source."${pkg.source}"]
+        #pkg: ${pkg.name}
+        #version: ${pkg.version}
+        [source."${gitParts.url}${lib.optionalString (gitParts ? type) "?${gitParts.type}=$gitPartsValue"}"]
         git = "${gitParts.url}"
         ${lib.optionalString (gitParts ? type) "${gitParts.type} = \"$gitPartsValue\""}
-        replace-with = "vendored-sources"
+        replace-with = "vendored-sources-git-${gitParts.sha}"
+        [source.vendored-sources-git-${gitParts.sha}]
+        directory = "cargo-vendor-dir/git-${gitParts.sha}"
         EOF
       ''
       else throw "Cannot handle crate source: ${pkg.source}";
@@ -247,7 +274,7 @@ let
 replace-with = "vendored-sources"
 
 [source.vendored-sources]
-directory = "cargo-vendor-dir"
+directory = "cargo-vendor-dir/registry"
 EOF
 
     declare -A keysSeen
@@ -260,17 +287,31 @@ registry = "$registry"
 replace-with = "vendored-sources"
 EOF
     done
+    mkdir $out/registry
 
     for crate in ${toString depCrates}; do
       # Link the crate directory, removing the output path hash from the destination.
-      ln -s "$crate" $out/$(basename "$crate" | cut -c 34-)
-
+      # When the crate directory has a directory directive, putting it to git-* directory.
       if [ -e "$crate/.cargo-config" ]; then
         key=$(sed 's/\[source\."\(.*\)"\]/\1/; t; d' < "$crate/.cargo-config")
+        directory=$(sed 's/directory = "\(.*\)"/\1/; t; d' < "$crate/.cargo-config")
+        package_name=$(sed 's/#pkg: \(.*\)/\1/; t; d' < "$crate/.cargo-config")
+        if [[ ! -z "$directory" ]]; then
+          gitdir=$(basename "$directory")
+          if [ ! -d $out/$gitdir ] ; then
+            mkdir $out/$gitdir
+          fi
+          ln -s "$crate" $out/$gitdir/$(basename "$crate" | cut -c 34-)
+        else
+          >&2 echo "The key of 'directory' is empty in $crate/.cargo-config."
+          exit 1
+        fi
         if [[ -z ''${keysSeen[$key]} ]]; then
           keysSeen[$key]=1
           cat "$crate/.cargo-config" >> $out/.cargo/config.toml
         fi
+      else
+        ln -s "$crate" $out/registry/$(basename "$crate" | cut -c 34-)
       fi
     done
   '';

pkgs/build-support/rust/replace-workspace-values.py

@@ -31,34 +31,36 @@ def replace_key(
         local_dep = table[key]
         del local_dep["workspace"]
 
-        workspace_dep = workspace_manifest[section][key]
-
         if section == "dependencies":
-            if isinstance(workspace_dep, str):
-                workspace_dep = {"version": workspace_dep}
+            if key in workspace_manifest[section]:
+                workspace_dep = workspace_manifest[section][key]
+                if isinstance(workspace_dep, str):
+                    workspace_dep = {"version": workspace_dep}
 
-            final: dict[str, Any] = workspace_dep.copy()
+                final: dict[str, Any] = workspace_dep.copy()
 
-            merged_features = local_dep.pop("features", []) + workspace_dep.get("features", [])
-            if merged_features:
-                final["features"] = merged_features
+                merged_features = local_dep.pop("features", []) + workspace_dep.get("features", [])
+                if merged_features:
+                    final["features"] = merged_features
 
-            local_default_features = local_dep.pop("default-features", None)
-            workspace_default_features = workspace_dep.get("default-features")
+                local_default_features = local_dep.pop("default-features", None)
+                workspace_default_features = workspace_dep.get("default-features")
 
-            if not workspace_default_features and local_default_features:
-                final["default-features"] = True
+                if not workspace_default_features and local_default_features:
+                    final["default-features"] = True
 
-            optional = local_dep.pop("optional", False)
-            if optional:
-                final["optional"] = True
+                optional = local_dep.pop("optional", False)
+                if optional:
+                    final["optional"] = True
 
-            if local_dep:
-                raise Exception(f"Unhandled keys in inherited dependency {key}: {local_dep}")
+                if local_dep:
+                    raise Exception(f"Unhandled keys in inherited dependency {key}: {local_dep}")
 
-            table[key] = final
+                table[key] = final
         elif section == "package":
-            table[key] = workspace_dep
+            if key in workspace_manifest[section]:
+                workspace_dep = workspace_manifest[section][key]
+                table[key] = workspace_dep
 
         return True
 

pkgs/build-support/rust/test/import-cargo-lock/default.nix

@@ -7,6 +7,7 @@
   basicDynamic = callPackage ./basic-dynamic { };
   gitDependency = callPackage ./git-dependency { };
   gitDependencyRev = callPackage ./git-dependency-rev { };
+  gitDependencyRevWithHash = callPackage ./git-dependency-rev-with-hash { };
   gitDependencyRevNonWorkspaceNestedCrate = callPackage ./git-dependency-rev-non-workspace-nested-crate { };
   gitDependencyTag = callPackage ./git-dependency-tag { };
   gitDependencyBranch = callPackage ./git-dependency-branch { };

pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-non-workspace-nested-crate/default.nix

@@ -2,6 +2,7 @@
   rustPlatform,
   pkg-config,
   openssl,
+  zlib,
   lib,
   darwin,
   stdenv,
@@ -29,6 +30,7 @@ rustPlatform.buildRustPackage {
   buildInputs =
     [
       openssl
+      zlib
     ]
     ++ lib.optionals stdenv.hostPlatform.isDarwin [
       darwin.apple_sdk.frameworks.Security

pkgs/build-support/rust/test/import-cargo-lock/git-dependency-rev-with-hash/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "git-dependency-rev-with-hash"
+version = "0.1.0"
+authors = ["Daniël de Kok <me@danieldk.eu>"]
+edition = "2018"
+
+[dependencies]
+rand = { git = "https://github.com/rust-random/rand.git", rev = "0.8.3" }