Skip to content

Commit

Permalink
Cache submodules between different checkouts of the same git dep
Browse files Browse the repository at this point in the history
This base64-encodes the URLs to avoid errors like the following:
```
error: failed to get `dep1` as a dependency of package `foo v0.5.0 (D:/a/cargo/cargo/target/tmp/cit/t1035/foo)`

Caused by:
  failed to load source for dependency `dep1`

Caused by:
  Unable to update file:///D:/a/cargo/cargo/target/tmp/cit/t1035/dep1

Caused by:
  failed to update submodule `src`

Caused by:
  failed to make directory 'D:/a/cargo/cargo/target/tmp/cit/t1035/home/.cargo/git/checkouts/submodules/file:': The filename, directory name, or volume label syntax is incorrect.
  ; class=Os (2)
', tests\testsuite\git.rs:2515:10
```

It uses bare checkouts instead of symbolic links to avoid permission errors on Windows.
  • Loading branch information
jyn514 committed Feb 15, 2022
1 parent 28fffd6 commit 9a8816c
Show file tree
Hide file tree
Showing 3 changed files with 46 additions and 15 deletions.
1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ path = "src/cargo/lib.rs"

[dependencies]
atty = "0.2"
base64 = "0.13"
bytesize = "1.0"
cargo-platform = { path = "crates/cargo-platform", version = "0.1.2" }
cargo-util = { path = "crates/cargo-util", version = "0.1.2" }
Expand Down
57 changes: 43 additions & 14 deletions src/cargo/sources/git/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -344,18 +344,32 @@ impl<'a> GitCheckout<'a> {
}

fn update_submodules(&self, cargo_config: &Config) -> CargoResult<()> {
return update_submodules(&self.repo, cargo_config);

fn update_submodules(repo: &git2::Repository, cargo_config: &Config) -> CargoResult<()> {
// `location` looks like `target/git/checkouts/crate-name-checksum/shorthash`
// `submodule_root` looks like `target/git/checkouts/submodules`
let checkout_root = self.location.parent().unwrap().parent().unwrap();
// Share the same submodules between all checkouts. Without a shared path,
// cargo would reclone the submodule for each commit that's checked out,
// even if the submodule itself hasn't changed.
let submodule_root = checkout_root.join("submodules");

return update_submodules(&self.repo, cargo_config, &submodule_root);

fn update_submodules(
repo: &git2::Repository,
cargo_config: &Config,
submodule_root: &Path,
) -> CargoResult<()> {
debug!("update submodules for: {:?}", repo.workdir().unwrap());

for mut child in repo.submodules()? {
update_submodule(repo, &mut child, cargo_config).with_context(|| {
format!(
"failed to update submodule `{}`",
child.name().unwrap_or("")
)
})?;
update_submodule(repo, &mut child, cargo_config, submodule_root).with_context(
|| {
format!(
"failed to update submodule `{}`",
child.name().unwrap_or("")
)
},
)?;
}
Ok(())
}
Expand All @@ -364,6 +378,7 @@ impl<'a> GitCheckout<'a> {
parent: &git2::Repository,
child: &mut git2::Submodule<'_>,
cargo_config: &Config,
submodule_root: &Path,
) -> CargoResult<()> {
child.init(false)?;
let url = child.url().ok_or_else(|| {
Expand All @@ -388,14 +403,28 @@ impl<'a> GitCheckout<'a> {
let mut repo = match head_and_repo {
Ok((head, repo)) => {
if child.head_id() == head {
return update_submodules(&repo, cargo_config);
debug!(
"saw up-to-date oid={:?} for submodule {}; skipping update",
head, url
);
return update_submodules(&repo, cargo_config, submodule_root);
}
repo
}
Err(..) => {
let path = parent.workdir().unwrap().join(child.path());
let _ = paths::remove_dir_all(&path);
init(&path, false)?
// NOTE: most URLs are invalid file paths on Windows.
// Base64-encode them to avoid FS errors.
let config = base64::Config::new(base64::CharacterSet::UrlSafe, true);
let encoded = base64::encode_config(url, config);
let shared_submodule_path = submodule_root.join(encoded);
std::fs::create_dir_all(&shared_submodule_path)?;
let submodule = init(&shared_submodule_path, true)?;

let checkout_path = parent.workdir().unwrap().join(child.path());
let _ = paths::remove_dir_all(&checkout_path);
std::fs::create_dir_all(&checkout_path)?;
submodule.set_workdir(&checkout_path, false)?;
submodule
}
};
// Fetch data from origin and reset to the head commit
Expand All @@ -413,7 +442,7 @@ impl<'a> GitCheckout<'a> {

let obj = repo.find_object(head, None)?;
reset(&repo, &obj, cargo_config)?;
update_submodules(&repo, cargo_config)
update_submodules(&repo, cargo_config, submodule_root)
}
}
}
Expand Down
3 changes: 2 additions & 1 deletion tests/testsuite/git.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1287,7 +1287,8 @@ fn dep_with_changed_submodule() {
println!("last run");
p.cargo("run")
.with_stderr(
"[COMPILING] dep1 v0.5.0 ([..])\n\
"[UPDATING] git submodule `file://[..]/dep3`\n\
[COMPILING] dep1 v0.5.0 ([..])\n\
[COMPILING] foo v0.5.0 ([..])\n\
[FINISHED] dev [unoptimized + debuginfo] target(s) in \
[..]\n\
Expand Down

0 comments on commit 9a8816c

Please sign in to comment.