From d25305b6ea3feebed5f720e86384cc4bcb04f1da Mon Sep 17 00:00:00 2001 From: Eh2406 Date: Sat, 22 Dec 2018 13:03:47 -0500 Subject: [PATCH 1/5] adds a timestamp file in the fingerprint folder --- src/cargo/core/compiler/fingerprint.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index 0b34da51fd7..6f6b3394b85 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -681,6 +681,10 @@ pub fn dep_info_loc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> Pa fn compare_old_fingerprint(loc: &Path, new_fingerprint: &Fingerprint) -> CargoResult<()> { let old_fingerprint_short = paths::read(loc)?; + let _ = paths::write( + &loc.with_file_name("last-used.timestamp"), + b"This file has an mtime of when cargo last used this fingerprint.", + ); let new_hash = new_fingerprint.hash(); if util::to_hex(new_hash) == old_fingerprint_short { From 3eaa70e22e48ee939fe4f8e55dacb10a333f0b62 Mon Sep 17 00:00:00 2001 From: Eh2406 Date: Thu, 10 Jan 2019 14:24:29 -0500 Subject: [PATCH 2/5] just touch some of the files we use. --- src/cargo/core/compiler/fingerprint.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index 6f6b3394b85..083196828bd 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -3,6 +3,7 @@ use std::fs; use std::hash::{self, Hasher}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use std::time::SystemTime; use filetime::FileTime; use log::{debug, info}; @@ -88,6 +89,7 @@ pub fn prepare_target<'a, 'cfg>( let root = cx.files().out_dir(unit); let missing_outputs = { + let t = FileTime::from_system_time(SystemTime::now()); if unit.mode.is_doc() { !root .join(unit.target.crate_name()) @@ -98,8 +100,15 @@ pub fn prepare_target<'a, 'cfg>( .outputs(unit)? .iter() .filter(|output| output.flavor != FileFlavor::DebugInfo) - .find(|output| !output.path.exists()) - { + .find(|output| { + if output.path.exists() { + // update the mtime so other cleaners know we used it + let _ = filetime::set_file_times(&output.path, t, t); + false + } else { + true + } + }) { None => false, Some(output) => { info!("missing output path {:?}", output.path); @@ -681,10 +690,11 @@ pub fn dep_info_loc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> Pa fn compare_old_fingerprint(loc: &Path, new_fingerprint: &Fingerprint) -> CargoResult<()> { let old_fingerprint_short = paths::read(loc)?; - let _ = paths::write( - &loc.with_file_name("last-used.timestamp"), - b"This file has an mtime of when cargo last used this fingerprint.", - ); + + // update the mtime so other cleaners know we used it + let t = FileTime::from_system_time(SystemTime::now()); + filetime::set_file_times(loc, t, t)?; + let new_hash = new_fingerprint.hash(); if util::to_hex(new_hash) == old_fingerprint_short { From 80f7b901cb88325a82bef7cad02cec78c3eba5a1 Mon Sep 17 00:00:00 2001 From: Eh2406 Date: Sat, 12 Jan 2019 22:49:46 -0500 Subject: [PATCH 3/5] add a test for touching deps --- tests/testsuite/freshness.rs | 79 ++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/tests/testsuite/freshness.rs b/tests/testsuite/freshness.rs index 38a434b605a..049b3e00e7d 100644 --- a/tests/testsuite/freshness.rs +++ b/tests/testsuite/freshness.rs @@ -1,7 +1,9 @@ use std::fs::{self, File, OpenOptions}; use std::io::prelude::*; use std::net::TcpListener; +use std::path::PathBuf; use std::thread; +use std::time::SystemTime; use crate::support::paths::CargoPathExt; use crate::support::registry::Package; @@ -1178,6 +1180,83 @@ fn changing_rustflags_is_cached() { .run(); } +fn simple_deps_cleaner(mut dir: PathBuf, timestamp: filetime::FileTime) { + // Cargo is experimenting with letting outside projects develop some + // limited forms of GC for target_dir. This is one of the forms. + // Specifically, Cargo is updating the mtime of files in + // target/profile/deps each time it uses the file. + // So a cleaner can remove files older then a time stamp without + // effecting any builds that happened since that time stamp. + let mut cleand = false; + dir.push("deps"); + for dep in fs::read_dir(&dir).unwrap() { + let dep = dep.unwrap(); + if filetime::FileTime::from_last_modification_time(&dep.metadata().unwrap()) <= timestamp { + fs::remove_file(dep.path()).unwrap(); + cleand = true; + } + } + assert!( + cleand, + "called simple_deps_cleaner, but there was nothing to remove" + ); +} + +#[test] +fn simple_deps_cleaner_dose_not_rebuild() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + + [dependencies] + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "") + .file("bar/Cargo.toml", &basic_manifest("bar", "0.0.1")) + .file("bar/src/lib.rs", "") + .build(); + + p.cargo("build").run(); + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr( + "\ +[COMPILING] bar v0.0.1 ([..]) +[COMPILING] foo v0.0.1 ([..]) +[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]", + ) + .run(); + if is_coarse_mtime() { + sleep_ms(1000); + } + let timestamp = filetime::FileTime::from_system_time(SystemTime::now()); + // This dose not make new files, but it dose update the mtime. + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr("[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]") + .run(); + simple_deps_cleaner(p.target_debug_dir(), timestamp); + // This should not recompile! + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr("[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]") + .run(); + // But this should be cleaned and so need a rebuild + p.cargo("build") + .with_stderr( + "\ +[COMPILING] bar v0.0.1 ([..]) +[COMPILING] foo v0.0.1 ([..]) +[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]", + ) + .run(); +} + #[test] fn reuse_panic_build_dep_test() { let p = project() From e31003af8f9d64f4e45aa5f3d080dd2e3d139e57 Mon Sep 17 00:00:00 2001 From: Eh2406 Date: Sat, 12 Jan 2019 23:06:47 -0500 Subject: [PATCH 4/5] add a test for touching fingerprint --- tests/testsuite/freshness.rs | 85 ++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/tests/testsuite/freshness.rs b/tests/testsuite/freshness.rs index 049b3e00e7d..2814966b51f 100644 --- a/tests/testsuite/freshness.rs +++ b/tests/testsuite/freshness.rs @@ -1257,6 +1257,91 @@ fn simple_deps_cleaner_dose_not_rebuild() { .run(); } +fn fingerprint_cleaner(mut dir: PathBuf, timestamp: filetime::FileTime) { + // Cargo is experimenting with letting outside projects develop some + // limited forms of GC for target_dir. This is one of the forms. + // Specifically, Cargo is updating the mtime of a file in + // target/profile/.fingerprint each time it uses the fingerprint. + // So a cleaner can remove files associated with a fingerprint + // if all the files in the fingerprint's folder are older then a time stamp without + // effecting any builds that happened since that time stamp. + let mut cleand = false; + dir.push(".fingerprint"); + for fing in fs::read_dir(&dir).unwrap() { + let fing = fing.unwrap(); + + if fs::read_dir(fing.path()).unwrap().all(|f| { + filetime::FileTime::from_last_modification_time(&f.unwrap().metadata().unwrap()) + <= timestamp + }) { + fs::remove_dir_all(fing.path()).unwrap(); + // a real cleaner would remove the big files in deps and build as well + // but fingerprint is sufficient for our tests + cleand = true; + } else { + } + } + assert!( + cleand, + "called fingerprint_cleaner, but there was nothing to remove" + ); +} + +#[test] +fn fingerprint_cleaner_dose_not_rebuild() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + + [dependencies] + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "") + .file("bar/Cargo.toml", &basic_manifest("bar", "0.0.1")) + .file("bar/src/lib.rs", "") + .build(); + + p.cargo("build").run(); + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr( + "\ +[COMPILING] bar v0.0.1 ([..]) +[COMPILING] foo v0.0.1 ([..]) +[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]", + ) + .run(); + if is_coarse_mtime() { + sleep_ms(1000); + } + let timestamp = filetime::FileTime::from_system_time(SystemTime::now()); + // This dose not make new files, but it dose update the mtime. + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr("[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]") + .run(); + fingerprint_cleaner(p.target_debug_dir(), timestamp); + // This should not recompile! + p.cargo("build") + .env("RUSTFLAGS", "-C target-cpu=native") + .with_stderr("[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]") + .run(); + // But this should be cleaned and so need a rebuild + p.cargo("build") + .with_stderr( + "\ +[COMPILING] bar v0.0.1 ([..]) +[COMPILING] foo v0.0.1 ([..]) +[FINISHED] dev [unoptimized + debuginfo] target(s) in [..]", + ) + .run(); +} + #[test] fn reuse_panic_build_dep_test() { let p = project() From 97363ca7a7378492dc8e91739cd020df4b0eb624 Mon Sep 17 00:00:00 2001 From: Eh2406 Date: Sun, 13 Jan 2019 09:31:22 -0500 Subject: [PATCH 5/5] fix tests on HFS --- tests/testsuite/freshness.rs | 8 ++++++++ tests/testsuite/support/mod.rs | 3 +++ 2 files changed, 11 insertions(+) diff --git a/tests/testsuite/freshness.rs b/tests/testsuite/freshness.rs index 2814966b51f..e21ded04345 100644 --- a/tests/testsuite/freshness.rs +++ b/tests/testsuite/freshness.rs @@ -1193,6 +1193,7 @@ fn simple_deps_cleaner(mut dir: PathBuf, timestamp: filetime::FileTime) { let dep = dep.unwrap(); if filetime::FileTime::from_last_modification_time(&dep.metadata().unwrap()) <= timestamp { fs::remove_file(dep.path()).unwrap(); + println!("remove: {:?}", dep.path()); cleand = true; } } @@ -1235,6 +1236,9 @@ fn simple_deps_cleaner_dose_not_rebuild() { sleep_ms(1000); } let timestamp = filetime::FileTime::from_system_time(SystemTime::now()); + if is_coarse_mtime() { + sleep_ms(1000); + } // This dose not make new files, but it dose update the mtime. p.cargo("build") .env("RUSTFLAGS", "-C target-cpu=native") @@ -1275,6 +1279,7 @@ fn fingerprint_cleaner(mut dir: PathBuf, timestamp: filetime::FileTime) { <= timestamp }) { fs::remove_dir_all(fing.path()).unwrap(); + println!("remove: {:?}", fing.path()); // a real cleaner would remove the big files in deps and build as well // but fingerprint is sufficient for our tests cleand = true; @@ -1320,6 +1325,9 @@ fn fingerprint_cleaner_dose_not_rebuild() { sleep_ms(1000); } let timestamp = filetime::FileTime::from_system_time(SystemTime::now()); + if is_coarse_mtime() { + sleep_ms(1000); + } // This dose not make new files, but it dose update the mtime. p.cargo("build") .env("RUSTFLAGS", "-C target-cpu=native") diff --git a/tests/testsuite/support/mod.rs b/tests/testsuite/support/mod.rs index 47e681460fb..404439ea416 100644 --- a/tests/testsuite/support/mod.rs +++ b/tests/testsuite/support/mod.rs @@ -1603,6 +1603,9 @@ pub fn sleep_ms(ms: u64) { /// Returns true if the local filesystem has low-resolution mtimes. pub fn is_coarse_mtime() -> bool { + // If the filetime crate is being used to emulate HFS then + // return true, without looking at the actual hardware. + cfg!(emulate_second_only_system) || // This should actually be a test that $CARGO_TARGET_DIR is on an HFS // filesystem, (or any filesystem with low-resolution mtimes). However, // that's tricky to detect, so for now just deal with CI.