Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add global_cache_tracker stability tests. #13467

Merged
merged 6 commits into from
Feb 21, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
@@ -116,6 +116,8 @@ jobs:
CARGO_PROFILE_TEST_DEBUG: 1
CARGO_INCREMENTAL: 0
CARGO_PUBLIC_NETWORK_TESTS: 1
# Workaround for https://github.com/rust-lang/rustup/issues/3036
RUSTUP_WINDOWS_PATH_ADD_BIN: 0
strategy:
matrix:
include:
@@ -156,6 +158,10 @@ jobs:
- uses: actions/checkout@v4
- name: Dump Environment
run: ci/dump-environment.sh
# Some tests require stable. Make sure it is set to the most recent stable
# so that we can predictably handle updates if necessary (and not randomly
# when GitHub updates its image).
- run: rustup update --no-self-update stable
- run: rustup update --no-self-update ${{ matrix.rust }} && rustup default ${{ matrix.rust }}
- run: rustup target add ${{ matrix.other }}
- run: rustup component add rustc-dev llvm-tools-preview rust-docs
@@ -166,7 +172,6 @@ jobs:
- name: Configure extra test environment
run: echo CARGO_CONTAINER_TESTS=1 >> $GITHUB_ENV
if: matrix.os == 'ubuntu-latest'

- run: cargo test -p cargo
- name: Clear intermediate test output
run: ci/clean-test-output.sh
52 changes: 43 additions & 9 deletions crates/cargo-test-macro/src/lib.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use proc_macro::*;
use std::path::Path;
use std::process::Command;
use std::sync::Once;

@@ -74,6 +75,12 @@ pub fn cargo_test(attr: TokenStream, item: TokenStream) -> TokenStream {
requires_reason = true;
set_ignore!(is_not_nightly, "requires nightly");
}
"requires_rustup_stable" => {
set_ignore!(
!has_rustup_stable(),
"rustup or stable toolchain not installed"
);
}
s if s.starts_with("requires_") => {
let command = &s[9..];
set_ignore!(!has_command(command), "{command} not installed");
@@ -204,29 +211,28 @@ fn version() -> (u32, bool) {
unsafe { VERSION }
}

fn has_command(command: &str) -> bool {
let output = match Command::new(command).arg("--version").output() {
fn check_command(command_path: &Path, args: &[&str]) -> bool {
let mut command = Command::new(command_path);
let command_name = command.get_program().to_str().unwrap().to_owned();
command.args(args);
let output = match command.output() {
Ok(output) => output,
Err(e) => {
// * hg is not installed on GitHub macOS or certain constrained
// environments like Docker. Consider installing it if Cargo
// gains more hg support, but otherwise it isn't critical.
// * lldb is not pre-installed on Ubuntu and Windows, so skip.
if is_ci() && !["hg", "lldb"].contains(&command) {
panic!(
"expected command `{}` to be somewhere in PATH: {}",
command, e
);
if is_ci() && !matches!(command_name.as_str(), "hg" | "lldb") {
panic!("expected command `{command_name}` to be somewhere in PATH: {e}",);
}
return false;
}
};
if !output.status.success() {
panic!(
"expected command `{}` to be runnable, got error {}:\n\
"expected command `{command_name}` to be runnable, got error {}:\n\
stderr:{}\n\
stdout:{}\n",
command,
output.status,
String::from_utf8_lossy(&output.stderr),
String::from_utf8_lossy(&output.stdout)
@@ -235,6 +241,34 @@ fn has_command(command: &str) -> bool {
true
}

fn has_command(command: &str) -> bool {
check_command(Path::new(command), &["--version"])
}

fn has_rustup_stable() -> bool {
if option_env!("CARGO_TEST_DISABLE_NIGHTLY").is_some() {
// This cannot run on rust-lang/rust CI due to the lack of rustup.
return false;
}
if cfg!(windows) && !is_ci() && option_env!("RUSTUP_WINDOWS_PATH_ADD_BIN").is_none() {
// There is an issue with rustup that doesn't allow recursive cargo
// invocations. Disable this on developer machines if the environment
// variable is not enabled. This can be removed once
// https://github.com/rust-lang/rustup/issues/3036 is resolved.
return false;
}
// Cargo mucks with PATH on Windows, adding sysroot host libdir, which is
// "bin", which circumvents the rustup wrapper. Use the path directly from
// CARGO_HOME.
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would #13468 fix this issue?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe! It will need to wait until that hits stable, though.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh, yeah. We need it in stable :)

let home = match option_env!("CARGO_HOME") {
Some(home) => home,
None if is_ci() => panic!("expected to run under rustup"),
None => return false,
};
let cargo = Path::new(home).join("bin/cargo");
check_command(&cargo, &["+stable", "--version"])
}

/// Whether or not this running in a Continuous Integration environment.
fn is_ci() -> bool {
// Consider using `tracked_env` instead of option_env! when it is stabilized.
7 changes: 7 additions & 0 deletions crates/cargo-test-support/src/lib.rs
Original file line number Diff line number Diff line change
@@ -764,6 +764,13 @@ impl Execs {
self
}

pub fn args<T: AsRef<OsStr>>(&mut self, args: &[T]) -> &mut Self {
if let Some(ref mut p) = self.process_builder {
p.args(args);
}
self
}

pub fn cwd<T: AsRef<OsStr>>(&mut self, path: T) -> &mut Self {
if let Some(ref mut p) = self.process_builder {
if let Some(cwd) = p.get_cwd() {
2 changes: 1 addition & 1 deletion src/cargo/core/global_cache_tracker.rs
Original file line number Diff line number Diff line change
@@ -499,7 +499,7 @@ impl GlobalCacheTracker {
let mut stmt = self.conn.prepare_cached(
"SELECT git_db.name, git_checkout.name, git_checkout.size, git_checkout.timestamp
FROM git_db, git_checkout
WHERE git_checkout.registry_id = git_db.id",
WHERE git_checkout.git_id = git_db.id",
)?;
let rows = stmt
.query_map([], |row| {
160 changes: 158 additions & 2 deletions tests/testsuite/global_cache_tracker.rs
Original file line number Diff line number Diff line change
@@ -14,11 +14,12 @@ use cargo::GlobalContext;
use cargo_test_support::paths::{self, CargoPathExt};
use cargo_test_support::registry::{Package, RegistryBuilder};
use cargo_test_support::{
basic_manifest, cargo_process, execs, git, project, retry, sleep_ms, thread_wait_timeout,
Project,
basic_manifest, cargo_process, execs, git, process, project, retry, sleep_ms,
thread_wait_timeout, Execs, Project,
};
use itertools::Itertools;
use std::fmt::Write;
use std::path::Path;
use std::path::PathBuf;
use std::process::Stdio;
use std::time::{Duration, SystemTime};
@@ -153,6 +154,14 @@ fn populate_cache(
(cache_dir, src_dir)
}

fn rustup_cargo() -> Execs {
// Get the path to the rustup cargo wrapper. This is necessary because
// cargo adds the "deps" directory into PATH on Windows, which points to
// the wrong cargo.
let rustup_cargo = Path::new(&std::env::var_os("CARGO_HOME").unwrap()).join("bin/cargo");
execs().with_process_builder(process(rustup_cargo))
}

#[cargo_test]
fn auto_gc_gated() {
// Requires -Zgc to both track last-use data and to run auto-gc.
@@ -1863,3 +1872,150 @@ fn clean_gc_quiet_is_quiet() {
)
.run();
}

#[cargo_test(requires_rustup_stable)]
fn compatible_with_older_cargo() {
// Ensures that db stays backwards compatible across versions.

// T-4 months: Current version, build the database.
Package::new("old", "1.0.0").publish();
Package::new("middle", "1.0.0").publish();
Package::new("new", "1.0.0").publish();
let p = project()
.file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"

[dependencies]
old = "1.0"
middle = "1.0"
new = "1.0"
"#,
)
.file("src/lib.rs", "")
.build();
// Populate the last-use data.
p.cargo("check -Zgc")
.masquerade_as_nightly_cargo(&["gc"])
.env("__CARGO_TEST_LAST_USE_NOW", months_ago_unix(4))
.run();
assert_eq!(
get_registry_names("src"),
["middle-1.0.0", "new-1.0.0", "old-1.0.0"]
);
assert_eq!(
get_registry_names("cache"),
["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"]
);

// T-2 months: Stable version, make sure it reads and deletes old src.
p.change_file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"

[dependencies]
new = "1.0"
middle = "1.0"
"#,
);
rustup_cargo()
.args(&["+stable", "check", "-Zgc"])
.cwd(p.root())
.masquerade_as_nightly_cargo(&["gc"])
.env("__CARGO_TEST_LAST_USE_NOW", months_ago_unix(2))
.run();
assert_eq!(get_registry_names("src"), ["middle-1.0.0", "new-1.0.0"]);
assert_eq!(
get_registry_names("cache"),
["middle-1.0.0.crate", "new-1.0.0.crate", "old-1.0.0.crate"]
);

// T-0 months: Current version, make sure it can read data from stable,
// deletes old crate and middle src.
p.change_file(
"Cargo.toml",
r#"
[package]
name = "foo"
version = "0.1.0"

[dependencies]
new = "1.0"
"#,
);
p.cargo("check -Zgc")
.masquerade_as_nightly_cargo(&["gc"])
.run();
assert_eq!(get_registry_names("src"), ["new-1.0.0"]);
assert_eq!(
get_registry_names("cache"),
["middle-1.0.0.crate", "new-1.0.0.crate"]
);
}

#[cargo_test(requires_rustup_stable)]
fn forward_compatible() {
// Checks that db created in an older version can be read in a newer version.
Package::new("bar", "1.0.0").publish();
let git_project = git::new("from_git", |p| {
p.file("Cargo.toml", &basic_manifest("from_git", "1.0.0"))
.file("src/lib.rs", "")
});

let p = project()
.file(
"Cargo.toml",
&format!(
r#"
[package]
name = "foo"

[dependencies]
bar = "1.0.0"
from_git = {{ git = '{}' }}
"#,
git_project.url()
),
)
.file("src/lib.rs", "")
.build();

rustup_cargo()
.args(&["+stable", "check", "-Zgc"])
.cwd(p.root())
.masquerade_as_nightly_cargo(&["gc"])
.run();

let config = GlobalContextBuilder::new().unstable_flag("gc").build();
let lock = config
.acquire_package_cache_lock(CacheLockMode::MutateExclusive)
.unwrap();
let tracker = GlobalCacheTracker::new(&config).unwrap();
// Don't want to check the actual index name here, since although the
// names are semi-stable, they might change over long periods of time.
let indexes = tracker.registry_index_all().unwrap();
assert_eq!(indexes.len(), 1);
let crates = tracker.registry_crate_all().unwrap();
let names: Vec<_> = crates
.iter()
.map(|(krate, _timestamp)| krate.crate_filename)
.collect();
assert_eq!(names, &["bar-1.0.0.crate"]);
let srcs = tracker.registry_src_all().unwrap();
let names: Vec<_> = srcs
.iter()
.map(|(src, _timestamp)| src.package_dir)
.collect();
assert_eq!(names, &["bar-1.0.0"]);
let dbs: Vec<_> = tracker.git_db_all().unwrap();
assert_eq!(dbs.len(), 1);
let cos: Vec<_> = tracker.git_checkout_all().unwrap();
assert_eq!(cos.len(), 1);
drop(lock);
}