From 6e96095cfcf3fd9ce7d6265526d9ef9b939a28ec Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 4 Mar 2023 11:21:55 +0100 Subject: [PATCH 1/2] Adjust git::fetch() progress bar to deal with gitoxide-specifics The `git2` implementation can leverage that `git2` provides a consistent view on the objects to be index, so it looks like 33% of the time is spent receiving objects, and the rest of the time is used resolving them. In `gitoxide`, there are two distinct phases and these are exposed by the way we obtain progress for two separate phases. We have to do some math to renormalize those to a single, continuous progress by mapping the values for 'amount of objects' to the first half and second half of the progress bar respectively. This has the advantage of having the first phase (receiving objects) end at 50% and the second phase (resolving deltas) at 100%. --- Cargo.toml | 4 ++-- src/cargo/sources/git/oxide.rs | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 4534cf15df0..f65fb7f5556 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,8 +30,8 @@ filetime = "0.2.9" flate2 = { version = "1.0.3", default-features = false, features = ["zlib"] } git2 = "0.16.0" git2-curl = "0.17.0" -gix = { version = "0.38.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] } -gix-features-for-configuration-only = { version = "0.27.0", package = "gix-features", features = [ "parallel" ] } +gix = { version = "0.39.0", default-features = false, features = ["blocking-http-transport-curl", "progress-tree"] } +gix-features-for-configuration-only = { version = "0.28.0", package = "gix-features", features = [ "parallel" ] } glob = "0.3.0" hex = "0.4" hmac = "0.12.1" diff --git a/src/cargo/sources/git/oxide.rs b/src/cargo/sources/git/oxide.rs index cccd127c43d..cf301bbd011 100644 --- a/src/cargo/sources/git/oxide.rs +++ b/src/cargo/sources/git/oxide.rs @@ -96,13 +96,14 @@ fn translate_progress_to_bar( tasks.iter().find_map(|(_, t)| cb(t)) } + const NUM_PHASES: usize = 2; // indexing + delta-resolution, both with same amount of objects to handle if let Some(objs) = find_in(&tasks, |t| progress_by_id(resolve_objects, t)) { // Resolving deltas. let objects = objs.step.load(Ordering::Relaxed); let total_objects = objs.done_at.expect("known amount of objects"); let msg = format!(", ({objects}/{total_objects}) resolving deltas"); - progress_bar.tick(objects, total_objects, &msg)?; + progress_bar.tick(total_objects + objects, total_objects * NUM_PHASES, &msg)?; } else if let Some((objs, read_pack)) = find_in(&tasks, |t| progress_by_id(read_pack_bytes, t)).and_then(|read| { find_in(&tasks, |t| progress_by_id(delta_index_objects, t)) @@ -122,7 +123,7 @@ fn translate_progress_to_bar( let (rate, unit) = human_readable_bytes(counter.rate() as u64); let msg = format!(", {rate:.2}{unit}/s"); - progress_bar.tick(objects, total_objects, &msg)?; + progress_bar.tick(objects, total_objects * NUM_PHASES, &msg)?; } } Ok(()) From 789efe36b9ab970b11c6b937c55af28e2a5b9574 Mon Sep 17 00:00:00 2001 From: Sebastian Thiel Date: Sat, 4 Mar 2023 11:51:02 +0100 Subject: [PATCH 2/2] Cool down progress loop. Previously it actually was very hot and needed an entire CPU for itself due to constant querying of progress information. Now it's slowed down by consistently sleeping for a short amount of time. This time should be short enough to not let the progress bar hold up the overall progress of the fetch operation, hence the 10ms sleep time, reducing the worst-case hold-up to 10ms. --- src/cargo/sources/git/oxide.rs | 36 ++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/cargo/sources/git/oxide.rs b/src/cargo/sources/git/oxide.rs index cf301bbd011..56d7f820b9e 100644 --- a/src/cargo/sources/git/oxide.rs +++ b/src/cargo/sources/git/oxide.rs @@ -69,18 +69,34 @@ fn translate_progress_to_bar( // We choose `N=10` here to make a `300ms * 10slots ~= 3000ms` // sliding window for tracking the data transfer rate (in bytes/s). - let mut last_update = Instant::now(); - let mut counter = MetricsCounter::<10>::new(0, last_update); + let mut last_percentage_update = Instant::now(); + let mut last_fast_update = Instant::now(); + let mut counter = MetricsCounter::<10>::new(0, last_percentage_update); let mut tasks = Vec::with_capacity(10); - let update_interval = std::time::Duration::from_millis(300); - let short_check_interval = Duration::from_millis(50); + let slow_check_interval = std::time::Duration::from_millis(300); + let fast_check_interval = Duration::from_millis(50); + let sleep_interval = Duration::from_millis(10); + debug_assert_eq!( + slow_check_interval.as_millis() % fast_check_interval.as_millis(), + 0, + "progress should be smoother by keeping these as multiples of each other" + ); + debug_assert_eq!( + fast_check_interval.as_millis() % sleep_interval.as_millis(), + 0, + "progress should be smoother by keeping these as multiples of each other" + ); while let Some(root) = root.upgrade() { - let not_yet = last_update.elapsed() < update_interval; - if not_yet { - std::thread::sleep(short_check_interval); + std::thread::sleep(sleep_interval); + let needs_update = last_fast_update.elapsed() >= fast_check_interval; + if !needs_update { + continue; } + let now = Instant::now(); + last_fast_update = now; + root.sorted_snapshot(&mut tasks); fn progress_by_id( @@ -115,10 +131,10 @@ fn translate_progress_to_bar( let total_objects = objs.done_at.expect("known amount of objects"); let received_bytes = read_pack.step.load(Ordering::Relaxed); - let now = Instant::now(); - if !not_yet { + let needs_percentage_update = last_percentage_update.elapsed() >= slow_check_interval; + if needs_percentage_update { counter.add(received_bytes, now); - last_update = now; + last_percentage_update = now; } let (rate, unit) = human_readable_bytes(counter.rate() as u64); let msg = format!(", {rate:.2}{unit}/s");