diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs
index 9ed6fd11f7a..917b2a8caf5 100644
--- a/src/cargo/core/compiler/context/compilation_files.rs
+++ b/src/cargo/core/compiler/context/compilation_files.rs
@@ -519,6 +519,7 @@ fn compute_metadata(
     // settings like debuginfo and whatnot.
     unit.profile.hash(&mut hasher);
     unit.mode.hash(&mut hasher);
+    cx.lto[unit].hash(&mut hasher);
 
     // Artifacts compiled for the host should have a different metadata
     // piece than those compiled for the target, so make sure we throw in
diff --git a/src/cargo/core/compiler/context/mod.rs b/src/cargo/core/compiler/context/mod.rs
index 82831f42314..4d1e7fdccf1 100644
--- a/src/cargo/core/compiler/context/mod.rs
+++ b/src/cargo/core/compiler/context/mod.rs
@@ -126,10 +126,10 @@ impl<'a, 'cfg> Context<'a, 'cfg> {
         let mut queue = JobQueue::new(self.bcx);
         let mut plan = BuildPlan::new();
         let build_plan = self.bcx.build_config.build_plan;
+        self.lto = super::lto::generate(&self.bcx)?;
         self.prepare_units()?;
         self.prepare()?;
         custom_build::build_map(&mut self)?;
-        super::lto::generate(&mut self)?;
         self.check_collistions()?;
 
         for unit in &self.bcx.roots {
diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs
index 0fb1175c70c..de4bcab26b3 100644
--- a/src/cargo/core/compiler/fingerprint.rs
+++ b/src/cargo/core/compiler/fingerprint.rs
@@ -72,7 +72,7 @@
 //! -C incremental=… flag                      | ✓           |
 //! mtime of sources                           | ✓[^3]       |
 //! RUSTFLAGS/RUSTDOCFLAGS                     | ✓           |
-//! LTO flags                                  | ✓           |
+//! LTO flags                                  | ✓           | ✓
 //! config settings[^5]                        | ✓           |
 //! is_std                                     |             | ✓
 //!
diff --git a/src/cargo/core/compiler/lto.rs b/src/cargo/core/compiler/lto.rs
index 222578918e9..71e135566da 100644
--- a/src/cargo/core/compiler/lto.rs
+++ b/src/cargo/core/compiler/lto.rs
@@ -1,4 +1,4 @@
-use crate::core::compiler::{CompileMode, Context, CrateType, Unit};
+use crate::core::compiler::{BuildContext, CompileMode, CrateType, Unit};
 use crate::core::profiles;
 use crate::util::interning::InternedString;
 
@@ -40,9 +40,9 @@ pub enum Lto {
     OnlyObject,
 }
 
-pub fn generate(cx: &mut Context<'_, '_>) -> CargoResult<()> {
+pub fn generate(bcx: &BuildContext<'_, '_>) -> CargoResult<HashMap<Unit, Lto>> {
     let mut map = HashMap::new();
-    for unit in cx.bcx.roots.iter() {
+    for unit in bcx.roots.iter() {
         let root_lto = match unit.profile.lto {
             // LTO not requested, no need for bitcode.
             profiles::Lto::Bool(false) | profiles::Lto::Off => Lto::OnlyObject,
@@ -60,10 +60,9 @@ pub fn generate(cx: &mut Context<'_, '_>) -> CargoResult<()> {
                 }
             }
         };
-        calculate(cx, &mut map, unit, root_lto)?;
+        calculate(bcx, &mut map, unit, root_lto)?;
     }
-    cx.lto = map;
-    Ok(())
+    Ok(map)
 }
 
 /// Whether or not any of these crate types need object code.
@@ -87,7 +86,7 @@ fn lto_when_needs_object(crate_types: &[CrateType]) -> Lto {
 }
 
 fn calculate(
-    cx: &Context<'_, '_>,
+    bcx: &BuildContext<'_, '_>,
     map: &mut HashMap<Unit, Lto>,
     unit: &Unit,
     parent_lto: Lto,
@@ -185,8 +184,8 @@ fn calculate(
         }
     };
 
-    for dep in cx.unit_deps(unit) {
-        calculate(cx, map, &dep.unit, merged_lto)?;
+    for dep in &bcx.unit_graph[unit] {
+        calculate(bcx, map, &dep.unit, merged_lto)?;
     }
     Ok(())
 }
diff --git a/tests/testsuite/lto.rs b/tests/testsuite/lto.rs
index 9e2f9a42835..0c771329f00 100644
--- a/tests/testsuite/lto.rs
+++ b/tests/testsuite/lto.rs
@@ -778,3 +778,78 @@ fn dylib_rlib_bin() {
     );
     verify_lto(&output, "foo", "--crate-type bin", Lto::Run(None));
 }
+
+#[cargo_test]
+fn fresh_swapping_commands() {
+    // In some rare cases, different commands end up building dependencies
+    // with different LTO settings. This checks that it doesn't cause the
+    // cache to thrash in that scenario.
+    Package::new("bar", "1.0.0").publish();
+
+    let p = project()
+        .file(
+            "Cargo.toml",
+            r#"
+                [package]
+                name = "foo"
+                version = "0.1.0"
+
+                [dependencies]
+                bar = "1.0"
+
+                [profile.release]
+                lto = true
+            "#,
+        )
+        .file("src/lib.rs", "pub fn foo() { println!(\"hi!\"); }")
+        .build();
+
+    p.cargo("build --release -v")
+        .with_stderr(
+            "\
+[UPDATING] [..]
+[DOWNLOADING] crates ...
+[DOWNLOADED] bar v1.0.0 [..]
+[COMPILING] bar v1.0.0
+[RUNNING] `rustc --crate-name bar [..]-C linker-plugin-lto[..]
+[COMPILING] foo v0.1.0 [..]
+[RUNNING] `rustc --crate-name foo src/lib.rs [..]-C linker-plugin-lto[..]
+[FINISHED] [..]
+",
+        )
+        .run();
+    p.cargo("test --release -v")
+        .with_stderr_unordered(
+            "\
+[COMPILING] bar v1.0.0
+[RUNNING] `rustc --crate-name bar [..]-C embed-bitcode=no[..]
+[COMPILING] foo v0.1.0 [..]
+[RUNNING] `rustc --crate-name foo src/lib.rs [..]--crate-type lib[..]-C embed-bitcode=no[..]
+[RUNNING] `rustc --crate-name foo src/lib.rs [..]-C embed-bitcode=no[..]--test[..]
+[FINISHED] [..]
+[RUNNING] `[..]/foo[..]`
+[DOCTEST] foo
+[RUNNING] `rustdoc [..]-C embed-bitcode=no[..]
+",
+        )
+        .run();
+
+    p.cargo("build --release -v")
+        .with_stderr(
+            "\
+[FRESH] bar v1.0.0
+[FRESH] foo [..]
+[FINISHED] [..]
+",
+        )
+        .run();
+    p.cargo("test --release -v --no-run -v")
+        .with_stderr(
+            "\
+[FRESH] bar v1.0.0
+[FRESH] foo [..]
+[FINISHED] [..]
+",
+        )
+        .run();
+}