diff --git a/src/compiler/builderState.ts b/src/compiler/builderState.ts
index 6922045498930..231dec0b5509e 100644
--- a/src/compiler/builderState.ts
+++ b/src/compiler/builderState.ts
@@ -48,6 +48,11 @@ namespace ts {
          */
         readonly exportedModulesMap: BuilderState.ManyToManyPathMap | undefined;
 
+        previousCache?: {
+            id: number,
+            version: number,
+        };
+
         /**
          * true if file version is used as signature
          * This helps in delaying the calculation of the d.ts hash as version for the file till reasonable time
@@ -80,6 +85,7 @@ namespace ts {
         }
 
         export interface ReadonlyManyToManyPathMap {
+            readonly id: number;
             clone(): ManyToManyPathMap;
             forEach(action: (v: ReadonlySet<Path>, k: Path) => void): void;
             getKeys(v: Path): ReadonlySet<Path> | undefined;
@@ -96,13 +102,18 @@ namespace ts {
         }
 
         export interface ManyToManyPathMap extends ReadonlyManyToManyPathMap {
+            version(): number; // Incremented each time the contents are changed
             deleteKey(k: Path): boolean;
             set(k: Path, v: ReadonlySet<Path>): void;
         }
 
+        let manyToManyPathMapCount = 0;
         export function createManyToManyPathMap(): ManyToManyPathMap {
             function create(forward: ESMap<Path, ReadonlySet<Path>>, reverse: ESMap<Path, Set<Path>>, deleted: Set<Path> | undefined): ManyToManyPathMap {
+                let version = 0;
                 const map: ManyToManyPathMap = {
+                    id: manyToManyPathMapCount++,
+                    version: () => version,
                     clone: () => create(new Map(forward), new Map(reverse), deleted && new Set(deleted)),
                     forEach: fn => forward.forEach(fn),
                     getKeys: v => reverse.get(v),
@@ -121,26 +132,33 @@ namespace ts {
 
                         set.forEach(v => deleteFromMultimap(reverse, v, k));
                         forward.delete(k);
+                        version++;
                         return true;
                     },
                     set: (k, vSet) => {
-                        deleted?.delete(k);
+                        let changed = !!deleted?.delete(k);
 
                         const existingVSet = forward.get(k);
                         forward.set(k, vSet);
 
                         existingVSet?.forEach(v => {
                             if (!vSet.has(v)) {
+                                changed = true;
                                 deleteFromMultimap(reverse, v, k);
                             }
                         });
 
                         vSet.forEach(v => {
                             if (!existingVSet?.has(v)) {
+                                changed = true;
                                 addToMultimap(reverse, v, k);
                             }
                         });
 
+                        if (changed) {
+                            version++;
+                        }
+
                         return map;
                     },
                 };
@@ -475,6 +493,22 @@ namespace ts {
         export function updateExportedFilesMapFromCache(state: BuilderState, exportedModulesMapCache: ManyToManyPathMap | undefined) {
             if (exportedModulesMapCache) {
                 Debug.assert(!!state.exportedModulesMap);
+
+                const cacheId = exportedModulesMapCache.id;
+                const cacheVersion = exportedModulesMapCache.version();
+                if (state.previousCache) {
+                    if (state.previousCache.id === cacheId && state.previousCache.version === cacheVersion) {
+                        // If this is the same cache at the same version as last time this BuilderState
+                        // was updated, there's no need to update again
+                        return;
+                    }
+                    state.previousCache.id = cacheId;
+                    state.previousCache.version = cacheVersion;
+                }
+                else {
+                    state.previousCache = { id: cacheId, version: cacheVersion };
+                }
+
                 exportedModulesMapCache.deletedKeys()?.forEach(path => state.exportedModulesMap!.deleteKey(path));
                 exportedModulesMapCache.forEach((exportedModules, path) => state.exportedModulesMap!.set(path, exportedModules));
             }