diff --git a/src/main/java/com/google/devtools/build/lib/remote/AbstractActionInputPrefetcher.java b/src/main/java/com/google/devtools/build/lib/remote/AbstractActionInputPrefetcher.java
index 78bbfd750abd34..37ac36303c7995 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/AbstractActionInputPrefetcher.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/AbstractActionInputPrefetcher.java
@@ -60,7 +60,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Deque;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
@@ -95,7 +94,7 @@ public abstract class AbstractActionInputPrefetcher implements ActionInputPrefet
 
   /** Keeps track of output directories written to by a single prefetcher call. */
   private final class DirectoryContext {
-    private final HashSet<Path> dirs = new HashSet<>();
+    private final Set<Path> dirs = Sets.newConcurrentHashSet();
 
     /**
      * Adds to the set of directories written to by the prefetcher call associated with this