diff --git a/changelog/6044.bugfix.rst b/changelog/6044.bugfix.rst
new file mode 100644
index 00000000000..36d38e9c85b
--- /dev/null
+++ b/changelog/6044.bugfix.rst
@@ -0,0 +1,3 @@
+Properly ignore ``FileNotFoundError`` exceptions when trying to remove old temporary directories,
+for instance when multiple processes try to remove the same directory (common with ``pytest-xdist``
+for example).
diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py
index 19f9c062f89..f45b0bab705 100644
--- a/src/_pytest/pathlib.py
+++ b/src/_pytest/pathlib.py
@@ -38,21 +38,35 @@ def ensure_reset_dir(path):
     path.mkdir()
 
 
-def on_rm_rf_error(func, path: str, exc, *, start_path):
-    """Handles known read-only errors during rmtree."""
-    excvalue = exc[1]
+def on_rm_rf_error(func, path: str, exc, *, start_path) -> bool:
+    """Handles known read-only errors during rmtree.
+
+    The returned value is used only by our own tests.
+    """
+    exctype, excvalue = exc[:2]
+
+    # another process removed the file in the middle of the "rm_rf" (xdist for example)
+    # more context: https://github.com/pytest-dev/pytest/issues/5974#issuecomment-543799018
+    if isinstance(excvalue, FileNotFoundError):
+        return False
 
     if not isinstance(excvalue, PermissionError):
         warnings.warn(
-            PytestWarning("(rm_rf) error removing {}: {}".format(path, excvalue))
+            PytestWarning(
+                "(rm_rf) error removing {}\n{}: {}".format(path, exctype, excvalue)
+            )
         )
-        return
+        return False
 
     if func not in (os.rmdir, os.remove, os.unlink):
         warnings.warn(
-            PytestWarning("(rm_rf) error removing {}: {}".format(path, excvalue))
+            PytestWarning(
+                "(rm_rf) unknown function {} when removing {}:\n{}: {}".format(
+                    path, func, exctype, excvalue
+                )
+            )
         )
-        return
+        return False
 
     # Chmod + retry.
     import stat
@@ -73,6 +87,7 @@ def chmod_rw(p: str):
     chmod_rw(str(path))
 
     func(path)
+    return True
 
 
 def rm_rf(path: Path):
diff --git a/testing/test_tmpdir.py b/testing/test_tmpdir.py
index ebde9044c05..0ebed22ac45 100644
--- a/testing/test_tmpdir.py
+++ b/testing/test_tmpdir.py
@@ -383,6 +383,10 @@ def test_on_rm_rf_error(self, tmp_path):
             on_rm_rf_error(os.unlink, str(fn), exc_info, start_path=tmp_path)
             assert fn.is_file()
 
+        # we ignore FileNotFoundError
+        exc_info = (None, FileNotFoundError(), None)
+        assert not on_rm_rf_error(None, str(fn), exc_info, start_path=tmp_path)
+
         # unknown function
         with pytest.warns(pytest.PytestWarning):
             exc_info = (None, PermissionError(), None)