Auto merge of #39799 - dpc:create_dir_all, r=alexcrichton

Fix race condition in fs::create_dir_all

The code would crash if the directory was created after create_dir_all
checked whether the directory already existed.  This was contrary to
the documentation which claimed to create the directory if it doesn't
exist, implying (but not stating) that there would not be a failure
due to the directory existing.

Author: bors

src/librustc/util/fs.rs

@@ -109,23 +109,3 @@ pub fn rename_or_copy_remove<P: AsRef<Path>, Q: AsRef<Path>>(p: P,
         }
     }
 }
-
-// Like std::fs::create_dir_all, except handles concurrent calls among multiple
-// threads or processes.
-pub fn create_dir_racy(path: &Path) -> io::Result<()> {
-    match fs::create_dir(path) {
-        Ok(()) => return Ok(()),
-        Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => return Ok(()),
-        Err(ref e) if e.kind() == io::ErrorKind::NotFound => (),
-        Err(e) => return Err(e),
-    }
-    match path.parent() {
-        Some(p) => try!(create_dir_racy(p)),
-        None => return Err(io::Error::new(io::ErrorKind::Other, "failed to create whole tree")),
-    }
-    match fs::create_dir(path) {
-        Ok(()) => Ok(()),
-        Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => Ok(()),
-        Err(e) => Err(e),
-    }
-}

src/librustc_incremental/persist/fs.rs

@@ -461,7 +461,7 @@ fn generate_session_dir_path(crate_dir: &Path) -> PathBuf {
 }
 
 fn create_dir(sess: &Session, path: &Path, dir_tag: &str) -> Result<(),()> {
-    match fs_util::create_dir_racy(path) {
+    match std_fs::create_dir_all(path) {
         Ok(()) => {
             debug!("{} directory created successfully", dir_tag);
             Ok(())

src/librustc_save_analysis/lib.rs

@@ -884,7 +884,7 @@ pub fn process_crate<'l, 'tcx>(tcx: TyCtxt<'l, 'tcx, 'tcx>,
         },
     };
 
-    if let Err(e) = rustc::util::fs::create_dir_racy(&root_path) {
+    if let Err(e) = std::fs::create_dir_all(&root_path) {
         tcx.sess.err(&format!("Could not create directory {}: {}",
                               root_path.display(),
                               e));

src/libstd/fs.rs

@@ -1534,6 +1534,12 @@ pub fn create_dir<P: AsRef<Path>>(path: P) -> io::Result<()> {
 /// error conditions for when a directory is being created (after it is
 /// determined to not exist) are outlined by `fs::create_dir`.
 ///
+/// Notable exception is made for situations where any of the directories
+/// specified in the `path` could not be created as it was created concurrently.
+/// Such cases are considered success. In other words: calling `create_dir_all`
+/// concurrently from multiple threads or processes is guaranteed to not fail
+/// due to race itself.
+///
 /// # Examples
 ///
 /// ```
@@ -1769,11 +1775,25 @@ impl DirBuilder {
     }
 
     fn create_dir_all(&self, path: &Path) -> io::Result<()> {
-        if path == Path::new("") || path.is_dir() { return Ok(()) }
-        if let Some(p) = path.parent() {
-            self.create_dir_all(p)?
+        if path == Path::new("") {
+            return Ok(())
+        }
+
+        match self.inner.mkdir(path) {
+            Ok(()) => return Ok(()),
+            Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
+            Err(_) if path.is_dir() => return Ok(()),
+            Err(e) => return Err(e),
+        }
+        match path.parent() {
+            Some(p) => try!(self.create_dir_all(p)),
+            None => return Err(io::Error::new(io::ErrorKind::Other, "failed to create whole tree")),
+        }
+        match self.inner.mkdir(path) {
+            Ok(()) => Ok(()),
+            Err(_) if path.is_dir() => Ok(()),
+            Err(e) => Err(e),
         }
-        self.inner.mkdir(path)
     }
 }
 
@@ -1793,6 +1813,7 @@ mod tests {
     use rand::{StdRng, Rng};
     use str;
     use sys_common::io::test::{TempDir, tmpdir};
+    use thread;
 
     #[cfg(windows)]
     use os::windows::fs::{symlink_dir, symlink_file};
@@ -2260,11 +2281,42 @@ mod tests {
         assert!(result.is_err());
     }
 
+    #[test]
+    fn concurrent_recursive_mkdir() {
+        for _ in 0..100 {
+            let dir = tmpdir();
+            let mut dir = dir.join("a");
+            for _ in 0..40 {
+                dir = dir.join("a");
+            }
+            let mut join = vec!();
+            for _ in 0..8 {
+                let dir = dir.clone();
+                join.push(thread::spawn(move || {
+                    check!(fs::create_dir_all(&dir));
+                }))
+            }
+
+            // No `Display` on result of `join()`
+            join.drain(..).map(|join| join.join().unwrap()).count();
+        }
+    }
+
     #[test]
     fn recursive_mkdir_slash() {
         check!(fs::create_dir_all(&Path::new("/")));
     }
 
+    #[test]
+    fn recursive_mkdir_dot() {
+        check!(fs::create_dir_all(&Path::new(".")));
+    }
+
+    #[test]
+    fn recursive_mkdir_empty() {
+        check!(fs::create_dir_all(&Path::new("")));
+    }
+
     #[test]
     fn recursive_rmdir() {
         let tmpdir = tmpdir();

src/tools/compiletest/src/runtest.rs

@@ -25,7 +25,7 @@ use util::logv;
 use std::collections::HashSet;
 use std::env;
 use std::fmt;
-use std::fs::{self, File};
+use std::fs::{self, File, create_dir_all};
 use std::io::prelude::*;
 use std::io::{self, BufReader};
 use std::path::{Path, PathBuf};
@@ -395,7 +395,7 @@ actual:\n\
 
         let out_dir = self.output_base_name().with_extension("pretty-out");
         let _ = fs::remove_dir_all(&out_dir);
-        self.create_dir_racy(&out_dir);
+        create_dir_all(&out_dir).unwrap();
 
         // FIXME (#9639): This needs to handle non-utf8 paths
         let mut args = vec!["-".to_owned(),
@@ -1269,7 +1269,7 @@ actual:\n\
 
     fn compose_and_run_compiler(&self, args: ProcArgs, input: Option<String>) -> ProcRes {
         if !self.props.aux_builds.is_empty() {
-            self.create_dir_racy(&self.aux_output_dir_name());
+            create_dir_all(&self.aux_output_dir_name()).unwrap();
         }
 
         let aux_dir = self.aux_output_dir_name();
@@ -1340,22 +1340,6 @@ actual:\n\
                              input)
     }
 
-    // Like std::fs::create_dir_all, except handles concurrent calls among multiple
-    // threads or processes.
-    fn create_dir_racy(&self, path: &Path) {
-        match fs::create_dir(path) {
-            Ok(()) => return,
-            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => return,
-            Err(ref e) if e.kind() == io::ErrorKind::NotFound => {}
-            Err(e) => panic!("failed to create dir {:?}: {}", path, e),
-        }
-        self.create_dir_racy(path.parent().unwrap());
-        match fs::create_dir(path) {
-            Ok(()) => {}
-            Err(ref e) if e.kind() == io::ErrorKind::AlreadyExists => {}
-            Err(e) => panic!("failed to create dir {:?}: {}", path, e),
-        }
-    }
 
     fn compose_and_run(&self,
                        ProcArgs{ args, prog }: ProcArgs,
@@ -1435,7 +1419,7 @@ actual:\n\
 
 
                 let mir_dump_dir = self.get_mir_dump_dir();
-                self.create_dir_racy(mir_dump_dir.as_path());
+                create_dir_all(mir_dump_dir.as_path()).unwrap();
                 let mut dir_opt = "dump-mir-dir=".to_string();
                 dir_opt.push_str(mir_dump_dir.to_str().unwrap());
                 debug!("dir_opt: {:?}", dir_opt);
@@ -1923,7 +1907,7 @@ actual:\n\
 
         let out_dir = self.output_base_name();
         let _ = fs::remove_dir_all(&out_dir);
-        self.create_dir_racy(&out_dir);
+        create_dir_all(&out_dir).unwrap();
 
         let proc_res = self.document(&out_dir);
         if !proc_res.status.success() {
@@ -2299,7 +2283,7 @@ actual:\n\
         if tmpdir.exists() {
             self.aggressive_rm_rf(&tmpdir).unwrap();
         }
-        self.create_dir_racy(&tmpdir);
+        create_dir_all(&tmpdir).unwrap();
 
         let host = &self.config.host;
         let make = if host.contains("bitrig") || host.contains("dragonfly") ||