HADOOP-16769. LocalDirAllocator to provide diagnostics when file creation fails

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/LocalDirAllocator.java

@@ -393,6 +393,7 @@ public Path getLocalPathForWrite(String pathStr, long size,
       Context ctx = confChanged(conf);
       int numDirs = ctx.localDirs.length;
       int numDirsSearched = 0;
+      long maxCapacity = 0;
       //remove the leading slash from the path (to make sure that the uri
       //resolution results in a valid path on the dir being checked)
       if (pathStr.startsWith("/")) {
@@ -441,6 +442,9 @@ public Path getLocalPathForWrite(String pathStr, long size,
         int dirNum = ctx.getAndIncrDirNumLastAccessed(randomInc);
         while (numDirsSearched < numDirs) {
           long capacity = ctx.dirDF[dirNum].getAvailable();
+          if (capacity > maxCapacity) {
+            maxCapacity = capacity;
+          }
           if (capacity > size) {
             returnPath =
                 createPath(ctx.localDirs[dirNum], pathStr, checkWrite);
@@ -460,7 +464,8 @@ public Path getLocalPathForWrite(String pathStr, long size,
 
       //no path found
       throw new DiskErrorException("Could not find any valid local " +
-          "directory for " + pathStr);
+          "directory for " + pathStr + " with requested size " + size +
+          " as the max capacity in any directory is " + maxCapacity);
     }
 
     /** Creates a file on the local FS. Pass size as 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java

@@ -532,4 +532,23 @@ public void testGetLocalPathForWriteForInvalidPaths() throws Exception {
     }
   }
 
+  /**
+   * Test to check the LocalDirAllocation for the less space HADOOP-16769
+   *
+   * @throws Exception
+   */
+  @Test(timeout = 30000)
+  public void testGetLocalPathForWriteForLessSpace() throws Exception {
+    String dir0 = buildBufferDir(ROOT, 0);
+    String dir1 = buildBufferDir(ROOT, 1);
+    conf.set(CONTEXT, dir0 + "," + dir1);
+    try {
+      dirAllocator.getLocalPathForWrite("p1/x", 3_000_000_000_000L, conf);
+      fail("not throwing the exception");
+    } catch (IOException e) {
+      assertTrue(e.getMessage().matches("Could not find any valid local directory for p1/x with requested " +
+          "size 3000000000000 as the max capacity in any directory is [0-9]*"));
+    }
+  }
+
 }