Log more precise representation of bytes in spilling code

apache · Nov 19, 2014 · 36ec152 · 36ec152
1 parent aa9ebda
commit 36ec152
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala
@@ -174,8 +174,9 @@ class ExternalAppendOnlyMap[K, V, C](
   private def spill(mapSize: Long): Unit = {
     spillCount += 1
     val threadId = Thread.currentThread().getId
-    logInfo("Thread %d spilling in-memory map of %d MB to disk (%d time%s so far)"
-      .format(threadId, mapSize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else ""))
+    logInfo("Thread %d spilling in-memory batch of %s to disk (%d times%s so far)"
+      .format(threadId, org.apache.spark.util.Utils.bytesToString(mapSize),
+        spillCount, if (spillCount > 1) "s" else ""))
     val (blockId, file) = diskBlockManager.createTempLocalBlock()
     curWriteMetrics = new ShuffleWriteMetrics()
     var writer = blockManager.getDiskWriter(blockId, file, serializer, fileBufferSize,

diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalSorter.scala
@@ -268,8 +268,9 @@ private[spark] class ExternalSorter[K, V, C](
 
     spillCount += 1
     val threadId = Thread.currentThread().getId
-    logInfo("Thread %d spilling in-memory batch of %d MB to disk (%d spill%s so far)"
-      .format(threadId, memorySize / (1024 * 1024), spillCount, if (spillCount > 1) "s" else ""))
+    logInfo("Thread %d spilling in-memory batch of %s to disk (%d spill%s so far)"
+      .format(threadId, org.apache.spark.util.Utils.bytesToString(memorySize),
+        spillCount, if (spillCount > 1) "s" else ""))
 
     if (bypassMergeSort) {
       spillToPartitionFiles(collection)