Skip to content

Commit

Permalink
Merge pull request apache#503 from pwendell/master
Browse files Browse the repository at this point in the history
Fix bug on read-side of external sort when using Snappy.

This case wasn't handled correctly and this patch fixes it.
  • Loading branch information
pwendell committed Jan 24, 2014
2 parents ff44732 + 0213b40 commit 3d6e754
Showing 1 changed file with 9 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -358,7 +358,15 @@ private[spark] class ExternalAppendOnlyMap[K, V, C](
private class DiskMapIterator(file: File, blockId: BlockId) extends Iterator[(K, C)] {
val fileStream = new FileInputStream(file)
val bufferedStream = new FastBufferedInputStream(fileStream, fileBufferSize)
val compressedStream = blockManager.wrapForCompression(blockId, bufferedStream)

val shouldCompress = blockManager.shouldCompress(blockId)
val compressionCodec = new LZFCompressionCodec(sparkConf)
val compressedStream =
if (shouldCompress) {
compressionCodec.compressedInputStream(bufferedStream)
} else {
bufferedStream
}
var deserializeStream = ser.deserializeStream(compressedStream)
var objectsRead = 0

Expand Down

0 comments on commit 3d6e754

Please sign in to comment.