diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala index c6a80611ea7c4..2e5539a90c65d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/csv/CSVOptions.scala @@ -211,6 +211,8 @@ class CSVOptions( } val lineSeparatorInWrite: Option[String] = lineSeparator + val inputBufferSize: Option[Int] = parameters.get("inputBufferSize").map(_.toInt) + /** * The handling method to be used when unescaped quotes are found in the input. */ @@ -257,6 +259,7 @@ class CSVOptions( settings.setIgnoreLeadingWhitespaces(ignoreLeadingWhiteSpaceInRead) settings.setIgnoreTrailingWhitespaces(ignoreTrailingWhiteSpaceInRead) settings.setReadInputOnSeparateThread(false) + inputBufferSize.foreach(settings.setInputBufferSize) settings.setMaxColumns(maxColumns) settings.setNullValue(nullValue) settings.setEmptyValue(emptyValueInRead)