Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Kernel][Defaults] Support reading parquet files with legacy 3-level repeated types #3083

Merged
merged 3 commits into from
May 10, 2024
+87 −35
Merged

[Kernel][Defaults] Support reading parquet files with legacy 3-level repeated types #3083

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4addd13
[Kernel][Defaults] Support reading parquet files with legacy 3-level …
vkorukanti May 10, 2024
8f1c14b
clean up
vkorukanti May 10, 2024
a2a3f91
fix tests
vkorukanti May 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added ...es-legacy-format/.part-00000-5afb67f1-094a-4a15-922e-c1eb96683964-c000.snappy.parquet.crc
View file
Open in desktop
Binary file not shown.
Binary file added ...esources/golden/parquet-all-types-legacy-format/_delta_log/.00000000000000000000.json.crc
View file
Open in desktop
Binary file not shown.
4 changes: 4 additions & 0 deletions ...ain/resources/golden/parquet-all-types-legacy-format/_delta_log/00000000000000000000.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{"commitInfo":{"timestamp":1715358358979,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"200","numOutputBytes":"20934"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"01d57c6f-6073-484f-b832-5cf368644e4b"}}
{"metaData":{"id":"fb723383-fb90-4346-a846-ccdac9a3204b","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampNTZType\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1715358356845}}
{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz"],"writerFeatures":["timestampNtz"]}}
{"add":{"path":"part-00000-5afb67f1-094a-4a15-922e-c1eb96683964-c000.snappy.parquet","partitionValues":{},"size":20934,"modificationTime":1715358358904,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"ByteType\":-128,\"ShortType\":1,\"IntegerType\":1,\"LongType\":2,\"FloatType\":0.234,\"DoubleType\":234234.23,\"decimal\":123.52,\"StringType\":\"1\",\"DateType\":\"1970-01-01\",\"TimestampType\":\"1970-01-01T06:30:23.523Z\",\"TimestampNTZType\":\"1970-01-03T17:03:54.000\",\"nested_struct\":{\"aa\":\"1\",\"ac\":{\"aca\":1}}},\"maxValues\":{\"ByteType\":127,\"ShortType\":199,\"IntegerType\":199,\"LongType\":200,\"FloatType\":46.566,\"DoubleType\":4.661261177E7,\"decimal\":24580.48,\"StringType\":\"99\",\"DateType\":\"1970-02-16\",\"TimestampType\":\"1970-02-23T22:48:01.077Z\",\"TimestampNTZType\":\"1971-06-24T11:56:06.000\",\"nested_struct\":{\"aa\":\"99\",\"ac\":{\"aca\":199}}},\"nullCount\":{\"ByteType\":3,\"ShortType\":4,\"IntegerType\":9,\"LongType\":8,\"FloatType\":8,\"DoubleType\":4,\"decimal\":3,\"BooleanType\":3,\"StringType\":4,\"BinaryType\":4,\"DateType\":4,\"TimestampType\":4,\"TimestampNTZType\":3,\"nested_struct\":{\"aa\":14,\"ac\":{\"aca\":22}},\"array_of_prims\":200,\"array_of_arrays\":200,\"array_of_structs\":200,\"map_of_prims\":200,\"map_of_rows\":200,\"map_of_arrays\":200}}"}}
Binary file added ...l-types-legacy-format/part-00000-5afb67f1-094a-4a15-922e-c1eb96683964-c000.snappy.parquet
View file
Open in desktop
Binary file not shown.
Binary file removed ...arquet-all-types/.part-00000-981a72ea-9b1a-4bf3-8c3c-3d7e56a7bb45-c000.snappy.parquet.crc
View file
Open in desktop
Binary file not shown.
Binary file added ...arquet-all-types/.part-00000-bf6680d4-5e83-4fce-8ebb-d2b60d7e69c9-c000.snappy.parquet.crc
View file
Open in desktop
Binary file not shown.
Binary file modified ...les/src/main/resources/golden/parquet-all-types/_delta_log/.00000000000000000000.json.crc
View file
Open in desktop
Binary file not shown.
6 changes: 3 additions & 3 deletions ...n-tables/src/main/resources/golden/parquet-all-types/_delta_log/00000000000000000000.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
{"commitInfo":{"timestamp":1713368423544,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"200","numOutputBytes":"30312"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"d711414e-08c1-46d0-a5c5-d5faad64d59e"}}
{"metaData":{"id":"7027c1fa-69c4-4867-ace8-6f8c44a022d0","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampNTZType\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1713368421437}}
{"commitInfo":{"timestamp":1715358308005,"operation":"WRITE","operationParameters":{"mode":"Append","partitionBy":"[]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"1","numOutputRows":"200","numOutputBytes":"21057"},"engineInfo":"Apache-Spark/3.5.0 Delta-Lake/3.2.0-SNAPSHOT","txnId":"c84f1a78-0895-4f01-b00e-f3a984c8afca"}}
{"metaData":{"id":"ab49cd9e-a908-4aad-a15b-9dd117d3e0ab","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"ByteType\",\"type\":\"byte\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ShortType\",\"type\":\"short\",\"nullable\":true,\"metadata\":{}},{\"name\":\"IntegerType\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}},{\"name\":\"LongType\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"FloatType\",\"type\":\"float\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DoubleType\",\"type\":\"double\",\"nullable\":true,\"metadata\":{}},{\"name\":\"decimal\",\"type\":\"decimal(10,2)\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BooleanType\",\"type\":\"boolean\",\"nullable\":true,\"metadata\":{}},{\"name\":\"StringType\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"BinaryType\",\"type\":\"binary\",\"nullable\":true,\"metadata\":{}},{\"name\":\"DateType\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampType\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"TimestampNTZType\",\"type\":\"timestamp_ntz\",\"nullable\":true,\"metadata\":{}},{\"name\":\"nested_struct\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aa\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}},{\"name\":\"ac\",\"type\":{\"type\":\"struct\",\"fields\":[{\"name\":\"aca\",\"type\":\"integer\",\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}}]},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_prims\",\"type\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_arrays\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"array_of_structs\",\"type\":{\"type\":\"array\",\"elementType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"containsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_prims\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":\"long\",\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_rows\",\"type\":{\"type\":\"map\",\"keyType\":\"integer\",\"valueType\":{\"type\":\"struct\",\"fields\":[{\"name\":\"ab\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}}]},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}},{\"name\":\"map_of_arrays\",\"type\":{\"type\":\"map\",\"keyType\":\"long\",\"valueType\":{\"type\":\"array\",\"elementType\":\"integer\",\"containsNull\":true},\"valueContainsNull\":true},\"nullable\":true,\"metadata\":{}}]}","partitionColumns":[],"configuration":{},"createdTime":1715358307675}}
{"protocol":{"minReaderVersion":3,"minWriterVersion":7,"readerFeatures":["timestampNtz"],"writerFeatures":["timestampNtz"]}}
{"add":{"path":"part-00000-981a72ea-9b1a-4bf3-8c3c-3d7e56a7bb45-c000.snappy.parquet","partitionValues":{},"size":30312,"modificationTime":1713368423449,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"ByteType\":-128,\"ShortType\":1,\"IntegerType\":1,\"LongType\":2,\"FloatType\":0.234,\"DoubleType\":234234.23,\"decimal\":123.52,\"StringType\":\"1\",\"DateType\":\"1970-01-01\",\"TimestampType\":\"1970-01-01T06:30:23.523Z\",\"TimestampNTZType\":\"1970-01-03T17:03:54.000\",\"nested_struct\":{\"aa\":\"1\",\"ac\":{\"aca\":1}}},\"maxValues\":{\"ByteType\":127,\"ShortType\":199,\"IntegerType\":199,\"LongType\":200,\"FloatType\":46.566,\"DoubleType\":4.661261177E7,\"decimal\":24580.48,\"StringType\":\"99\",\"DateType\":\"1970-02-16\",\"TimestampType\":\"1970-02-23T22:48:01.077Z\",\"TimestampNTZType\":\"1971-06-24T11:56:06.000\",\"nested_struct\":{\"aa\":\"99\",\"ac\":{\"aca\":199}}},\"nullCount\":{\"ByteType\":3,\"ShortType\":4,\"IntegerType\":9,\"LongType\":8,\"FloatType\":8,\"DoubleType\":4,\"decimal\":3,\"BooleanType\":3,\"StringType\":4,\"BinaryType\":4,\"DateType\":4,\"TimestampType\":4,\"TimestampNTZType\":3,\"nested_struct\":{\"aa\":14,\"ac\":{\"aca\":22}},\"array_of_prims\":8,\"array_of_arrays\":25,\"array_of_structs\":0,\"map_of_prims\":8,\"map_of_rows\":0,\"map_of_arrays\":7}}"}}
{"add":{"path":"part-00000-bf6680d4-5e83-4fce-8ebb-d2b60d7e69c9-c000.snappy.parquet","partitionValues":{},"size":21057,"modificationTime":1715358307997,"dataChange":true,"stats":"{\"numRecords\":200,\"minValues\":{\"ByteType\":-128,\"ShortType\":1,\"IntegerType\":1,\"LongType\":2,\"FloatType\":0.234,\"DoubleType\":234234.23,\"decimal\":123.52,\"StringType\":\"1\",\"DateType\":\"1970-01-01\",\"TimestampType\":\"1970-01-01T06:30:23.523Z\",\"TimestampNTZType\":\"1970-01-03T17:03:54.000\",\"nested_struct\":{\"aa\":\"1\",\"ac\":{\"aca\":1}}},\"maxValues\":{\"ByteType\":127,\"ShortType\":199,\"IntegerType\":199,\"LongType\":200,\"FloatType\":46.566,\"DoubleType\":4.661261177E7,\"decimal\":24580.48,\"StringType\":\"99\",\"DateType\":\"1970-02-16\",\"TimestampType\":\"1970-02-23T22:48:01.077Z\",\"TimestampNTZType\":\"1971-06-24T11:56:06.000\",\"nested_struct\":{\"aa\":\"99\",\"ac\":{\"aca\":199}}},\"nullCount\":{\"ByteType\":3,\"ShortType\":4,\"IntegerType\":9,\"LongType\":8,\"FloatType\":8,\"DoubleType\":4,\"decimal\":3,\"BooleanType\":3,\"StringType\":4,\"BinaryType\":4,\"DateType\":4,\"TimestampType\":4,\"TimestampNTZType\":3,\"nested_struct\":{\"aa\":14,\"ac\":{\"aca\":22}},\"array_of_prims\":200,\"array_of_arrays\":200,\"array_of_structs\":200,\"map_of_prims\":200,\"map_of_rows\":200,\"map_of_arrays\":200}}"}}
Binary file renamed ...bf3-8c3c-3d7e56a7bb45-c000.snappy.parquet → ...fce-8ebb-d2b60d7e69c9-c000.snappy.parquet
View file
Open in desktop
Binary file not shown.
27 changes: 24 additions & 3 deletions connectors/golden-tables/src/test/scala/io/delta/golden/GoldenTables.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1038,7 +1038,18 @@ class GoldenTables extends QueryTest with SharedSparkSession {
writeBasicTimestampTable(tablePath, TimeZone.getTimeZone("PST"))
}

generateGoldenTable("parquet-all-types-legacy-format") { tablePath =>
withSQLConf(("spark.sql.parquet.writeLegacyFormat", "true")) {
generateAllTypesTable(tablePath)
}
}

generateGoldenTable("parquet-all-types") { tablePath =>
// generating using the standard parquet format
generateAllTypesTable(tablePath)
}

def generateAllTypesTable(tablePath: String): Unit = {
val timeZone = java.util.TimeZone.getTimeZone("UTC")
java.util.TimeZone.setDefault(timeZone)
import java.sql._
Expand Down Expand Up @@ -1113,6 +1124,7 @@ class GoldenTables extends QueryTest with SharedSparkSession {
if (i % 61 != 0) new java.sql.Date(i * 20000000L) else null,
if (i % 62 != 0) new Timestamp(i * 23423523L) else null,
if (i % 69 != 0) LocalDateTime.ofEpochSecond(i * 234234L, 200012, UTC) else null,
// nested_struct
if (i % 63 != 0) {
if (i % 19 == 0) {
// write a struct with all fields null
Expand All @@ -1121,13 +1133,15 @@ class GoldenTables extends QueryTest with SharedSparkSession {
Row(i.toString, if (i % 23 != 0) Row(i) else null)
}
} else null,
// array_of_prims
if (i % 25 != 0) {
if (i % 29 == 0) {
scala.Array()
} else {
scala.Array(i, null, i + 1)
}
} else null,
// array_of_arrays
if (i % 8 != 0) {
val singleElemArray = scala.Array(i)
val doubleElemArray = scala.Array(i + 10, i + 20)
Expand All @@ -1144,7 +1158,11 @@ class GoldenTables extends QueryTest with SharedSparkSession {
case 6 => scala.Array()
}
} else null,
scala.Array(Row(i.longValue()), null),
// array_of_structs
if (i % 10 != 0) {
scala.Array(Row(i.longValue()), null)
} else null,
// map_of_prims
if (i % 28 != 0) {
if (i % 30 == 0) {
Map()
Expand All @@ -1155,7 +1173,11 @@ class GoldenTables extends QueryTest with SharedSparkSession {
)
}
} else null,
Map(i + 1 -> (if (i % 10 == 0) Row((i * 20).longValue()) else null)),
// map_of_rows
if (i % 25 != 0) {
Map(i + 1 -> (if (i % 10 == 0) Row((i * 20).longValue()) else null))
} else null,
// map_of_arrays
if (i % 30 != 0) {
if (i % 24 == 0) {
Map()
Expand All @@ -1181,7 +1203,6 @@ class GoldenTables extends QueryTest with SharedSparkSession {
.save(tablePath)
}


def writeBasicDecimalTable(tablePath: String): Unit = {
val data = Seq(
Seq("234", "1", "2", "3"),
Expand Down
36 changes: 31 additions & 5 deletions ...l-defaults/src/main/java/io/delta/kernel/defaults/internal/parquet/ArrayColumnReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@

import org.apache.parquet.io.api.Converter;
import org.apache.parquet.schema.GroupType;
import org.apache.parquet.schema.Type;

import io.delta.kernel.data.ColumnVector;
import io.delta.kernel.types.ArrayType;

import static io.delta.kernel.internal.util.Preconditions.checkArgument;

import io.delta.kernel.defaults.internal.data.vector.DefaultArrayVector;
import static io.delta.kernel.defaults.internal.parquet.ParquetColumnReaders.createConverter;

Expand Down Expand Up @@ -52,15 +55,38 @@ public ColumnVector getDataColumnVector(int batchSize) {
return arrayVector;
}

/**
Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same code in Spark Parquet reader.

* Currently, support for 3-level nested arrays only.
* <p>
* optional group readerFeatures (LIST) {
* repeated group list {
* optional binary element (STRING);
* }
* }
* <p>
* optional group readerFeatures (LIST) {
* repeated group bag {
* optional binary array (STRING);
* }
* }
*
* TODO: Add support for 2-level nested arrays.
*/
private static Converter createElementConverter(
int initialBatchSize,
ArrayType typeFromClient,
GroupType typeFromFile) {

final GroupType innerElementType = (GroupType) typeFromFile.getType("list");
return createConverter(
initialBatchSize,
typeFromClient.getElementType(),
innerElementType.getType("element"));
checkArgument(
typeFromFile.getFieldCount() == 1, "Expected exactly one field in the array type");
GroupType repeatedGroup = typeFromFile.getType(0).asGroupType();

// TODO: handle the legacy 2-level list physical format
checkArgument(repeatedGroup.getFieldCount() == 1,
"Expected exactly one field in the repeated group");

Type elmentType = repeatedGroup.getType(0);

return createConverter(initialBatchSize, typeFromClient.getElementType(), elmentType);
}
}
Loading
Loading