diff --git a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala index 0898e068b7b6..a53c464b65fb 100644 --- a/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala +++ b/gluten-iceberg/src-iceberg/main/scala/org/apache/iceberg/spark/source/GlutenIcebergSourceUtil.scala @@ -111,12 +111,24 @@ object GlutenIcebergSourceUtil { // Iceberg will generate some non-table fields as partition fields, such as x_bucket, // which will not appear in readFields, they also cannot be filtered. val tableFields = spec.schema().columns().asScala.map(_.name()).toSet + val voidTransformFields = scan + .table() + .spec() + .fields() + .asScala + .filter( + f => { + f.transform().isVoid + }) + .map(_.name()) + .toSet val partitionFields = spec .partitionType() .fields() .asScala .filter(f => !tableFields.contains(f.name) || readFields.contains(f.name())) + .filter(f => !voidTransformFields.contains(f.name())) partitionFields.foreach { field => TypeUtil.validatePartitionColumnType(field.`type`().typeId()) } diff --git a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala index cb630a147539..170be9746b8c 100644 --- a/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala +++ b/gluten-iceberg/src-iceberg/test/scala/org/apache/gluten/execution/IcebergSuite.scala @@ -489,4 +489,30 @@ abstract class IcebergSuite extends WholeStageTransformerSuite { } } } + + test("test read v1 iceberg with partition drop") { + val testTable = "test_table_with_partition" + withTable(testTable) { + spark.sql(s""" + |CREATE TABLE $testTable (id INT, data STRING, p1 STRING, p2 STRING) + |USING iceberg + |tblproperties ( + | 'format-version' = '1' + |) + |PARTITIONED BY (p1, p2); + |""".stripMargin) + spark.sql(s""" + |INSERT INTO $testTable VALUES + |(1, 'test_data', 'test_p1', 'test_p2'); + |""".stripMargin) + spark.sql(s""" + |ALTER TABLE $testTable DROP PARTITION FIELD p2 + |""".stripMargin) + val resultDf = spark.sql(s"SELECT id, data, p1, p2 FROM $testTable") + val result = resultDf.collect() + + assert(result.length == 1) + assert(result.head.getString(3) == "test_p2") + } + } }