[HUDI-1375] Fix bug in HoodieAvroUtils.removeMetadataFields() method (#…

…2232) Co-authored-by: Wenning Ding <wenningd@amazon.com>
apache · Nov 6, 2020 · 0364498 · 0364498
1 parent 33ec88f
commit 0364498
Show file tree

Hide file tree

Showing 2 changed files with 11 additions and 0 deletions.
diff --git a/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java b/hudi-common/src/main/java/org/apache/hudi/avro/HoodieAvroUtils.java
@@ -204,6 +204,7 @@ public static Schema removeMetadataFields(Schema schema) {
     List<Schema.Field> filteredFields = schema.getFields()
                                               .stream()
                                               .filter(field -> !HoodieRecord.HOODIE_META_COLUMNS.contains(field.name()))
+                                              .map(field -> new Schema.Field(field.name(), field.schema(), field.doc(), field.defaultVal()))
                                               .collect(Collectors.toList());
     Schema filteredSchema = Schema.createRecord(schema.getName(), schema.getDoc(), schema.getNamespace(), false);
     filteredSchema.setFields(filteredFields);

diff --git a/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java b/hudi-common/src/test/java/org/apache/hudi/avro/TestHoodieAvroUtils.java
@@ -55,6 +55,8 @@ public class TestHoodieAvroUtils {
       + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
       + "{\"name\": \"pii_col\", \"type\": \"string\", \"column_category\": \"user_profile\"}]}";
 
+  private static int NUM_FIELDS_IN_EXAMPLE_SCHEMA = 4;
+
   private static String SCHEMA_WITH_METADATA_FIELD = "{\"type\": \"record\",\"name\": \"testrec2\",\"fields\": [ "
       + "{\"name\": \"timestamp\",\"type\": \"double\"},{\"name\": \"_row_key\", \"type\": \"string\"},"
       + "{\"name\": \"non_pii_col\", \"type\": \"string\"},"
@@ -197,4 +199,12 @@ public void testJsonNodeNullWithDefaultValues() {
     //evolvedField5.defaultVal() returns null.
     assertNull(rec1.get("evolved_field1"));
   }
+
+  @Test
+  public void testAddingAndRemovingMetadataFields() {
+    Schema schemaWithMetaCols = HoodieAvroUtils.addMetadataFields(new Schema.Parser().parse(EXAMPLE_SCHEMA));
+    assertEquals(schemaWithMetaCols.getFields().size(), NUM_FIELDS_IN_EXAMPLE_SCHEMA + HoodieRecord.HOODIE_META_COLUMNS.size());
+    Schema schemaWithoutMetaCols = HoodieAvroUtils.removeMetadataFields(schemaWithMetaCols);
+    assertEquals(schemaWithoutMetaCols.getFields().size(), NUM_FIELDS_IN_EXAMPLE_SCHEMA);
+  }
 }