-
Notifications
You must be signed in to change notification settings - Fork 80
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added support to write metadata files in parquet (#5105)
Also, - Added support to do partitioned parquet writing. - Offset index information will be read from parquet files on demand and not while reading column chunks.
- Loading branch information
1 parent
765934b
commit a43948f
Showing
41 changed files
with
2,564 additions
and
344 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
157 changes: 157 additions & 0 deletions
157
...table/src/main/java/io/deephaven/engine/table/impl/locations/util/PartitionFormatter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| // | ||
| // Copyright (c) 2016-2024 Deephaven Data Labs and Patent Pending | ||
| // | ||
| package io.deephaven.engine.table.impl.locations.util; | ||
|
|
||
| import io.deephaven.time.DateTimeUtils; | ||
| import org.jetbrains.annotations.NotNull; | ||
| import org.jetbrains.annotations.Nullable; | ||
|
|
||
| import java.math.BigDecimal; | ||
| import java.math.BigInteger; | ||
| import java.time.Instant; | ||
| import java.time.LocalDate; | ||
| import java.time.LocalTime; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * This class takes a partition value object and formats it to a {@link String}. Useful when generating partitioning | ||
| * paths for table. Complementary to {@link PartitionParser}. | ||
| */ | ||
| public enum PartitionFormatter { | ||
| ForString { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return (String) value; | ||
| } | ||
| }, | ||
| ForBoolean { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Boolean) value).toString(); | ||
| } | ||
| }, | ||
| ForChar { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Character) value).toString(); | ||
| } | ||
| }, | ||
| ForByte { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Byte) value).toString(); | ||
| } | ||
| }, | ||
| ForShort { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Short) value).toString(); | ||
| } | ||
| }, | ||
| ForInt { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Integer) value).toString(); | ||
| } | ||
| }, | ||
| ForLong { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Long) value).toString(); | ||
| } | ||
| }, | ||
| ForFloat { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Float) value).toString(); | ||
| } | ||
| }, | ||
| ForDouble { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Double) value).toString(); | ||
| } | ||
| }, | ||
| ForBigInteger { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((BigInteger) value).toString(); | ||
| } | ||
| }, | ||
| ForBigDecimal { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((BigDecimal) value).toString(); | ||
| } | ||
| }, | ||
| ForInstant { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((Instant) value).toString(); | ||
| } | ||
| }, | ||
| ForLocalDate { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return DateTimeUtils.formatDate((LocalDate) value); | ||
| } | ||
| }, | ||
| ForLocalTime { | ||
| @Override | ||
| public String formatObject(@NotNull final Object value) { | ||
| return ((LocalTime) value).toString(); | ||
| } | ||
| }; | ||
|
|
||
| private static final Map<Class<?>, PartitionFormatter> typeMap = new HashMap<>(); | ||
| static { | ||
| typeMap.put(String.class, ForString); | ||
| typeMap.put(Boolean.class, ForBoolean); | ||
| typeMap.put(boolean.class, ForBoolean); | ||
| typeMap.put(Character.class, ForChar); | ||
| typeMap.put(char.class, ForChar); | ||
| typeMap.put(Byte.class, ForByte); | ||
| typeMap.put(byte.class, ForByte); | ||
| typeMap.put(Short.class, ForShort); | ||
| typeMap.put(short.class, ForShort); | ||
| typeMap.put(Integer.class, ForInt); | ||
| typeMap.put(int.class, ForInt); | ||
| typeMap.put(Long.class, ForLong); | ||
| typeMap.put(long.class, ForLong); | ||
| typeMap.put(Float.class, ForFloat); | ||
| typeMap.put(float.class, ForFloat); | ||
| typeMap.put(Double.class, ForDouble); | ||
| typeMap.put(double.class, ForDouble); | ||
| typeMap.put(BigInteger.class, ForBigInteger); | ||
| typeMap.put(BigDecimal.class, ForBigDecimal); | ||
| typeMap.put(Instant.class, ForInstant); | ||
| typeMap.put(LocalDate.class, ForLocalDate); | ||
| typeMap.put(LocalTime.class, ForLocalTime); | ||
| } | ||
|
|
||
| abstract String formatObject(@NotNull final Object value); | ||
|
|
||
| /** | ||
| * Takes a partition value object and returns a formatted string. Returns an empty string if the object is null. | ||
| */ | ||
| public String format(@Nullable final Object value) { | ||
| if (value == null) { | ||
| return ""; | ||
| } | ||
| return formatObject(value); | ||
| } | ||
|
|
||
| /** | ||
| * Takes a partitioning column type and returns the corresponding formatter. | ||
| */ | ||
| public static PartitionFormatter getFormatterForType(@NotNull final Class<?> clazz) { | ||
| final PartitionFormatter formatter = typeMap.get(clazz); | ||
| if (formatter != null) { | ||
| return formatter; | ||
| } else { | ||
| throw new UnsupportedOperationException("Unsupported type: " + clazz.getSimpleName()); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.