Introduced BigDecimalConverter

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/DataType.java

@@ -7,6 +7,7 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 
+import java.math.BigDecimal;
 import java.util.Arrays;
 import java.util.ArrayList;
 import java.util.Map;
@@ -49,6 +50,13 @@ public enum DataType {
      */
     DOUBLE("double"),
 
+    /**
+     * Type of <i>BigDecimal</i>. No precision loss possible type. Compatible with the Java <b>BigDecimal</b> primitive data type.
+     *
+     * @since 2.8
+     */
+    BIGDECIMAL("bigdecimal"),
+
     /**
      * Type of <i>map</i>. Compatible with the Java <b>map</b> primitive data type.
      *
@@ -96,20 +104,22 @@ public static boolean isSameType(final Object object, final String option) {
         if (type == null)
             throw new IllegalArgumentException("Unknown DataType");
         switch (type) {
-          case MAP:
-            return (object instanceof Map);
-          case ARRAY:
-            return (object instanceof ArrayList || object.getClass().isArray());
-          case DOUBLE:
-            return (object instanceof Double);
-          case BOOLEAN:
-            return (object instanceof Boolean);
-          case INTEGER:
-            return (object instanceof Integer);
-          case LONG:
-            return (object instanceof Long);
-          default: // STRING
-            return (object instanceof String);
+            case MAP:
+                return (object instanceof Map);
+            case ARRAY:
+                return (object instanceof ArrayList || object.getClass().isArray());
+            case DOUBLE:
+                return (object instanceof Double);
+            case BOOLEAN:
+                return (object instanceof Boolean);
+            case INTEGER:
+                return (object instanceof Integer);
+            case LONG:
+                return (object instanceof Long);
+            case BIGDECIMAL:
+                return (object instanceof BigDecimal);
+            default: // STRING
+                return (object instanceof String);
         }
     }
 }

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java

@@ -11,6 +11,8 @@
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.cfg.JsonNodeFeature;
+import com.fasterxml.jackson.databind.json.JsonMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.type.TypeFactory;
@@ -61,11 +63,14 @@ public class JacksonEvent implements Event {
 
     private static final String SEPARATOR = "/";
 
-    private static final ObjectMapper mapper = new ObjectMapper()
+    private static final ObjectMapper mapper = JsonMapper.builder()
+            .disable(JsonNodeFeature.STRIP_TRAILING_BIGDECIMAL_ZEROES)
+            .build()
             .registerModule(new JavaTimeModule())
             .registerModule(new Jdk8Module()); // required for using Optional with Jackson. Ref: https://github.com/FasterXML/jackson-modules-java8
 
-    private static final TypeReference<Map<String, Object>> MAP_TYPE_REFERENCE = new TypeReference<Map<String, Object>>() {
+
+    private static final TypeReference<Map<String, Object>> MAP_TYPE_REFERENCE = new TypeReference<>() {
     };
 
     private final EventMetadata eventMetadata;

data-prepper-api/src/main/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverter.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+ package org.opensearch.dataprepper.typeconverter;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+
+/**
+ * Converter class for BigDecimal data type. By default, it applies zero scaling keeping the original value as it is.
+ * If required, the scale can be set using the setScale method.
+ */
+public class BigDecimalConverter implements TypeConverter<BigDecimal> {
+    // Scale declared here becomes the state of this converter.
+    // If the same instance of the converter is used for multiple conversions,
+    // scale value set will be preserved and applied on the next conversion.
+    private int scale = 0;
+
+    public void setScale(int scale) {
+        this.scale = scale;
+    }
+
+    public BigDecimal convert(Object source) throws IllegalArgumentException {
+        BigDecimal result = null;
+        if (source instanceof String) {
+            result = new BigDecimal((String)source);
+        }
+        else if (source instanceof Float) {
+            result = BigDecimal.valueOf((Float)source);
+        }
+        else if (source instanceof Double) {
+            result = BigDecimal.valueOf((Double)source);
+        }
+        else if (source instanceof Boolean) {
+            result = ((Boolean)source) ? BigDecimal.valueOf(1L) : BigDecimal.valueOf(0L);
+        }
+        else if (source instanceof Integer) {
+            result = BigDecimal.valueOf((Integer)source);
+        }
+        else if (source instanceof Long) {
+            result = BigDecimal.valueOf((Long)source);
+        }
+        else if (source instanceof BigDecimal) {
+            result = ((BigDecimal)source);
+        }
+
+        if(result!=null) {
+            if(scale!=0) {
+                result = result.setScale(scale, RoundingMode.HALF_EVEN);
+            }
+            return result;
+        }
+        throw new IllegalArgumentException("Unsupported type conversion. From Source class: " + source.getClass() + " to BigDecimal");
+    }
+}
+

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/DataTypeTest.java

@@ -15,6 +15,7 @@
 import static org.hamcrest.MatcherAssert.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
 
+import java.math.BigDecimal;
 import java.util.Map;
 import java.util.stream.Stream;
 
@@ -37,12 +38,13 @@ void test_isSameType(Object object, String type, boolean expectedResult) {
     }
 
     private static Stream<Arguments> getSameTypeTestData() {
-        int testArray[] = {1,2};
+        int[] testArray = {1,2};
         return Stream.of(
             Arguments.of(2, "integer", true),
             Arguments.of("testString", "string", true),
             Arguments.of(2L, "long", true),
             Arguments.of(2.0, "double", true),
+            Arguments.of(BigDecimal.valueOf(2.34567), "bigdecimal", true),
             Arguments.of(true, "boolean", true),
             Arguments.of(Map.of("k","v"), "map", true),
             Arguments.of(testArray, "array", true),

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java

@@ -9,11 +9,14 @@
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.CsvSource;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.ValueSource;
 import org.opensearch.dataprepper.expression.ExpressionEvaluator;
 import org.opensearch.dataprepper.model.event.exceptions.EventKeyNotFoundException;
 
+import java.math.BigDecimal;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Collections;
@@ -22,6 +25,7 @@
 import java.util.Map;
 import java.util.Random;
 import java.util.UUID;
+import java.util.stream.Stream;
 
 import static org.hamcrest.CoreMatchers.containsStringIgnoringCase;
 import static org.hamcrest.CoreMatchers.equalTo;
@@ -882,4 +886,24 @@ private static Map<String, Object> createComplexDataMap() {
         return dataObject;
     }
 
+    @ParameterizedTest
+    @MethodSource("getBigDecimalPutTestData")
+    void testPutAndGet_withBigDecimal(final String value) {
+        final String key = "bigDecimalKey";
+        event.put(key, new BigDecimal(value));
+        final Object result = event.get(key, Object.class);
+        assertThat(result, is(notNullValue()));
+        assertThat(result.toString(), is(equalTo(value)));
+    }
+
+    private static Stream<Arguments> getBigDecimalPutTestData() {
+        return Stream.of(
+                Arguments.of("702062202420"),
+                Arguments.of("1.23345E+9"),
+                Arguments.of("1.2345E+60"),
+                Arguments.of("1.2345E+6"),
+                Arguments.of("1.000")
+        );
+    }
+
 }

data-prepper-api/src/test/java/org/opensearch/dataprepper/typeconverter/BigDecimalConverterTests.java

@@ -0,0 +1,102 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.typeconverter;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+public class BigDecimalConverterTests {
+    @Test
+    void testStringToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final String stringConstant = "12345678912.12345";
+        assertThat(converter.convert(stringConstant), equalTo(new BigDecimal(stringConstant)));
+    }
+
+    @Test
+    void testIntegerToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final int intConstant = 12345;
+        assertThat(converter.convert(intConstant), equalTo(BigDecimal.valueOf(intConstant)));
+    }
+
+    @Test
+    void testLongToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final long longConstant = 123456789012L;
+        assertThat(converter.convert(longConstant).longValue(), equalTo(longConstant));
+    }
+
+    @Test
+    void testBooleanToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final Boolean boolFalseConstant = false;
+        assertThat(converter.convert(boolFalseConstant), equalTo(BigDecimal.valueOf(0)));
+        final Boolean boolTrueConstant = true;
+        assertThat(converter.convert(boolTrueConstant), equalTo(BigDecimal.valueOf(1)));
+    }
+
+    @Test
+    void testFloatToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final float fval = 12345.6789f;
+        assertThat(converter.convert(fval).floatValue(), equalTo(fval));
+    }
+
+    @Test
+    void testBigDecimalToBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        BigDecimal bigDecimal = new BigDecimal("12345.6789");
+        assertThat(converter.convert(bigDecimal), equalTo(bigDecimal));
+    }
+
+    @ParameterizedTest
+    @MethodSource("decimalToBigDecimalValueProvider")
+    void testDoubleToBigDecimalConversion(BigDecimal expectedBigDecimal, double actualValue, int scale) {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        if(scale!=0) {
+            converter.setScale(scale);
+            expectedBigDecimal = expectedBigDecimal.setScale(scale, RoundingMode.HALF_EVEN);
+        }
+        assertThat(converter.convert(actualValue), equalTo(expectedBigDecimal));
+    }
+
+    private static Stream<Arguments> decimalToBigDecimalValueProvider() {
+        return Stream.of(
+            Arguments.of(new BigDecimal ("0.0"), 0, 1),
+            Arguments.of(new BigDecimal ("0.0"), 0.0, 1),
+            Arguments.of(new BigDecimal ("0.00000000000000000000000"), 0.00000000000000000000000, 1),
+            Arguments.of(BigDecimal.ZERO, BigDecimal.ZERO.doubleValue(), 1),
+            Arguments.of(new BigDecimal ("1"), (double)1, 1),
+            Arguments.of(new BigDecimal ("1703908514.045833"), 1703908514.045833, 6),
+            Arguments.of(new BigDecimal ("1.00000000000000000000000"), 1.00000000000000000000000, 1),
+            Arguments.of(new BigDecimal ("-12345678912.12345"), -12345678912.12345, 1),
+            Arguments.of(BigDecimal.ONE, BigDecimal.ONE.doubleValue(), 1),
+            Arguments.of(new BigDecimal("1.7976931348623157E+308"), 1.7976931348623157E+308, 0),
+            Arguments.of(BigDecimal.valueOf(Double.MAX_VALUE), Double.MAX_VALUE, 0),
+            Arguments.of(BigDecimal.valueOf(Double.MIN_VALUE), Double.MIN_VALUE, 0)
+        );
+    }
+
+    @Test
+    void testInvalidBigDecimalConversion() {
+        BigDecimalConverter converter = new BigDecimalConverter();
+        final Map<Object, Object> map = Collections.emptyMap();
+        assertThrows(IllegalArgumentException.class, () -> converter.convert(map));
+    }
+}

data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DataPrepperScalarTypeDeserializerTest.java

@@ -16,6 +16,7 @@
 import org.mockito.junit.jupiter.MockitoExtension;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.time.Duration;
 import java.util.stream.Stream;
 
@@ -53,6 +54,7 @@ private static Stream<Arguments> getScalarTypeArguments() {
                 Arguments.of(Long.class, 200L),
                 Arguments.of(Double.class, 1.23d),
                 Arguments.of(Float.class, 2.15f),
-                Arguments.of(Character.class, 'c'));
+                Arguments.of(Character.class, 'c'),
+                Arguments.of(BigDecimal.class, 1.2345E+5));
     }
 }

data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java

@@ -21,6 +21,7 @@
 import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator;
 
 import java.io.IOException;
+import java.math.BigDecimal;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.Map;
@@ -113,6 +114,7 @@ private static Stream<Arguments> getNonStringTypeArguments() {
                 Arguments.of(Long.class, "200", 200L),
                 Arguments.of(Double.class, "1.23", 1.23d),
                 Arguments.of(Float.class, "2.15", 2.15f),
+                Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)),
                 Arguments.of(Map.class, "{}", Collections.emptyMap()));
     }
 
@@ -127,6 +129,7 @@ private static Stream<Arguments> getStringTypeArguments() {
                 Arguments.of(Long.class, "\"200\"", 200L),
                 Arguments.of(Double.class, "\"1.23\"", 1.23d),
                 Arguments.of(Float.class, "\"2.15\"", 2.15f),
+                Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)),
                 Arguments.of(Character.class, "\"c\"", 'c'));
     }
 }

data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/ConvertEntryTypeProcessorConfig.java

@@ -6,6 +6,7 @@
 package org.opensearch.dataprepper.plugins.processor.mutateevent;
 
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.opensearch.dataprepper.typeconverter.BigDecimalConverter;
 
 import java.util.List;
 import java.util.Optional;
@@ -20,6 +21,12 @@ public class ConvertEntryTypeProcessorConfig  {
     @JsonProperty("type")
     private TargetType type = TargetType.INTEGER;
 
+    /**
+     * Optional scale value used only in the case of BigDecimal converter
+     */
+    @JsonProperty("scale")
+    private int scale = 0;
+
     @JsonProperty("convert_when")
     private String convertWhen;
 
@@ -36,6 +43,9 @@ public String getKey() {
     public List<String> getKeys() { return keys; }
 
     public TargetType getType() {
+        if(type == TargetType.BIGDECIMAL && scale!=0) {
+            ((BigDecimalConverter) type.getTargetConverter()).setScale(scale);
+        }
         return type;
     }