From 64692b171b254557430116d9eaf760be512bb7f2 Mon Sep 17 00:00:00 2001 From: Andriy Plokhotnyuk Date: Tue, 18 Feb 2025 14:24:05 +0100 Subject: [PATCH] More efficient encoding of `BigDecimal` and `java.math.BigDecimal` values --- .../scala/zio/json/internal/SafeNumbers.scala | 126 +++++++++++++++++- .../scala/zio/json/internal/SafeNumbers.scala | 123 ++++++++++++++++- .../src/main/scala/zio/json/JsonEncoder.scala | 16 ++- .../src/test/scala/zio/json/EncoderSpec.scala | 32 +++-- .../shared/src/test/scala/zio/json/Gens.scala | 11 +- .../test/scala/zio/json/RoundTripSpec.scala | 3 + 6 files changed, 287 insertions(+), 24 deletions(-) diff --git a/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala b/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala index 761310ed7..61fba7803 100644 --- a/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala +++ b/zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala @@ -78,6 +78,12 @@ object SafeNumbers { try Some(UnsafeNumbers.bigDecimal(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } + def toString(x: java.math.BigDecimal): String = { + val out = writes.get + write(x, out) + out.buffer.toString + } + def toString(x: java.math.BigInteger): String = { val out = writes.get write(x, out) @@ -102,6 +108,121 @@ object SafeNumbers { out.buffer.toString } + def write(x: java.math.BigDecimal, out: Write): Unit = { + var exp = writeBigDecimal(x.unscaledValue, x.scale, 0, null, out) + if (exp != 0) { + var sc = '+' + if (exp < 0) { + sc = '-' + exp = -exp + } + out.write('E', sc) + writeMantissa(exp, out) + } + } + + private[this] def writeBigDecimal( + x: java.math.BigInteger, + scale: Int, + blockScale: Int, + ss: Array[java.math.BigInteger], + out: Write + ): Int = { + val bitLen = x.bitLength + if (bitLen < 64) { + val v = x.longValue + val pv = Math.abs(v) + val digits = + if (pv >= 100000000000000000L) { + if (pv >= 1000000000000000000L) 19 + else 18 + } else digitCount(pv) + val dotOff = scale - blockScale + val exp = (digits - 1) - dotOff + if (scale >= 0 && exp >= -6) { + if (exp < 0) { + out.write('0', '.') + var zeros = -exp - 1 + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + write(v, out) + } else if (dotOff > 0) writeLongWithDot(v, dotOff, out) + else write(v, out) + 0 + } else { + if (digits > 1) writeLongWithDot(v, digits - 1, out) + else { + write(v, out) + if (blockScale > 0) out.write('.') + } + exp + } + } else { + val n = calculateTenPow18SquareNumber(bitLen) + val ss1 = + if (ss eq null) getTenPow18Squares(n) + else ss + val qr = x.divideAndRemainder(ss1(n)) + val exp = writeBigDecimal(qr(0), scale, (18 << n) + blockScale, ss1, out) + writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss1, out) + exp + } + } + + @inline private[this] def writeLongWithDot(v: Long, dotOff: Int, out: Write): Unit = { + val pow10 = pow10longs(dotOff) + val q = v / pow10 + val r = Math.abs(v - q * pow10) + write(q, out) + out.write('.') + var zeros = dotOff - digitCount(r) + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + write(r, out) + } + + private[this] def writeBigDecimalRemainder( + x: java.math.BigInteger, + scale: Int, + blockScale: Int, + n: Int, + ss: Array[java.math.BigInteger], + out: Write + ): Unit = + if (n < 0) { + val v = Math.abs(x.longValue) + var dotOff = scale - blockScale + if (dotOff > 0 && dotOff < 18) { + val pow10 = pow10longs(dotOff) + val q = v / pow10 + val r = v - q * pow10 + var zeros = 18 - dotOff - digitCount(q) + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + writeMantissa(q, out) + out.write('.') + dotOff -= digitCount(r) + while (dotOff > 0) { + out.write('0') + dotOff -= 1 + } + writeMantissa(r, out) + } else { + if (dotOff == 18) out.write('.') + write18Digits(v, out) + } + } else { + val qr = x.divideAndRemainder(ss(n)) + writeBigDecimalRemainder(qr(0), scale, (18 << n) + blockScale, n - 1, ss, out) + writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss, out) + } + def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out) private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = { @@ -660,8 +781,7 @@ object SafeNumbers { @inline private[json] def write2Digits(x: Int, out: Write): Unit = out.write(digits(x)) - @inline - private[this] def digitCount(x: Long): Int = + @inline private[this] def digitCount(x: Long): Int = if (x >= 1000000000000000L) { if (x >= 10000000000000000L) 17 else 16 @@ -975,7 +1095,7 @@ object SafeNumbers { private[this] final val pow10longs: Array[Long] = Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L, - 100000000000000000L) + 100000000000000000L, 1000000000000000000L) @volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] = Array(java.math.BigInteger.valueOf(1000000000000000000L)) diff --git a/zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala b/zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala index 9dc3b3ea3..2d05488c1 100644 --- a/zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala +++ b/zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala @@ -78,6 +78,12 @@ object SafeNumbers { try Some(UnsafeNumbers.bigDecimal(num, max_bits)) catch { case _: UnexpectedEnd | UnsafeNumber => None } + def toString(x: java.math.BigDecimal): String = { + val out = writes.get + write(x, out) + out.buffer.toString + } + def toString(x: java.math.BigInteger): String = { val out = writes.get write(x, out) @@ -102,6 +108,121 @@ object SafeNumbers { out.buffer.toString } + def write(x: java.math.BigDecimal, out: Write): Unit = { + var exp = writeBigDecimal(x.unscaledValue, x.scale, 0, null, out) + if (exp != 0) { + var sc = '+' + if (exp < 0) { + sc = '-' + exp = -exp + } + out.write('E', sc) + writeMantissa(exp, out) + } + } + + private[this] def writeBigDecimal( + x: java.math.BigInteger, + scale: Int, + blockScale: Int, + ss: Array[java.math.BigInteger], + out: Write + ): Int = { + val bitLen = x.bitLength + if (bitLen < 64) { + val v = x.longValue + val pv = Math.abs(v) + val digits = + if (pv >= 100000000000000000L) { + if (pv >= 1000000000000000000L) 19 + else 18 + } else digitCount(pv) + val dotOff = scale - blockScale + val exp = (digits - 1) - dotOff + if (scale >= 0 && exp >= -6) { + if (exp < 0) { + out.write('0', '.') + var zeros = -exp - 1 + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + write(v, out) + } else if (dotOff > 0) writeLongWithDot(v, dotOff, out) + else write(v, out) + 0 + } else { + if (digits > 1) writeLongWithDot(v, digits - 1, out) + else { + write(v, out) + if (blockScale > 0) out.write('.') + } + exp + } + } else { + val n = calculateTenPow18SquareNumber(bitLen) + val ss1 = + if (ss eq null) getTenPow18Squares(n) + else ss + val qr = x.divideAndRemainder(ss1(n)) + val exp = writeBigDecimal(qr(0), scale, (18 << n) + blockScale, ss1, out) + writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss1, out) + exp + } + } + + private[this] def writeLongWithDot(v: Long, dotOff: Int, out: Write): Unit = { + val pow10 = pow10longs(dotOff) + val q = v / pow10 + val r = Math.abs(v - q * pow10) + write(q, out) + out.write('.') + var zeros = dotOff - digitCount(r) + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + write(r, out) + } + + private[this] def writeBigDecimalRemainder( + x: java.math.BigInteger, + scale: Int, + blockScale: Int, + n: Int, + ss: Array[java.math.BigInteger], + out: Write + ): Unit = + if (n < 0) { + val v = Math.abs(x.longValue) + var dotOff = scale - blockScale + if (dotOff > 0 && dotOff < 18) { + val pow10 = pow10longs(dotOff) + val q = v / pow10 + val r = v - q * pow10 + var zeros = 18 - dotOff - digitCount(q) + while (zeros > 0) { + out.write('0') + zeros -= 1 + } + writeMantissa(q, out) + out.write('.') + dotOff -= digitCount(r) + while (dotOff > 0) { + out.write('0') + dotOff -= 1 + } + writeMantissa(r, out) + } else { + if (dotOff == 18) out.write('.') + write18Digits(v, out) + } + } else { + val qr = x.divideAndRemainder(ss(n)) + writeBigDecimalRemainder(qr(0), scale, (18 << n) + blockScale, n - 1, ss, out) + writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss, out) + } + def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out) private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = { @@ -911,7 +1032,7 @@ object SafeNumbers { private[this] final val pow10longs: Array[Long] = Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L, 100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L, - 100000000000000000L) + 100000000000000000L, 1000000000000000000L) @volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] = Array(java.math.BigInteger.valueOf(1000000000000000000L)) diff --git a/zio-json/shared/src/main/scala/zio/json/JsonEncoder.scala b/zio-json/shared/src/main/scala/zio/json/JsonEncoder.scala index d41c38d09..273dd9f5c 100644 --- a/zio-json/shared/src/main/scala/zio/json/JsonEncoder.scala +++ b/zio-json/shared/src/main/scala/zio/json/JsonEncoder.scala @@ -215,12 +215,12 @@ object JsonEncoder extends GeneratedTupleEncoders with EncoderLowPriority1 with implicit val byte: JsonEncoder[Byte] = new JsonEncoder[Byte] { def unsafeEncode(a: Byte, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.toInt, out) - override def toJsonAST(a: Byte): Either[String, Json] = new Right(Json.Num(a)) + override def toJsonAST(a: Byte): Either[String, Json] = new Right(Json.Num(a.toInt)) } implicit val short: JsonEncoder[Short] = new JsonEncoder[Short] { def unsafeEncode(a: Short, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.toInt, out) - override def toJsonAST(a: Short): Either[String, Json] = new Right(Json.Num(a)) + override def toJsonAST(a: Short): Either[String, Json] = new Right(Json.Num(a.toInt)) } implicit val int: JsonEncoder[Int] = new JsonEncoder[Int] { def unsafeEncode(a: Int, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a, out) @@ -254,8 +254,16 @@ object JsonEncoder extends GeneratedTupleEncoders with EncoderLowPriority1 with override def toJsonAST(a: Float): Either[String, Json] = new Right(Json.Num(a)) } - implicit val bigDecimal: JsonEncoder[java.math.BigDecimal] = explicit(_.toString, n => new Json.Num(n)) - implicit val scalaBigDecimal: JsonEncoder[BigDecimal] = explicit(_.toString, Json.Num.apply) + implicit val bigDecimal: JsonEncoder[java.math.BigDecimal] = new JsonEncoder[java.math.BigDecimal] { + def unsafeEncode(a: java.math.BigDecimal, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a, out) + + override def toJsonAST(a: java.math.BigDecimal): Either[String, Json] = new Right(new Json.Num(a)) + } + implicit val scalaBigDecimal: JsonEncoder[BigDecimal] = new JsonEncoder[BigDecimal] { + def unsafeEncode(a: BigDecimal, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.bigDecimal, out) + + override def toJsonAST(a: BigDecimal): Either[String, Json] = new Right(new Json.Num(a.bigDecimal)) + } implicit def option[A](implicit A: JsonEncoder[A]): JsonEncoder[Option[A]] = new JsonEncoder[Option[A]] { def unsafeEncode(oa: Option[A], indent: Option[Int], out: Write): Unit = diff --git a/zio-json/shared/src/test/scala/zio/json/EncoderSpec.scala b/zio-json/shared/src/test/scala/zio/json/EncoderSpec.scala index 278512cd2..d498e4989 100644 --- a/zio-json/shared/src/test/scala/zio/json/EncoderSpec.scala +++ b/zio-json/shared/src/test/scala/zio/json/EncoderSpec.scala @@ -245,20 +245,28 @@ object EncoderSpec extends ZIOSpecDefault { assert((-6939.0464d).toJson)( equalTo("-6939.0464") ) // See the issue: https://github.com/zio/zio-json/pull/375 - }, - test("other numerics") { - val exampleBigIntStr = "170141183460469231731687303715884105728" - val exampleBigDecimalStr = "170141183460469231731687303715884105728.4433" - assert((1: Byte).toJson)(equalTo("1")) && - assert((1: Short).toJson)(equalTo("1")) && - assert((1: Int).toJson)(equalTo("1")) && - assert(1L.toJson)(equalTo("1")) && - assert(new java.math.BigInteger("1").toJson)(equalTo("1")) && - assert(new java.math.BigInteger(exampleBigIntStr).toJson)(equalTo(exampleBigIntStr)) && - assert(BigInt(exampleBigIntStr).toJson)(equalTo(exampleBigIntStr)) && - assert(BigDecimal(exampleBigDecimalStr).toJson)(equalTo(exampleBigDecimalStr)) } ), + test("BigInt") { + assert(BigInt("-1").toJson)(equalTo("-1")) && + assert(BigInt("-316873037158841").toJson)(equalTo("-316873037158841")) && + assert(BigInt("1701411834604692317316873037158841").toJson)(equalTo("1701411834604692317316873037158841")) + }, + test("BigDecimal") { + assert(BigDecimal("-1.0").toJson)(equalTo("-1.0")) && + assert(BigDecimal("1.0E+5").toJson)(equalTo("1.0E+5")) && + assert(BigDecimal("0.000100").toJson)(equalTo("0.000100")) && + assert(BigDecimal("0.000001").toJson)(equalTo("0.000001")) && + assert(BigDecimal("100000.00").toJson)(equalTo("100000.00")) && + assert(BigDecimal("1E-2147483647").toJson)(equalTo("1E-2147483647")) && + assert(BigDecimal("1E+2147483647").toJson)(equalTo("1E+2147483647")) && + assert(BigDecimal("-234316873037.008841").toJson)(equalTo("-234316873037.008841")) && + assert(BigDecimal("141183460469231731687303715.8841").toJson)(equalTo("141183460469231731687303715.8841")) && + assert(BigDecimal("1.7014118346046923173168730E+119").toJson)(equalTo("1.7014118346046923173168730E+119")) && + assert( + BigDecimal("-9.999999999999874791608720182523363282786709588281885514820801359042815031E-4571018").toJson + )(equalTo("-9.999999999999874791608720182523363282786709588281885514820801359042815031E-4571018")) + }, test("options") { assert((None: Option[Int]).toJson)(equalTo("null")) && assert((Some(1): Option[Int]).toJson)(equalTo("1")) diff --git a/zio-json/shared/src/test/scala/zio/json/Gens.scala b/zio-json/shared/src/test/scala/zio/json/Gens.scala index 417956559..1e72df0a3 100644 --- a/zio-json/shared/src/test/scala/zio/json/Gens.scala +++ b/zio-json/shared/src/test/scala/zio/json/Gens.scala @@ -14,10 +14,13 @@ object Gens { .filter(_.bitLength < 256) val genBigDecimal = - Gen - .bigDecimal((BigDecimal(2).pow(256) - 1) * -1, BigDecimal(2).pow(256) - 1) - .map(_.bigDecimal) - .filter(_.unscaledValue.bitLength < 256) + for { + unscaled <- Gen + .bigInt((BigInt(2).pow(256) - 1) * -1, BigInt(2).pow(256) - 1) + .map(_.bigInteger) + .filter(_.bitLength < 256) + scale <- Gen.oneOf(Gen.int(-20, 20), Gen.int(-1000000000, 1000000000)) + } yield new java.math.BigDecimal(unscaled, scale) val genUsAsciiString = Gen.string(Gen.oneOf(Gen.char('!', '~'))) diff --git a/zio-json/shared/src/test/scala/zio/json/RoundTripSpec.scala b/zio-json/shared/src/test/scala/zio/json/RoundTripSpec.scala index dcca2473e..3e62faf98 100644 --- a/zio-json/shared/src/test/scala/zio/json/RoundTripSpec.scala +++ b/zio-json/shared/src/test/scala/zio/json/RoundTripSpec.scala @@ -30,6 +30,9 @@ object RoundTripSpec extends ZIOSpecDefault { test("bigInts") { check(genBigInteger)(assertRoundtrips[java.math.BigInteger]) } @@ jvm(samples(10000)), + test("bigDecimals") { + check(genBigDecimal)(assertRoundtrips[java.math.BigDecimal]) + } @@ jvm(samples(10000)), test("floats") { // NaN / Infinity is tested manually, because of == semantics check(Gen.float.filter(java.lang.Float.isFinite))(assertRoundtrips[Float])