Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

More efficient encoding of BigDecimal and java.math.BigDecimal values #1324

Merged
merged 1 commit into from
Feb 18, 2025
Merged

More efficient encoding of BigDecimal and java.math.BigDecimal values #1324

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
126 changes: 123 additions & 3 deletions zio-json/js/src/main/scala/zio/json/internal/SafeNumbers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ object SafeNumbers {
try Some(UnsafeNumbers.bigDecimal(num, max_bits))
catch { case _: UnexpectedEnd | UnsafeNumber => None }

def toString(x: java.math.BigDecimal): String = {
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: java.math.BigInteger): String = {
val out = writes.get
write(x, out)
Expand All @@ -102,6 +108,121 @@ object SafeNumbers {
out.buffer.toString
}

def write(x: java.math.BigDecimal, out: Write): Unit = {
var exp = writeBigDecimal(x.unscaledValue, x.scale, 0, null, out)
if (exp != 0) {
var sc = '+'
if (exp < 0) {
sc = '-'
exp = -exp
}
out.write('E', sc)
writeMantissa(exp, out)
}
}

private[this] def writeBigDecimal(
x: java.math.BigInteger,
scale: Int,
blockScale: Int,
ss: Array[java.math.BigInteger],
out: Write
): Int = {
val bitLen = x.bitLength
if (bitLen < 64) {
val v = x.longValue
val pv = Math.abs(v)
val digits =
if (pv >= 100000000000000000L) {
if (pv >= 1000000000000000000L) 19
else 18
} else digitCount(pv)
val dotOff = scale - blockScale
val exp = (digits - 1) - dotOff
if (scale >= 0 && exp >= -6) {
if (exp < 0) {
out.write('0', '.')
var zeros = -exp - 1
while (zeros > 0) {
out.write('0')
zeros -= 1
}
write(v, out)
} else if (dotOff > 0) writeLongWithDot(v, dotOff, out)
else write(v, out)
0
} else {
if (digits > 1) writeLongWithDot(v, digits - 1, out)
else {
write(v, out)
if (blockScale > 0) out.write('.')
}
exp
}
} else {
val n = calculateTenPow18SquareNumber(bitLen)
val ss1 =
if (ss eq null) getTenPow18Squares(n)
else ss
val qr = x.divideAndRemainder(ss1(n))
val exp = writeBigDecimal(qr(0), scale, (18 << n) + blockScale, ss1, out)
writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss1, out)
exp
}
}

@inline private[this] def writeLongWithDot(v: Long, dotOff: Int, out: Write): Unit = {
val pow10 = pow10longs(dotOff)
val q = v / pow10
val r = Math.abs(v - q * pow10)
write(q, out)
out.write('.')
var zeros = dotOff - digitCount(r)
while (zeros > 0) {
out.write('0')
zeros -= 1
}
write(r, out)
}

private[this] def writeBigDecimalRemainder(
x: java.math.BigInteger,
scale: Int,
blockScale: Int,
n: Int,
ss: Array[java.math.BigInteger],
out: Write
): Unit =
if (n < 0) {
val v = Math.abs(x.longValue)
var dotOff = scale - blockScale
if (dotOff > 0 && dotOff < 18) {
val pow10 = pow10longs(dotOff)
val q = v / pow10
val r = v - q * pow10
var zeros = 18 - dotOff - digitCount(q)
while (zeros > 0) {
out.write('0')
zeros -= 1
}
writeMantissa(q, out)
out.write('.')
dotOff -= digitCount(r)
while (dotOff > 0) {
out.write('0')
dotOff -= 1
}
writeMantissa(r, out)
} else {
if (dotOff == 18) out.write('.')
write18Digits(v, out)
}
} else {
val qr = x.divideAndRemainder(ss(n))
writeBigDecimalRemainder(qr(0), scale, (18 << n) + blockScale, n - 1, ss, out)
writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss, out)
}

def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out)

private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = {
Expand Down Expand Up @@ -660,8 +781,7 @@ object SafeNumbers {
@inline private[json] def write2Digits(x: Int, out: Write): Unit =
out.write(digits(x))

@inline
private[this] def digitCount(x: Long): Int =
@inline private[this] def digitCount(x: Long): Int =
if (x >= 1000000000000000L) {
if (x >= 10000000000000000L) 17
else 16
Expand Down Expand Up @@ -975,7 +1095,7 @@ object SafeNumbers {
private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)
100000000000000000L, 1000000000000000000L)

@volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] =
Array(java.math.BigInteger.valueOf(1000000000000000000L))
Expand Down
123 changes: 122 additions & 1 deletion zio-json/jvm-native/src/main/scala/zio/json/internal/SafeNumbers.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ object SafeNumbers {
try Some(UnsafeNumbers.bigDecimal(num, max_bits))
catch { case _: UnexpectedEnd | UnsafeNumber => None }

def toString(x: java.math.BigDecimal): String = {
val out = writes.get
write(x, out)
out.buffer.toString
}

def toString(x: java.math.BigInteger): String = {
val out = writes.get
write(x, out)
Expand All @@ -102,6 +108,121 @@ object SafeNumbers {
out.buffer.toString
}

def write(x: java.math.BigDecimal, out: Write): Unit = {
var exp = writeBigDecimal(x.unscaledValue, x.scale, 0, null, out)
if (exp != 0) {
var sc = '+'
if (exp < 0) {
sc = '-'
exp = -exp
}
out.write('E', sc)
writeMantissa(exp, out)
}
}

private[this] def writeBigDecimal(
x: java.math.BigInteger,
scale: Int,
blockScale: Int,
ss: Array[java.math.BigInteger],
out: Write
): Int = {
val bitLen = x.bitLength
if (bitLen < 64) {
val v = x.longValue
val pv = Math.abs(v)
val digits =
if (pv >= 100000000000000000L) {
if (pv >= 1000000000000000000L) 19
else 18
} else digitCount(pv)
val dotOff = scale - blockScale
val exp = (digits - 1) - dotOff
if (scale >= 0 && exp >= -6) {
if (exp < 0) {
out.write('0', '.')
var zeros = -exp - 1
while (zeros > 0) {
out.write('0')
zeros -= 1
}
write(v, out)
} else if (dotOff > 0) writeLongWithDot(v, dotOff, out)
else write(v, out)
0
} else {
if (digits > 1) writeLongWithDot(v, digits - 1, out)
else {
write(v, out)
if (blockScale > 0) out.write('.')
}
exp
}
} else {
val n = calculateTenPow18SquareNumber(bitLen)
val ss1 =
if (ss eq null) getTenPow18Squares(n)
else ss
val qr = x.divideAndRemainder(ss1(n))
val exp = writeBigDecimal(qr(0), scale, (18 << n) + blockScale, ss1, out)
writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss1, out)
exp
}
}

private[this] def writeLongWithDot(v: Long, dotOff: Int, out: Write): Unit = {
val pow10 = pow10longs(dotOff)
val q = v / pow10
val r = Math.abs(v - q * pow10)
write(q, out)
out.write('.')
var zeros = dotOff - digitCount(r)
while (zeros > 0) {
out.write('0')
zeros -= 1
}
write(r, out)
}

private[this] def writeBigDecimalRemainder(
x: java.math.BigInteger,
scale: Int,
blockScale: Int,
n: Int,
ss: Array[java.math.BigInteger],
out: Write
): Unit =
if (n < 0) {
val v = Math.abs(x.longValue)
var dotOff = scale - blockScale
if (dotOff > 0 && dotOff < 18) {
val pow10 = pow10longs(dotOff)
val q = v / pow10
val r = v - q * pow10
var zeros = 18 - dotOff - digitCount(q)
while (zeros > 0) {
out.write('0')
zeros -= 1
}
writeMantissa(q, out)
out.write('.')
dotOff -= digitCount(r)
while (dotOff > 0) {
out.write('0')
dotOff -= 1
}
writeMantissa(r, out)
} else {
if (dotOff == 18) out.write('.')
write18Digits(v, out)
}
} else {
val qr = x.divideAndRemainder(ss(n))
writeBigDecimalRemainder(qr(0), scale, (18 << n) + blockScale, n - 1, ss, out)
writeBigDecimalRemainder(qr(1), scale, blockScale, n - 1, ss, out)
}

def write(x: java.math.BigInteger, out: Write): Unit = writeBigInteger(x, null, out)

private[this] def writeBigInteger(x: java.math.BigInteger, ss: Array[java.math.BigInteger], out: Write): Unit = {
Expand Down Expand Up @@ -911,7 +1032,7 @@ object SafeNumbers {
private[this] final val pow10longs: Array[Long] =
Array(1L, 10L, 100L, 1000L, 10000L, 100000L, 1000000L, 10000000L, 100000000L, 1000000000L, 10000000000L,
100000000000L, 1000000000000L, 10000000000000L, 100000000000000L, 1000000000000000L, 10000000000000000L,
100000000000000000L)
100000000000000000L, 1000000000000000000L)

@volatile private[this] var tenPow18Squares: Array[java.math.BigInteger] =
Array(java.math.BigInteger.valueOf(1000000000000000000L))
Expand Down
16 changes: 12 additions & 4 deletions zio-json/shared/src/main/scala/zio/json/JsonEncoder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -215,12 +215,12 @@ object JsonEncoder extends GeneratedTupleEncoders with EncoderLowPriority1 with
implicit val byte: JsonEncoder[Byte] = new JsonEncoder[Byte] {
def unsafeEncode(a: Byte, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.toInt, out)

override def toJsonAST(a: Byte): Either[String, Json] = new Right(Json.Num(a))
override def toJsonAST(a: Byte): Either[String, Json] = new Right(Json.Num(a.toInt))
}
implicit val short: JsonEncoder[Short] = new JsonEncoder[Short] {
def unsafeEncode(a: Short, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.toInt, out)

override def toJsonAST(a: Short): Either[String, Json] = new Right(Json.Num(a))
override def toJsonAST(a: Short): Either[String, Json] = new Right(Json.Num(a.toInt))
}
implicit val int: JsonEncoder[Int] = new JsonEncoder[Int] {
def unsafeEncode(a: Int, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a, out)
Expand Down Expand Up @@ -254,8 +254,16 @@ object JsonEncoder extends GeneratedTupleEncoders with EncoderLowPriority1 with

override def toJsonAST(a: Float): Either[String, Json] = new Right(Json.Num(a))
}
implicit val bigDecimal: JsonEncoder[java.math.BigDecimal] = explicit(_.toString, n => new Json.Num(n))
implicit val scalaBigDecimal: JsonEncoder[BigDecimal] = explicit(_.toString, Json.Num.apply)
implicit val bigDecimal: JsonEncoder[java.math.BigDecimal] = new JsonEncoder[java.math.BigDecimal] {
def unsafeEncode(a: java.math.BigDecimal, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a, out)

override def toJsonAST(a: java.math.BigDecimal): Either[String, Json] = new Right(new Json.Num(a))
}
implicit val scalaBigDecimal: JsonEncoder[BigDecimal] = new JsonEncoder[BigDecimal] {
def unsafeEncode(a: BigDecimal, indent: Option[Int], out: Write): Unit = SafeNumbers.write(a.bigDecimal, out)

override def toJsonAST(a: BigDecimal): Either[String, Json] = new Right(new Json.Num(a.bigDecimal))
}

implicit def option[A](implicit A: JsonEncoder[A]): JsonEncoder[Option[A]] = new JsonEncoder[Option[A]] {
def unsafeEncode(oa: Option[A], indent: Option[Int], out: Write): Unit =
Expand Down
32 changes: 20 additions & 12 deletions zio-json/shared/src/test/scala/zio/json/EncoderSpec.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,20 +245,28 @@ object EncoderSpec extends ZIOSpecDefault {
assert((-6939.0464d).toJson)(
equalTo("-6939.0464")
) // See the issue: https://github.com/zio/zio-json/pull/375
},
test("other numerics") {
val exampleBigIntStr = "170141183460469231731687303715884105728"
val exampleBigDecimalStr = "170141183460469231731687303715884105728.4433"
assert((1: Byte).toJson)(equalTo("1")) &&
assert((1: Short).toJson)(equalTo("1")) &&
assert((1: Int).toJson)(equalTo("1")) &&
assert(1L.toJson)(equalTo("1")) &&
assert(new java.math.BigInteger("1").toJson)(equalTo("1")) &&
assert(new java.math.BigInteger(exampleBigIntStr).toJson)(equalTo(exampleBigIntStr)) &&
assert(BigInt(exampleBigIntStr).toJson)(equalTo(exampleBigIntStr)) &&
assert(BigDecimal(exampleBigDecimalStr).toJson)(equalTo(exampleBigDecimalStr))
}
),
test("BigInt") {
assert(BigInt("-1").toJson)(equalTo("-1")) &&
assert(BigInt("-316873037158841").toJson)(equalTo("-316873037158841")) &&
assert(BigInt("1701411834604692317316873037158841").toJson)(equalTo("1701411834604692317316873037158841"))
},
test("BigDecimal") {
assert(BigDecimal("-1.0").toJson)(equalTo("-1.0")) &&
assert(BigDecimal("1.0E+5").toJson)(equalTo("1.0E+5")) &&
assert(BigDecimal("0.000100").toJson)(equalTo("0.000100")) &&
assert(BigDecimal("0.000001").toJson)(equalTo("0.000001")) &&
assert(BigDecimal("100000.00").toJson)(equalTo("100000.00")) &&
assert(BigDecimal("1E-2147483647").toJson)(equalTo("1E-2147483647")) &&
assert(BigDecimal("1E+2147483647").toJson)(equalTo("1E+2147483647")) &&
assert(BigDecimal("-234316873037.008841").toJson)(equalTo("-234316873037.008841")) &&
assert(BigDecimal("141183460469231731687303715.8841").toJson)(equalTo("141183460469231731687303715.8841")) &&
assert(BigDecimal("1.7014118346046923173168730E+119").toJson)(equalTo("1.7014118346046923173168730E+119")) &&
assert(
BigDecimal("-9.999999999999874791608720182523363282786709588281885514820801359042815031E-4571018").toJson
)(equalTo("-9.999999999999874791608720182523363282786709588281885514820801359042815031E-4571018"))
},
test("options") {
assert((None: Option[Int]).toJson)(equalTo("null")) &&
assert((Some(1): Option[Int]).toJson)(equalTo("1"))
Expand Down
11 changes: 7 additions & 4 deletions zio-json/shared/src/test/scala/zio/json/Gens.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,13 @@ object Gens {
.filter(_.bitLength < 256)

val genBigDecimal =
Gen
.bigDecimal((BigDecimal(2).pow(256) - 1) * -1, BigDecimal(2).pow(256) - 1)
.map(_.bigDecimal)
.filter(_.unscaledValue.bitLength < 256)
for {
unscaled <- Gen
.bigInt((BigInt(2).pow(256) - 1) * -1, BigInt(2).pow(256) - 1)
.map(_.bigInteger)
.filter(_.bitLength < 256)
scale <- Gen.oneOf(Gen.int(-20, 20), Gen.int(-1000000000, 1000000000))
} yield new java.math.BigDecimal(unscaled, scale)

val genUsAsciiString =
Gen.string(Gen.oneOf(Gen.char('!', '~')))
Expand Down
Loading
Loading