From bb5f382703d5e8a429793314c8e3eef344f50466 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Sep 2022 06:32:02 -0500 Subject: [PATCH 01/50] relaxed marshaling, slow base 10 parsing --- conversion.go | 28 +++++++++- go.mod | 2 +- stringscan.go | 99 +++++++++++++++++++++++++++++++++++ stringscan_test.go | 125 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 252 insertions(+), 2 deletions(-) create mode 100644 stringscan.go create mode 100644 stringscan_test.go diff --git a/conversion.go b/conversion.go index f1530f1..c010af7 100644 --- a/conversion.go +++ b/conversion.go @@ -5,6 +5,7 @@ package uint256 import ( + "database/sql/driver" "encoding/binary" "errors" "fmt" @@ -98,6 +99,9 @@ func (z *Int) UnmarshalText(input []byte) error { // SetFromBig converts a big.Int to Int and sets the value to z. // TODO: Ensure we have sufficient testing, esp for negative bigints. func (z *Int) SetFromBig(b *big.Int) bool { + if b == nil { + return true + } z.Clear() words := b.Bits() overflow := len(words) > maxWords @@ -147,7 +151,6 @@ func (z *Int) SetFromBig(b *big.Int) bool { // specification of minimum digits precision, output field // width, space or zero padding, and '-' for left or right // justification. -// func (z *Int) Format(s fmt.State, ch rune) { z.ToBig().Format(s, ch) } @@ -525,6 +528,29 @@ func (z *Int) Hex() string { return string(output[64-nibbles:]) } +// Scan implements the database/sql Scanner interface. +// It decodes a string, because that is what postgres uses for its numeric type +func (dst *Int) Scan(src interface{}) error { + if src == nil { + *dst = Int{} + } + + switch src := src.(type) { + case string: + return dst.SetString(src, 0) + case []byte: + return dst.SetString(string(src), 0) + } + + return fmt.Errorf("cannot scan %T", src) +} + +// Value implements the database/sql/driver Valuer interface. +// It encodes a string, because that is what postgres uses for its numeric type +func (src Int) Value() (driver.Value, error) { + return string(src.ToBig().String()) + "e0", nil +} + var ( ErrEmptyString = errors.New("empty hex string") ErrSyntax = errors.New("invalid hex string") diff --git a/go.mod b/go.mod index a38ade7..92c444c 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/holiman/uint256 -go 1.13 +go 1.19 diff --git a/stringscan.go b/stringscan.go new file mode 100644 index 0000000..a898365 --- /dev/null +++ b/stringscan.go @@ -0,0 +1,99 @@ +package uint256 + +import ( + "strconv" + "strings" +) + +const twoPow256 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" +const twoPow128 = "340282366920938463463374607431768211456" +const twoPow64 = "18446744073709551616" + +func (z *Int) SetString(b string, base int) (err error) { + switch base { + case 0: + if strings.HasPrefix(b, "0x") { + return z.fromHex(b) + } + return z.FromBase10(b) + case 10: + return z.FromBase10(b) + case 16: + return z.fromHex(b) + } + return ErrSyntax +} +func (z *Int) FromBase10(s string) (err error) { + if len(s) < len(twoPow256) { + return z.fromBase10Long(s) + } + if len(s) == len(twoPow256) { + if s[0] > '1' { + return ErrBig256Range + } + return z.fromBase10Long(s) + } + return ErrBig256Range +} + +var scaleTable10 [78]*Int + +func init() { + for k := range scaleTable10 { + scaleTable10[k] = new(Int) + scaleTable10[k] = scaleTable10[k].Exp(NewInt(10), NewInt(uint64(k))) + } +} + +func (z *Int) fromBase10Long(bs string) error { + z[0] = 0 + z[1] = 0 + z[2] = 0 + z[3] = 0 + if len(bs) == 0 { + return nil + } + iv := 19 + c := 0 + if len(bs) >= (iv * 4) { + nm, err := strconv.Atoi(bs[c:(c + iv)]) + if err != nil { + return err + } + z.Add(z, new(Int).Mul(scaleTable10[len(bs)-iv], NewInt(uint64(nm)))) + c = c + iv + } + if len(bs) >= (iv * 3) { + nm, err := strconv.Atoi(bs[c:(c + iv)]) + if err != nil { + return err + } + z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + c = c + iv + } + if len(bs) >= (iv * 2) { + nm, err := strconv.Atoi(bs[c:(c + iv)]) + if err != nil { + return err + } + z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + c = c + iv + } + if len(bs) >= (iv * 1) { + nm, err := strconv.Atoi(bs[c:(c + iv)]) + if err != nil { + return err + } + z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + c = c + iv + } + if len(bs) == c { + return nil + } + nm, err := strconv.Atoi(bs[c:]) + if err != nil { + return err + } + z.AddUint64(z, uint64(nm)) + return nil +} diff --git a/stringscan_test.go b/stringscan_test.go new file mode 100644 index 0000000..98bc19b --- /dev/null +++ b/stringscan_test.go @@ -0,0 +1,125 @@ +package uint256 + +import ( + "errors" + "testing" +) + +func TestStringScanBase10(t *testing.T) { + z := new(Int) + + type testCase struct { + i string + err error + } + + cases := []testCase{ + {i: twoPow256 + "1", err: ErrBig256Range}, + {i: "2" + twoPow256[1:], err: ErrBig256Range}, + {i: twoPow256[1:]}, + {i: twoPow128}, + {i: twoPow128 + "1"}, + {i: twoPow128[1:]}, + {i: twoPow64 + "1"}, + {i: twoPow64[1:]}, + } + + for _, v := range cases { + err := z.FromBase10(v.i) + if !errors.Is(err, v.err) { + t.Errorf("expect err %s, got %s", v.err, err) + } + if err == nil { + got := z.ToBig().String() + if got != v.i { + t.Errorf("expect val %s, got %s", v.i, got) + } + } + } +} + +func BenchmarkStringBase10(b *testing.B) { + val := new(Int) + bytearr := twoPow256 + b.Run("generic", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + val.SetString(bytearr[:2], 10) + val.SetString(bytearr[:4], 10) + val.SetString(bytearr[:6], 10) + val.SetString(bytearr[:8], 10) + val.SetString(bytearr[:16], 10) + val.SetString(bytearr[:12], 10) + val.SetString(bytearr[:14], 10) + val.SetString(bytearr[:16], 10) + val.SetString(bytearr[:18], 10) + val.SetString(bytearr[:20], 10) + val.SetString(bytearr[:22], 10) + val.SetString(bytearr[:24], 10) + val.SetString(bytearr[:26], 10) + val.SetString(bytearr[:28], 10) + val.SetString(bytearr[:30], 10) + val.SetString(bytearr[:32], 10) + val.SetString(bytearr[:34], 10) + val.SetString(bytearr[:36], 10) + val.SetString(bytearr[:38], 10) + val.SetString(bytearr[:40], 10) + val.SetString(bytearr[:42], 10) + val.SetString(bytearr[:44], 10) + val.SetString(bytearr[:46], 10) + val.SetString(bytearr[:48], 10) + val.SetString(bytearr[:50], 10) + val.SetString(bytearr[:52], 10) + val.SetString(bytearr[:54], 10) + val.SetString(bytearr[:56], 10) + val.SetString(bytearr[:58], 10) + val.SetString(bytearr[:60], 10) + val.SetString(bytearr[:62], 10) + val.SetString(bytearr[:64], 10) + } + }) +} + +func BenchmarkStringBase16(b *testing.B) { + val := new(Int) + bytearr := "aaaa12131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031bbbb" + b.Run("generic", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + val.SetString(bytearr[:1], 16) + val.SetString(bytearr[:2], 16) + val.SetString(bytearr[:3], 16) + val.SetString(bytearr[:4], 16) + val.SetString(bytearr[:5], 16) + val.SetString(bytearr[:6], 16) + val.SetString(bytearr[:7], 16) + val.SetString(bytearr[:8], 16) + val.SetString(bytearr[:9], 16) + val.SetString(bytearr[:10], 16) + val.SetString(bytearr[:11], 16) + val.SetString(bytearr[:12], 16) + val.SetString(bytearr[:13], 16) + val.SetString(bytearr[:14], 16) + val.SetString(bytearr[:15], 16) + val.SetString(bytearr[:16], 16) + val.SetString(bytearr[:17], 16) + val.SetString(bytearr[:18], 16) + val.SetString(bytearr[:19], 16) + val.SetString(bytearr[:20], 16) + val.SetString(bytearr[:21], 16) + val.SetString(bytearr[:22], 16) + val.SetString(bytearr[:23], 16) + val.SetString(bytearr[:24], 16) + val.SetString(bytearr[:25], 16) + val.SetString(bytearr[:26], 16) + val.SetString(bytearr[:27], 16) + val.SetString(bytearr[:28], 16) + val.SetString(bytearr[:29], 16) + val.SetString(bytearr[:20], 16) + val.SetString(bytearr[:31], 16) + val.SetString(bytearr[:32], 16) + } + }) +} From fb1229552ed3a87e51b7b3f6f030aff6e7ae5239 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Sep 2022 06:44:23 -0500 Subject: [PATCH 02/50] update version --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 92c444c..a38ade7 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/holiman/uint256 -go 1.19 +go 1.13 From 68669651f568732c24826b36ded89bbbf7a20579 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Sep 2022 06:51:40 -0500 Subject: [PATCH 03/50] add control to benchmark --- stringscan.go => base10.go | 0 stringscan_test.go => base10_test.go | 45 +++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 1 deletion(-) rename stringscan.go => base10.go (100%) rename stringscan_test.go => base10_test.go (71%) diff --git a/stringscan.go b/base10.go similarity index 100% rename from stringscan.go rename to base10.go diff --git a/stringscan_test.go b/base10_test.go similarity index 71% rename from stringscan_test.go rename to base10_test.go index 98bc19b..e220a4e 100644 --- a/stringscan_test.go +++ b/base10_test.go @@ -2,6 +2,7 @@ package uint256 import ( "errors" + "math/big" "testing" ) @@ -38,6 +39,49 @@ func TestStringScanBase10(t *testing.T) { } } +func BenchmarkStringBase10BigInt(b *testing.B) { + val := new(big.Int) + bytearr := twoPow256 + b.Run("generic", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + val.SetString(bytearr[:2], 10) + val.SetString(bytearr[:4], 10) + val.SetString(bytearr[:6], 10) + val.SetString(bytearr[:8], 10) + val.SetString(bytearr[:16], 10) + val.SetString(bytearr[:12], 10) + val.SetString(bytearr[:14], 10) + val.SetString(bytearr[:16], 10) + val.SetString(bytearr[:18], 10) + val.SetString(bytearr[:20], 10) + val.SetString(bytearr[:22], 10) + val.SetString(bytearr[:24], 10) + val.SetString(bytearr[:26], 10) + val.SetString(bytearr[:28], 10) + val.SetString(bytearr[:30], 10) + val.SetString(bytearr[:32], 10) + val.SetString(bytearr[:34], 10) + val.SetString(bytearr[:36], 10) + val.SetString(bytearr[:38], 10) + val.SetString(bytearr[:40], 10) + val.SetString(bytearr[:42], 10) + val.SetString(bytearr[:44], 10) + val.SetString(bytearr[:46], 10) + val.SetString(bytearr[:48], 10) + val.SetString(bytearr[:50], 10) + val.SetString(bytearr[:52], 10) + val.SetString(bytearr[:54], 10) + val.SetString(bytearr[:56], 10) + val.SetString(bytearr[:58], 10) + val.SetString(bytearr[:60], 10) + val.SetString(bytearr[:62], 10) + val.SetString(bytearr[:64], 10) + } + }) +} + func BenchmarkStringBase10(b *testing.B) { val := new(Int) bytearr := twoPow256 @@ -80,7 +124,6 @@ func BenchmarkStringBase10(b *testing.B) { } }) } - func BenchmarkStringBase16(b *testing.B) { val := new(Int) bytearr := "aaaa12131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031bbbb" From 55f7ffc4a6567a5a43579040fb7c0e912246ce60 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Sep 2022 07:15:49 -0500 Subject: [PATCH 04/50] make the ci thing less angry --- base10.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base10.go b/base10.go index a898365..cf4e0c9 100644 --- a/base10.go +++ b/base10.go @@ -50,7 +50,7 @@ func (z *Int) fromBase10Long(bs string) error { z[1] = 0 z[2] = 0 z[3] = 0 - if len(bs) == 0 { + if bs == "" { return nil } iv := 19 From 56893159aa81a1b9189e377381068f761af6c314 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Sep 2022 07:17:48 -0500 Subject: [PATCH 05/50] make the ci even less angy --- base10.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/base10.go b/base10.go index cf4e0c9..96d966d 100644 --- a/base10.go +++ b/base10.go @@ -56,7 +56,7 @@ func (z *Int) fromBase10Long(bs string) error { iv := 19 c := 0 if len(bs) >= (iv * 4) { - nm, err := strconv.Atoi(bs[c:(c + iv)]) + nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { return err } @@ -64,7 +64,7 @@ func (z *Int) fromBase10Long(bs string) error { c = c + iv } if len(bs) >= (iv * 3) { - nm, err := strconv.Atoi(bs[c:(c + iv)]) + nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { return err } @@ -72,7 +72,7 @@ func (z *Int) fromBase10Long(bs string) error { c = c + iv } if len(bs) >= (iv * 2) { - nm, err := strconv.Atoi(bs[c:(c + iv)]) + nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { return err } @@ -80,7 +80,7 @@ func (z *Int) fromBase10Long(bs string) error { c = c + iv } if len(bs) >= (iv * 1) { - nm, err := strconv.Atoi(bs[c:(c + iv)]) + nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { return err } @@ -90,7 +90,7 @@ func (z *Int) fromBase10Long(bs string) error { if len(bs) == c { return nil } - nm, err := strconv.Atoi(bs[c:]) + nm, err := strconv.ParseUint(bs[c:], 10, 64) if err != nil { return err } From 3c7131f2fe312f3da49f1879e94ad7c1f68e3aff Mon Sep 17 00:00:00 2001 From: a Date: Sun, 27 Nov 2022 19:49:09 -0600 Subject: [PATCH 06/50] add fuzz tests --- base10.go | 54 +++++++++++++++++------- base10_test.go | 110 ++++++++++++++++++++++++++++++++++++++++++++++--- conversion.go | 33 +++++++++++++-- 3 files changed, 171 insertions(+), 26 deletions(-) diff --git a/base10.go b/base10.go index 96d966d..aa3fafa 100644 --- a/base10.go +++ b/base10.go @@ -5,30 +5,52 @@ import ( "strings" ) -const twoPow256 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" +const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" const twoPow128 = "340282366920938463463374607431768211456" const twoPow64 = "18446744073709551616" -func (z *Int) SetString(b string, base int) (err error) { +func (z *Int) Base10() string { + return z.ToBig().String() +} + +// SetString implements a subset of (*big.Int).SetString +// ok will be true iff i == nil +func (z *Int) SetString(s string, base int) (i *Int, ok bool) { switch base { case 0: - if strings.HasPrefix(b, "0x") { - return z.fromHex(b) + if strings.HasPrefix(s, "0x") { + err := z.fromHex(s) + if err != nil { + return nil, false + } + return z, true + } + err := z.FromBase10(s) + if err != nil { + return nil, false } - return z.FromBase10(b) + return z, true case 10: - return z.FromBase10(b) + err := z.FromBase10(s) + if err != nil { + return nil, false + } + return z, true case 16: - return z.fromHex(b) + err := z.fromHex(s) + if err != nil { + return nil, false + } + return z, true } - return ErrSyntax + return nil, false } func (z *Int) FromBase10(s string) (err error) { - if len(s) < len(twoPow256) { + if len(s) < len(twoPow256Sub1) { return z.fromBase10Long(s) } - if len(s) == len(twoPow256) { - if s[0] > '1' { + if len(s) == len(twoPow256Sub1) { + if s > twoPow256Sub1 { return ErrBig256Range } return z.fromBase10Long(s) @@ -58,7 +80,7 @@ func (z *Int) fromBase10Long(bs string) error { if len(bs) >= (iv * 4) { nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { - return err + return ErrSyntaxBase10 } z.Add(z, new(Int).Mul(scaleTable10[len(bs)-iv], NewInt(uint64(nm)))) c = c + iv @@ -66,7 +88,7 @@ func (z *Int) fromBase10Long(bs string) error { if len(bs) >= (iv * 3) { nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { - return err + return ErrSyntaxBase10 } z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv @@ -74,7 +96,7 @@ func (z *Int) fromBase10Long(bs string) error { if len(bs) >= (iv * 2) { nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { - return err + return ErrSyntaxBase10 } z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv @@ -82,7 +104,7 @@ func (z *Int) fromBase10Long(bs string) error { if len(bs) >= (iv * 1) { nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) if err != nil { - return err + return ErrSyntaxBase10 } z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv @@ -92,7 +114,7 @@ func (z *Int) fromBase10Long(bs string) error { } nm, err := strconv.ParseUint(bs[c:], 10, 64) if err != nil { - return err + return ErrSyntaxBase10 } z.AddUint64(z, uint64(nm)) return nil diff --git a/base10_test.go b/base10_test.go index e220a4e..996da66 100644 --- a/base10_test.go +++ b/base10_test.go @@ -2,6 +2,7 @@ package uint256 import ( "errors" + "fmt" "math/big" "testing" ) @@ -12,17 +13,29 @@ func TestStringScanBase10(t *testing.T) { type testCase struct { i string err error + val string } cases := []testCase{ - {i: twoPow256 + "1", err: ErrBig256Range}, - {i: "2" + twoPow256[1:], err: ErrBig256Range}, - {i: twoPow256[1:]}, + {i: twoPow256Sub1 + "1", err: ErrBig256Range}, + {i: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, + {i: twoPow256Sub1[1:]}, {i: twoPow128}, {i: twoPow128 + "1"}, {i: twoPow128[1:]}, {i: twoPow64 + "1"}, {i: twoPow64[1:]}, + {i: "banana", err: ErrSyntaxBase10}, + {i: "0xab", err: ErrSyntaxBase10}, + {i: "ab", err: ErrSyntaxBase10}, + {i: "0"}, + {i: "000", val: "0"}, + {i: "010", val: "10"}, + {i: "01", val: "1"}, + {i: "-0", err: ErrSyntaxBase10}, + {i: "-10", err: ErrSyntaxBase10}, + {i: "115792089237316195423570985008687907853269984665640564039457584007913129639936", err: ErrBig256Range}, + {i: "115792089237316195423570985008687907853269984665640564039457584007913129639935"}, } for _, v := range cases { @@ -32,16 +45,101 @@ func TestStringScanBase10(t *testing.T) { } if err == nil { got := z.ToBig().String() - if got != v.i { + want := v.i + if v.val != "" { + want = v.val + } + if got != want { t.Errorf("expect val %s, got %s", v.i, got) } } } } +func FuzzBase10StringCompare(f *testing.F) { + bi := new(big.Int) + z := new(Int) + max256 := new(Int) + max256.FromBase10(twoPow256Sub1) + testcase := []string{ + twoPow256Sub1 + "1", + "2" + twoPow256Sub1[1:], + twoPow256Sub1, + twoPow128, + twoPow128, + twoPow128, + twoPow64, + twoPow64, + "banana", + "0xab", + "ab", + "0", + "000", + "010", + "01", + "-0", + "-10", + "115792089237316195423570985008687907853269984665640564039457584007913129639936", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "apple", + "04112401274120741204712xxxxxz00", + "0x10101011010", + "熊熊熊熊熊熊熊熊", + } + for _, tc := range testcase { + f.Add(tc) + } + f.Fuzz(func(t *testing.T, orig string) { + err := z.FromBase10(orig) + val, ok := bi.SetString(orig, 10) + // if fail, make sure that we failed too + if !ok { + if err == nil { + t.Errorf("expected base 10 parse to fail: %s", orig) + } + return + } + // if its negative number, we should err + if len(orig) > 0 && (orig[0] == '-') { + if !errors.Is(err, ErrSyntaxBase10) { + t.Errorf("should have errored at negative number: %s", orig) + } + return + } + // if its too large, ignore it also + if val.Cmp(max256.ToBig()) > 0 { + return + } + // so here, if it errors, it means that we failed + if err != nil { + t.Errorf("should have parsed %s to %s, but err'd instead", orig, val.String()) + return + } + // otherwise, make sure that the values are equal + if z.ToBig().Cmp(bi) != 0 { + t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Base10()) + return + } + // make sure that bigint base 10 string is equal to base10 string + if z.Base10() != bi.String() { + t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Base10()) + return + } + value, err := z.Value() + if err != nil { + t.Errorf("fail to Value() %s, got err %s", val, err) + return + } + if z.Base10()+"e0" != fmt.Sprint(value) { + t.Errorf("value of %s did not match base 10 encoding %s", value, z.Base10()) + return + } + }) +} + func BenchmarkStringBase10BigInt(b *testing.B) { val := new(big.Int) - bytearr := twoPow256 + bytearr := twoPow256Sub1 b.Run("generic", func(b *testing.B) { b.ReportAllocs() b.ResetTimer() @@ -84,7 +182,7 @@ func BenchmarkStringBase10BigInt(b *testing.B) { func BenchmarkStringBase10(b *testing.B) { val := new(Int) - bytearr := twoPow256 + bytearr := twoPow256Sub1 b.Run("generic", func(b *testing.B) { b.ReportAllocs() b.ResetTimer() diff --git a/conversion.go b/conversion.go index c010af7..3632cb2 100644 --- a/conversion.go +++ b/conversion.go @@ -5,8 +5,11 @@ package uint256 import ( + "database/sql" "database/sql/driver" + "encoding" "encoding/binary" + "encoding/json" "errors" "fmt" "io" @@ -25,6 +28,16 @@ const ( _ uint = -(maxWords & ^(4 | 8)) // maxWords is 4 or 8. ) +// Compile time interface checks +var ( + _ driver.Value = (*Int)(nil) + _ sql.Scanner = (*Int)(nil) + _ encoding.TextMarshaler = (*Int)(nil) + _ encoding.TextUnmarshaler = (*Int)(nil) + _ json.Marshaler = (*Int)(nil) + _ json.Unmarshaler = (*Int)(nil) +) + // ToBig returns a big.Int version of z. func (z *Int) ToBig() *big.Int { b := new(big.Int) @@ -473,6 +486,11 @@ func (z *Int) MarshalText() ([]byte, error) { return []byte(z.Hex()), nil } +// MarshalJSON implements json.Marshaler. +func (z *Int) MarshalJSON() ([]byte, error) { + return []byte(`"` + z.Hex() + `"`), nil +} + // UnmarshalJSON implements json.Unmarshaler. func (z *Int) UnmarshalJSON(input []byte) error { if len(input) < 2 || input[0] != '"' || input[len(input)-1] != '"' { @@ -537,23 +555,30 @@ func (dst *Int) Scan(src interface{}) error { switch src := src.(type) { case string: - return dst.SetString(src, 0) + _, ok := dst.SetString(src, 0) + if !ok { + return fmt.Errorf("cannot scan %T", src) + } case []byte: - return dst.SetString(string(src), 0) + _, ok := dst.SetString(string(src), 0) + if !ok { + return fmt.Errorf("cannot scan %T", src) + } } return fmt.Errorf("cannot scan %T", src) } // Value implements the database/sql/driver Valuer interface. -// It encodes a string, because that is what postgres uses for its numeric type +// It encodes a string with an e0 suffix, telling postgresql that it is of the numeric/decimal type. func (src Int) Value() (driver.Value, error) { - return string(src.ToBig().String()) + "e0", nil + return src.ToBig().String() + "e0", nil } var ( ErrEmptyString = errors.New("empty hex string") ErrSyntax = errors.New("invalid hex string") + ErrSyntaxBase10 = errors.New("invalid base 10 string") ErrMissingPrefix = errors.New("hex string without 0x prefix") ErrEmptyNumber = errors.New("hex string \"0x\"") ErrLeadingZero = errors.New("hex number with leading zero digits") From 5a106b0ec0397625a896564b67225433bcf66cdc Mon Sep 17 00:00:00 2001 From: a Date: Sun, 27 Nov 2022 19:56:23 -0600 Subject: [PATCH 07/50] change map --- base10.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/base10.go b/base10.go index aa3fafa..a28c12e 100644 --- a/base10.go +++ b/base10.go @@ -58,12 +58,12 @@ func (z *Int) FromBase10(s string) (err error) { return ErrBig256Range } -var scaleTable10 [78]*Int +var scaleTable10 [78]Int func init() { for k := range scaleTable10 { - scaleTable10[k] = new(Int) - scaleTable10[k] = scaleTable10[k].Exp(NewInt(10), NewInt(uint64(k))) + scaleTable10[k] = Int{} + scaleTable10[k].Exp(NewInt(10), NewInt(uint64(k))) } } @@ -82,7 +82,7 @@ func (z *Int) fromBase10Long(bs string) error { if err != nil { return ErrSyntaxBase10 } - z.Add(z, new(Int).Mul(scaleTable10[len(bs)-iv], NewInt(uint64(nm)))) + z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-iv], NewInt(uint64(nm)))) c = c + iv } if len(bs) >= (iv * 3) { @@ -90,7 +90,7 @@ func (z *Int) fromBase10Long(bs string) error { if err != nil { return ErrSyntaxBase10 } - z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv } if len(bs) >= (iv * 2) { @@ -98,7 +98,7 @@ func (z *Int) fromBase10Long(bs string) error { if err != nil { return ErrSyntaxBase10 } - z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv } if len(bs) >= (iv * 1) { @@ -106,7 +106,7 @@ func (z *Int) fromBase10Long(bs string) error { if err != nil { return ErrSyntaxBase10 } - z.Add(z, new(Int).Mul(scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) + z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) c = c + iv } if len(bs) == c { From b296df729f60d07f8dcc55f17efe855a79bfdc32 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 27 Nov 2022 19:58:02 -0600 Subject: [PATCH 08/50] magic --- base10.go | 1 - 1 file changed, 1 deletion(-) diff --git a/base10.go b/base10.go index a28c12e..f1588d5 100644 --- a/base10.go +++ b/base10.go @@ -62,7 +62,6 @@ var scaleTable10 [78]Int func init() { for k := range scaleTable10 { - scaleTable10[k] = Int{} scaleTable10[k].Exp(NewInt(10), NewInt(uint64(k))) } } From cb2c9100c1c7abcf837cedda272fedcce5d4d0ce Mon Sep 17 00:00:00 2001 From: a Date: Mon, 28 Nov 2022 14:47:21 -0600 Subject: [PATCH 09/50] clear z on SetFrombig, even if b is nil --- conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index 3632cb2..f4fd1fa 100644 --- a/conversion.go +++ b/conversion.go @@ -112,10 +112,10 @@ func (z *Int) UnmarshalText(input []byte) error { // SetFromBig converts a big.Int to Int and sets the value to z. // TODO: Ensure we have sufficient testing, esp for negative bigints. func (z *Int) SetFromBig(b *big.Int) bool { + z.Clear() if b == nil { return true } - z.Clear() words := b.Bits() overflow := len(words) > maxWords From df6b9262af75a615a6482ef33ddbcfdf5510d353 Mon Sep 17 00:00:00 2001 From: a Date: Tue, 6 Dec 2022 06:28:50 -0600 Subject: [PATCH 10/50] Update conversion.go --- conversion.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/conversion.go b/conversion.go index f4fd1fa..62f3e7b 100644 --- a/conversion.go +++ b/conversion.go @@ -113,9 +113,6 @@ func (z *Int) UnmarshalText(input []byte) error { // TODO: Ensure we have sufficient testing, esp for negative bigints. func (z *Int) SetFromBig(b *big.Int) bool { z.Clear() - if b == nil { - return true - } words := b.Bits() overflow := len(words) > maxWords From bd117d9657b0d86dcf41d063312323ff9b07cc47 Mon Sep 17 00:00:00 2001 From: Tjudice <43015609+Tjudice@users.noreply.github.com> Date: Wed, 14 Dec 2022 13:13:36 -0600 Subject: [PATCH 11/50] Update conversion.go fix scan --- conversion.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conversion.go b/conversion.go index 62f3e7b..3ff4539 100644 --- a/conversion.go +++ b/conversion.go @@ -556,11 +556,13 @@ func (dst *Int) Scan(src interface{}) error { if !ok { return fmt.Errorf("cannot scan %T", src) } + return nil case []byte: _, ok := dst.SetString(string(src), 0) if !ok { return fmt.Errorf("cannot scan %T", src) } + return nil } return fmt.Errorf("cannot scan %T", src) From 9a76ab7ae667761410a97ef7592f5abd476184a1 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 15 Dec 2022 09:32:59 +0100 Subject: [PATCH 12/50] base10: minor api change, added docs --- base10.go | 23 ++++++++++++++++------- base10_test.go | 18 +++++++++--------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/base10.go b/base10.go index f1588d5..77fbda9 100644 --- a/base10.go +++ b/base10.go @@ -25,13 +25,13 @@ func (z *Int) SetString(s string, base int) (i *Int, ok bool) { } return z, true } - err := z.FromBase10(s) + err := z.SetFromBase10(s) if err != nil { return nil, false } return z, true case 10: - err := z.FromBase10(s) + err := z.SetFromBase10(s) if err != nil { return nil, false } @@ -45,7 +45,19 @@ func (z *Int) SetString(s string, base int) (i *Int, ok bool) { } return nil, false } -func (z *Int) FromBase10(s string) (err error) { + +// FromBase10 is a convenience-constructor to create an Int from a +// decimal (base 10) string. Numbers larger than 256 bits are not accepted. +func FromBase10(hex string) (*Int, error) { + var z Int + if err := z.SetFromBase10(hex); err != nil { + return nil, err + } + return &z, nil +} + +// SetFromBase10 sets z from the given string, interpreted as a decimal number. +func (z *Int) SetFromBase10(s string) (err error) { if len(s) < len(twoPow256Sub1) { return z.fromBase10Long(s) } @@ -67,10 +79,7 @@ func init() { } func (z *Int) fromBase10Long(bs string) error { - z[0] = 0 - z[1] = 0 - z[2] = 0 - z[3] = 0 + z.Clear() if bs == "" { return nil } diff --git a/base10_test.go b/base10_test.go index 996da66..7dc40ea 100644 --- a/base10_test.go +++ b/base10_test.go @@ -39,7 +39,7 @@ func TestStringScanBase10(t *testing.T) { } for _, v := range cases { - err := z.FromBase10(v.i) + err := z.SetFromBase10(v.i) if !errors.Is(err, v.err) { t.Errorf("expect err %s, got %s", v.err, err) } @@ -57,11 +57,12 @@ func TestStringScanBase10(t *testing.T) { } func FuzzBase10StringCompare(f *testing.F) { - bi := new(big.Int) - z := new(Int) - max256 := new(Int) - max256.FromBase10(twoPow256Sub1) - testcase := []string{ + var ( + bi = new(big.Int) + z = new(Int) + max256, _ = FromBase10(twoPow256Sub1) + ) + for _, tc := range []string{ twoPow256Sub1 + "1", "2" + twoPow256Sub1[1:], twoPow256Sub1, @@ -85,12 +86,11 @@ func FuzzBase10StringCompare(f *testing.F) { "04112401274120741204712xxxxxz00", "0x10101011010", "熊熊熊熊熊熊熊熊", - } - for _, tc := range testcase { + } { f.Add(tc) } f.Fuzz(func(t *testing.T, orig string) { - err := z.FromBase10(orig) + err := z.SetFromBase10(orig) val, ok := bi.SetString(orig, 10) // if fail, make sure that we failed too if !ok { From 42313ad260feb24000df810ad409af58650793ac Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 15 Dec 2022 09:59:19 +0100 Subject: [PATCH 13/50] ci, go.mod: require go 1.18+ --- circle.yml | 46 +++++++--------------------------------------- go.mod | 2 +- 2 files changed, 8 insertions(+), 40 deletions(-) diff --git a/circle.yml b/circle.yml index effc4cb..7f8197e 100644 --- a/circle.yml +++ b/circle.yml @@ -19,14 +19,14 @@ commands: jobs: - go117: + go119: docker: - - image: cimg/go:1.17 + - image: cimg/go:1.19 steps: - run: name: "Install tools" command: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.42.0 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.50.1 - checkout - run: name: "Lint" @@ -78,40 +78,13 @@ jobs: name: "Test (PPC64 emulation)" command: qemu-ppc64-static uint256.test.ppc64 -test.v - go116: + go118: docker: - - image: cimg/go:1.16 + - image: cimg/go:1.18 steps: - checkout - test - go115: - docker: - - image: cimg/go:1.15 - steps: - - checkout - - test - - go114: - docker: - - image: cimg/go:1.14 - steps: - - checkout - - test - - go113: - docker: - - image: cimg/go:1.13 - steps: - - checkout - - test - - go112: - docker: - - image: cimg/go:1.12 - steps: - - checkout - - test @@ -119,12 +92,7 @@ workflows: version: 2 uint256: jobs: - - go117 - - go116 - - go115 - - go114 - - go113 - - go112 + - go118 - bigendian: requires: - - go117 + - go119 diff --git a/go.mod b/go.mod index a38ade7..26425ac 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/holiman/uint256 -go 1.13 +go 1.18 From 7e0467ab1543aa0bbf5a3fe911d460ac11aca073 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 15 Dec 2022 10:01:04 +0100 Subject: [PATCH 14/50] squashme: circle fixes --- circle.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/circle.yml b/circle.yml index 7f8197e..4470b49 100644 --- a/circle.yml +++ b/circle.yml @@ -92,6 +92,7 @@ workflows: version: 2 uint256: jobs: + - go119 - go118 - bigendian: requires: From 7f61ee94799c226d83cf26a27a579fb594ae09b8 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 10:18:28 -0600 Subject: [PATCH 15/50] properly cgheck for overflow in fuzz --- base10_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/base10_test.go b/base10_test.go index 7dc40ea..f2ca872 100644 --- a/base10_test.go +++ b/base10_test.go @@ -108,6 +108,9 @@ func FuzzBase10StringCompare(f *testing.F) { } // if its too large, ignore it also if val.Cmp(max256.ToBig()) > 0 { + if !errors.Is(err, ErrBig256Range) { + t.Errorf("should have errored at negative number: %s", orig) + } return } // so here, if it errors, it means that we failed From 1d8f7261042a0f6274fd98e287f38e9025d8367c Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 10:19:00 -0600 Subject: [PATCH 16/50] number overflow test --- base10_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base10_test.go b/base10_test.go index f2ca872..3f1b5b8 100644 --- a/base10_test.go +++ b/base10_test.go @@ -109,7 +109,7 @@ func FuzzBase10StringCompare(f *testing.F) { // if its too large, ignore it also if val.Cmp(max256.ToBig()) > 0 { if !errors.Is(err, ErrBig256Range) { - t.Errorf("should have errored at negative number: %s", orig) + t.Errorf("should have errored at number overflow: %s", orig) } return } From f4e6c19c447b83a5227e54f48e08b83177086a83 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 10:34:53 -0600 Subject: [PATCH 17/50] added comments to explain fromBase10Long --- base10.go | 71 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 34 deletions(-) diff --git a/base10.go b/base10.go index 77fbda9..d14f742 100644 --- a/base10.go +++ b/base10.go @@ -78,52 +78,55 @@ func init() { } } +// helper function to only ever be called via SetFromBase10 +// this function takes a string and chunks it up, calling ParseUint on it up to 5 times +// these chunks are then multiplied by the proper power of 10, then added together. func (z *Int) fromBase10Long(bs string) error { + // first clear the input z.Clear() + // if the input value is empty string, just do nothing. effectively, empty string sets to 0 if bs == "" { return nil } - iv := 19 - c := 0 - if len(bs) >= (iv * 4) { - nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) - if err != nil { - return ErrSyntaxBase10 - } - z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-iv], NewInt(uint64(nm)))) - c = c + iv - } - if len(bs) >= (iv * 3) { - nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) - if err != nil { - return ErrSyntaxBase10 - } - z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) - c = c + iv - } - if len(bs) >= (iv * 2) { - nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) - if err != nil { - return ErrSyntaxBase10 - } - z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) - c = c + iv - } - if len(bs) >= (iv * 1) { - nm, err := strconv.ParseUint(bs[c:(c+iv)], 10, 64) - if err != nil { - return ErrSyntaxBase10 + // the maximum value of uint64 is 18446744073709551615, which is 20 characters + // one less means that a string of 19 9's is always within the uint64 limit + cutLength := 19 + // cutStart tracks the current position of the string that we are in + cutStart := 0 + // start iterating from 4 to 1. This is because the maximum value of uint256 is 78 characters, + // which can be divided into 5 integers of up to 19 characters. + // however, the last number will always be below 19 characters, so i=0 is dealt with as special case + for i := 4; i >= 1; i-- { + // check if the length of the string is larger than cutLength * i + if len(bs) >= (cutLength * i) { + // cut the string from the cutStart to the cutLength. + nm, err := strconv.ParseUint(bs[cutStart:(cutStart+cutLength)], 10, 64) + if err != nil { + return ErrSyntaxBase10 + } + // create a new int with that number as the value + base := NewInt(nm) + // pointer to the exponent. We need to multiply our number by 10^(len-cutStart-cutLength) + // len-cutStart-cutLength is index of the last character in our cutset, counting from the right. + exp := &scaleTable10[len(bs)-cutStart-cutLength] + // add that number to our running total + z.Add(z, base.Mul(exp, base)) + // increase the cut start point, since we have now read from cutStart to cutStart + length + cutStart = cutStart + cutLength } - z.Add(z, new(Int).Mul(&scaleTable10[len(bs)-c-iv], NewInt(uint64(nm)))) - c = c + iv } - if len(bs) == c { + // if we have read every character of the string, we are done, and can return + // this is a short circuit that we can do if the length of the string is a multiple of 19 + if len(bs) == cutStart { return nil } - nm, err := strconv.ParseUint(bs[c:], 10, 64) + // finally, there are a remaining set of characters. + // SetFromBase10 already did the check that this remaining cutset, after 4 cuts, will be lower than 19 charactes + nm, err := strconv.ParseUint(bs[cutStart:], 10, 64) if err != nil { return ErrSyntaxBase10 } + // and add it! no need to multiply by 10^0 z.AddUint64(z, uint64(nm)) return nil } From 84510f7d7139c6b65c7443875d9586bbb6fa8a1d Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 12:39:17 -0600 Subject: [PATCH 18/50] Update base10.go Co-authored-by: Martin Holst Swende --- base10.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/base10.go b/base10.go index d14f742..ee9bf61 100644 --- a/base10.go +++ b/base10.go @@ -70,6 +70,11 @@ func (z *Int) SetFromBase10(s string) (err error) { return ErrBig256Range } +// scaleTable10 contains the 10-exponents, +// 0: 10 ^ 0 +// 1: 10 ^ 1 +// .. +// 77: 10 ^ 77 var scaleTable10 [78]Int func init() { From 6e4f91445c0f3fbb12f2f7f12d1e4527ac7048ab Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 12:39:42 -0600 Subject: [PATCH 19/50] Update base10.go Co-authored-by: Martin Holst Swende --- base10.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base10.go b/base10.go index ee9bf61..a43254a 100644 --- a/base10.go +++ b/base10.go @@ -83,7 +83,7 @@ func init() { } } -// helper function to only ever be called via SetFromBase10 +// fromBase10Long is a helper function to only ever be called via SetFromBase10 // this function takes a string and chunks it up, calling ParseUint on it up to 5 times // these chunks are then multiplied by the proper power of 10, then added together. func (z *Int) fromBase10Long(bs string) error { From e25ce45ec1bfdde5d750b76ee7a75ebf334a67f9 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 12:41:20 -0600 Subject: [PATCH 20/50] lift up --- base10.go | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/base10.go b/base10.go index a43254a..0158668 100644 --- a/base10.go +++ b/base10.go @@ -70,10 +70,10 @@ func (z *Int) SetFromBase10(s string) (err error) { return ErrBig256Range } -// scaleTable10 contains the 10-exponents, +// scaleTable10 contains the 10-exponents, // 0: 10 ^ 0 // 1: 10 ^ 1 -// .. +// .. // 77: 10 ^ 77 var scaleTable10 [78]Int @@ -103,22 +103,23 @@ func (z *Int) fromBase10Long(bs string) error { // however, the last number will always be below 19 characters, so i=0 is dealt with as special case for i := 4; i >= 1; i-- { // check if the length of the string is larger than cutLength * i - if len(bs) >= (cutLength * i) { - // cut the string from the cutStart to the cutLength. - nm, err := strconv.ParseUint(bs[cutStart:(cutStart+cutLength)], 10, 64) - if err != nil { - return ErrSyntaxBase10 - } - // create a new int with that number as the value - base := NewInt(nm) - // pointer to the exponent. We need to multiply our number by 10^(len-cutStart-cutLength) - // len-cutStart-cutLength is index of the last character in our cutset, counting from the right. - exp := &scaleTable10[len(bs)-cutStart-cutLength] - // add that number to our running total - z.Add(z, base.Mul(exp, base)) - // increase the cut start point, since we have now read from cutStart to cutStart + length - cutStart = cutStart + cutLength + if len(bs) < (cutLength * i) { + continue + } + // cut the string from the cutStart to the cutLength. + nm, err := strconv.ParseUint(bs[cutStart:(cutStart+cutLength)], 10, 64) + if err != nil { + return ErrSyntaxBase10 } + // create a new int with that number as the value + base := NewInt(nm) + // pointer to the exponent. We need to multiply our number by 10^(len-cutStart-cutLength) + // len-cutStart-cutLength is index of the last character in our cutset, counting from the right. + exp := &scaleTable10[len(bs)-cutStart-cutLength] + // add that number to our running total + z.Add(z, base.Mul(exp, base)) + // increase the cut start point, since we have now read from cutStart to cutStart + length + cutStart = cutStart + cutLength } // if we have read every character of the string, we are done, and can return // this is a short circuit that we can do if the length of the string is a multiple of 19 From 8420f460acafa89d41f7c290a04de3c0e63bd535 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 12:50:45 -0600 Subject: [PATCH 21/50] change cutLength to local constant --- base10.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/base10.go b/base10.go index 0158668..74cc584 100644 --- a/base10.go +++ b/base10.go @@ -95,13 +95,15 @@ func (z *Int) fromBase10Long(bs string) error { } // the maximum value of uint64 is 18446744073709551615, which is 20 characters // one less means that a string of 19 9's is always within the uint64 limit - cutLength := 19 + const cutLength = 19 // cutStart tracks the current position of the string that we are in cutStart := 0 - // start iterating from 4 to 1. This is because the maximum value of uint256 is 78 characters, + // startPoint is equal to the length of string / cutLength + startPoint := len(bs) / cutLength + // start iterating from startPoint to 1. // which can be divided into 5 integers of up to 19 characters. // however, the last number will always be below 19 characters, so i=0 is dealt with as special case - for i := 4; i >= 1; i-- { + for i := startPoint; i >= 1; i-- { // check if the length of the string is larger than cutLength * i if len(bs) < (cutLength * i) { continue From 3fc5f0848d405c2f4e404fa29c12982f59ded36b Mon Sep 17 00:00:00 2001 From: a Date: Thu, 15 Dec 2022 13:10:25 -0600 Subject: [PATCH 22/50] better doc --- base10.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base10.go b/base10.go index 74cc584..f3c75a9 100644 --- a/base10.go +++ b/base10.go @@ -101,7 +101,7 @@ func (z *Int) fromBase10Long(bs string) error { // startPoint is equal to the length of string / cutLength startPoint := len(bs) / cutLength // start iterating from startPoint to 1. - // which can be divided into 5 integers of up to 19 characters. + // a uint256 sized string can be divided into startPoint+1 integers of up to 19 characters. // however, the last number will always be below 19 characters, so i=0 is dealt with as special case for i := startPoint; i >= 1; i-- { // check if the length of the string is larger than cutLength * i From d6f32f6ac204f69b0e47a847d64b488899cd9791 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 15 Dec 2022 23:04:12 +0100 Subject: [PATCH 23/50] base10: modify algorithm + fixup tests a bit This commit changes the algorithm so it traverses right-to-left, meaning that we know exactly what multipliers are used. --- base10.go | 88 +++++++++++++++++++++----------------------------- base10_test.go | 81 +++++++++++++++++++++++----------------------- conversion.go | 1 - 3 files changed, 77 insertions(+), 93 deletions(-) diff --git a/base10.go b/base10.go index f3c75a9..70be94a 100644 --- a/base10.go +++ b/base10.go @@ -70,17 +70,13 @@ func (z *Int) SetFromBase10(s string) (err error) { return ErrBig256Range } -// scaleTable10 contains the 10-exponents, -// 0: 10 ^ 0 -// 1: 10 ^ 1 -// .. -// 77: 10 ^ 77 -var scaleTable10 [78]Int - -func init() { - for k := range scaleTable10 { - scaleTable10[k].Exp(NewInt(10), NewInt(uint64(k))) - } +// multipliers holds the values that are needed for fromBase10Long +var multipliers = [5]*Int{ + nil, // represents first round, no multiplication needed + &Int{10000000000000000000, 0, 0, 0}, // 10 ^ 19 + &Int{687399551400673280, 5421010862427522170, 0, 0}, // 10 ^ 38 + &Int{5332261958806667264, 17004971331911604867, 2938735877055718769, 0}, // 10 ^ 57 + &Int{0, 8607968719199866880, 532749306367912313, 1593091911132452277}, // 10 ^ 76 } // fromBase10Long is a helper function to only ever be called via SetFromBase10 @@ -89,52 +85,42 @@ func init() { func (z *Int) fromBase10Long(bs string) error { // first clear the input z.Clear() - // if the input value is empty string, just do nothing. effectively, empty string sets to 0 - if bs == "" { - return nil - } // the maximum value of uint64 is 18446744073709551615, which is 20 characters // one less means that a string of 19 9's is always within the uint64 limit - const cutLength = 19 - // cutStart tracks the current position of the string that we are in - cutStart := 0 - // startPoint is equal to the length of string / cutLength - startPoint := len(bs) / cutLength - // start iterating from startPoint to 1. - // a uint256 sized string can be divided into startPoint+1 integers of up to 19 characters. - // however, the last number will always be below 19 characters, so i=0 is dealt with as special case - for i := startPoint; i >= 1; i-- { - // check if the length of the string is larger than cutLength * i - if len(bs) < (cutLength * i) { - continue + var ( + num uint64 + err error + remaining = len(bs) + ) + // We proceed in steps of 19 characters (nibbles), from least significant to most significant. + // This means that the first (up to) 19 characters do not need to be multiplied. + // In the second iteration, our slice of 19 characters needs to be multipleied + // by a factor of 10^19. Et cetera. + + for i, mult := range multipliers { + if remaining <= 0 { + return nil // Done + } else if remaining > 19 { + num, err = strconv.ParseUint(bs[remaining-19:remaining], 10, 64) + } else { + // Final round + num, err = strconv.ParseUint(bs, 10, 64) } - // cut the string from the cutStart to the cutLength. - nm, err := strconv.ParseUint(bs[cutStart:(cutStart+cutLength)], 10, 64) if err != nil { - return ErrSyntaxBase10 + return err } - // create a new int with that number as the value - base := NewInt(nm) - // pointer to the exponent. We need to multiply our number by 10^(len-cutStart-cutLength) - // len-cutStart-cutLength is index of the last character in our cutset, counting from the right. - exp := &scaleTable10[len(bs)-cutStart-cutLength] // add that number to our running total - z.Add(z, base.Mul(exp, base)) - // increase the cut start point, since we have now read from cutStart to cutStart + length - cutStart = cutStart + cutLength - } - // if we have read every character of the string, we are done, and can return - // this is a short circuit that we can do if the length of the string is a multiple of 19 - if len(bs) == cutStart { - return nil - } - // finally, there are a remaining set of characters. - // SetFromBase10 already did the check that this remaining cutset, after 4 cuts, will be lower than 19 charactes - nm, err := strconv.ParseUint(bs[cutStart:], 10, 64) - if err != nil { - return ErrSyntaxBase10 + if i != 0 { + base := NewInt(num) + z.Add(z, base.Mul(base, mult)) + } else { + z.SetUint64(num) + } + // Chop off another 19 characters + if remaining > 19 { + bs = bs[0 : remaining-19] + } + remaining -= 19 } - // and add it! no need to multiply by 10^0 - z.AddUint64(z, uint64(nm)) return nil } diff --git a/base10_test.go b/base10_test.go index 3f1b5b8..2f4c98b 100644 --- a/base10_test.go +++ b/base10_test.go @@ -4,54 +4,53 @@ import ( "errors" "fmt" "math/big" + "strconv" "testing" ) func TestStringScanBase10(t *testing.T) { z := new(Int) - type testCase struct { - i string - err error - val string - } - - cases := []testCase{ - {i: twoPow256Sub1 + "1", err: ErrBig256Range}, - {i: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, - {i: twoPow256Sub1[1:]}, - {i: twoPow128}, - {i: twoPow128 + "1"}, - {i: twoPow128[1:]}, - {i: twoPow64 + "1"}, - {i: twoPow64[1:]}, - {i: "banana", err: ErrSyntaxBase10}, - {i: "0xab", err: ErrSyntaxBase10}, - {i: "ab", err: ErrSyntaxBase10}, - {i: "0"}, - {i: "000", val: "0"}, - {i: "010", val: "10"}, - {i: "01", val: "1"}, - {i: "-0", err: ErrSyntaxBase10}, - {i: "-10", err: ErrSyntaxBase10}, - {i: "115792089237316195423570985008687907853269984665640564039457584007913129639936", err: ErrBig256Range}, - {i: "115792089237316195423570985008687907853269984665640564039457584007913129639935"}, + input string + err error } - - for _, v := range cases { - err := z.SetFromBase10(v.i) - if !errors.Is(err, v.err) { - t.Errorf("expect err %s, got %s", v.err, err) + for i, tc := range []testCase{ + {input: twoPow256Sub1 + "1", err: ErrBig256Range}, + {input: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, + {input: twoPow256Sub1[1:]}, + {input: twoPow128}, + {input: twoPow128 + "1"}, + {input: twoPow128[1:]}, + {input: twoPow64 + "1"}, + {input: twoPow64[1:]}, + {input: "banana", err: strconv.ErrSyntax}, + {input: "0xab", err: strconv.ErrSyntax}, + {input: "ab", err: strconv.ErrSyntax}, + {input: "0"}, + {input: "000"}, + {input: "010"}, + {input: "01"}, + {input: "-0", err: strconv.ErrSyntax}, + {input: "-10", err: strconv.ErrSyntax}, + {input: "115792089237316195423570985008687907853269984665640564039457584007913129639936", err: ErrBig256Range}, + {input: "115792089237316195423570985008687907853269984665640564039457584007913129639935"}, + } { + z.SetAllOne() // Set to ensure all bits are cleared after + err := z.SetFromBase10(tc.input) + if !errors.Is(err, tc.err) { + t.Errorf("test %d, input %v: want err %s, have %s", i, tc.input, tc.err, err) } - if err == nil { - got := z.ToBig().String() - want := v.i - if v.val != "" { - want = v.val - } - if got != want { - t.Errorf("expect val %s, got %s", v.i, got) - } + if err != nil { + continue + } + var want string + if w, ok := big.NewInt(0).SetString(tc.input, 10); !ok { + panic(fmt.Sprintf("test %d error", i)) + } else { + want = w.String() + } + if have := z.ToBig().String(); have != want { + t.Errorf("test %d: input %v, want %v: have %s", i, tc.input, want, have) } } } @@ -101,7 +100,7 @@ func FuzzBase10StringCompare(f *testing.F) { } // if its negative number, we should err if len(orig) > 0 && (orig[0] == '-') { - if !errors.Is(err, ErrSyntaxBase10) { + if !errors.Is(err, strconv.ErrSyntax) { t.Errorf("should have errored at negative number: %s", orig) } return diff --git a/conversion.go b/conversion.go index 3ff4539..e09de73 100644 --- a/conversion.go +++ b/conversion.go @@ -577,7 +577,6 @@ func (src Int) Value() (driver.Value, error) { var ( ErrEmptyString = errors.New("empty hex string") ErrSyntax = errors.New("invalid hex string") - ErrSyntaxBase10 = errors.New("invalid base 10 string") ErrMissingPrefix = errors.New("hex string without 0x prefix") ErrEmptyNumber = errors.New("hex string \"0x\"") ErrLeadingZero = errors.New("hex number with leading zero digits") From 8c858a1ddad54cf2360741a03a756bd1eb01b9d5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 15 Dec 2022 23:24:08 +0100 Subject: [PATCH 24/50] base10: fuzzing found some bugs, which were fixed --- base10.go | 20 +++++++++++++++++++- base10_test.go | 21 +++++++++++++++------ 2 files changed, 34 insertions(+), 7 deletions(-) diff --git a/base10.go b/base10.go index 70be94a..081fbc3 100644 --- a/base10.go +++ b/base10.go @@ -1,6 +1,7 @@ package uint256 import ( + "io" "strconv" "strings" ) @@ -58,6 +59,21 @@ func FromBase10(hex string) (*Int, error) { // SetFromBase10 sets z from the given string, interpreted as a decimal number. func (z *Int) SetFromBase10(s string) (err error) { + // Remove max one leading + + if len(s) > 0 && s[0] == '+' { + s = s[1:] + } + // Remove any number of leading zeroes + if len(s) > 0 && s[0] == '0' { + var i int + var c rune + for i, c = range s { + if c != '0' { + break + } + } + s = s[i:] + } if len(s) < len(twoPow256Sub1) { return z.fromBase10Long(s) } @@ -92,11 +108,13 @@ func (z *Int) fromBase10Long(bs string) error { err error remaining = len(bs) ) + if remaining == 0 { + return io.EOF + } // We proceed in steps of 19 characters (nibbles), from least significant to most significant. // This means that the first (up to) 19 characters do not need to be multiplied. // In the second iteration, our slice of 19 characters needs to be multipleied // by a factor of 10^19. Et cetera. - for i, mult := range multipliers { if remaining <= 0 { return nil // Done diff --git a/base10_test.go b/base10_test.go index 2f4c98b..ecfaad7 100644 --- a/base10_test.go +++ b/base10_test.go @@ -3,6 +3,7 @@ package uint256 import ( "errors" "fmt" + "io" "math/big" "strconv" "testing" @@ -15,9 +16,12 @@ func TestStringScanBase10(t *testing.T) { err error } for i, tc := range []testCase{ + {input: "0000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {input: "-000000000000000000000000000000000000000000000000000000000000000000000000000000"}, {input: twoPow256Sub1 + "1", err: ErrBig256Range}, {input: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, {input: twoPow256Sub1[1:]}, + {input: "+" + twoPow256Sub1}, {input: twoPow128}, {input: twoPow128 + "1"}, {input: twoPow128[1:]}, @@ -27,7 +31,9 @@ func TestStringScanBase10(t *testing.T) { {input: "0xab", err: strconv.ErrSyntax}, {input: "ab", err: strconv.ErrSyntax}, {input: "0"}, + {input: "", err: io.EOF}, {input: "000"}, + {input: "+000"}, {input: "010"}, {input: "01"}, {input: "-0", err: strconv.ErrSyntax}, @@ -56,11 +62,7 @@ func TestStringScanBase10(t *testing.T) { } func FuzzBase10StringCompare(f *testing.F) { - var ( - bi = new(big.Int) - z = new(Int) - max256, _ = FromBase10(twoPow256Sub1) - ) + for _, tc := range []string{ twoPow256Sub1 + "1", "2" + twoPow256Sub1[1:], @@ -78,6 +80,7 @@ func FuzzBase10StringCompare(f *testing.F) { "010", "01", "-0", + "+0", "-10", "115792089237316195423570985008687907853269984665640564039457584007913129639936", "115792089237316195423570985008687907853269984665640564039457584007913129639935", @@ -89,6 +92,11 @@ func FuzzBase10StringCompare(f *testing.F) { f.Add(tc) } f.Fuzz(func(t *testing.T, orig string) { + var ( + bi = new(big.Int) + z = new(Int) + max256, _ = FromBase10(twoPow256Sub1) + ) err := z.SetFromBase10(orig) val, ok := bi.SetString(orig, 10) // if fail, make sure that we failed too @@ -100,7 +108,8 @@ func FuzzBase10StringCompare(f *testing.F) { } // if its negative number, we should err if len(orig) > 0 && (orig[0] == '-') { - if !errors.Is(err, strconv.ErrSyntax) { + // We may error either on too large OR negative + if err == nil { t.Errorf("should have errored at negative number: %s", orig) } return From 078a8e620249e2de8af1c6eaaa31ba79505d369d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 00:02:11 +0100 Subject: [PATCH 25/50] fuzzing: some improvements to the fuzzer --- base10.go | 15 +++++++++------ base10_test.go | 36 +++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/base10.go b/base10.go index 081fbc3..1c92604 100644 --- a/base10.go +++ b/base10.go @@ -16,6 +16,12 @@ func (z *Int) Base10() string { // SetString implements a subset of (*big.Int).SetString // ok will be true iff i == nil +// The SetString method on uint256.Int differs from big.Int when it comes to +// base 16, since uint256.Int is stricter, requiring: +// - 0x or 0X prefix +// - Non-empty string after prefix +// - No leading zeroes +// The base10 version allows leading zeroes. func (z *Int) SetString(s string, base int) (i *Int, ok bool) { switch base { case 0: @@ -26,20 +32,17 @@ func (z *Int) SetString(s string, base int) (i *Int, ok bool) { } return z, true } - err := z.SetFromBase10(s) - if err != nil { + if err := z.SetFromBase10(s); err != nil { return nil, false } return z, true case 10: - err := z.SetFromBase10(s) - if err != nil { + if err := z.SetFromBase10(s); err != nil { return nil, false } return z, true case 16: - err := z.fromHex(s) - if err != nil { + if err := z.fromHex(s); err != nil { return nil, false } return z, true diff --git a/base10_test.go b/base10_test.go index ecfaad7..0d8243c 100644 --- a/base10_test.go +++ b/base10_test.go @@ -88,6 +88,13 @@ func FuzzBase10StringCompare(f *testing.F) { "04112401274120741204712xxxxxz00", "0x10101011010", "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x00000000000000000000000000000000000000000000000000000000000000000", + "-0x00000000000000000000000000000000000000000000000000000000000000000", } { f.Add(tc) } @@ -97,33 +104,32 @@ func FuzzBase10StringCompare(f *testing.F) { z = new(Int) max256, _ = FromBase10(twoPow256Sub1) ) - err := z.SetFromBase10(orig) - val, ok := bi.SetString(orig, 10) - // if fail, make sure that we failed too - if !ok { - if err == nil { - t.Errorf("expected base 10 parse to fail: %s", orig) + z, haveOk := z.SetString(orig, 10) + bi, wantOk := bi.SetString(orig, 10) + // if bigint parsing fail, make sure that we failed too + if !wantOk { + if haveOk { + t.Errorf("parsing status, want ok=%v, have ok=%v. Input: %s", haveOk, wantOk, orig) } return } - // if its negative number, we should err + // if its a negative number, we should err if len(orig) > 0 && (orig[0] == '-') { - // We may error either on too large OR negative - if err == nil { + if haveOk { t.Errorf("should have errored at negative number: %s", orig) } return } // if its too large, ignore it also - if val.Cmp(max256.ToBig()) > 0 { - if !errors.Is(err, ErrBig256Range) { + if bi.Cmp(max256.ToBig()) > 0 { + if haveOk { t.Errorf("should have errored at number overflow: %s", orig) } return } - // so here, if it errors, it means that we failed - if err != nil { - t.Errorf("should have parsed %s to %s, but err'd instead", orig, val.String()) + // No more reasons not to succeed + if !haveOk { + t.Errorf("should have parsed %s to %s, but errored instead", orig, bi.String()) return } // otherwise, make sure that the values are equal @@ -138,7 +144,7 @@ func FuzzBase10StringCompare(f *testing.F) { } value, err := z.Value() if err != nil { - t.Errorf("fail to Value() %s, got err %s", val, err) + t.Errorf("fail to Value() %s, got err %s", bi, err) return } if z.Base10()+"e0" != fmt.Sprint(value) { From 2413444705e6268072d7848f148ef017fd5abf7a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 00:23:23 +0100 Subject: [PATCH 26/50] add oss-fuzz fuzzer, rename base10 to decimal --- base10.go => decimal.go | 30 +++++++++-------- base10_test.go => decimal_test.go | 27 ++++++++------- fuzz.go | 55 +++++++++++++++++++++++++++++++ oss-fuzz.sh | 1 + 4 files changed, 88 insertions(+), 25 deletions(-) rename base10.go => decimal.go (81%) rename base10_test.go => decimal_test.go (94%) diff --git a/base10.go b/decimal.go similarity index 81% rename from base10.go rename to decimal.go index 1c92604..37f435b 100644 --- a/base10.go +++ b/decimal.go @@ -1,3 +1,7 @@ +// uint256: Fixed size 256-bit math library +// Copyright 2020 uint256 Authors +// SPDX-License-Identifier: BSD-3-Clause + package uint256 import ( @@ -10,7 +14,7 @@ const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039 const twoPow128 = "340282366920938463463374607431768211456" const twoPow64 = "18446744073709551616" -func (z *Int) Base10() string { +func (z *Int) Dec() string { return z.ToBig().String() } @@ -32,12 +36,12 @@ func (z *Int) SetString(s string, base int) (i *Int, ok bool) { } return z, true } - if err := z.SetFromBase10(s); err != nil { + if err := z.SetFromDecimal(s); err != nil { return nil, false } return z, true case 10: - if err := z.SetFromBase10(s); err != nil { + if err := z.SetFromDecimal(s); err != nil { return nil, false } return z, true @@ -50,18 +54,18 @@ func (z *Int) SetString(s string, base int) (i *Int, ok bool) { return nil, false } -// FromBase10 is a convenience-constructor to create an Int from a +// FromDecimal is a convenience-constructor to create an Int from a // decimal (base 10) string. Numbers larger than 256 bits are not accepted. -func FromBase10(hex string) (*Int, error) { +func FromDecimal(hex string) (*Int, error) { var z Int - if err := z.SetFromBase10(hex); err != nil { + if err := z.SetFromDecimal(hex); err != nil { return nil, err } return &z, nil } -// SetFromBase10 sets z from the given string, interpreted as a decimal number. -func (z *Int) SetFromBase10(s string) (err error) { +// SetFromDecimal sets z from the given string, interpreted as a decimal number. +func (z *Int) SetFromDecimal(s string) (err error) { // Remove max one leading + if len(s) > 0 && s[0] == '+' { s = s[1:] @@ -78,18 +82,18 @@ func (z *Int) SetFromBase10(s string) (err error) { s = s[i:] } if len(s) < len(twoPow256Sub1) { - return z.fromBase10Long(s) + return z.fromDecimal(s) } if len(s) == len(twoPow256Sub1) { if s > twoPow256Sub1 { return ErrBig256Range } - return z.fromBase10Long(s) + return z.fromDecimal(s) } return ErrBig256Range } -// multipliers holds the values that are needed for fromBase10Long +// multipliers holds the values that are needed for fromDecimal var multipliers = [5]*Int{ nil, // represents first round, no multiplication needed &Int{10000000000000000000, 0, 0, 0}, // 10 ^ 19 @@ -98,10 +102,10 @@ var multipliers = [5]*Int{ &Int{0, 8607968719199866880, 532749306367912313, 1593091911132452277}, // 10 ^ 76 } -// fromBase10Long is a helper function to only ever be called via SetFromBase10 +// fromDecimal is a helper function to only ever be called via SetFromDecimal // this function takes a string and chunks it up, calling ParseUint on it up to 5 times // these chunks are then multiplied by the proper power of 10, then added together. -func (z *Int) fromBase10Long(bs string) error { +func (z *Int) fromDecimal(bs string) error { // first clear the input z.Clear() // the maximum value of uint64 is 18446744073709551615, which is 20 characters diff --git a/base10_test.go b/decimal_test.go similarity index 94% rename from base10_test.go rename to decimal_test.go index 0d8243c..e91aa7b 100644 --- a/base10_test.go +++ b/decimal_test.go @@ -1,3 +1,7 @@ +// uint256: Fixed size 256-bit math library +// Copyright 2020 uint256 Authors +// SPDX-License-Identifier: BSD-3-Clause + package uint256 import ( @@ -9,7 +13,7 @@ import ( "testing" ) -func TestStringScanBase10(t *testing.T) { +func TestStringScanDecimal(t *testing.T) { z := new(Int) type testCase struct { input string @@ -17,7 +21,7 @@ func TestStringScanBase10(t *testing.T) { } for i, tc := range []testCase{ {input: "0000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {input: "-000000000000000000000000000000000000000000000000000000000000000000000000000000"}, + {input: "-000000000000000000000000000000000000000000000000000000000000000000000000000000", err: ErrBig256Range}, {input: twoPow256Sub1 + "1", err: ErrBig256Range}, {input: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, {input: twoPow256Sub1[1:]}, @@ -42,7 +46,7 @@ func TestStringScanBase10(t *testing.T) { {input: "115792089237316195423570985008687907853269984665640564039457584007913129639935"}, } { z.SetAllOne() // Set to ensure all bits are cleared after - err := z.SetFromBase10(tc.input) + err := z.SetFromDecimal(tc.input) if !errors.Is(err, tc.err) { t.Errorf("test %d, input %v: want err %s, have %s", i, tc.input, tc.err, err) } @@ -100,9 +104,8 @@ func FuzzBase10StringCompare(f *testing.F) { } f.Fuzz(func(t *testing.T, orig string) { var ( - bi = new(big.Int) - z = new(Int) - max256, _ = FromBase10(twoPow256Sub1) + bi = new(big.Int) + z = new(Int) ) z, haveOk := z.SetString(orig, 10) bi, wantOk := bi.SetString(orig, 10) @@ -121,7 +124,7 @@ func FuzzBase10StringCompare(f *testing.F) { return } // if its too large, ignore it also - if bi.Cmp(max256.ToBig()) > 0 { + if bi.BitLen() > 256 { if haveOk { t.Errorf("should have errored at number overflow: %s", orig) } @@ -134,12 +137,12 @@ func FuzzBase10StringCompare(f *testing.F) { } // otherwise, make sure that the values are equal if z.ToBig().Cmp(bi) != 0 { - t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Base10()) + t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec()) return } // make sure that bigint base 10 string is equal to base10 string - if z.Base10() != bi.String() { - t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Base10()) + if z.Dec() != bi.String() { + t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec()) return } value, err := z.Value() @@ -147,8 +150,8 @@ func FuzzBase10StringCompare(f *testing.F) { t.Errorf("fail to Value() %s, got err %s", bi, err) return } - if z.Base10()+"e0" != fmt.Sprint(value) { - t.Errorf("value of %s did not match base 10 encoding %s", value, z.Base10()) + if z.Dec()+"e0" != fmt.Sprint(value) { + t.Errorf("value of %s did not match base 10 encoding %s", value, z.Dec()) return } }) diff --git a/fuzz.go b/fuzz.go index 790d6bf..e8ad547 100644 --- a/fuzz.go +++ b/fuzz.go @@ -304,3 +304,58 @@ func fuzzTernaryOp(data []byte) int { } return 1 } + +func FuzzSetString(data []byte) int { + if len(data) > 512 { + // Too large, makes no sense + return -1 + } + var ( + orig = string(data) + bi = new(big.Int) + z = new(Int) + ) + z, haveOk := z.SetString(orig, 10) + bi, wantOk := bi.SetString(orig, 10) + // if bigint parsing fail, make sure that we failed too + if !wantOk { + if haveOk { + panic(fmt.Sprintf("parsing status, want ok=%v, have ok=%v. Input: %s", haveOk, wantOk, orig)) + } + return 1 + } + // if its a negative number, we should err + if len(orig) > 0 && (orig[0] == '-') { + if haveOk { + panic(fmt.Sprintf("should have errored at negative number: %s", orig)) + } + return 1 + } + // if its too large, ignore it also + if bi.BitLen() > 256 { + if haveOk { + panic(fmt.Sprintf("should have errored at number overflow: %s", orig)) + } + return 1 + } + // No more reasons not to succeed + if !haveOk { + panic(fmt.Sprintf("should have parsed '%s' to '%s', but errored instead", orig, bi.String())) + } + // otherwise, make sure that the values are equal + if z.ToBig().Cmp(bi) != 0 { + panic(fmt.Sprintf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec())) + } + // make sure that bigint base 10 string is equal to base10 string + if z.Dec() != bi.String() { + panic(fmt.Sprintf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec())) + } + value, err := z.Value() + if err != nil { + panic(fmt.Sprintf("fail to Value() %s, got err %s", bi, err)) + } + if z.Dec()+"e0" != fmt.Sprint(value) { + panic(fmt.Sprintf("value of %s did not match base 10 encoding %s", value, z.Dec())) + } + return 1 +} diff --git a/oss-fuzz.sh b/oss-fuzz.sh index 7364dc7..ca516c0 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -1,2 +1,3 @@ #!/bin/bash -eu compile_go_fuzzer github.com/holiman/uint256 Fuzz uint256Fuzz +compile_go_fuzzer github.com/holiman/uint256 FuzzSetString uint256FuzzSetString From 2e6c49ff156e8c40c6ceab16d3e8bacf7f9b44d2 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 10:29:25 +0100 Subject: [PATCH 27/50] decimal: fix conversion on base 0, fallback to big.Int --- conversion.go | 4 +- decimal.go | 26 +++++++-- decimal_test.go | 152 +++++++++++++++++++++++++++++++++--------------- 3 files changed, 129 insertions(+), 53 deletions(-) diff --git a/conversion.go b/conversion.go index e09de73..aca9c2f 100644 --- a/conversion.go +++ b/conversion.go @@ -30,7 +30,7 @@ const ( // Compile time interface checks var ( - _ driver.Value = (*Int)(nil) + _ driver.Valuer = (*Int)(nil) _ sql.Scanner = (*Int)(nil) _ encoding.TextMarshaler = (*Int)(nil) _ encoding.TextUnmarshaler = (*Int)(nil) @@ -70,7 +70,6 @@ func (z *Int) fromHex(hex string) error { if err := checkNumberS(hex); err != nil { return err } - if len(hex) > 66 { return ErrBig256Range } @@ -106,6 +105,7 @@ func FromHex(hex string) (*Int, error) { // UnmarshalText implements encoding.TextUnmarshaler func (z *Int) UnmarshalText(input []byte) error { + z.Clear() return z.fromHex(string(input)) } diff --git a/decimal.go b/decimal.go index 37f435b..5e12b30 100644 --- a/decimal.go +++ b/decimal.go @@ -6,8 +6,8 @@ package uint256 import ( "io" + "math/big" "strconv" - "strings" ) const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" @@ -27,11 +27,29 @@ func (z *Int) Dec() string { // - No leading zeroes // The base10 version allows leading zeroes. func (z *Int) SetString(s string, base int) (i *Int, ok bool) { + z.Clear() switch base { case 0: - if strings.HasPrefix(s, "0x") { - err := z.fromHex(s) - if err != nil { + // From documentation at big.Int: + // For base 0, the number prefix determines the actual base: A prefix of + // “0b” or “0B” selects base 2, + // “0”, “0o” or “0O” selects base 8, + // and “0x” or “0X” selects base 16. + // Otherwise, the selected base is 10 and no prefix is accepted. + if len(s) > 1 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') { + // base 16 + if err := z.fromHex(s); err != nil { + return nil, false + } + return z, true + } + if len(s) > 0 && s[0] == '0' { + // base 2 or base 8 + // Not implemented here, use big.Int + // @TODO + if b, ok := big.NewInt(0).SetString(s, 0); !ok { + return nil, false + } else if overflow := z.SetFromBig(b); overflow { return nil, false } return z, true diff --git a/decimal_test.go b/decimal_test.go index e91aa7b..03e665a 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -5,63 +5,121 @@ package uint256 import ( - "errors" "fmt" - "io" "math/big" - "strconv" "testing" ) -func TestStringScanDecimal(t *testing.T) { - z := new(Int) - type testCase struct { - input string - err error +// Test SetFromDecimal +func testSetFromDec(tc string) error { + a := new(Int).SetAllOne() + err := a.SetFromDecimal(tc) + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if err == nil { + return fmt.Errorf("want error on negative input") + } + return nil } - for i, tc := range []testCase{ - {input: "0000000000000000000000000000000000000000000000000000000000000000000000000000000"}, - {input: "-000000000000000000000000000000000000000000000000000000000000000000000000000000", err: ErrBig256Range}, - {input: twoPow256Sub1 + "1", err: ErrBig256Range}, - {input: "2" + twoPow256Sub1[1:], err: ErrBig256Range}, - {input: twoPow256Sub1[1:]}, - {input: "+" + twoPow256Sub1}, - {input: twoPow128}, - {input: twoPow128 + "1"}, - {input: twoPow128[1:]}, - {input: twoPow64 + "1"}, - {input: twoPow64[1:]}, - {input: "banana", err: strconv.ErrSyntax}, - {input: "0xab", err: strconv.ErrSyntax}, - {input: "ab", err: strconv.ErrSyntax}, - {input: "0"}, - {input: "", err: io.EOF}, - {input: "000"}, - {input: "+000"}, - {input: "010"}, - {input: "01"}, - {input: "-0", err: strconv.ErrSyntax}, - {input: "-10", err: strconv.ErrSyntax}, - {input: "115792089237316195423570985008687907853269984665640564039457584007913129639936", err: ErrBig256Range}, - {input: "115792089237316195423570985008687907853269984665640564039457584007913129639935"}, - } { - z.SetAllOne() // Set to ensure all bits are cleared after - err := z.SetFromDecimal(tc.input) - if !errors.Is(err, tc.err) { - t.Errorf("test %d, input %v: want err %s, have %s", i, tc.input, tc.err, err) + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 10) + if !ok { + if err == nil { + return fmt.Errorf("want error") } - if err != nil { - continue + return nil // both agree that input is bad + } + if bigA.BitLen() > 256 { + if err == nil { + return fmt.Errorf("want error (bitlen > 256)") + } + return nil + } + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) + } + return nil +} + +// Test SetString base 0 +func testSetFromBase0(tc string) error { + a := new(Int).SetAllOne() + a, haveOk := a.SetString(tc, 0) + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if haveOk { + return fmt.Errorf("want error on negative input") } - var want string - if w, ok := big.NewInt(0).SetString(tc.input, 10); !ok { - panic(fmt.Sprintf("test %d error", i)) - } else { - want = w.String() + return nil + } + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 0) + if !ok { + if haveOk { + return fmt.Errorf("want error") + } + return nil // both agree that input is bad + } + if bigA.BitLen() > 256 { + if haveOk { + return fmt.Errorf("want error (bitlen > 256)") + } + return nil + } + if !haveOk { + return fmt.Errorf("want no err, have err") + } + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) + } + return nil +} + +func TestStringScan(t *testing.T) { + for i, tc := range []string{ + "0000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000000000000000097", + "-000000000000000000000000000000000000000000000000000000000000000000000000000000", + "1157920892373161954235709850086879078532699846656405640394575840079131296399351", + "215792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "15792089237316195423570985008687907853269984665640564039457584007913129639935", + "+115792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639936", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "340282366920938463463374607431768211456", + "3402823669209384634633746074317682114561", + "40282366920938463463374607431768211456", + "00000000000000000000000097", + "184467440737095516161", + "8446744073709551616", + "banana", + "000", + "+000", + "010", + "0xab", + "-10", + "01", + "ab", + "0", + "-0", + "+0", + "", + "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + } { + if err := testSetFromDec(tc); err != nil { + t.Errorf("test %d, input '%s', SetFromDecimal err: %v", i, tc, err) } - if have := z.ToBig().String(); have != want { - t.Errorf("test %d: input %v, want %v: have %s", i, tc.input, want, have) + if err := testSetFromBase0(tc); err != nil { + t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) } + // TODO test SetString(.., 16) } } From 765b02b82b250a3201458e401057e140943a4b81 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 10:32:30 +0100 Subject: [PATCH 28/50] decimal: move some unused constants --- decimal.go | 2 -- decimal_test.go | 5 +++++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/decimal.go b/decimal.go index 5e12b30..8716d91 100644 --- a/decimal.go +++ b/decimal.go @@ -11,8 +11,6 @@ import ( ) const twoPow256Sub1 = "115792089237316195423570985008687907853269984665640564039457584007913129639935" -const twoPow128 = "340282366920938463463374607431768211456" -const twoPow64 = "18446744073709551616" func (z *Int) Dec() string { return z.ToBig().String() diff --git a/decimal_test.go b/decimal_test.go index 03e665a..00a1736 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -10,6 +10,11 @@ import ( "testing" ) +const ( + twoPow64 = "18446744073709551616" + twoPow128 = "340282366920938463463374607431768211456" +) + // Test SetFromDecimal func testSetFromDec(tc string) error { a := new(Int).SetAllOne() From 3430ee59d0a99555664114c09d21500fd88df811 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 10:46:27 +0100 Subject: [PATCH 29/50] more failing tests --- decimal_test.go | 112 +++++++++++++++++++++++++++--------------------- 1 file changed, 62 insertions(+), 50 deletions(-) diff --git a/decimal_test.go b/decimal_test.go index 00a1736..1f5aaac 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -84,6 +84,42 @@ func testSetFromBase0(tc string) error { return nil } +// Test SetString base 10 +func testSetFromBase10(tc string) error { + a := new(Int).SetAllOne() + a, haveOk := a.SetString(tc, 10) + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if haveOk { + return fmt.Errorf("want error on negative input") + } + return nil + } + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 10) + if !ok { + if haveOk { + return fmt.Errorf("want error") + } + return nil // both agree that input is bad + } + if bigA.BitLen() > 256 { + if haveOk { + return fmt.Errorf("want error (bitlen > 256)") + } + return nil + } + if !haveOk { + return fmt.Errorf("want no err, have err") + } + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) + } + return nil +} + func TestStringScan(t *testing.T) { for i, tc := range []string{ "0000000000000000000000000000000000000000000000000000000000000000000000000000000", @@ -96,13 +132,17 @@ func TestStringScan(t *testing.T) { "+115792089237316195423570985008687907853269984665640564039457584007913129639935", "115792089237316195423570985008687907853269984665640564039457584007913129639936", "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "+0b00000000000000000000000000000000000000000000000000000000000000010", "340282366920938463463374607431768211456", "3402823669209384634633746074317682114561", + "+3402823669209384634633746074317682114561", + "+-3402823669209384634633746074317682114561", "40282366920938463463374607431768211456", "00000000000000000000000097", "184467440737095516161", "8446744073709551616", "banana", + "+0x10", "000", "+000", "010", @@ -124,6 +164,9 @@ func TestStringScan(t *testing.T) { if err := testSetFromBase0(tc); err != nil { t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) } + if err := testSetFromBase10(tc); err != nil { + t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) + } // TODO test SetString(.., 16) } } @@ -165,57 +208,18 @@ func FuzzBase10StringCompare(f *testing.F) { } { f.Add(tc) } - f.Fuzz(func(t *testing.T, orig string) { - var ( - bi = new(big.Int) - z = new(Int) - ) - z, haveOk := z.SetString(orig, 10) - bi, wantOk := bi.SetString(orig, 10) - // if bigint parsing fail, make sure that we failed too - if !wantOk { - if haveOk { - t.Errorf("parsing status, want ok=%v, have ok=%v. Input: %s", haveOk, wantOk, orig) - } - return - } - // if its a negative number, we should err - if len(orig) > 0 && (orig[0] == '-') { - if haveOk { - t.Errorf("should have errored at negative number: %s", orig) - } - return - } - // if its too large, ignore it also - if bi.BitLen() > 256 { - if haveOk { - t.Errorf("should have errored at number overflow: %s", orig) - } - return - } - // No more reasons not to succeed - if !haveOk { - t.Errorf("should have parsed %s to %s, but errored instead", orig, bi.String()) - return - } - // otherwise, make sure that the values are equal - if z.ToBig().Cmp(bi) != 0 { - t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec()) - return - } - // make sure that bigint base 10 string is equal to base10 string - if z.Dec() != bi.String() { - t.Errorf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec()) - return - } - value, err := z.Value() - if err != nil { - t.Errorf("fail to Value() %s, got err %s", bi, err) - return + f.Fuzz(func(t *testing.T, tc string) { + if err := testSetFromDec(tc); err != nil { + t.Errorf("input '%s', SetFromDecimal err: %v", tc, err) } - if z.Dec()+"e0" != fmt.Sprint(value) { - t.Errorf("value of %s did not match base 10 encoding %s", value, z.Dec()) - return + // Our base16 parsing differs, so inputs like this would be rejected + // where big.Int accepts them: + // +0x00000000000000000000000000000000000000000000000000000000000000000 + //if err := testSetFromBase0(tc); err != nil { + // t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) + //} + if err := testSetFromBase10(tc); err != nil { + t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) } }) } @@ -347,3 +351,11 @@ func BenchmarkStringBase16(b *testing.B) { } }) } + +func TestFoo(t *testing.T) { + s := "+0b00000000000000000000000000000000000000000000000000000000000000010" + b, ok := new(big.Int).SetString(s, 0) + fmt.Printf("b: %v, ok : %v\n", b, ok) + z, ok := new(Int).SetString(s, 0) + fmt.Printf("z: %v, ok : %v\n", z, ok) +} From cbdcdcb271b0b67ad8ef227a87d2add675e48978 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 15:26:05 +0100 Subject: [PATCH 30/50] decimal: remove SetString method --- conversion.go | 26 +++--- decimal.go | 60 ++------------ decimal_test.go | 207 ++++++++++-------------------------------------- 3 files changed, 62 insertions(+), 231 deletions(-) diff --git a/conversion.go b/conversion.go index aca9c2f..004a458 100644 --- a/conversion.go +++ b/conversion.go @@ -65,6 +65,18 @@ func FromBig(b *big.Int) (*Int, bool) { return z, overflow } +// SetFromDecimal sets z from the given string, interpreted as a decimal number. +// OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 16) method. +// Notable differences: +// - This method does not accept zero-prefixed hex, e.g. "0x0001" +// - This method does not accept underscore input, e.g. "100_000", +// - This method does not accept negative zero as valid, e.g "-0x0", +// - (this method does not accept any negative input as valid) +func (z *Int) SetFromHex(hex string) error { + z.Clear() + return z.fromHex(hex) +} + // fromHex is the internal implementation of parsing a hex-string. func (z *Int) fromHex(hex string) error { if err := checkNumberS(hex); err != nil { @@ -549,22 +561,12 @@ func (dst *Int) Scan(src interface{}) error { if src == nil { *dst = Int{} } - switch src := src.(type) { case string: - _, ok := dst.SetString(src, 0) - if !ok { - return fmt.Errorf("cannot scan %T", src) - } - return nil + return dst.SetFromDecimal(src) case []byte: - _, ok := dst.SetString(string(src), 0) - if !ok { - return fmt.Errorf("cannot scan %T", src) - } - return nil + return dst.SetFromDecimal(string(src)) } - return fmt.Errorf("cannot scan %T", src) } diff --git a/decimal.go b/decimal.go index 8716d91..9702361 100644 --- a/decimal.go +++ b/decimal.go @@ -6,7 +6,6 @@ package uint256 import ( "io" - "math/big" "strconv" ) @@ -16,60 +15,6 @@ func (z *Int) Dec() string { return z.ToBig().String() } -// SetString implements a subset of (*big.Int).SetString -// ok will be true iff i == nil -// The SetString method on uint256.Int differs from big.Int when it comes to -// base 16, since uint256.Int is stricter, requiring: -// - 0x or 0X prefix -// - Non-empty string after prefix -// - No leading zeroes -// The base10 version allows leading zeroes. -func (z *Int) SetString(s string, base int) (i *Int, ok bool) { - z.Clear() - switch base { - case 0: - // From documentation at big.Int: - // For base 0, the number prefix determines the actual base: A prefix of - // “0b” or “0B” selects base 2, - // “0”, “0o” or “0O” selects base 8, - // and “0x” or “0X” selects base 16. - // Otherwise, the selected base is 10 and no prefix is accepted. - if len(s) > 1 && s[0] == '0' && (s[1] == 'x' || s[1] == 'X') { - // base 16 - if err := z.fromHex(s); err != nil { - return nil, false - } - return z, true - } - if len(s) > 0 && s[0] == '0' { - // base 2 or base 8 - // Not implemented here, use big.Int - // @TODO - if b, ok := big.NewInt(0).SetString(s, 0); !ok { - return nil, false - } else if overflow := z.SetFromBig(b); overflow { - return nil, false - } - return z, true - } - if err := z.SetFromDecimal(s); err != nil { - return nil, false - } - return z, true - case 10: - if err := z.SetFromDecimal(s); err != nil { - return nil, false - } - return z, true - case 16: - if err := z.fromHex(s); err != nil { - return nil, false - } - return z, true - } - return nil, false -} - // FromDecimal is a convenience-constructor to create an Int from a // decimal (base 10) string. Numbers larger than 256 bits are not accepted. func FromDecimal(hex string) (*Int, error) { @@ -81,6 +26,11 @@ func FromDecimal(hex string) (*Int, error) { } // SetFromDecimal sets z from the given string, interpreted as a decimal number. +// OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 10) method. +// Notable differences: +// - This method does not accept underscore input, e.g. "100_000", +// - This method does not accept negative zero as valid, e.g "-0", +// - (this method does not accept any negative input as valid)) func (z *Int) SetFromDecimal(s string) (err error) { // Remove max one leading + if len(s) > 0 && s[0] == '+' { diff --git a/decimal_test.go b/decimal_test.go index 1f5aaac..5a9d097 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -48,46 +48,11 @@ func testSetFromDec(tc string) error { return nil } -// Test SetString base 0 -func testSetFromBase0(tc string) error { - a := new(Int).SetAllOne() - a, haveOk := a.SetString(tc, 0) - // If input is negative, we should eror - if len(tc) > 0 && tc[0] == '-' { - if haveOk { - return fmt.Errorf("want error on negative input") - } - return nil - } - // Need to compare with big.Int - bigA, ok := big.NewInt(0).SetString(tc, 0) - if !ok { - if haveOk { - return fmt.Errorf("want error") - } - return nil // both agree that input is bad - } - if bigA.BitLen() > 256 { - if haveOk { - return fmt.Errorf("want error (bitlen > 256)") - } - return nil - } - if !haveOk { - return fmt.Errorf("want no err, have err") - } - want := bigA.String() - have := a.Dec() - if want != have { - return fmt.Errorf("want %v, have %v", want, have) - } - return nil -} - -// Test SetString base 10 -func testSetFromBase10(tc string) error { +// Test SetFromDecimal +func testSetFromDecimal(tc string) error { a := new(Int).SetAllOne() - a, haveOk := a.SetString(tc, 10) + err := a.SetFromDecimal(tc) + haveOk := (err == nil) // If input is negative, we should eror if len(tc) > 0 && tc[0] == '-' { if haveOk { @@ -161,12 +126,12 @@ func TestStringScan(t *testing.T) { if err := testSetFromDec(tc); err != nil { t.Errorf("test %d, input '%s', SetFromDecimal err: %v", i, tc, err) } - if err := testSetFromBase0(tc); err != nil { - t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) - } - if err := testSetFromBase10(tc); err != nil { - t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) - } + //if err := testSetFromBase0(tc); err != nil { + // t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) + //} + //if err := testSetFromBase10(tc); err != nil { + // t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) + //} // TODO test SetString(.., 16) } } @@ -218,144 +183,58 @@ func FuzzBase10StringCompare(f *testing.F) { //if err := testSetFromBase0(tc); err != nil { // t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) //} - if err := testSetFromBase10(tc); err != nil { - t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) - } + //if err := testSetFromBase10(tc); err != nil { + // t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) + //} }) } -func BenchmarkStringBase10BigInt(b *testing.B) { - val := new(big.Int) - bytearr := twoPow256Sub1 - b.Run("generic", func(b *testing.B) { +func BenchmarkStringBase10(b *testing.B) { + input := twoPow256Sub1 + + b.Run("bigint", func(b *testing.B) { + val := new(big.Int) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - val.SetString(bytearr[:2], 10) - val.SetString(bytearr[:4], 10) - val.SetString(bytearr[:6], 10) - val.SetString(bytearr[:8], 10) - val.SetString(bytearr[:16], 10) - val.SetString(bytearr[:12], 10) - val.SetString(bytearr[:14], 10) - val.SetString(bytearr[:16], 10) - val.SetString(bytearr[:18], 10) - val.SetString(bytearr[:20], 10) - val.SetString(bytearr[:22], 10) - val.SetString(bytearr[:24], 10) - val.SetString(bytearr[:26], 10) - val.SetString(bytearr[:28], 10) - val.SetString(bytearr[:30], 10) - val.SetString(bytearr[:32], 10) - val.SetString(bytearr[:34], 10) - val.SetString(bytearr[:36], 10) - val.SetString(bytearr[:38], 10) - val.SetString(bytearr[:40], 10) - val.SetString(bytearr[:42], 10) - val.SetString(bytearr[:44], 10) - val.SetString(bytearr[:46], 10) - val.SetString(bytearr[:48], 10) - val.SetString(bytearr[:50], 10) - val.SetString(bytearr[:52], 10) - val.SetString(bytearr[:54], 10) - val.SetString(bytearr[:56], 10) - val.SetString(bytearr[:58], 10) - val.SetString(bytearr[:60], 10) - val.SetString(bytearr[:62], 10) - val.SetString(bytearr[:64], 10) + for j := 1; j < len(input); j++ { + val.SetString(input[:j], 10) + } } }) -} -func BenchmarkStringBase10(b *testing.B) { - val := new(Int) - bytearr := twoPow256Sub1 - b.Run("generic", func(b *testing.B) { + b.Run("u256", func(b *testing.B) { + val := new(Int) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - val.SetString(bytearr[:2], 10) - val.SetString(bytearr[:4], 10) - val.SetString(bytearr[:6], 10) - val.SetString(bytearr[:8], 10) - val.SetString(bytearr[:16], 10) - val.SetString(bytearr[:12], 10) - val.SetString(bytearr[:14], 10) - val.SetString(bytearr[:16], 10) - val.SetString(bytearr[:18], 10) - val.SetString(bytearr[:20], 10) - val.SetString(bytearr[:22], 10) - val.SetString(bytearr[:24], 10) - val.SetString(bytearr[:26], 10) - val.SetString(bytearr[:28], 10) - val.SetString(bytearr[:30], 10) - val.SetString(bytearr[:32], 10) - val.SetString(bytearr[:34], 10) - val.SetString(bytearr[:36], 10) - val.SetString(bytearr[:38], 10) - val.SetString(bytearr[:40], 10) - val.SetString(bytearr[:42], 10) - val.SetString(bytearr[:44], 10) - val.SetString(bytearr[:46], 10) - val.SetString(bytearr[:48], 10) - val.SetString(bytearr[:50], 10) - val.SetString(bytearr[:52], 10) - val.SetString(bytearr[:54], 10) - val.SetString(bytearr[:56], 10) - val.SetString(bytearr[:58], 10) - val.SetString(bytearr[:60], 10) - val.SetString(bytearr[:62], 10) - val.SetString(bytearr[:64], 10) + for j := 1; j < len(input); j++ { + val.SetFromDecimal(input[:j]) + } } }) } + func BenchmarkStringBase16(b *testing.B) { - val := new(Int) - bytearr := "aaaa12131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031bbbb" - b.Run("generic", func(b *testing.B) { + input := "aaaa12131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031bbbb" + b.Run("bigint", func(b *testing.B) { + val := new(big.Int) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - val.SetString(bytearr[:1], 16) - val.SetString(bytearr[:2], 16) - val.SetString(bytearr[:3], 16) - val.SetString(bytearr[:4], 16) - val.SetString(bytearr[:5], 16) - val.SetString(bytearr[:6], 16) - val.SetString(bytearr[:7], 16) - val.SetString(bytearr[:8], 16) - val.SetString(bytearr[:9], 16) - val.SetString(bytearr[:10], 16) - val.SetString(bytearr[:11], 16) - val.SetString(bytearr[:12], 16) - val.SetString(bytearr[:13], 16) - val.SetString(bytearr[:14], 16) - val.SetString(bytearr[:15], 16) - val.SetString(bytearr[:16], 16) - val.SetString(bytearr[:17], 16) - val.SetString(bytearr[:18], 16) - val.SetString(bytearr[:19], 16) - val.SetString(bytearr[:20], 16) - val.SetString(bytearr[:21], 16) - val.SetString(bytearr[:22], 16) - val.SetString(bytearr[:23], 16) - val.SetString(bytearr[:24], 16) - val.SetString(bytearr[:25], 16) - val.SetString(bytearr[:26], 16) - val.SetString(bytearr[:27], 16) - val.SetString(bytearr[:28], 16) - val.SetString(bytearr[:29], 16) - val.SetString(bytearr[:20], 16) - val.SetString(bytearr[:31], 16) - val.SetString(bytearr[:32], 16) + for j := 1; j < len(input); j++ { + val.SetString(input[:j], 16) + } + } + }) + b.Run("u256", func(b *testing.B) { + val := new(Int) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 1; j < len(input); j++ { + val.SetFromHex(input[:j]) + } } }) -} - -func TestFoo(t *testing.T) { - s := "+0b00000000000000000000000000000000000000000000000000000000000000010" - b, ok := new(big.Int).SetString(s, 0) - fmt.Printf("b: %v, ok : %v\n", b, ok) - z, ok := new(Int).SetString(s, 0) - fmt.Printf("z: %v, ok : %v\n", z, ok) } From 38d53f3929507b4cac50da6ad134a9c62277c4d9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 15:32:45 +0100 Subject: [PATCH 31/50] decimal: simplify tests --- decimal_test.go | 171 ++++++++++++++---------------------------------- 1 file changed, 49 insertions(+), 122 deletions(-) diff --git a/decimal_test.go b/decimal_test.go index 5a9d097..d00b03f 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -48,144 +48,71 @@ func testSetFromDec(tc string) error { return nil } -// Test SetFromDecimal -func testSetFromDecimal(tc string) error { - a := new(Int).SetAllOne() - err := a.SetFromDecimal(tc) - haveOk := (err == nil) - // If input is negative, we should eror - if len(tc) > 0 && tc[0] == '-' { - if haveOk { - return fmt.Errorf("want error on negative input") - } - return nil - } - // Need to compare with big.Int - bigA, ok := big.NewInt(0).SetString(tc, 10) - if !ok { - if haveOk { - return fmt.Errorf("want error") - } - return nil // both agree that input is bad - } - if bigA.BitLen() > 256 { - if haveOk { - return fmt.Errorf("want error (bitlen > 256)") - } - return nil - } - if !haveOk { - return fmt.Errorf("want no err, have err") - } - want := bigA.String() - have := a.Dec() - if want != have { - return fmt.Errorf("want %v, have %v", want, have) - } - return nil +var cases = []string{ + "0000000000000000000000000000000000000000000000000000000000000000000000000000000", + "0000000000000000000000000000000000000000000000000000000000000000000000000000097", + "-000000000000000000000000000000000000000000000000000000000000000000000000000000", + "1157920892373161954235709850086879078532699846656405640394575840079131296399351", + "215792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "15792089237316195423570985008687907853269984665640564039457584007913129639935", + "+115792089237316195423570985008687907853269984665640564039457584007913129639935", + "115792089237316195423570985008687907853269984665640564039457584007913129639936", + "115792089237316195423570985008687907853269984665640564039457584007913129639935", + "+0b00000000000000000000000000000000000000000000000000000000000000010", + "340282366920938463463374607431768211456", + "3402823669209384634633746074317682114561", + "+3402823669209384634633746074317682114561", + "+-3402823669209384634633746074317682114561", + "40282366920938463463374607431768211456", + "00000000000000000000000097", + "184467440737095516161", + "8446744073709551616", + "banana", + "+0x10", + "000", + "+000", + "010", + "0xab", + "-10", + "01", + "ab", + "0", + "-0", + "+0", + "", + "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x10101011010", + "熊熊熊熊熊熊熊熊", + "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x10000000000000000000000000000000000000000000000000000000000000000", + "+0x00000000000000000000000000000000000000000000000000000000000000000", + "-0x00000000000000000000000000000000000000000000000000000000000000000", } func TestStringScan(t *testing.T) { - for i, tc := range []string{ - "0000000000000000000000000000000000000000000000000000000000000000000000000000000", - "0000000000000000000000000000000000000000000000000000000000000000000000000000097", - "-000000000000000000000000000000000000000000000000000000000000000000000000000000", - "1157920892373161954235709850086879078532699846656405640394575840079131296399351", - "215792089237316195423570985008687907853269984665640564039457584007913129639935", - "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "15792089237316195423570985008687907853269984665640564039457584007913129639935", - "+115792089237316195423570985008687907853269984665640564039457584007913129639935", - "115792089237316195423570985008687907853269984665640564039457584007913129639936", - "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "+0b00000000000000000000000000000000000000000000000000000000000000010", - "340282366920938463463374607431768211456", - "3402823669209384634633746074317682114561", - "+3402823669209384634633746074317682114561", - "+-3402823669209384634633746074317682114561", - "40282366920938463463374607431768211456", - "00000000000000000000000097", - "184467440737095516161", - "8446744073709551616", - "banana", - "+0x10", - "000", - "+000", - "010", - "0xab", - "-10", - "01", - "ab", - "0", - "-0", - "+0", - "", - "熊熊熊熊熊熊熊熊", - "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - } { + for i, tc := range cases { if err := testSetFromDec(tc); err != nil { t.Errorf("test %d, input '%s', SetFromDecimal err: %v", i, tc, err) } - //if err := testSetFromBase0(tc); err != nil { - // t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) - //} - //if err := testSetFromBase10(tc); err != nil { - // t.Errorf("test %d, input '%s', SetString(..,0) err: %v", i, tc, err) - //} - // TODO test SetString(.., 16) + // TODO testSetFromHex(tc) } } func FuzzBase10StringCompare(f *testing.F) { - - for _, tc := range []string{ - twoPow256Sub1 + "1", - "2" + twoPow256Sub1[1:], - twoPow256Sub1, - twoPow128, - twoPow128, - twoPow128, - twoPow64, - twoPow64, - "banana", - "0xab", - "ab", - "0", - "000", - "010", - "01", - "-0", - "+0", - "-10", - "115792089237316195423570985008687907853269984665640564039457584007913129639936", - "115792089237316195423570985008687907853269984665640564039457584007913129639935", - "apple", - "04112401274120741204712xxxxxz00", - "0x10101011010", - "熊熊熊熊熊熊熊熊", - "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "-0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "0x10000000000000000000000000000000000000000000000000000000000000000", - "+0x10000000000000000000000000000000000000000000000000000000000000000", - "+0x00000000000000000000000000000000000000000000000000000000000000000", - "-0x00000000000000000000000000000000000000000000000000000000000000000", - } { + for _, tc := range cases { f.Add(tc) } f.Fuzz(func(t *testing.T, tc string) { if err := testSetFromDec(tc); err != nil { t.Errorf("input '%s', SetFromDecimal err: %v", tc, err) } - // Our base16 parsing differs, so inputs like this would be rejected - // where big.Int accepts them: - // +0x00000000000000000000000000000000000000000000000000000000000000000 - //if err := testSetFromBase0(tc); err != nil { - // t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) - //} - //if err := testSetFromBase10(tc); err != nil { - // t.Errorf("input '%s', SetString(..,0) err: %v", tc, err) - //} + // TODO testSetFromHex(tc) }) } From ca9a5ea19cc72e73574868fa1ca01872c903801e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 16 Dec 2022 15:45:42 +0100 Subject: [PATCH 32/50] rm unused constants, fix benchmarks --- conversion.go | 1 + decimal_test.go | 31 +++++++++++++++++-------------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/conversion.go b/conversion.go index 004a458..862471d 100644 --- a/conversion.go +++ b/conversion.go @@ -68,6 +68,7 @@ func FromBig(b *big.Int) (*Int, bool) { // SetFromDecimal sets z from the given string, interpreted as a decimal number. // OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 16) method. // Notable differences: +// - This method _require_ "0x" or "0X" prefix. // - This method does not accept zero-prefixed hex, e.g. "0x0001" // - This method does not accept underscore input, e.g. "100_000", // - This method does not accept negative zero as valid, e.g "-0x0", diff --git a/decimal_test.go b/decimal_test.go index d00b03f..beb3901 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -10,11 +10,6 @@ import ( "testing" ) -const ( - twoPow64 = "18446744073709551616" - twoPow128 = "340282366920938463463374607431768211456" -) - // Test SetFromDecimal func testSetFromDec(tc string) error { a := new(Int).SetAllOne() @@ -116,7 +111,7 @@ func FuzzBase10StringCompare(f *testing.F) { }) } -func BenchmarkStringBase10(b *testing.B) { +func BenchmarkFromDecimalString(b *testing.B) { input := twoPow256Sub1 b.Run("bigint", func(b *testing.B) { @@ -125,7 +120,9 @@ func BenchmarkStringBase10(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 1; j < len(input); j++ { - val.SetString(input[:j], 10) + if _, ok := val.SetString(input[:j], 10); !ok { + b.Fatalf("Error on %v", string(input[:j])) + } } } }) @@ -136,21 +133,25 @@ func BenchmarkStringBase10(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { for j := 1; j < len(input); j++ { - val.SetFromDecimal(input[:j]) + if err := val.SetFromDecimal(input[:j]); err != nil { + b.Fatalf("%v: %v", err, string(input[:j])) + } } } }) } -func BenchmarkStringBase16(b *testing.B) { - input := "aaaa12131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f3031bbbb" +func BenchmarkFromHexString(b *testing.B) { + input := "0xf131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303" b.Run("bigint", func(b *testing.B) { val := new(big.Int) b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - for j := 1; j < len(input); j++ { - val.SetString(input[:j], 16) + for j := 3; j < len(input); j++ { + if _, ok := val.SetString(input[:j], 0); !ok { + b.Fatalf("Error on %v", string(input[:j])) + } } } }) @@ -159,8 +160,10 @@ func BenchmarkStringBase16(b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - for j := 1; j < len(input); j++ { - val.SetFromHex(input[:j]) + for j := 3; j < len(input); j++ { + if err := val.SetFromHex(input[:j]); err != nil { + b.Fatalf("%v: %v", err, string(input[:j])) + } } } }) From 95188318bcbaa53cfac510d1eda6239103e0b15a Mon Sep 17 00:00:00 2001 From: a Date: Fri, 16 Dec 2022 22:56:44 -0600 Subject: [PATCH 33/50] Update decimal.go Co-authored-by: Martin Holst Swende --- decimal.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/decimal.go b/decimal.go index 9702361..0b2a820 100644 --- a/decimal.go +++ b/decimal.go @@ -101,11 +101,11 @@ func (z *Int) fromDecimal(bs string) error { return err } // add that number to our running total - if i != 0 { + if i == 0 { + z.SetUint64(num) + } else { base := NewInt(num) z.Add(z, base.Mul(base, mult)) - } else { - z.SetUint64(num) } // Chop off another 19 characters if remaining > 19 { From ad5ba70bdff264b4c33d3907fdedbe5437f55b45 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 16 Dec 2022 22:56:54 -0600 Subject: [PATCH 34/50] Update conversion.go Co-authored-by: Martin Holst Swende --- conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index 862471d..2b1364a 100644 --- a/conversion.go +++ b/conversion.go @@ -65,7 +65,7 @@ func FromBig(b *big.Int) (*Int, bool) { return z, overflow } -// SetFromDecimal sets z from the given string, interpreted as a decimal number. +// SetFromHex sets z from the given string, interpreted as a hexadecimal number. // OBS! This method is _not_ strictly identical to the (*big.Int).SetString(..., 16) method. // Notable differences: // - This method _require_ "0x" or "0X" prefix. From 301af32f3ec7a713a246774a4a7467c33a89c86d Mon Sep 17 00:00:00 2001 From: a Date: Tue, 20 Dec 2022 16:14:41 -0600 Subject: [PATCH 35/50] change sql valuer --- conversion.go | 7 +++++-- decimal.go | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/conversion.go b/conversion.go index 2b1364a..09fec1c 100644 --- a/conversion.go +++ b/conversion.go @@ -572,9 +572,12 @@ func (dst *Int) Scan(src interface{}) error { } // Value implements the database/sql/driver Valuer interface. -// It encodes a string with an e0 suffix, telling postgresql that it is of the numeric/decimal type. +// It encodes a base 10 string. +// In Postgres, this will work with both integer and the Numeric/Decimal types +// In MariaDB/MySQL, this will work with the Numeric/Decimal types up to 65 digits, however any more and you should use either VarChar or Char(79) +// In SqLite, use TEXT func (src Int) Value() (driver.Value, error) { - return src.ToBig().String() + "e0", nil + return src.ToBig().String(), nil } var ( diff --git a/decimal.go b/decimal.go index 0b2a820..ac05134 100644 --- a/decimal.go +++ b/decimal.go @@ -17,9 +17,9 @@ func (z *Int) Dec() string { // FromDecimal is a convenience-constructor to create an Int from a // decimal (base 10) string. Numbers larger than 256 bits are not accepted. -func FromDecimal(hex string) (*Int, error) { +func FromDecimal(decimal string) (*Int, error) { var z Int - if err := z.SetFromDecimal(hex); err != nil { + if err := z.SetFromDecimal(decimal); err != nil { return nil, err } return &z, nil From b5a490679ce1eda20c0d0c9bca187a8b479e6826 Mon Sep 17 00:00:00 2001 From: a Date: Wed, 28 Dec 2022 21:36:32 -0600 Subject: [PATCH 36/50] remove extra struct def --- decimal.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/decimal.go b/decimal.go index ac05134..3390a08 100644 --- a/decimal.go +++ b/decimal.go @@ -61,11 +61,11 @@ func (z *Int) SetFromDecimal(s string) (err error) { // multipliers holds the values that are needed for fromDecimal var multipliers = [5]*Int{ - nil, // represents first round, no multiplication needed - &Int{10000000000000000000, 0, 0, 0}, // 10 ^ 19 - &Int{687399551400673280, 5421010862427522170, 0, 0}, // 10 ^ 38 - &Int{5332261958806667264, 17004971331911604867, 2938735877055718769, 0}, // 10 ^ 57 - &Int{0, 8607968719199866880, 532749306367912313, 1593091911132452277}, // 10 ^ 76 + nil, // represents first round, no multiplication needed + {10000000000000000000, 0, 0, 0}, // 10 ^ 19 + {687399551400673280, 5421010862427522170, 0, 0}, // 10 ^ 38 + {5332261958806667264, 17004971331911604867, 2938735877055718769, 0}, // 10 ^ 57 + {0, 8607968719199866880, 532749306367912313, 1593091911132452277}, // 10 ^ 76 } // fromDecimal is a helper function to only ever be called via SetFromDecimal From fb7ceef62716a7666d2103320d5c2a6d0b79c7aa Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 08:29:44 +0100 Subject: [PATCH 37/50] testing: try to get 100% coverage again --- conversion_test.go | 83 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/conversion_test.go b/conversion_test.go index 7e7c82e..8d51ed7 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -720,16 +720,20 @@ func TestEnDecode(t *testing.T) { } var testSample = func(i int, bigSample big.Int, intSample Int) { // Encoding - exp := fmt.Sprintf("0x%s", bigSample.Text(16)) + wantHex := fmt.Sprintf("0x%s", bigSample.Text(16)) + wantDec := fmt.Sprintf("%s", bigSample.Text(10)) - if got := intSample.Hex(); exp != got { - t.Fatalf("test %d #1, got %v, exp %v", i, got, exp) + if got := intSample.Hex(); wantHex != got { + t.Fatalf("test %d #1, got %v, exp %v", i, got, wantHex) } - if got := intSample.String(); exp != got { - t.Fatalf("test %d #2, got %v, exp %v", i, got, exp) + if got := intSample.String(); wantHex != got { + t.Fatalf("test %d #2, got %v, exp %v", i, got, wantHex) } - if got, _ := intSample.MarshalText(); exp != string(got) { - t.Fatalf("test %d #3, got %v, exp %v", i, got, exp) + if got, _ := intSample.MarshalText(); wantHex != string(got) { + t.Fatalf("test %d #3, got %v, exp %v", i, got, wantHex) + } + if got, _ := intSample.Value(); wantDec != got.(string) { + t.Fatalf("test %d #4, got %v, exp %v", i, got, wantHex) } { // Json jsonEncoded, err := json.Marshal(&jsonStruct{&intSample}) @@ -746,21 +750,64 @@ func TestEnDecode(t *testing.T) { } } // Decoding - dec, err := FromHex(exp) - if err != nil { - t.Fatalf("test %d #5, err: %v", i, err) + // + // FromHex + decoded, err := FromHex(wantHex) + { + if err != nil { + t.Fatalf("test %d #5, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + } + } + // UnmarshalText + decoded = new(Int) + { + if err := decoded.UnmarshalText([]byte(wantHex)); err != nil { + t.Fatalf("test %d #7, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #8, got %v, exp %v", i, decoded, intSample) + } + } + // FromDecimal + decoded, err = FromDecimal(wantDec) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } } - if dec.Cmp(&intSample) != 0 { - t.Fatalf("test %d #6, got %v, exp %v", i, dec, intSample) + // Scan w string + err = decoded.Scan(wantDec) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } } - dec = new(Int) - if err := dec.UnmarshalText([]byte(exp)); err != nil { - t.Fatalf("test %d #7, err: %v", i, err) + // Scan w byte slice + err = decoded.Scan([]byte(wantDec)) + { + if err != nil { + t.Fatalf("test %d #9, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #10, got %v, exp %v", i, decoded, intSample) + } } - if dec.Cmp(&intSample) != 0 { - t.Fatalf("test %d #8, got %v, exp %v", i, dec, intSample) + // Scan with neither string nor byte + err = decoded.Scan(5) + { + if err == nil { + t.Fatalf("test %d #11, want error", i) + } } - } for i, bigSample := range big256Samples { intSample := int256Samples[i] From 4ef03d763cec1e68540a1c830c089ed539b64473 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 08:31:27 +0100 Subject: [PATCH 38/50] lintfix --- conversion_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion_test.go b/conversion_test.go index 8d51ed7..a8b5a5a 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -721,7 +721,7 @@ func TestEnDecode(t *testing.T) { var testSample = func(i int, bigSample big.Int, intSample Int) { // Encoding wantHex := fmt.Sprintf("0x%s", bigSample.Text(16)) - wantDec := fmt.Sprintf("%s", bigSample.Text(10)) + wantDec := bigSample.Text(10) if got := intSample.Hex(); wantHex != got { t.Fatalf("test %d #1, got %v, exp %v", i, got, wantHex) From 5587dd69110b8cf0f706afaadc3d31076de35c1c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 08:38:48 +0100 Subject: [PATCH 39/50] fuzzing: fix up fuzzer for string conversion --- fuzz.go | 76 ++++++++++++++++++++++++++------------------------------- 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/fuzz.go b/fuzz.go index e8ad547..e085644 100644 --- a/fuzz.go +++ b/fuzz.go @@ -305,57 +305,49 @@ func fuzzTernaryOp(data []byte) int { return 1 } -func FuzzSetString(data []byte) int { - if len(data) > 512 { - // Too large, makes no sense - return -1 - } - var ( - orig = string(data) - bi = new(big.Int) - z = new(Int) - ) - z, haveOk := z.SetString(orig, 10) - bi, wantOk := bi.SetString(orig, 10) - // if bigint parsing fail, make sure that we failed too - if !wantOk { - if haveOk { - panic(fmt.Sprintf("parsing status, want ok=%v, have ok=%v. Input: %s", haveOk, wantOk, orig)) +// Test SetFromDecimal +func testSetFromDecForFuzzing(tc string) error { + a := new(Int).SetAllOne() + err := a.SetFromDecimal(tc) + // If input is negative, we should eror + if len(tc) > 0 && tc[0] == '-' { + if err == nil { + return fmt.Errorf("want error on negative input") } - return 1 + return nil } - // if its a negative number, we should err - if len(orig) > 0 && (orig[0] == '-') { - if haveOk { - panic(fmt.Sprintf("should have errored at negative number: %s", orig)) + // Need to compare with big.Int + bigA, ok := big.NewInt(0).SetString(tc, 10) + if !ok { + if err == nil { + return fmt.Errorf("want error") } - return 1 + return nil // both agree that input is bad } - // if its too large, ignore it also - if bi.BitLen() > 256 { - if haveOk { - panic(fmt.Sprintf("should have errored at number overflow: %s", orig)) + if bigA.BitLen() > 256 { + if err == nil { + return fmt.Errorf("want error (bitlen > 256)") } - return 1 - } - // No more reasons not to succeed - if !haveOk { - panic(fmt.Sprintf("should have parsed '%s' to '%s', but errored instead", orig, bi.String())) + return nil } - // otherwise, make sure that the values are equal - if z.ToBig().Cmp(bi) != 0 { - panic(fmt.Sprintf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec())) + want := bigA.String() + have := a.Dec() + if want != have { + return fmt.Errorf("want %v, have %v", want, have) } - // make sure that bigint base 10 string is equal to base10 string - if z.Dec() != bi.String() { - panic(fmt.Sprintf("should have parsed %s to %s, but got %s", orig, bi.String(), z.Dec())) + if _, err := a.Value(); err != nil { + return fmt.Errorf("fail to Value() %s, got err %s", tc, err) } - value, err := z.Value() - if err != nil { - panic(fmt.Sprintf("fail to Value() %s, got err %s", bi, err)) + return nil +} + +func FuzzSetString(data []byte) int { + if len(data) > 512 { + // Too large, makes no sense + return -1 } - if z.Dec()+"e0" != fmt.Sprint(value) { - panic(fmt.Sprintf("value of %s did not match base 10 encoding %s", value, z.Dec())) + if err := testSetFromDecForFuzzing(string(data)); err != nil { + panic(err) } return 1 } From 7ff0d8fcf7b45deca051fd46c4f697374b18ff0f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 08:43:32 +0100 Subject: [PATCH 40/50] circle: try to get circleci fuzzing going --- circle.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circle.yml b/circle.yml index 4470b49..1b4b31d 100644 --- a/circle.yml +++ b/circle.yml @@ -44,7 +44,7 @@ jobs: - run: name: "Fuzzing" command: | - go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build + go install github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build go-fuzz-build timeout --preserve-status --signal INT 1m go-fuzz -procs=2 test ! "$(ls crashers)" From 5019647223f5e0134868be88450d56f81f532729 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 08:50:17 +0100 Subject: [PATCH 41/50] conversion: more coverage + fix in Scan --- conversion.go | 3 ++- conversion_test.go | 20 ++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index 09fec1c..afc9c29 100644 --- a/conversion.go +++ b/conversion.go @@ -560,7 +560,8 @@ func (z *Int) Hex() string { // It decodes a string, because that is what postgres uses for its numeric type func (dst *Int) Scan(src interface{}) error { if src == nil { - *dst = Int{} + dst.Clear() + return nil } switch src := src.(type) { case string: diff --git a/conversion_test.go b/conversion_test.go index a8b5a5a..33ad989 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -761,6 +761,16 @@ func TestEnDecode(t *testing.T) { t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) } } + // z.SetFromHex + err = decoded.SetFromHex(wantHex) + { + if err != nil { + t.Fatalf("test %d #5, err: %v", i, err) + } + if decoded.Cmp(&intSample) != 0 { + t.Fatalf("test %d #6, got %v, exp %v", i, decoded, intSample) + } + } // UnmarshalText decoded = new(Int) { @@ -819,3 +829,13 @@ func TestEnDecode(t *testing.T) { testSample(i, bigSample, intSample) } } + +func TestNil(t *testing.T) { + a := NewInt(1337) + if err := a.Scan(nil); err != nil { + t.Fatal(err) + } + if !a.IsZero() { + t.Fatal("want zero") + } +} From 2f0b0b7dadbfac7967a1e3a90a1f72397398a0d7 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 09:03:21 +0100 Subject: [PATCH 42/50] decimal_test: more coverage --- decimal_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/decimal_test.go b/decimal_test.go index beb3901..be10668 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -14,6 +14,17 @@ import ( func testSetFromDec(tc string) error { a := new(Int).SetAllOne() err := a.SetFromDecimal(tc) + { // Check the FromDecimal too + b, err2 := FromDecimal(tc) + if (err == nil) != (err2 == nil) { + return fmt.Errorf("err != err2: %v %v", err, err2) + } + if err == nil { + if a.Cmp(b) != 0 { + return fmt.Errorf("a != b: %v %v", a, b) + } + } + } // If input is negative, we should eror if len(tc) > 0 && tc[0] == '-' { if err == nil { From dcd75fd237508e2be2b2d4eedf660848ec3e4143 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 09:04:51 +0100 Subject: [PATCH 43/50] go.mod: fuzzing dep --- circle.yml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/circle.yml b/circle.yml index 1b4b31d..6c3a263 100644 --- a/circle.yml +++ b/circle.yml @@ -40,18 +40,16 @@ jobs: command: bash <(curl -s https://codecov.io/bash) - restore_cache: keys: - - corpus-v2 + - corpus-v3 - run: name: "Fuzzing" command: | - go install github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build - go-fuzz-build - timeout --preserve-status --signal INT 1m go-fuzz -procs=2 - test ! "$(ls crashers)" + GOCACHE=`pwd`/corpus-v3 go test . -run - -fuzz . -fuzztime 1m +# TODO - save_cache: - key: corpus-v2-{{ epoch }} + key: corpus-v3 paths: - - corpus + - corpus-v3 - run: name: "Benchmark" command: go test -run=- -bench=. -benchmem From 0472e9b9f59c16fc7a936a5f9d784c3b8f06989d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 09:45:22 +0100 Subject: [PATCH 44/50] conversion: consistent use of ptr receiver --- conversion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index afc9c29..3752432 100644 --- a/conversion.go +++ b/conversion.go @@ -577,7 +577,7 @@ func (dst *Int) Scan(src interface{}) error { // In Postgres, this will work with both integer and the Numeric/Decimal types // In MariaDB/MySQL, this will work with the Numeric/Decimal types up to 65 digits, however any more and you should use either VarChar or Char(79) // In SqLite, use TEXT -func (src Int) Value() (driver.Value, error) { +func (src *Int) Value() (driver.Value, error) { return src.ToBig().String(), nil } From d55274d77c21d7e01acce3c906181ef6b87d78e1 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 29 Dec 2022 09:49:32 +0100 Subject: [PATCH 45/50] circle: make use of restored corpus --- circle.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index 6c3a263..7a4d20d 100644 --- a/circle.yml +++ b/circle.yml @@ -44,10 +44,9 @@ jobs: - run: name: "Fuzzing" command: | - GOCACHE=`pwd`/corpus-v3 go test . -run - -fuzz . -fuzztime 1m -# TODO + GOCACHE=/home/circleci/project/corpus-v3 go test . -run - -fuzz . -fuzztime 1m - save_cache: - key: corpus-v3 + key: corpus-v3-{{ epoch }} paths: - corpus-v3 - run: From 82fd325f742e3813628e1428f903c23a566636f1 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 12 Jan 2023 00:22:35 -0600 Subject: [PATCH 46/50] properly parse scientific notation --- conversion.go | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index 3752432..f1bcd70 100644 --- a/conversion.go +++ b/conversion.go @@ -15,6 +15,7 @@ import ( "io" "math/big" "math/bits" + "strings" ) const ( @@ -565,7 +566,25 @@ func (dst *Int) Scan(src interface{}) error { } switch src := src.(type) { case string: - return dst.SetFromDecimal(src) + splt := strings.SplitN(src, "e", 2) + if len(splt) < 2 { + return dst.SetFromDecimal(src) + } + err := dst.SetFromDecimal(splt[0]) + if err != nil { + return err + } + if splt[1] == "0" { + return nil + } + exp := new(Int) + err = exp.SetFromDecimal(splt[1]) + if err != nil { + return err + } + exp.Exp(NewInt(10), exp) + dst.Mul(dst, exp) + return nil case []byte: return dst.SetFromDecimal(string(src)) } From 35d62f2e1b6cb83c6caf02bfb50b80cd7e575540 Mon Sep 17 00:00:00 2001 From: a Date: Thu, 26 Jan 2023 19:47:42 -0600 Subject: [PATCH 47/50] add some overflow cases for scan, and error on overflow --- conversion.go | 8 ++++++- conversion_test.go | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/conversion.go b/conversion.go index f1bcd70..88354f3 100644 --- a/conversion.go +++ b/conversion.go @@ -582,8 +582,14 @@ func (dst *Int) Scan(src interface{}) error { if err != nil { return err } + if exp.Uint64() > uint64(len(twoPow256Sub1)) { + return ErrBig256Range + } exp.Exp(NewInt(10), exp) - dst.Mul(dst, exp) + _, overflow := dst.MulOverflow(dst, exp) + if overflow { + return ErrBig256Range + } return nil case []byte: return dst.SetFromDecimal(string(src)) diff --git a/conversion_test.go b/conversion_test.go index 33ad989..b6f12d6 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -8,6 +8,7 @@ import ( "bufio" "bytes" "encoding/json" + "errors" "fmt" "math/big" "testing" @@ -74,6 +75,60 @@ func TestFromBig(t *testing.T) { t.Fatalf("got %x exp %x", got, exp) } } +func TestScanScientific(t *testing.T) { + intsub1 := new(Int) + intsub1.fromDecimal(twoPow256Sub1) + cases := []struct { + in string + exp *Int + err error + }{ + { + "14e30", + new(Int).Mul(NewInt(14), new(Int).Exp(NewInt(10), NewInt(30))), + nil, + }, + { + "1455522523e31", + new(Int).Mul(NewInt(1455522523), new(Int).Exp(NewInt(10), NewInt(31))), + nil, + }, + { + twoPow256Sub1 + "e0", + intsub1, + nil, + }, + { + "1e25352", + nil, + ErrBig256Range, + }, + { + "1213128763127863781263781263781263781263781263871263871268371268371263781627836128736128736127836127836127863781e0", + nil, + ErrBig256Range, + }, + { + twoPow256Sub1 + "e1", + nil, + ErrBig256Range, + }, + } + for _, v := range cases { + i := new(Int) + err := i.Scan(v.in) + if v.err == nil { + if err != nil { + t.Fatalf("expected no error, got %s", err) + } + if !v.exp.Eq(i) { + t.Fatalf("got %x exp %x", i, v.exp) + } + } else if !errors.Is(err, v.err) { + t.Fatalf("expected error %s got %s", v.err, err) + } + } +} func TestFromBigOverflow(t *testing.T) { _, o := FromBig(new(big.Int).SetBytes(hex2Bytes("ababee444444444444ffcc333333333333ddaa222222222222bb8811111111111199"))) From 88226a1a621a021462c87432ed22d3aac213bf28 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 8 Feb 2023 13:56:54 +0100 Subject: [PATCH 48/50] conversion minor nitpicks --- conversion.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/conversion.go b/conversion.go index 88354f3..6a23441 100644 --- a/conversion.go +++ b/conversion.go @@ -567,22 +567,20 @@ func (dst *Int) Scan(src interface{}) error { switch src := src.(type) { case string: splt := strings.SplitN(src, "e", 2) - if len(splt) < 2 { + if len(splt) == 1 { return dst.SetFromDecimal(src) } - err := dst.SetFromDecimal(splt[0]) - if err != nil { + if err := dst.SetFromDecimal(splt[0]); err != nil { return err } if splt[1] == "0" { return nil } exp := new(Int) - err = exp.SetFromDecimal(splt[1]) - if err != nil { + if err := exp.SetFromDecimal(splt[1]); err != nil { return err } - if exp.Uint64() > uint64(len(twoPow256Sub1)) { + if !exp.IsUint64() || exp.Uint64() > uint64(len(twoPow256Sub1)) { return ErrBig256Range } exp.Exp(NewInt(10), exp) From 5ff06b00093626005cb22f603489ded3ec559f9c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 8 Feb 2023 14:00:11 +0100 Subject: [PATCH 49/50] conversion: fix test --- conversion_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conversion_test.go b/conversion_test.go index b6f12d6..d0eddaf 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -75,9 +75,10 @@ func TestFromBig(t *testing.T) { t.Fatalf("got %x exp %x", got, exp) } } + func TestScanScientific(t *testing.T) { intsub1 := new(Int) - intsub1.fromDecimal(twoPow256Sub1) + _ = intsub1.fromDecimal(twoPow256Sub1) cases := []struct { in string exp *Int From 091c3f9912b296e1920bdd6a08d75d849d6d3b2f Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 8 Feb 2023 14:37:49 +0100 Subject: [PATCH 50/50] conversion_test: bring coverage back to 100 --- conversion_test.go | 65 ++++++++++++++++++++++++---------------------- 1 file changed, 34 insertions(+), 31 deletions(-) diff --git a/conversion_test.go b/conversion_test.go index d0eddaf..b8e35af 100644 --- a/conversion_test.go +++ b/conversion_test.go @@ -8,7 +8,6 @@ import ( "bufio" "bytes" "encoding/json" - "errors" "fmt" "math/big" "testing" @@ -82,51 +81,55 @@ func TestScanScientific(t *testing.T) { cases := []struct { in string exp *Int - err error + err string }{ { - "14e30", - new(Int).Mul(NewInt(14), new(Int).Exp(NewInt(10), NewInt(30))), - nil, + in: "14e30", + exp: new(Int).Mul(NewInt(14), new(Int).Exp(NewInt(10), NewInt(30))), }, { - "1455522523e31", - new(Int).Mul(NewInt(1455522523), new(Int).Exp(NewInt(10), NewInt(31))), - nil, + in: "1455522523e31", + exp: new(Int).Mul(NewInt(1455522523), new(Int).Exp(NewInt(10), NewInt(31))), }, { - twoPow256Sub1 + "e0", - intsub1, - nil, + in: twoPow256Sub1 + "e0", + exp: intsub1, }, { - "1e25352", - nil, - ErrBig256Range, + in: "1e25352", + err: ErrBig256Range.Error(), }, { - "1213128763127863781263781263781263781263781263871263871268371268371263781627836128736128736127836127836127863781e0", - nil, - ErrBig256Range, + in: "1213128763127863781263781263781263781263781263871263871268371268371263781627836128736128736127836127836127863781e0", + err: ErrBig256Range.Error(), }, { - twoPow256Sub1 + "e1", - nil, - ErrBig256Range, + in: twoPow256Sub1 + "e1", + err: ErrBig256Range.Error(), + }, + { + in: "1e253e52", + err: `strconv.ParseUint: parsing "253e52": invalid syntax`, + }, + { + in: "1e00000000000000000", + exp: NewInt(1), }, } - for _, v := range cases { + for tc, v := range cases { + have := "" i := new(Int) - err := i.Scan(v.in) - if v.err == nil { - if err != nil { - t.Fatalf("expected no error, got %s", err) - } - if !v.exp.Eq(i) { - t.Fatalf("got %x exp %x", i, v.exp) - } - } else if !errors.Is(err, v.err) { - t.Fatalf("expected error %s got %s", v.err, err) + if err := i.Scan(v.in); err != nil { + have = err.Error() + } + if want := v.err; have != want { + t.Fatalf("test %d: wrong error, have '%s', want '%s'", tc, have, want) + } + if len(v.err) > 0 { + continue + } + if !v.exp.Eq(i) { + t.Fatalf("test %d: got %#x exp %#x", tc, i, v.exp) } } }