From 86ed7d4d7f30b8d344e00d0a1b0bf9bae216297a Mon Sep 17 00:00:00 2001 From: Loris Leiva Date: Wed, 21 Aug 2024 18:26:17 +0100 Subject: [PATCH] Implements a JSON.parse that transforms unsafe numbers to bigints --- .../src/__tests__/get-account-info-test.ts | 2 +- .../__tests__/get-multiple-accounts-test.ts | 6 +- .../__tests__/get-program-accounts-test.ts | 6 +- packages/rpc-transport-http/package.json | 1 + .../src/__benchmarks__/json-parse.ts | 50 +++++ .../__tests__/http-transport-abort-test.ts | 2 +- .../src/__tests__/http-transport-test.ts | 2 +- .../__tests__/json-parse-with-bigint-test.ts | 104 ++++++++++ .../src/__tests__/large-json-file.json | 191 ++++++++++++++++++ .../rpc-transport-http/src/http-transport.ts | 3 +- .../src/json-parse-with-bigint.ts | 187 +++++++++++++++++ packages/rpc-transport-http/tsconfig.json | 2 +- 12 files changed, 545 insertions(+), 11 deletions(-) create mode 100755 packages/rpc-transport-http/src/__benchmarks__/json-parse.ts create mode 100644 packages/rpc-transport-http/src/__tests__/json-parse-with-bigint-test.ts create mode 100644 packages/rpc-transport-http/src/__tests__/large-json-file.json create mode 100644 packages/rpc-transport-http/src/json-parse-with-bigint.ts diff --git a/packages/rpc-api/src/__tests__/get-account-info-test.ts b/packages/rpc-api/src/__tests__/get-account-info-test.ts index 01b26852ad22..07d2fc832766 100644 --- a/packages/rpc-api/src/__tests__/get-account-info-test.ts +++ b/packages/rpc-api/src/__tests__/get-account-info-test.ts @@ -37,7 +37,7 @@ describe('getAccountInfo', () => { executable: false, lamports: 5000000n, owner: '11111111111111111111111111111111', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 9n, }, }); diff --git a/packages/rpc-api/src/__tests__/get-multiple-accounts-test.ts b/packages/rpc-api/src/__tests__/get-multiple-accounts-test.ts index 438cd696404e..2330f96c6ea2 100644 --- a/packages/rpc-api/src/__tests__/get-multiple-accounts-test.ts +++ b/packages/rpc-api/src/__tests__/get-multiple-accounts-test.ts @@ -43,7 +43,7 @@ describe('getMultipleAccounts', () => { executable: false, lamports: 5000000n, owner: '11111111111111111111111111111111', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 9n, }, { @@ -51,7 +51,7 @@ describe('getMultipleAccounts', () => { executable: false, lamports: 5000000n, owner: '11111111111111111111111111111111', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 0n, }, ], @@ -114,7 +114,7 @@ describe('getMultipleAccounts', () => { executable: false, lamports: 5000000n, owner: '11111111111111111111111111111111', - rentEpoch: 18446744073709551616n, + rentEpoch: 18446744073709551615n, space: 9n, }, null, diff --git a/packages/rpc-api/src/__tests__/get-program-accounts-test.ts b/packages/rpc-api/src/__tests__/get-program-accounts-test.ts index 3cef63a74e66..853832851ecf 100644 --- a/packages/rpc-api/src/__tests__/get-program-accounts-test.ts +++ b/packages/rpc-api/src/__tests__/get-program-accounts-test.ts @@ -65,7 +65,7 @@ describe('getProgramAccounts', () => { executable: false, lamports: 5000000n, owner: 'DXngmJfjurhnAwbMPgpUGPH6qNvetCKRJ6PiD4ag4PTj', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 9n, }, pubkey: 'CcYNb7WqpjaMrNr7B1mapaNfWctZRH7LyAjWRLBGt1Fk', @@ -126,7 +126,7 @@ describe('getProgramAccounts', () => { executable: false, lamports: 5000000n, owner: 'AmtpVzo6H6qQCP9dH9wfu5hfa8kKaAFpTJ4aamPYR6V6', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 9n, }, pubkey: 'C5q1p5UiCVrt6vcLJDGcS4AZ98fahKyb9XkDRdqATK17', @@ -137,7 +137,7 @@ describe('getProgramAccounts', () => { executable: false, lamports: 5000000n, owner: 'AmtpVzo6H6qQCP9dH9wfu5hfa8kKaAFpTJ4aamPYR6V6', - rentEpoch: 18446744073709551616n, // TODO: This number loses precision + rentEpoch: 18446744073709551615n, space: 9n, }, pubkey: 'Hhsoev7Apk5dMbktzLUrsTHuMq9e9GSYBaLcnN2PfdKS', diff --git a/packages/rpc-transport-http/package.json b/packages/rpc-transport-http/package.json index 38b7cea54d26..b1a29524d915 100644 --- a/packages/rpc-transport-http/package.json +++ b/packages/rpc-transport-http/package.json @@ -34,6 +34,7 @@ ], "scripts": { "benchmark": "./src/__benchmarks__/run.ts", + "benchmark:json-parse": "./src/__benchmarks__/json-parse.ts", "compile:js": "tsup --config build-scripts/tsup.config.package.ts", "compile:typedefs": "tsc -p ./tsconfig.declarations.json", "dev": "jest -c ../../node_modules/@solana/test-config/jest-dev.config.ts --rootDir . --watch", diff --git a/packages/rpc-transport-http/src/__benchmarks__/json-parse.ts b/packages/rpc-transport-http/src/__benchmarks__/json-parse.ts new file mode 100755 index 000000000000..494be6ff8f10 --- /dev/null +++ b/packages/rpc-transport-http/src/__benchmarks__/json-parse.ts @@ -0,0 +1,50 @@ +#!/usr/bin/env -S pnpm dlx tsx -- + +import { Bench } from 'tinybench'; +import { fs, path } from 'zx'; + +import { + jsonParseWithLargeIntegersAsBigInts, + wrapLargeIntegersUsingParser, + wrapLargeIntegersUsingParserAndRegex, + wrapLargeIntegersUsingRegex, +} from '../json-parse-with-bigint'; + +Object.assign(globalThis, { + __BROWSER__: false, + __DEV__: false, + __NODEJS__: true, + __REACTNATIVE____: false, +}); + +const bench = new Bench({ + throws: true, +}); + +const largeJsonPath = path.join(__dirname, '..', '__tests__', 'large-json-file.json'); +const largeJsonString = fs.readFileSync(largeJsonPath, 'utf8'); + +bench + .add('JSON.parse', () => { + return JSON.parse(largeJsonString); + }) + .add('JSON.parse with noop reviver', () => { + return jsonParseWithLargeIntegersAsBigInts(largeJsonString, x => x); + }) + .add('jsonParseWithLargeIntegersAsBigInts (parser)', () => { + return jsonParseWithLargeIntegersAsBigInts(largeJsonString); + }) + .add('jsonParseWithLargeIntegersAsBigInts (parser and regex)', () => { + return jsonParseWithLargeIntegersAsBigInts(largeJsonString, wrapLargeIntegersUsingParserAndRegex); + }) + .add('jsonParseWithLargeIntegersAsBigInts (parser and noop)', () => { + return jsonParseWithLargeIntegersAsBigInts(largeJsonString, x => wrapLargeIntegersUsingParser(x, () => null)); + }) + .add('jsonParseWithLargeIntegersAsBigInts (regex)', () => { + return jsonParseWithLargeIntegersAsBigInts(largeJsonString, wrapLargeIntegersUsingRegex); + }); + +(async () => { + await bench.run(); + console.table(bench.table()); +})(); diff --git a/packages/rpc-transport-http/src/__tests__/http-transport-abort-test.ts b/packages/rpc-transport-http/src/__tests__/http-transport-abort-test.ts index 07779f87e636..030de71971c7 100644 --- a/packages/rpc-transport-http/src/__tests__/http-transport-abort-test.ts +++ b/packages/rpc-transport-http/src/__tests__/http-transport-abort-test.ts @@ -71,8 +71,8 @@ describe('createHttpTransport and `AbortSignal`', () => { it('resolves with the response', async () => { expect.assertions(1); jest.mocked(fetchSpy).mockResolvedValueOnce({ - json: () => ({ ok: true }), ok: true, + text: () => '{"ok":true}', } as unknown as Response); const sendPromise = makeHttpRequest({ payload: 123, signal: abortSignal }); abortController.abort('I got bored waiting'); diff --git a/packages/rpc-transport-http/src/__tests__/http-transport-test.ts b/packages/rpc-transport-http/src/__tests__/http-transport-test.ts index 7ff9b3321f28..f80df201f5cd 100644 --- a/packages/rpc-transport-http/src/__tests__/http-transport-test.ts +++ b/packages/rpc-transport-http/src/__tests__/http-transport-test.ts @@ -49,8 +49,8 @@ describe('createHttpTransport', () => { describe('when the endpoint returns a well-formed JSON response', () => { beforeEach(() => { fetchSpy.mockResolvedValue({ - json: () => ({ ok: true }), ok: true, + text: () => '{"ok":true}', }); }); it('calls fetch with the specified URL', () => { diff --git a/packages/rpc-transport-http/src/__tests__/json-parse-with-bigint-test.ts b/packages/rpc-transport-http/src/__tests__/json-parse-with-bigint-test.ts new file mode 100644 index 000000000000..8df87016c5dc --- /dev/null +++ b/packages/rpc-transport-http/src/__tests__/json-parse-with-bigint-test.ts @@ -0,0 +1,104 @@ +import fs from 'fs'; +import path from 'path'; + +import { jsonParseWithLargeIntegersAsBigInts, wrapLargeIntegers } from '../json-parse-with-bigint'; + +const MAX_SAFE_INTEGER: number = Number.MAX_SAFE_INTEGER; +const MAX_SAFE_INTEGER_PLUS_ONE: bigint = BigInt(Number.MAX_SAFE_INTEGER) + 1n; + +describe('jsonParseWithLargeNumbersAsBigInts', () => { + it('parses large integers as bigints', () => { + const result = jsonParseWithLargeIntegersAsBigInts(`{"value":${MAX_SAFE_INTEGER_PLUS_ONE}}`) as { + value: unknown; + }; + expect(result).toEqual({ value: MAX_SAFE_INTEGER_PLUS_ONE }); + expect(typeof result.value).toBe('bigint'); + }); + it('keeps safe integers as numbers', () => { + const result = jsonParseWithLargeIntegersAsBigInts(`{"value":${MAX_SAFE_INTEGER}}`) as { value: unknown }; + expect(result).toEqual({ value: MAX_SAFE_INTEGER }); + expect(typeof result.value).toBe('number'); + }); + it('does not affect numbers in strings', () => { + expect(jsonParseWithLargeIntegersAsBigInts(`{"value":"Hello 123 World"}`)).toEqual({ + value: 'Hello 123 World', + }); + }); + it('does not affect numbers in strings with escaped double quotes', () => { + expect(jsonParseWithLargeIntegersAsBigInts(`{"value":"Hello\\" 123 World"}`)).toEqual({ + value: 'Hello" 123 World', + }); + }); + it('parses large integers with exponents as bigints', () => { + expect(jsonParseWithLargeIntegersAsBigInts(`1e32`)).toBe(100000000000000000000000000000000n); + expect(jsonParseWithLargeIntegersAsBigInts(`-1189e+32`)).toBe(-118900000000000000000000000000000000n); + }); + it('does not affect negative exponents', () => { + expect(jsonParseWithLargeIntegersAsBigInts(`1e-32`)).toBe(1e-32); + expect(jsonParseWithLargeIntegersAsBigInts(`-1189e-32`)).toBe(-1189e-32); + }); + it('can parse complex JSON files', () => { + const largeJsonPath = path.join(__dirname, 'large-json-file.json'); + const largeJsonString = fs.readFileSync(largeJsonPath, 'utf8'); + const expectedResult = JSON.parse(largeJsonString, (key, value) => + // eslint-disable-next-line jest/no-conditional-in-test + key === 'lamports' ? 142302234983644260n : value, + ); + expect(jsonParseWithLargeIntegersAsBigInts(largeJsonString)).toEqual(expectedResult); + }); +}); + +describe('wrapLargeIntegers', () => { + it('wraps large integers as bigint strings', () => { + expect(wrapLargeIntegers(`{"value":${MAX_SAFE_INTEGER_PLUS_ONE}}`)).toBe( + `{"value":{"$n":"${MAX_SAFE_INTEGER_PLUS_ONE}"}}`, + ); + }); + it('keeps safe integers as numbers', () => { + expect(wrapLargeIntegers(`{"value":${MAX_SAFE_INTEGER}}`)).toBe(`{"value":${MAX_SAFE_INTEGER}}`); + }); + it('does not wrap numbers in strings', () => { + expect(wrapLargeIntegers(`{"value":"Hello 123 World"}`)).toBe(`{"value":"Hello 123 World"}`); + }); + it('does not wrap numbers in strings with escaped double quotes', () => { + expect(wrapLargeIntegers(`{"value":"Hello\\" 123 World"}`)).toBe(`{"value":"Hello\\" 123 World"}`); + }); + it('does not alter special keywords', () => { + expect(wrapLargeIntegers('true')).toBe('true'); + expect(wrapLargeIntegers('false')).toBe('false'); + expect(wrapLargeIntegers('null')).toBe('null'); + }); + it('does not alter safe numbers', () => { + expect(wrapLargeIntegers('0')).toBe('0'); + expect(wrapLargeIntegers('123456')).toBe('123456'); + expect(wrapLargeIntegers('-123456')).toBe('-123456'); + expect(wrapLargeIntegers('3.14')).toBe('3.14'); + expect(wrapLargeIntegers('-3.14')).toBe('-3.14'); + expect(wrapLargeIntegers('1e5')).toBe('1e5'); + expect(wrapLargeIntegers('-1e5')).toBe('-1e5'); + expect(wrapLargeIntegers('1E5')).toBe('1E5'); + expect(wrapLargeIntegers('-1E5')).toBe('-1E5'); + expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER}`)).toBe(`${MAX_SAFE_INTEGER}`); + expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER}`)).toBe(`-${MAX_SAFE_INTEGER}`); + }); + it('wraps unsafe large integers', () => { + expect(wrapLargeIntegers('1e32')).toBe('{"$n":"1e32"}'); + expect(wrapLargeIntegers('-1e32')).toBe('{"$n":"-1e32"}'); + expect(wrapLargeIntegers('1E32')).toBe('{"$n":"1E32"}'); + expect(wrapLargeIntegers('-1E32')).toBe('{"$n":"-1E32"}'); + expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER_PLUS_ONE}`)).toBe(`{"$n":"${MAX_SAFE_INTEGER_PLUS_ONE}"}`); + expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER_PLUS_ONE}`)).toBe(`{"$n":"-${MAX_SAFE_INTEGER_PLUS_ONE}"}`); + }); + it('does not alter unsafe large floating points', () => { + expect(wrapLargeIntegers('3.14e32')).toBe('3.14e32'); + expect(wrapLargeIntegers('-3.14e32')).toBe('-3.14e32'); + expect(wrapLargeIntegers('3.14E32')).toBe('3.14E32'); + expect(wrapLargeIntegers('-3.14E32')).toBe('-3.14E32'); + expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER_PLUS_ONE}.123`)).toBe(`${MAX_SAFE_INTEGER_PLUS_ONE}.123`); + expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER_PLUS_ONE}.123`)).toBe(`-${MAX_SAFE_INTEGER_PLUS_ONE}.123`); + }); + it('does not alter strings', () => { + expect(wrapLargeIntegers('""')).toBe('""'); + expect(wrapLargeIntegers('"Hello World"')).toBe('"Hello World"'); + }); +}); diff --git a/packages/rpc-transport-http/src/__tests__/large-json-file.json b/packages/rpc-transport-http/src/__tests__/large-json-file.json new file mode 100644 index 000000000000..713d47371e36 --- /dev/null +++ b/packages/rpc-transport-http/src/__tests__/large-json-file.json @@ -0,0 +1,191 @@ +[ + { + "_id": "66c71d2bd28af9a3c7f1a766", + "index": 0, + "guid": "b6465798-e542-4eb0-a5c1-59e3933290ee", + "isActive": false, + "balance": "$1,274.63", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 29, + "eyeColor": "blue", + "name": "Collier Carson", + "gender": "male", + "company": "NAMEBOX", + "email": "colliercarson@namebox.com", + "phone": "+1 (888) 428-2044", + "address": "826 Irving Place, Colton, Illinois, 4908", + "about": "Consectetur reprehenderit aliqua eu esse voluptate cupidatat sint anim ex ipsum pariatur eu laborum. Mollit exercitation excepteur consectetur exercitation. Duis non incididunt pariatur consectetur tempor esse nulla aute ad. Ut exercitation tempor duis mollit. Mollit labore sint non est cillum. In esse ullamco fugiat velit est amet ad laboris occaecat officia nisi.", + "registered": "2024-01-12T05:09:11 -00:00", + "latitude": 49.638368, + "longitude": -113.827497, + "tags": ["qui", "excepteur", "aliquip", "amet", "ipsum", "labore", "officia"], + "friends": [ + { "id": 0, "name": "Jeannette Galloway" }, + { "id": 1, "name": "Gordon Murray" }, + { "id": 2, "name": "Randolph Sullivan" } + ] + }, + { + "_id": "66c71d2b0a5ca9a164428ff8", + "index": 1, + "guid": "dafaa74b-5fa0-4181-9ba7-371aea1188a2", + "isActive": false, + "balance": "$2,863.05", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 38, + "eyeColor": "green", + "name": "Shields Fischer", + "gender": "male", + "company": "RONELON", + "email": "shieldsfischer@ronelon.com", + "phone": "+1 (988) 401-2468", + "address": "431 Norman Avenue, Innsbrook, New Mexico, 5118", + "about": "Magna velit tempor est sint elit commodo mollit mollit exercitation reprehenderit id in ullamco quis. Sint reprehenderit minim voluptate culpa fugiat et aliqua enim ipsum. Nostrud magna dolore excepteur occaecat non ex sint eiusmod. Occaecat deserunt labore nostrud veniam adipisicing ullamco esse. Aliqua sunt nostrud proident veniam cupidatat voluptate enim ipsum. Dolore consequat anim eiusmod laboris sunt pariatur anim fugiat sunt velit quis officia. Enim dolore laborum commodo eiusmod nisi nisi.", + "registered": "2019-11-17T03:40:22 -00:00", + "latitude": 11.324387, + "longitude": -80.832796, + "tags": ["eu", "voluptate", "id", "id", "dolor", "nostrud", "nulla"], + "friends": [ + { "id": 0, "name": "Vilma Fox" }, + { "id": 1, "name": "Concetta Ross" }, + { "id": 2, "name": "Wilkins Howe" } + ] + }, + { + "_id": "66c71d2bcdba189600f87199", + "index": 2, + "guid": "06835d94-0a4f-4e89-b889-798c59918fad", + "isActive": true, + "balance": "$2,307.46", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 33, + "eyeColor": "green", + "name": "Penelope Cabrera", + "gender": "female", + "company": "GLUKGLUK", + "email": "penelopecabrera@glukgluk.com", + "phone": "+1 (884) 419-2242", + "address": "743 Dooley Street, Jessie, Palau, 8602", + "about": "Culpa ut veniam reprehenderit do. Sunt ut excepteur cillum laboris esse occaecat officia amet et duis nostrud excepteur. Incididunt reprehenderit ullamco incididunt velit incididunt esse officia eu dolor irure nostrud eiusmod ut quis. Irure mollit consequat sunt commodo non exercitation. Aliquip labore aute sunt deserunt ullamco proident est esse. Minim velit voluptate consequat voluptate dolore. Eu commodo Lorem sint aliquip amet do do ad culpa Lorem nulla anim quis.", + "registered": "2019-10-10T10:38:26 -01:00", + "latitude": 34.456814, + "longitude": 60.745321, + "tags": ["est", "qui", "cupidatat", "excepteur", "ut", "ad", "veniam"], + "friends": [ + { "id": 0, "name": "Elizabeth Huber" }, + { "id": 1, "name": "Sally Robbins" }, + { "id": 2, "name": "Daisy Hunt" } + ] + }, + { + "_id": "66c71d2bf99c62818eb44055", + "index": 3, + "guid": "c496d7cc-eaf8-4b86-ab12-bb9fd9946d21", + "isActive": false, + "balance": "$1,885.35", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 30, + "eyeColor": "blue", + "name": "Mae Fulton", + "gender": "female", + "company": "COREPAN", + "email": "maefulton@corepan.com", + "phone": "+1 (898) 433-2640", + "address": "494 Greene Avenue, Dyckesville, South Carolina, 4544", + "about": "Id commodo quis sint sint voluptate culpa deserunt ad anim sint reprehenderit. Proident occaecat non et dolore magna aliquip ut adipisicing enim officia aliquip consequat. Ullamco reprehenderit cupidatat in est proident aliquip minim sint elit eu magna irure velit.", + "registered": "2022-12-24T04:14:42 -00:00", + "latitude": -78.635799, + "longitude": 53.578193, + "tags": ["minim", "eu", "sint", "culpa", "consequat", "ullamco", "exercitation"], + "friends": [ + { "id": 0, "name": "Patrick Middleton" }, + { "id": 1, "name": "Buchanan Patton" }, + { "id": 2, "name": "Marilyn Cash" } + ] + }, + { + "_id": "66c71d2b17723c12ff35509a", + "index": 4, + "guid": "892f741f-39a6-4157-8deb-3fa5cb7c89ac", + "isActive": false, + "balance": "$1,720.04", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 29, + "eyeColor": "blue", + "name": "Nell Newton", + "gender": "female", + "company": "ASIMILINE", + "email": "nellnewton@asimiline.com", + "phone": "+1 (973) 445-2478", + "address": "494 Elton Street, Alafaya, Arizona, 1347", + "about": "Aliquip consectetur dolor pariatur qui. Enim consequat adipisicing aliqua Lorem amet consectetur irure est laborum ipsum. Magna exercitation excepteur incididunt cupidatat aliquip do tempor non. Et mollit quis quis tempor enim cillum id fugiat. Nisi eiusmod cillum reprehenderit in eiusmod labore est fugiat et officia.", + "registered": "2014-06-19T08:01:53 -01:00", + "latitude": 59.571221, + "longitude": -78.053459, + "tags": ["fugiat", "deserunt", "laborum", "eu", "sit", "ex", "dolor"], + "friends": [ + { "id": 0, "name": "Cheryl Harding" }, + { "id": 1, "name": "Laurie Logan" }, + { "id": 2, "name": "Johns Moody" } + ] + }, + { + "_id": "66c71d2b5ee698e22b8e9925", + "index": 5, + "guid": "ba974954-e9e1-4d8c-8847-3f28e3a6175b", + "isActive": false, + "balance": "$3,203.47", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 22, + "eyeColor": "blue", + "name": "Albert Harmon", + "gender": "male", + "company": "EXOBLUE", + "email": "albertharmon@exoblue.com", + "phone": "+1 (954) 439-3813", + "address": "313 Fountain Avenue, Beason, Rhode Island, 9738", + "about": "Dolor excepteur nisi ut aliqua occaecat non ipsum irure id eu proident duis nulla. Ex dolor do excepteur aliqua officia consectetur ea reprehenderit occaecat sit qui aliquip aute. Occaecat proident cillum in ullamco nostrud laboris nisi excepteur. Incididunt velit cupidatat veniam fugiat.", + "registered": "2016-11-18T08:07:12 -00:00", + "latitude": 42.687475, + "longitude": 39.897468, + "tags": ["minim", "in", "adipisicing", "officia", "culpa", "in", "do"], + "friends": [ + { "id": 0, "name": "Macdonald Mckinney" }, + { "id": 1, "name": "Virginia Crawford" }, + { "id": 2, "name": "Jimmie Lawrence" } + ] + }, + { + "_id": "66c71d2b06572e5de0348e08", + "index": 6, + "guid": "59bd67a0-44e0-43dd-aebf-2b7422f273b5", + "isActive": true, + "balance": "$2,639.98", + "lamports": 142302234983644260, + "picture": "http://placehold.it/32x32", + "age": 38, + "eyeColor": "blue", + "name": "Jannie Case", + "gender": "female", + "company": "VALREDA", + "email": "janniecase@valreda.com", + "phone": "+1 (851) 464-3024", + "address": "201 Pierrepont Place, Whitehaven, Marshall Islands, 7595", + "about": "Dolore laboris anim non sint cillum cupidatat enim veniam aliqua amet ipsum anim sit ipsum. Id labore labore nostrud incididunt. Lorem culpa quis ad dolor cupidatat elit aliqua officia laborum. Reprehenderit irure dolore do sunt do dolor laborum officia Lorem anim cupidatat consequat. Sunt aute est ad laborum ea magna elit esse sit in nostrud laborum sint. Proident velit eu id laboris commodo aliqua nisi elit consectetur ut exercitation commodo ut ad. Id aute tempor laboris officia fugiat non consectetur nisi ipsum cillum sint anim anim consequat.", + "registered": "2014-10-10T02:16:44 -01:00", + "latitude": 11.356616, + "longitude": -64.69489, + "tags": ["culpa", "qui", "exercitation", "elit", "veniam", "aliquip", "Lorem"], + "friends": [ + { "id": 0, "name": "Bowers Britt" }, + { "id": 1, "name": "Tommie Morris" }, + { "id": 2, "name": "Olsen Pollard" } + ] + } +] diff --git a/packages/rpc-transport-http/src/http-transport.ts b/packages/rpc-transport-http/src/http-transport.ts index 68602c03fb1b..ceeba0d30fa4 100644 --- a/packages/rpc-transport-http/src/http-transport.ts +++ b/packages/rpc-transport-http/src/http-transport.ts @@ -7,6 +7,7 @@ import { assertIsAllowedHttpRequestHeaders, normalizeHeaders, } from './http-transport-headers'; +import { jsonParseWithLargeIntegersAsBigInts } from './json-parse-with-bigint'; type Config = Readonly<{ dispatcher_NODE_ONLY?: Dispatcher; @@ -66,6 +67,6 @@ export function createHttpTransport(config: Config): RpcTransport { statusCode: response.status, }); } - return (await response.json()) as TResponse; + return jsonParseWithLargeIntegersAsBigInts(await response.text()) as TResponse; }; } diff --git a/packages/rpc-transport-http/src/json-parse-with-bigint.ts b/packages/rpc-transport-http/src/json-parse-with-bigint.ts new file mode 100644 index 000000000000..ff6ab6192549 --- /dev/null +++ b/packages/rpc-transport-http/src/json-parse-with-bigint.ts @@ -0,0 +1,187 @@ +/** + * This function is a replacement for `JSON.parse` that can handle large integers by + * parsing them as BigInts. It will only transform integers that are too large to be + * represented as a regular JavaScript number. + */ +export function jsonParseWithLargeIntegersAsBigInts( + json: string, + wrap: (json: string) => string = wrapLargeIntegers, +): unknown { + return JSON.parse(wrap(json), (_, value) => { + return isBigIntValueObject(value) ? unwrapBigIntValueObject(value) : value; + }); +} + +/** + * This function takes a JSON string and wraps any large, + * unsafe integers in `BigIntValueObjects` that can then + * be recognized and unwrapped by a `JSON.parse` reviver. + */ +export function wrapLargeIntegers(json: string): string { + return wrapLargeIntegersUsingParser(json, consumeNumberUsingParser); +} + +export function wrapLargeIntegersUsingParser( + json: string, + consumeNumber: (json: string, ii: number) => string | null, +): string { + const out = []; + let inQuote = false; + for (let ii = 0; ii < json.length; ii++) { + let isEscaped = false; + if (json[ii] === '\\') { + out.push(json[ii++]); + isEscaped = !isEscaped; + } + if (json[ii] === '"') { + out.push(json[ii++]); + if (!isEscaped) { + inQuote = !inQuote; + } + } + if (!inQuote) { + const consumedNumber = consumeNumber(json, ii); + if (consumedNumber?.length) { + ii += consumedNumber.length; + if (consumedNumber.match(/\.|[eE]-/) || Number.isSafeInteger(Number(consumedNumber))) { + out.push(consumedNumber); + } else { + out.push(wrapBigIntValueObject(consumedNumber)); + } + } + } + out.push(json[ii]); + } + + return out.join(''); +} + +export function wrapLargeIntegersUsingParserAndRegex(json: string): string { + return wrapLargeIntegersUsingParser(json, consumeNumberUsingRegex); +} + +function consumeNumberUsingRegex(json: string, ii: number): string | null { + /** @see https://stackoverflow.com/a/13340826/11440277 */ + const JSON_NUMBER_REGEX = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/; + + // Stop early if the first character isn't a digit or a minus sign. + if (!json[ii]?.match(/[-\d]/)) { + return null; + } + + // Otherwise, check if the next characters form a valid JSON number. + const numberMatch = json.slice(ii).match(JSON_NUMBER_REGEX); + return numberMatch ? numberMatch[0] : null; +} + +function consumeNumberUsingParser(json: string, ii: number): string | null { + /** @see https://stackoverflow.com/a/13340826/11440277 */ + const consumed = []; + + // Consume the optional negative sign. + if (json[ii] === '-') { + consumed.push(json[ii++]); + } + + // Consume the units. + if (json[ii] === '0') { + consumed.push(json[ii++]); + } else if (json[ii]?.match(/[1-9]/)) { + consumed.push(json[ii++]); + while (json[ii]?.match(/\d/)) { + consumed.push(json[ii++]); + } + } else { + return null; + } + + // Consume the optional decimals. + if (json[ii] === '.') { + consumed.push(json[ii++]); + if (!json[ii]?.match(/\d/)) { + return null; + } + while (json[ii]?.match(/\d/)) { + consumed.push(json[ii++]); + } + } + + // Consume the optional exponent. + if (json[ii] === 'e' || json[ii] === 'E') { + consumed.push(json[ii++]); + if (json[ii] === '+' || json[ii] === '-') { + consumed.push(json[ii++]); + } + if (!json[ii]?.match(/\d/)) { + return null; + } + while (json[ii]?.match(/\d/)) { + consumed.push(json[ii++]); + } + } + + return consumed.join(''); +} + +export function wrapLargeIntegersUsingRegex(json: string): string { + return transformUnquotedSegments(json, unquotedSegment => { + return unquotedSegment.replaceAll(/(-?\d+)/g, (_, value: string) => { + return Number.isSafeInteger(Number(value)) ? value : wrapBigIntValueObject(value); + }); + }); +} + +/** + * This function takes a JSON string and transforms any unquoted segments using the provided + * `transform` function. This means we can be sure that our transformations will never occur + * inside a double quoted string — even if it contains escaped double quotes. + */ +function transformUnquotedSegments(json: string, transform: (value: string) => string): string { + /** + * This regex matches any part of a JSON string that isn't wrapped in double quotes. + * + * For instance, in the string `{"age":42,"name":"Alice \"The\" 2nd"}`, it would the + * following parts: `{`, `:42,`, `:`, `}`. Notice the whole "Alice \"The\" 2nd" string + * is not matched as it is wrapped in double quotes and contains escaped double quotes. + * + * The regex is composed of two parts: + * + * 1. The first part `^([^"]+)` matches any character until we reach the first double quote. + * 2. The second part `("(?:\\"|[^"])+")([^"]+)` matches any double quoted string that may exist + * after the first part and any unquoted segment that follows it. To match a double quoted string, + * we use the `(?:\\"|[^"])` regex to match any character that isn't a double quote whilst allowing + * escaped double quotes. + */ + const unquotedSegmentsRegex = /^([^"]+)|("(?:\\"|[^"])+")([^"]+)/g; + + return json.replaceAll(unquotedSegmentsRegex, (_, firstGroup, secondGroup, thirdGroup) => { + // If the first group is matched, it means we are at the + // beginning of the JSON string and we have an unquoted segment. + if (firstGroup) return transform(firstGroup); + + // Otherwise, we have a double quoted string followed by an unquoted segment. + return `${secondGroup}${transform(thirdGroup)}`; + }); +} + +type BigIntValueObject = { + // `$` implies 'this is a value object'. + // `n` implies 'interpret the value as a bigint'. + $n: string; +}; + +function wrapBigIntValueObject(value: string): string { + return `{"$n":"${value}"}`; +} + +function unwrapBigIntValueObject({ $n }: BigIntValueObject): bigint { + if ($n.match(/[eE]/)) { + const [units, exponent] = $n.split(/[eE]/); + return BigInt(units) * BigInt(10) ** BigInt(exponent); + } + return BigInt($n); +} + +function isBigIntValueObject(value: unknown): value is BigIntValueObject { + return !!value && typeof value === 'object' && '$n' in value && typeof value.$n === 'string'; +} diff --git a/packages/rpc-transport-http/tsconfig.json b/packages/rpc-transport-http/tsconfig.json index 5abbeea5baa6..af1a4e652975 100644 --- a/packages/rpc-transport-http/tsconfig.json +++ b/packages/rpc-transport-http/tsconfig.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/tsconfig", "compilerOptions": { - "lib": ["DOM", "ES2022.Error"] + "lib": ["DOM", "ES2021.String", "ES2022.Error"] }, "display": "@solana/rpc-transport-http", "extends": "../tsconfig/base.json",