Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implements a JSON.parse that transforms unsafe numbers to bigints #3133

Closed
wants to merge 1 commit into from
Closed

Implements a JSON.parse that transforms unsafe numbers to bigints #3133

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/rpc-api/src/__tests__/get-account-info-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ describe('getAccountInfo', () => {
executable: false,
lamports: 5000000n,
owner: '11111111111111111111111111111111',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 9n,
},
});
Expand Down
6 changes: 3 additions & 3 deletions packages/rpc-api/src/__tests__/get-multiple-accounts-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,15 +43,15 @@ describe('getMultipleAccounts', () => {
executable: false,
lamports: 5000000n,
owner: '11111111111111111111111111111111',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 9n,
},
{
data: ['', 'base64'],
executable: false,
lamports: 5000000n,
owner: '11111111111111111111111111111111',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 0n,
},
],
Expand Down Expand Up @@ -114,7 +114,7 @@ describe('getMultipleAccounts', () => {
executable: false,
lamports: 5000000n,
owner: '11111111111111111111111111111111',
rentEpoch: 18446744073709551616n,
rentEpoch: 18446744073709551615n,
space: 9n,
},
null,
Expand Down
6 changes: 3 additions & 3 deletions packages/rpc-api/src/__tests__/get-program-accounts-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ describe('getProgramAccounts', () => {
executable: false,
lamports: 5000000n,
owner: 'DXngmJfjurhnAwbMPgpUGPH6qNvetCKRJ6PiD4ag4PTj',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 9n,
},
pubkey: 'CcYNb7WqpjaMrNr7B1mapaNfWctZRH7LyAjWRLBGt1Fk',
Expand Down Expand Up @@ -126,7 +126,7 @@ describe('getProgramAccounts', () => {
executable: false,
lamports: 5000000n,
owner: 'AmtpVzo6H6qQCP9dH9wfu5hfa8kKaAFpTJ4aamPYR6V6',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 9n,
},
pubkey: 'C5q1p5UiCVrt6vcLJDGcS4AZ98fahKyb9XkDRdqATK17',
Expand All @@ -137,7 +137,7 @@ describe('getProgramAccounts', () => {
executable: false,
lamports: 5000000n,
owner: 'AmtpVzo6H6qQCP9dH9wfu5hfa8kKaAFpTJ4aamPYR6V6',
rentEpoch: 18446744073709551616n, // TODO: This number loses precision
rentEpoch: 18446744073709551615n,
space: 9n,
},
pubkey: 'Hhsoev7Apk5dMbktzLUrsTHuMq9e9GSYBaLcnN2PfdKS',
Expand Down
1 change: 1 addition & 0 deletions packages/rpc-transport-http/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
],
"scripts": {
"benchmark": "./src/__benchmarks__/run.ts",
"benchmark:json-parse": "./src/__benchmarks__/json-parse.ts",
"compile:js": "tsup --config build-scripts/tsup.config.package.ts",
"compile:typedefs": "tsc -p ./tsconfig.declarations.json",
"dev": "jest -c ../../node_modules/@solana/test-config/jest-dev.config.ts --rootDir . --watch",
Expand Down
57 changes: 57 additions & 0 deletions packages/rpc-transport-http/src/__benchmarks__/json-parse.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
#!/usr/bin/env -S pnpm dlx tsx --

import { Bench } from 'tinybench';
import { fs, path } from 'zx';

import {
jsonParseWithLargeIntegersAsBigInts,
wrapLargeIntegersUsingParser,
wrapLargeIntegersUsingParserAndRegex,
wrapLargeIntegersUsingRegex,
} from '../json-parse-with-bigint';

Object.assign(globalThis, {
__BROWSER__: false,
__DEV__: false,
__NODEJS__: true,
__REACTNATIVE____: false,
});

const bench = new Bench({
throws: true,
});

const largeJsonPath = path.join(__dirname, '..', '__tests__', 'large-json-file.json');
const largeJsonString = fs.readFileSync(largeJsonPath, 'utf8');

bench
.add('JSON.parse', () => {
return JSON.parse(largeJsonString);
})
.add('JSON.parse with noop pre-processing loop', () => {
const out = [];
for (let ii = 0; ii < largeJsonString.length; ii++) {
out.push(largeJsonString[ii]);
}
return JSON.parse(out.join(''));
})
.add('JSON.parse with noop reviver', () => {
return jsonParseWithLargeIntegersAsBigInts(largeJsonString, x => x);
})
.add('jsonParseWithLargeIntegersAsBigInts (parser)', () => {
return jsonParseWithLargeIntegersAsBigInts(largeJsonString);
})
.add('jsonParseWithLargeIntegersAsBigInts (parser and regex)', () => {
return jsonParseWithLargeIntegersAsBigInts(largeJsonString, wrapLargeIntegersUsingParserAndRegex);
})
.add('jsonParseWithLargeIntegersAsBigInts (parser and noop)', () => {
return jsonParseWithLargeIntegersAsBigInts(largeJsonString, x => wrapLargeIntegersUsingParser(x, () => null));
})
.add('jsonParseWithLargeIntegersAsBigInts (regex)', () => {
return jsonParseWithLargeIntegersAsBigInts(largeJsonString, wrapLargeIntegersUsingRegex);
});

(async () => {
await bench.run();
console.table(bench.table());
})();
2 changes: 1 addition & 1 deletion packages/rpc-transport-http/src/__tests__/http-transport-abort-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,8 +71,8 @@ describe('createHttpTransport and `AbortSignal`', () => {
it('resolves with the response', async () => {
expect.assertions(1);
jest.mocked(fetchSpy).mockResolvedValueOnce({
json: () => ({ ok: true }),
ok: true,
text: () => '{"ok":true}',
} as unknown as Response);
const sendPromise = makeHttpRequest({ payload: 123, signal: abortSignal });
abortController.abort('I got bored waiting');
Expand Down
2 changes: 1 addition & 1 deletion packages/rpc-transport-http/src/__tests__/http-transport-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ describe('createHttpTransport', () => {
describe('when the endpoint returns a well-formed JSON response', () => {
beforeEach(() => {
fetchSpy.mockResolvedValue({
json: () => ({ ok: true }),
ok: true,
text: () => '{"ok":true}',
});
});
it('calls fetch with the specified URL', () => {
Expand Down
104 changes: 104 additions & 0 deletions packages/rpc-transport-http/src/__tests__/json-parse-with-bigint-test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
import fs from 'fs';
import path from 'path';

import { jsonParseWithLargeIntegersAsBigInts, wrapLargeIntegers } from '../json-parse-with-bigint';

const MAX_SAFE_INTEGER: number = Number.MAX_SAFE_INTEGER;
const MAX_SAFE_INTEGER_PLUS_ONE: bigint = BigInt(Number.MAX_SAFE_INTEGER) + 1n;

describe('jsonParseWithLargeNumbersAsBigInts', () => {
it('parses large integers as bigints', () => {
const result = jsonParseWithLargeIntegersAsBigInts(`{"value":${MAX_SAFE_INTEGER_PLUS_ONE}}`) as {
value: unknown;
};
expect(result).toEqual({ value: MAX_SAFE_INTEGER_PLUS_ONE });
expect(typeof result.value).toBe('bigint');
});
it('keeps safe integers as numbers', () => {
const result = jsonParseWithLargeIntegersAsBigInts(`{"value":${MAX_SAFE_INTEGER}}`) as { value: unknown };
expect(result).toEqual({ value: MAX_SAFE_INTEGER });
expect(typeof result.value).toBe('number');
});
it('does not affect numbers in strings', () => {
expect(jsonParseWithLargeIntegersAsBigInts(`{"value":"Hello 123 World"}`)).toEqual({
value: 'Hello 123 World',
});
});
it('does not affect numbers in strings with escaped double quotes', () => {
expect(jsonParseWithLargeIntegersAsBigInts(`{"value":"Hello\\" 123 World"}`)).toEqual({
value: 'Hello" 123 World',
});
});
it('parses large integers with exponents as bigints', () => {
expect(jsonParseWithLargeIntegersAsBigInts(`1e32`)).toBe(100000000000000000000000000000000n);
expect(jsonParseWithLargeIntegersAsBigInts(`-1189e+32`)).toBe(-118900000000000000000000000000000000n);
});
it('does not affect negative exponents', () => {
expect(jsonParseWithLargeIntegersAsBigInts(`1e-32`)).toBe(1e-32);
expect(jsonParseWithLargeIntegersAsBigInts(`-1189e-32`)).toBe(-1189e-32);
});
it('can parse complex JSON files', () => {
const largeJsonPath = path.join(__dirname, 'large-json-file.json');
const largeJsonString = fs.readFileSync(largeJsonPath, 'utf8');
const expectedResult = JSON.parse(largeJsonString, (key, value) =>
// eslint-disable-next-line jest/no-conditional-in-test
key === 'lamports' ? 142302234983644260n : value,
);
expect(jsonParseWithLargeIntegersAsBigInts(largeJsonString)).toEqual(expectedResult);
});
});

describe('wrapLargeIntegers', () => {
it('wraps large integers as bigint strings', () => {
expect(wrapLargeIntegers(`{"value":${MAX_SAFE_INTEGER_PLUS_ONE}}`)).toBe(
`{"value":{"$n":"${MAX_SAFE_INTEGER_PLUS_ONE}"}}`,
);
});
it('keeps safe integers as numbers', () => {
expect(wrapLargeIntegers(`{"value":${MAX_SAFE_INTEGER}}`)).toBe(`{"value":${MAX_SAFE_INTEGER}}`);
});
it('does not wrap numbers in strings', () => {
expect(wrapLargeIntegers(`{"value":"Hello 123 World"}`)).toBe(`{"value":"Hello 123 World"}`);
});
it('does not wrap numbers in strings with escaped double quotes', () => {
expect(wrapLargeIntegers(`{"value":"Hello\\" 123 World"}`)).toBe(`{"value":"Hello\\" 123 World"}`);
});
it('does not alter special keywords', () => {
expect(wrapLargeIntegers('true')).toBe('true');
expect(wrapLargeIntegers('false')).toBe('false');
expect(wrapLargeIntegers('null')).toBe('null');
});
it('does not alter safe numbers', () => {
expect(wrapLargeIntegers('0')).toBe('0');
expect(wrapLargeIntegers('123456')).toBe('123456');
expect(wrapLargeIntegers('-123456')).toBe('-123456');
expect(wrapLargeIntegers('3.14')).toBe('3.14');
expect(wrapLargeIntegers('-3.14')).toBe('-3.14');
expect(wrapLargeIntegers('1e5')).toBe('1e5');
expect(wrapLargeIntegers('-1e5')).toBe('-1e5');
expect(wrapLargeIntegers('1E5')).toBe('1E5');
expect(wrapLargeIntegers('-1E5')).toBe('-1E5');
expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER}`)).toBe(`${MAX_SAFE_INTEGER}`);
expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER}`)).toBe(`-${MAX_SAFE_INTEGER}`);
});
it('wraps unsafe large integers', () => {
expect(wrapLargeIntegers('1e32')).toBe('{"$n":"1e32"}');
expect(wrapLargeIntegers('-1e32')).toBe('{"$n":"-1e32"}');
expect(wrapLargeIntegers('1E32')).toBe('{"$n":"1E32"}');
expect(wrapLargeIntegers('-1E32')).toBe('{"$n":"-1E32"}');
expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER_PLUS_ONE}`)).toBe(`{"$n":"${MAX_SAFE_INTEGER_PLUS_ONE}"}`);
expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER_PLUS_ONE}`)).toBe(`{"$n":"-${MAX_SAFE_INTEGER_PLUS_ONE}"}`);
});
it('does not alter unsafe large floating points', () => {
expect(wrapLargeIntegers('3.14e32')).toBe('3.14e32');
expect(wrapLargeIntegers('-3.14e32')).toBe('-3.14e32');
expect(wrapLargeIntegers('3.14E32')).toBe('3.14E32');
expect(wrapLargeIntegers('-3.14E32')).toBe('-3.14E32');
expect(wrapLargeIntegers(`${MAX_SAFE_INTEGER_PLUS_ONE}.123`)).toBe(`${MAX_SAFE_INTEGER_PLUS_ONE}.123`);
expect(wrapLargeIntegers(`-${MAX_SAFE_INTEGER_PLUS_ONE}.123`)).toBe(`-${MAX_SAFE_INTEGER_PLUS_ONE}.123`);
});
it('does not alter strings', () => {
expect(wrapLargeIntegers('""')).toBe('""');
expect(wrapLargeIntegers('"Hello World"')).toBe('"Hello World"');
});
});
Loading