From 56fbb0b056a815ffc970993699605b3b74a653d6 Mon Sep 17 00:00:00 2001 From: Ryan Ghods Date: Mon, 24 Jan 2022 11:26:14 -0800 Subject: [PATCH] rlp: v3 updates from integration (#1648) * rlp updates * util: add arrToBufArr and bufArrToArr and tests * util updates, add deprecation notice for re-exports * rlp tsconfig: include rootDir for composite option * remove util exports deprecation notices * rlp: add readme note for buffer compatibility * undo capitalization --- packages/rlp/CHANGELOG.md | 21 ++++++++++++++- packages/rlp/README.md | 17 ++++++++++++ packages/rlp/src/index.ts | 21 +++++---------- packages/rlp/tsconfig.prod.json | 5 +++- packages/util/package.json | 3 ++- packages/util/src/bytes.ts | 34 ++++++++++++++++++++++- packages/util/src/index.ts | 2 +- packages/util/src/types.ts | 3 +++ packages/util/test/bytes.spec.ts | 46 ++++++++++++++++++++++++++++++++ 9 files changed, 133 insertions(+), 19 deletions(-) diff --git a/packages/rlp/CHANGELOG.md b/packages/rlp/CHANGELOG.md index 56b0c3720f..2b59a24683 100644 --- a/packages/rlp/CHANGELOG.md +++ b/packages/rlp/CHANGELOG.md @@ -27,7 +27,26 @@ RLP.encode(1) ### Uint8Array -Buffers were replaced in favor of using Uint8Arrays for greater compatibility with browsers. +Buffers were replaced in favor of using Uint8Arrays for improved performance and greater compatibility with browsers. + +When upgrading from rlp v2 to v3, you must convert your Buffers to Uint8Arrays before passing in. To help, two new utility methods were added to `ethereumjs-util v7.1.4`: `arrToBufArr` and `bufArrToArr`. These will recursively step through your arrays to replace Buffers with Uint8Arrays, or vise versa. + +Example: + +```typescript +// Old, rlp v2 +import * as rlp from 'rlp' +const bufArr = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')] +const encoded = rlp.encode(bufArr) +const decoded = rlp.decode(encoded) + +// New, rlp v3 +import RLP from 'rlp' +const encoded: Uint8Array = RLP.encode(bufArrToArr(bufArr)) +const encodedAsBuffer = Buffer.from(encoded) +const decoded: Uint8Array[] = RLP.decode(encoded) +const decodedAsBuffers = arrToBufArr(decoded) +``` ### Invalid RLPs diff --git a/packages/rlp/README.md b/packages/rlp/README.md index 2fbdcb4abf..e877d06020 100644 --- a/packages/rlp/README.md +++ b/packages/rlp/README.md @@ -32,6 +32,23 @@ assert.deepEqual(nestedList, decoded) `RLP.decode(encoded, [stream=false])` - Decodes an RLP encoded `Uint8Array`, `Array` or `String` and returns a `Uint8Array` or `NestedUint8Array`. If `stream` is enabled, it will just decode the first rlp sequence in the Uint8Array. By default, it would throw an error if there are more bytes in Uint8Array than used by the rlp sequence. +### Buffer compatibility + +If you would like to continue using Buffers like in rlp v2, you can use: + +```typescript +import assert from 'assert' +import { arrToBufArr, bufArrToArr } from 'ethereumjs-util' +import RLP from 'rlp' + +const bufferList = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')] +const encoded = RLP.encode(bufArrToArr(bufferList)) +const encodedAsBuffer = Buffer.from(encoded) +const decoded = RLP.decode(Uint8Array.from(encodedAsBuffer)) // or RLP.decode(encoded) +const decodedAsBuffers = arrToBufArr(decoded) +assert.deepEqual(bufferList, decodedAsBuffers) +``` + ## CLI `rlp encode `\ diff --git a/packages/rlp/src/index.ts b/packages/rlp/src/index.ts index d1f00a494b..bc712bb873 100644 --- a/packages/rlp/src/index.ts +++ b/packages/rlp/src/index.ts @@ -1,16 +1,12 @@ -export type Input = string | number | bigint | Uint8Array | List | null | undefined +export type Input = string | number | bigint | Uint8Array | Array | null | undefined -// Use interface extension instead of type alias to -// make circular declaration possible. -export interface List extends Array {} +export type NestedUint8Array = Array export interface Decoded { data: Uint8Array | NestedUint8Array remainder: Uint8Array } -export interface NestedUint8Array extends Array {} - /** * RLP Encoding based on https://eth.wiki/en/fundamentals/rlp * This function takes in data, converts it to Uint8Array if not, @@ -51,13 +47,11 @@ function safeSlice(input: Uint8Array, start: number, end: number) { /** * Parse integers. Check if there is no leading zeros * @param v The value to parse - * @param base The base to parse the integer into */ function decodeLength(v: Uint8Array): number { - if (v[0] === 0 && v[1] === 0) { + if (v[0] === 0) { throw new Error('invalid RLP: extra zeros') } - return parseHexByte(bytesToHex(v)) } @@ -147,12 +141,12 @@ function _decode(input: Uint8Array): Decoded { remainder: input.slice(length + llength), } } else if (firstByte <= 0xf7) { - // a list between 0-55 bytes long + // a list between 0-55 bytes long length = firstByte - 0xbf innerRemainder = safeSlice(input, 1, length) while (innerRemainder.length) { d = _decode(innerRemainder) - decoded.push(d.data as Uint8Array) + decoded.push(d.data) innerRemainder = d.remainder } @@ -161,7 +155,7 @@ function _decode(input: Uint8Array): Decoded { remainder: input.slice(length), } } else { - // a list over 55 bytes long + // a list over 55 bytes long llength = firstByte - 0xf6 length = decodeLength(safeSlice(input, 1, llength)) if (length < 56) { @@ -176,7 +170,7 @@ function _decode(input: Uint8Array): Decoded { while (innerRemainder.length) { d = _decode(innerRemainder) - decoded.push(d.data as Uint8Array) + decoded.push(d.data) innerRemainder = d.remainder } @@ -198,7 +192,6 @@ function bytesToHex(uint8a: Uint8Array): string { } function parseHexByte(hexByte: string): number { - if (hexByte.length !== 2) throw new Error('Invalid byte sequence') const byte = Number.parseInt(hexByte, 16) if (Number.isNaN(byte)) throw new Error('Invalid byte sequence') return byte diff --git a/packages/rlp/tsconfig.prod.json b/packages/rlp/tsconfig.prod.json index 7a9cba178c..ab41edb0de 100644 --- a/packages/rlp/tsconfig.prod.json +++ b/packages/rlp/tsconfig.prod.json @@ -1,7 +1,10 @@ { "extends": "../../config/tsconfig.prod.json", "compilerOptions": { + "target": "es2020", + "rootDir": "./src", "outDir": "./dist", + "composite": true }, "include": ["src/*.ts"] -} +} \ No newline at end of file diff --git a/packages/util/package.json b/packages/util/package.json index f8aa9d3239..99664d0eb0 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -6,7 +6,8 @@ "author": "mjbecze ", "keywords": [ "ethereum", - "utilties" + "utilities", + "utils" ], "engines": { "node": ">=10.0.0" diff --git a/packages/util/src/bytes.ts b/packages/util/src/bytes.ts index 5940f41d71..0e8654c724 100644 --- a/packages/util/src/bytes.ts +++ b/packages/util/src/bytes.ts @@ -1,6 +1,12 @@ import { BN } from './externals' import { stripHexPrefix, padToEven, isHexString, isHexPrefixed } from './internal' -import { PrefixedHexString, TransformableToArray, TransformableToBuffer } from './types' +import { + PrefixedHexString, + TransformableToArray, + TransformableToBuffer, + NestedBufferArray, + NestedUint8Array, +} from './types' import { assertIsBuffer, assertIsArray, assertIsHexString } from './helpers' /** @@ -300,3 +306,29 @@ export const validateNoLeadingZeroes = function (values: { [key: string]: Buffer } } } + +/** + * Converts a {@link Uint8Array} or {@link NestedUint8Array} to {@link Buffer} or {@link NestedBufferArray} + */ +export function arrToBufArr(arr: Uint8Array): Buffer +export function arrToBufArr(arr: NestedUint8Array): NestedBufferArray +export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray +export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray { + if (!Array.isArray(arr)) { + return Buffer.from(arr) + } + return arr.map((a) => arrToBufArr(a)) +} + +/** + * Converts a {@link Buffer} or {@link NestedBufferArray} to {@link Uint8Array} or {@link NestedUint8Array} + */ +export function bufArrToArr(arr: Buffer): Uint8Array +export function bufArrToArr(arr: NestedBufferArray): NestedUint8Array +export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array +export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array { + if (!Array.isArray(arr)) { + return Uint8Array.from(arr ?? []) + } + return arr.map((a) => bufArrToArr(a)) +} diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index a25a036005..bda39a3f6c 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -34,7 +34,7 @@ export * from './bytes' export * from './object' /** - * External exports (BN, rlp, secp256k1) + * External exports (BN, rlp) */ export * from './externals' diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index f59d25c87a..3f84526559 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -47,6 +47,9 @@ export interface TransformableToBuffer { toArray?(): Uint8Array } +export type NestedUint8Array = Array +export type NestedBufferArray = Array + /** * Convert BN to 0x-prefixed hex string. */ diff --git a/packages/util/test/bytes.spec.ts b/packages/util/test/bytes.spec.ts index 75dc2961c3..31a63506f5 100644 --- a/packages/util/test/bytes.spec.ts +++ b/packages/util/test/bytes.spec.ts @@ -1,7 +1,9 @@ import tape from 'tape' import { + arrToBufArr, Address, BN, + bufArrToArr, zeros, zeroAddress, isZeroAddress, @@ -397,3 +399,47 @@ tape('validateNoLeadingZeroes', function (st) { st.throws(() => validateNoLeadingZeroes(onlyZeroes), 'throws when value has only zeroes') st.end() }) + +tape('arrToBufArr', function (st) { + const uint8 = Uint8Array.from([0, 1, 2]) + const uint8Arr = [ + Uint8Array.from([1, 2, 3]), + Uint8Array.from([4, 5, 6]), + [Uint8Array.from([7, 8, 9]), Uint8Array.from([1, 0, 0]), [Uint8Array.from([1, 1, 1])]], + ] + const buf = Buffer.from(uint8) + const bufArr = [ + Buffer.from(Uint8Array.from([1, 2, 3])), + Buffer.from(Uint8Array.from([4, 5, 6])), + [ + Buffer.from(Uint8Array.from([7, 8, 9])), + Buffer.from(Uint8Array.from([1, 0, 0])), + [Buffer.from(Uint8Array.from([1, 1, 1]))], + ], + ] + st.deepEqual(arrToBufArr(uint8), buf) + st.deepEqual(arrToBufArr(uint8Arr), bufArr) + st.end() +}) + +tape('bufArrToArr', function (st) { + const buf = Buffer.from('123', 'hex') + const bufArr = [ + Buffer.from('123', 'hex'), + Buffer.from('456', 'hex'), + [Buffer.from('789', 'hex'), Buffer.from('100', 'hex'), [Buffer.from('111', 'hex')]], + ] + const uint8 = Uint8Array.from(buf) + const uint8Arr = [ + Uint8Array.from(Buffer.from('123', 'hex')), + Uint8Array.from(Buffer.from('456', 'hex')), + [ + Uint8Array.from(Buffer.from('789', 'hex')), + Uint8Array.from(Buffer.from('100', 'hex')), + [Uint8Array.from(Buffer.from('111', 'hex'))], + ], + ] + st.deepEqual(bufArrToArr(buf), uint8) + st.deepEqual(bufArrToArr(bufArr), uint8Arr) + st.end() +})