From 56fbb0b056a815ffc970993699605b3b74a653d6 Mon Sep 17 00:00:00 2001 From: Ryan Ghods Date: Mon, 24 Jan 2022 11:26:14 -0800 Subject: [PATCH 1/2] rlp: v3 updates from integration (#1648) * rlp updates * util: add arrToBufArr and bufArrToArr and tests * util updates, add deprecation notice for re-exports * rlp tsconfig: include rootDir for composite option * remove util exports deprecation notices * rlp: add readme note for buffer compatibility * undo capitalization --- packages/rlp/CHANGELOG.md | 21 ++++++++++++++- packages/rlp/README.md | 17 ++++++++++++ packages/rlp/src/index.ts | 21 +++++---------- packages/rlp/tsconfig.prod.json | 5 +++- packages/util/package.json | 3 ++- packages/util/src/bytes.ts | 34 ++++++++++++++++++++++- packages/util/src/index.ts | 2 +- packages/util/src/types.ts | 3 +++ packages/util/test/bytes.spec.ts | 46 ++++++++++++++++++++++++++++++++ 9 files changed, 133 insertions(+), 19 deletions(-) diff --git a/packages/rlp/CHANGELOG.md b/packages/rlp/CHANGELOG.md index 56b0c3720f..2b59a24683 100644 --- a/packages/rlp/CHANGELOG.md +++ b/packages/rlp/CHANGELOG.md @@ -27,7 +27,26 @@ RLP.encode(1) ### Uint8Array -Buffers were replaced in favor of using Uint8Arrays for greater compatibility with browsers. +Buffers were replaced in favor of using Uint8Arrays for improved performance and greater compatibility with browsers. + +When upgrading from rlp v2 to v3, you must convert your Buffers to Uint8Arrays before passing in. To help, two new utility methods were added to `ethereumjs-util v7.1.4`: `arrToBufArr` and `bufArrToArr`. These will recursively step through your arrays to replace Buffers with Uint8Arrays, or vise versa. + +Example: + +```typescript +// Old, rlp v2 +import * as rlp from 'rlp' +const bufArr = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')] +const encoded = rlp.encode(bufArr) +const decoded = rlp.decode(encoded) + +// New, rlp v3 +import RLP from 'rlp' +const encoded: Uint8Array = RLP.encode(bufArrToArr(bufArr)) +const encodedAsBuffer = Buffer.from(encoded) +const decoded: Uint8Array[] = RLP.decode(encoded) +const decodedAsBuffers = arrToBufArr(decoded) +``` ### Invalid RLPs diff --git a/packages/rlp/README.md b/packages/rlp/README.md index 2fbdcb4abf..e877d06020 100644 --- a/packages/rlp/README.md +++ b/packages/rlp/README.md @@ -32,6 +32,23 @@ assert.deepEqual(nestedList, decoded) `RLP.decode(encoded, [stream=false])` - Decodes an RLP encoded `Uint8Array`, `Array` or `String` and returns a `Uint8Array` or `NestedUint8Array`. If `stream` is enabled, it will just decode the first rlp sequence in the Uint8Array. By default, it would throw an error if there are more bytes in Uint8Array than used by the rlp sequence. +### Buffer compatibility + +If you would like to continue using Buffers like in rlp v2, you can use: + +```typescript +import assert from 'assert' +import { arrToBufArr, bufArrToArr } from 'ethereumjs-util' +import RLP from 'rlp' + +const bufferList = [Buffer.from('123', 'hex'), Buffer.from('456', 'hex')] +const encoded = RLP.encode(bufArrToArr(bufferList)) +const encodedAsBuffer = Buffer.from(encoded) +const decoded = RLP.decode(Uint8Array.from(encodedAsBuffer)) // or RLP.decode(encoded) +const decodedAsBuffers = arrToBufArr(decoded) +assert.deepEqual(bufferList, decodedAsBuffers) +``` + ## CLI `rlp encode `\ diff --git a/packages/rlp/src/index.ts b/packages/rlp/src/index.ts index d1f00a494b..bc712bb873 100644 --- a/packages/rlp/src/index.ts +++ b/packages/rlp/src/index.ts @@ -1,16 +1,12 @@ -export type Input = string | number | bigint | Uint8Array | List | null | undefined +export type Input = string | number | bigint | Uint8Array | Array | null | undefined -// Use interface extension instead of type alias to -// make circular declaration possible. -export interface List extends Array {} +export type NestedUint8Array = Array export interface Decoded { data: Uint8Array | NestedUint8Array remainder: Uint8Array } -export interface NestedUint8Array extends Array {} - /** * RLP Encoding based on https://eth.wiki/en/fundamentals/rlp * This function takes in data, converts it to Uint8Array if not, @@ -51,13 +47,11 @@ function safeSlice(input: Uint8Array, start: number, end: number) { /** * Parse integers. Check if there is no leading zeros * @param v The value to parse - * @param base The base to parse the integer into */ function decodeLength(v: Uint8Array): number { - if (v[0] === 0 && v[1] === 0) { + if (v[0] === 0) { throw new Error('invalid RLP: extra zeros') } - return parseHexByte(bytesToHex(v)) } @@ -147,12 +141,12 @@ function _decode(input: Uint8Array): Decoded { remainder: input.slice(length + llength), } } else if (firstByte <= 0xf7) { - // a list between 0-55 bytes long + // a list between 0-55 bytes long length = firstByte - 0xbf innerRemainder = safeSlice(input, 1, length) while (innerRemainder.length) { d = _decode(innerRemainder) - decoded.push(d.data as Uint8Array) + decoded.push(d.data) innerRemainder = d.remainder } @@ -161,7 +155,7 @@ function _decode(input: Uint8Array): Decoded { remainder: input.slice(length), } } else { - // a list over 55 bytes long + // a list over 55 bytes long llength = firstByte - 0xf6 length = decodeLength(safeSlice(input, 1, llength)) if (length < 56) { @@ -176,7 +170,7 @@ function _decode(input: Uint8Array): Decoded { while (innerRemainder.length) { d = _decode(innerRemainder) - decoded.push(d.data as Uint8Array) + decoded.push(d.data) innerRemainder = d.remainder } @@ -198,7 +192,6 @@ function bytesToHex(uint8a: Uint8Array): string { } function parseHexByte(hexByte: string): number { - if (hexByte.length !== 2) throw new Error('Invalid byte sequence') const byte = Number.parseInt(hexByte, 16) if (Number.isNaN(byte)) throw new Error('Invalid byte sequence') return byte diff --git a/packages/rlp/tsconfig.prod.json b/packages/rlp/tsconfig.prod.json index 7a9cba178c..ab41edb0de 100644 --- a/packages/rlp/tsconfig.prod.json +++ b/packages/rlp/tsconfig.prod.json @@ -1,7 +1,10 @@ { "extends": "../../config/tsconfig.prod.json", "compilerOptions": { + "target": "es2020", + "rootDir": "./src", "outDir": "./dist", + "composite": true }, "include": ["src/*.ts"] -} +} \ No newline at end of file diff --git a/packages/util/package.json b/packages/util/package.json index f8aa9d3239..99664d0eb0 100644 --- a/packages/util/package.json +++ b/packages/util/package.json @@ -6,7 +6,8 @@ "author": "mjbecze ", "keywords": [ "ethereum", - "utilties" + "utilities", + "utils" ], "engines": { "node": ">=10.0.0" diff --git a/packages/util/src/bytes.ts b/packages/util/src/bytes.ts index 5940f41d71..0e8654c724 100644 --- a/packages/util/src/bytes.ts +++ b/packages/util/src/bytes.ts @@ -1,6 +1,12 @@ import { BN } from './externals' import { stripHexPrefix, padToEven, isHexString, isHexPrefixed } from './internal' -import { PrefixedHexString, TransformableToArray, TransformableToBuffer } from './types' +import { + PrefixedHexString, + TransformableToArray, + TransformableToBuffer, + NestedBufferArray, + NestedUint8Array, +} from './types' import { assertIsBuffer, assertIsArray, assertIsHexString } from './helpers' /** @@ -300,3 +306,29 @@ export const validateNoLeadingZeroes = function (values: { [key: string]: Buffer } } } + +/** + * Converts a {@link Uint8Array} or {@link NestedUint8Array} to {@link Buffer} or {@link NestedBufferArray} + */ +export function arrToBufArr(arr: Uint8Array): Buffer +export function arrToBufArr(arr: NestedUint8Array): NestedBufferArray +export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray +export function arrToBufArr(arr: Uint8Array | NestedUint8Array): Buffer | NestedBufferArray { + if (!Array.isArray(arr)) { + return Buffer.from(arr) + } + return arr.map((a) => arrToBufArr(a)) +} + +/** + * Converts a {@link Buffer} or {@link NestedBufferArray} to {@link Uint8Array} or {@link NestedUint8Array} + */ +export function bufArrToArr(arr: Buffer): Uint8Array +export function bufArrToArr(arr: NestedBufferArray): NestedUint8Array +export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array +export function bufArrToArr(arr: Buffer | NestedBufferArray): Uint8Array | NestedUint8Array { + if (!Array.isArray(arr)) { + return Uint8Array.from(arr ?? []) + } + return arr.map((a) => bufArrToArr(a)) +} diff --git a/packages/util/src/index.ts b/packages/util/src/index.ts index a25a036005..bda39a3f6c 100644 --- a/packages/util/src/index.ts +++ b/packages/util/src/index.ts @@ -34,7 +34,7 @@ export * from './bytes' export * from './object' /** - * External exports (BN, rlp, secp256k1) + * External exports (BN, rlp) */ export * from './externals' diff --git a/packages/util/src/types.ts b/packages/util/src/types.ts index f59d25c87a..3f84526559 100644 --- a/packages/util/src/types.ts +++ b/packages/util/src/types.ts @@ -47,6 +47,9 @@ export interface TransformableToBuffer { toArray?(): Uint8Array } +export type NestedUint8Array = Array +export type NestedBufferArray = Array + /** * Convert BN to 0x-prefixed hex string. */ diff --git a/packages/util/test/bytes.spec.ts b/packages/util/test/bytes.spec.ts index 75dc2961c3..31a63506f5 100644 --- a/packages/util/test/bytes.spec.ts +++ b/packages/util/test/bytes.spec.ts @@ -1,7 +1,9 @@ import tape from 'tape' import { + arrToBufArr, Address, BN, + bufArrToArr, zeros, zeroAddress, isZeroAddress, @@ -397,3 +399,47 @@ tape('validateNoLeadingZeroes', function (st) { st.throws(() => validateNoLeadingZeroes(onlyZeroes), 'throws when value has only zeroes') st.end() }) + +tape('arrToBufArr', function (st) { + const uint8 = Uint8Array.from([0, 1, 2]) + const uint8Arr = [ + Uint8Array.from([1, 2, 3]), + Uint8Array.from([4, 5, 6]), + [Uint8Array.from([7, 8, 9]), Uint8Array.from([1, 0, 0]), [Uint8Array.from([1, 1, 1])]], + ] + const buf = Buffer.from(uint8) + const bufArr = [ + Buffer.from(Uint8Array.from([1, 2, 3])), + Buffer.from(Uint8Array.from([4, 5, 6])), + [ + Buffer.from(Uint8Array.from([7, 8, 9])), + Buffer.from(Uint8Array.from([1, 0, 0])), + [Buffer.from(Uint8Array.from([1, 1, 1]))], + ], + ] + st.deepEqual(arrToBufArr(uint8), buf) + st.deepEqual(arrToBufArr(uint8Arr), bufArr) + st.end() +}) + +tape('bufArrToArr', function (st) { + const buf = Buffer.from('123', 'hex') + const bufArr = [ + Buffer.from('123', 'hex'), + Buffer.from('456', 'hex'), + [Buffer.from('789', 'hex'), Buffer.from('100', 'hex'), [Buffer.from('111', 'hex')]], + ] + const uint8 = Uint8Array.from(buf) + const uint8Arr = [ + Uint8Array.from(Buffer.from('123', 'hex')), + Uint8Array.from(Buffer.from('456', 'hex')), + [ + Uint8Array.from(Buffer.from('789', 'hex')), + Uint8Array.from(Buffer.from('100', 'hex')), + [Uint8Array.from(Buffer.from('111', 'hex'))], + ], + ] + st.deepEqual(bufArrToArr(buf), uint8) + st.deepEqual(bufArrToArr(bufArr), uint8Arr) + st.end() +}) From 5de8729e348e724bd762ae7cc5f3b72c9e093ee1 Mon Sep 17 00:00:00 2001 From: ScottyPoi <66335769+ScottyPoi@users.noreply.github.com> Date: Tue, 25 Jan 2022 02:26:58 -0700 Subject: [PATCH 2/2] Devp2p, Client: Fix duplicated debug messages (#1643) * devp2p: update debug package * blockchain: update debug package * client: update debug package * vm: update debug package * devp2p/dpt fix for #1485 * devp2p/eth: Fix for 485 * devp2p: Fix for #1485: add base debug * devp2p/dpt #1485: change to debug.extend() * devp2p:dpt:server change to degub.extend() * devp2p:eth change to debug.extend() * devp2p/les: change to debug.extend() * devp2p:rlpx: change to debug.extend() * rlps: change to debug.extend() * Update debug to use IP as first tag --- packages/blockchain/package.json | 2 +- packages/client/package.json | 2 +- packages/devp2p/package.json | 2 +- packages/devp2p/src/dpt/dpt.ts | 15 +++--- packages/devp2p/src/dpt/server.ts | 32 ++++-------- packages/devp2p/src/eth/index.ts | 30 +++++------ packages/devp2p/src/les/index.ts | 30 +++++------ packages/devp2p/src/rlpx/peer.ts | 82 ++++++++++--------------------- packages/devp2p/src/rlpx/rlpx.ts | 35 +++++++------ packages/devp2p/src/util.ts | 3 ++ packages/vm/package.json | 2 +- 11 files changed, 90 insertions(+), 145 deletions(-) diff --git a/packages/blockchain/package.json b/packages/blockchain/package.json index 671b2b705e..f5b099d15e 100644 --- a/packages/blockchain/package.json +++ b/packages/blockchain/package.json @@ -36,7 +36,7 @@ "@ethereumjs/block": "^3.6.0", "@ethereumjs/common": "^2.6.0", "@ethereumjs/ethash": "^1.1.0", - "debug": "^2.2.0", + "debug": "^4.3.3", "ethereumjs-util": "^7.1.3", "level-mem": "^5.0.1", "lru-cache": "^5.1.1", diff --git a/packages/client/package.json b/packages/client/package.json index 83281b206a..5f1d635d67 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -58,7 +58,7 @@ "@ethereumjs/tx": "^3.4.0", "@ethereumjs/vm": "^5.6.0", "chalk": "^4.1.2", - "debug": "^2.2.0", + "debug": "^4.3.3", "ethereumjs-util": "^7.1.3", "fs-extra": "^10.0.0", "it-pipe": "^1.1.0", diff --git a/packages/devp2p/package.json b/packages/devp2p/package.json index 38c5e42b04..77ca847a35 100644 --- a/packages/devp2p/package.json +++ b/packages/devp2p/package.json @@ -47,7 +47,7 @@ "@types/lru-cache": "^5.1.0", "base64url": "^3.0.1", "bl": "^1.1.2", - "debug": "^2.2.0", + "debug": "^4.3.3", "ethereumjs-util": "^7.1.3", "hi-base32": "^0.5.0", "inherits": "^2.0.1", diff --git a/packages/devp2p/src/dpt/dpt.ts b/packages/devp2p/src/dpt/dpt.ts index e8372aa076..274204d9dd 100644 --- a/packages/devp2p/src/dpt/dpt.ts +++ b/packages/devp2p/src/dpt/dpt.ts @@ -2,14 +2,16 @@ import ms from 'ms' import { EventEmitter } from 'events' import { publicKeyCreate } from 'secp256k1' import { randomBytes } from 'crypto' -import { debug as createDebugLogger } from 'debug' +// import { debug as createDebugLogger } from 'debug' +import { devp2pDebug } from '../util' import { buffer2int, pk2id } from '../util' import { KBucket } from './kbucket' import { BanList } from './ban-list' import { Server as DPTServer } from './server' import { DNS } from '../dns' +import { Debugger } from 'debug' -const debug = createDebugLogger('devp2p:dpt') +const DEBUG_BASE_NAME = 'dpt' export interface PeerInfo { id?: Uint8Array | Buffer @@ -87,6 +89,7 @@ export class DPT extends EventEmitter { privateKey: Buffer banlist: BanList dns: DNS + _debug: Debugger private _id: Buffer | undefined private _kbucket: KBucket @@ -127,7 +130,7 @@ export class DPT extends EventEmitter { this._server.once('listening', () => this.emit('listening')) this._server.once('close', () => this.emit('close')) this._server.on('error', (err) => this.emit('error', err)) - + this._debug = devp2pDebug.extend(DEBUG_BASE_NAME) // When not using peer neighbour discovery we don't add peers here // because it results in duplicate calls for the same targets this._server.on('peers', (peers) => { @@ -198,7 +201,7 @@ export class DPT extends EventEmitter { async addPeer(obj: PeerInfo): Promise { if (this.banlist.has(obj)) throw new Error('Peer is banned') - debug(`attempt adding peer ${obj.address}:${obj.udpPort}`) + this._debug(`attempt adding peer ${obj.address}:${obj.udpPort}`) // check k-bucket first const peer = this._kbucket.get(obj) @@ -247,7 +250,7 @@ export class DPT extends EventEmitter { this._refreshIntervalSelectionCounter = (this._refreshIntervalSelectionCounter + 1) % 10 const peers = this.getPeers() - debug( + this._debug( `call .refresh() (selector ${this._refreshIntervalSelectionCounter}) (${peers.length} peers in table)` ) @@ -264,7 +267,7 @@ export class DPT extends EventEmitter { if (this._shouldGetDnsPeers) { const dnsPeers = await this.getDnsPeers() - debug( + this._debug( `.refresh() Adding ${dnsPeers.length} from DNS tree, (${ this.getPeers().length } current peers in table)` diff --git a/packages/devp2p/src/dpt/server.ts b/packages/devp2p/src/dpt/server.ts index 900775da0d..096392312e 100644 --- a/packages/devp2p/src/dpt/server.ts +++ b/packages/devp2p/src/dpt/server.ts @@ -1,15 +1,14 @@ import { EventEmitter } from 'events' import * as dgram from 'dgram' import ms from 'ms' -import { debug as createDebugLogger } from 'debug' +import { debug as createDebugLogger, Debugger } from 'debug' import LRUCache = require('lru-cache') import { encode, decode } from './message' -import { keccak256, pk2id, createDeferred, formatLogId } from '../util' +import { keccak256, pk2id, createDeferred, formatLogId, devp2pDebug } from '../util' import { DPT, PeerInfo } from './dpt' import { Socket as DgramSocket, RemoteInfo } from 'dgram' -const DEBUG_BASE_NAME = 'devp2p:dpt:server' -const debug = createDebugLogger(DEBUG_BASE_NAME) +const DEBUG_BASE_NAME = 'dpt:server' const verbose = createDebugLogger('verbose').enabled const VERSION = 0x04 @@ -46,9 +45,7 @@ export class Server extends EventEmitter { _parityRequestMap: Map _requestsCache: LRUCache> _socket: DgramSocket | null - - // Message debuggers (e.g. { 'findneighbours': [debug Object], ...}) - private msgDebuggers: { [key: string]: (debug: string) => void } = {} + _debug: Debugger constructor(dpt: DPT, privateKey: Buffer, options: DPTServerOptions) { super() @@ -62,10 +59,9 @@ export class Server extends EventEmitter { this._parityRequestMap = new Map() this._requestsCache = new LRUCache({ max: 1000, maxAge: ms('1s'), stale: false }) - this.initMsgDebuggers() - const createSocket = options.createSocket ?? dgram.createSocket.bind(null, { type: 'udp4' }) this._socket = createSocket() + this._debug = devp2pDebug.extend(DEBUG_BASE_NAME) if (this._socket) { this._socket.once('listening', () => this.emit('listening')) this._socket.once('close', () => this.emit('close')) @@ -82,14 +78,14 @@ export class Server extends EventEmitter { bind(...args: any[]) { this._isAliveCheck() - debug('call .bind') + this._debug('call .bind') if (this._socket) this._socket.bind(...args) } destroy(...args: any[]) { this._isAliveCheck() - debug('call .destroy') + this._debug('call .destroy') if (this._socket) { this._socket.close(...args) @@ -117,7 +113,7 @@ export class Server extends EventEmitter { deferred, timeoutId: setTimeout(() => { if (this._requests.get(rkey) !== undefined) { - debug( + this._debug( `ping timeout: ${peer.address}:${peer.udpPort} ${ peer.id ? formatLogId(peer.id.toString('hex'), verbose) : '-' }` @@ -241,13 +237,6 @@ export class Server extends EventEmitter { } } - private initMsgDebuggers() { - const MESSAGE_NAMES = ['ping', 'pong', 'findneighbours', 'neighbours'] - for (const name of MESSAGE_NAMES) { - this.msgDebuggers[name] = createDebugLogger(`${DEBUG_BASE_NAME}:${name}`) - } - } - /** * Debug message both on the generic as well as the * per-message debug logger @@ -255,9 +244,6 @@ export class Server extends EventEmitter { * @param msg Message text to debug */ private debug(messageName: string, msg: string) { - debug(msg) - if (this.msgDebuggers[messageName]) { - this.msgDebuggers[messageName](msg) - } + this._debug.extend(messageName)(msg) } } diff --git a/packages/devp2p/src/eth/index.ts b/packages/devp2p/src/eth/index.ts index 91c24b3e82..9923b73543 100644 --- a/packages/devp2p/src/eth/index.ts +++ b/packages/devp2p/src/eth/index.ts @@ -2,13 +2,13 @@ import assert from 'assert' import { EventEmitter } from 'events' import ms from 'ms' import snappy from 'snappyjs' -import { debug as createDebugLogger } from 'debug' +import { debug as createDebugLogger, Debugger } from 'debug' +import { devp2pDebug } from '../util' import { BN, rlp } from 'ethereumjs-util' import { int2buffer, buffer2int, assertEq, formatLogId, formatLogData } from '../util' import { Peer, DISCONNECT_REASONS } from '../rlpx/peer' -const DEBUG_BASE_NAME = 'devp2p:eth' -const debug = createDebugLogger(DEBUG_BASE_NAME) +const DEBUG_BASE_NAME = 'eth' const verbose = createDebugLogger('verbose').enabled /** @@ -26,6 +26,7 @@ export class ETH extends EventEmitter { _peerStatus: ETH.StatusMsg | null _statusTimeoutId: NodeJS.Timeout _send: SendMethod + _debug: Debugger // Eth64 _hardfork: string = 'chainstart' @@ -42,7 +43,7 @@ export class ETH extends EventEmitter { this._version = version this._peer = peer this._send = send - + this._debug = devp2pDebug.extend(DEBUG_BASE_NAME) this._status = null this._peerStatus = null this._statusTimeoutId = setTimeout(() => { @@ -341,17 +342,10 @@ export class ETH extends EventEmitter { } private initMsgDebuggers() { - const MESSAGE_NAMES = Object.values(ETH.MESSAGE_CODES).filter( - (value) => typeof value === 'string' - ) as string[] - for (const name of MESSAGE_NAMES) { - this.msgDebuggers[name] = createDebugLogger(`${DEBUG_BASE_NAME}:${name}`) - } - // Remote Peer IP logger const ip = this._peer._socket.remoteAddress if (ip) { - this.msgDebuggers[ip] = createDebugLogger(`devp2p:${ip}`) + this._debug = devp2pDebug.extend(ip) } } @@ -364,7 +358,7 @@ export class ETH extends EventEmitter { _addFirstPeerDebugger() { const ip = this._peer._socket.remoteAddress if (ip) { - this.msgDebuggers[ip] = createDebugLogger(`devp2p:FIRST_PEER`) + this._debug = this._debug.extend(`FIRST_PEER`) this._peer._addFirstPeerDebugger() _firstPeer = ip } @@ -377,13 +371,11 @@ export class ETH extends EventEmitter { * @param msg Message text to debug */ private debug(messageName: string, msg: string) { - debug(msg) - if (this.msgDebuggers[messageName]) { - this.msgDebuggers[messageName](msg) - } const ip = this._peer._socket.remoteAddress - if (ip && this.msgDebuggers[ip]) { - this.msgDebuggers[ip](msg) + if (ip) { + this._debug.extend(ip).extend(messageName)(msg) + } else { + this._debug.extend(messageName)(msg) } } } diff --git a/packages/devp2p/src/les/index.ts b/packages/devp2p/src/les/index.ts index 43c5f3c93e..24b9e9a551 100644 --- a/packages/devp2p/src/les/index.ts +++ b/packages/devp2p/src/les/index.ts @@ -2,12 +2,12 @@ import { EventEmitter } from 'events' import { rlp } from 'ethereumjs-util' import ms from 'ms' import snappy from 'snappyjs' -import { debug as createDebugLogger } from 'debug' +import { debug as createDebugLogger, Debugger } from 'debug' +import { devp2pDebug } from '../util' import { int2buffer, buffer2int, assertEq, formatLogData } from '../util' import { Peer, DISCONNECT_REASONS } from '../rlpx/peer' -const DEBUG_BASE_NAME = 'devp2p:les' -const debug = createDebugLogger(DEBUG_BASE_NAME) +const DEBUG_BASE_NAME = 'les' const verbose = createDebugLogger('verbose').enabled export const DEFAULT_ANNOUNCE_TYPE = 1 @@ -27,6 +27,7 @@ export class LES extends EventEmitter { _status: LES.Status | null _peerStatus: LES.Status | null _statusTimeoutId: NodeJS.Timeout + _debug: Debugger // Message debuggers (e.g. { 'GET_BLOCK_HEADERS': [debug Object], ...}) private msgDebuggers: { [key: string]: (debug: string) => void } = {} @@ -37,14 +38,12 @@ export class LES extends EventEmitter { this._version = version this._peer = peer this._send = send - + this._debug = devp2pDebug this._status = null this._peerStatus = null this._statusTimeoutId = setTimeout(() => { this._peer.disconnect(DISCONNECT_REASONS.TIMEOUT) }, ms('5s')) - - this.initMsgDebuggers() } static les2 = { name: 'les', version: 2, length: 21, constructor: LES } @@ -272,15 +271,6 @@ export class LES extends EventEmitter { return LES.MESSAGE_CODES[msgCode] } - private initMsgDebuggers() { - const MESSAGE_NAMES = Object.values(LES.MESSAGE_CODES).filter( - (value) => typeof value === 'string' - ) as string[] - for (const name of MESSAGE_NAMES) { - this.msgDebuggers[name] = createDebugLogger(`${DEBUG_BASE_NAME}:${name}`) - } - } - /** * Called once on the peer where a first successful `STATUS` * msg exchange could be achieved. @@ -290,7 +280,7 @@ export class LES extends EventEmitter { _addFirstPeerDebugger() { const ip = this._peer._socket.remoteAddress if (ip) { - this.msgDebuggers[ip] = createDebugLogger(`devp2p:FIRST_PEER`) + this._debug = this._debug.extend('FIRST_PEER') this._peer._addFirstPeerDebugger() _firstPeer = ip } @@ -303,9 +293,11 @@ export class LES extends EventEmitter { * @param msg Message text to debug */ private debug(messageName: string, msg: string) { - debug(msg) - if (this.msgDebuggers[messageName]) { - this.msgDebuggers[messageName](msg) + const ip = this._peer._socket.remoteAddress + if (ip) { + this._debug.extend(ip).extend(DEBUG_BASE_NAME).extend(messageName)(msg) + } else { + this._debug.extend(DEBUG_BASE_NAME).extend(messageName)(msg) } } } diff --git a/packages/devp2p/src/rlpx/peer.ts b/packages/devp2p/src/rlpx/peer.ts index 06a1d85323..934593e299 100644 --- a/packages/devp2p/src/rlpx/peer.ts +++ b/packages/devp2p/src/rlpx/peer.ts @@ -3,15 +3,15 @@ import { EventEmitter } from 'events' import BufferList = require('bl') import ms from 'ms' import snappy from 'snappyjs' -import { debug as createDebugLogger } from 'debug' +import { debug as createDebugLogger, Debugger } from 'debug' +import { devp2pDebug } from '../util' import Common from '@ethereumjs/common' import { rlp } from 'ethereumjs-util' import { ETH, LES } from '../' import { int2buffer, buffer2int, formatLogData } from '../util' import { ECIES } from './ecies' -const DEBUG_BASE_NAME = 'devp2p:rlpx:peer' -const debug = createDebugLogger(DEBUG_BASE_NAME) +const DEBUG_BASE_NAME = 'rlpx:peer' const verbose = createDebugLogger('verbose').enabled export const BASE_PROTOCOL_VERSION = 5 @@ -99,9 +99,7 @@ export class Peer extends EventEmitter { _disconnectReason?: DISCONNECT_REASONS _disconnectWe: any _pingTimeout: number - - // Message debuggers (e.g. { 'HELLO': [debug Object], ...}) - private msgDebuggers: { [key: string]: (debug: string) => void } = {} + _logger: Debugger /** * Subprotocols (e.g. `ETH`) derived from the exchange on @@ -137,7 +135,9 @@ export class Peer extends EventEmitter { this._socket.on('data', this._onSocketData.bind(this)) this._socket.on('error', (err: Error) => this.emit('error', err)) this._socket.once('close', this._onSocketClose.bind(this)) - + this._logger = this._socket.remoteAddress + ? devp2pDebug.extend(this._socket.remoteAddress).extend(DEBUG_BASE_NAME) + : devp2pDebug.extend(DEBUG_BASE_NAME) this._connected = false this._closed = false this._disconnectWe = null @@ -148,8 +148,6 @@ export class Peer extends EventEmitter { // sub-protocols this._protocols = [] - this.initMsgDebuggers() - // send AUTH if outgoing connection if (this._remoteId !== null) { this._sendAuth() @@ -161,7 +159,7 @@ export class Peer extends EventEmitter { */ _sendAuth() { if (this._closed) return - debug( + this._logger( `Send auth (EIP8: ${this._EIP8}) to ${this._socket.remoteAddress}:${this._socket.remotePort}` ) if (this._EIP8) { @@ -182,7 +180,7 @@ export class Peer extends EventEmitter { */ _sendAck() { if (this._closed) return - debug( + this._logger( `Send ack (EIP8: ${this._eciesSession._gotEIP8Auth}) to ${this._socket.remoteAddress}:${this._socket.remotePort}` ) if (this._eciesSession._gotEIP8Auth) { @@ -328,7 +326,7 @@ export class Peer extends EventEmitter { if (!this._eciesSession._gotEIP8Ack) { if (parseData.slice(0, 1) === Buffer.from('04', 'hex')) { this._eciesSession.parseAckPlain(parseData) - debug( + this._logger( `Received ack (old format) from ${this._socket.remoteAddress}:${this._socket.remotePort}` ) } else { @@ -338,7 +336,9 @@ export class Peer extends EventEmitter { } } else { this._eciesSession.parseAckEIP8(parseData) - debug(`Received ack (EIP8) from ${this._socket.remoteAddress}:${this._socket.remotePort}`) + this._logger( + `Received ack (EIP8) from ${this._socket.remoteAddress}:${this._socket.remotePort}` + ) } this._state = 'Header' this._nextPacketSize = 32 @@ -473,10 +473,10 @@ export class Peer extends EventEmitter { _handleHeader() { const bytesCount = this._nextPacketSize const parseData = this._socketData.slice(0, bytesCount) - debug(`Received header ${this._socket.remoteAddress}:${this._socket.remotePort}`) + this._logger(`Received header ${this._socket.remoteAddress}:${this._socket.remotePort}`) const size = this._eciesSession.parseHeader(parseData) if (!size) { - debug('invalid header size!') + this._logger('invalid header size!') return } @@ -494,10 +494,10 @@ export class Peer extends EventEmitter { const parseData = this._socketData.slice(0, bytesCount) const body = this._eciesSession.parseBody(parseData) if (!body) { - debug('empty body!') + this._logger('empty body!') return } - debug( + this._logger( `Received body ${this._socket.remoteAddress}:${this._socket.remotePort} ${formatLogData( body.toString('hex'), verbose @@ -526,7 +526,7 @@ export class Peer extends EventEmitter { const messageName = this.getMsgPrefix(msgCode) this.debug(messageName, `Received ${messageName} message ${postAdd}`) } else { - debug(`Received ${protocolName} subprotocol message ${postAdd}`) + this._logger(`Received ${protocolName} subprotocol message ${postAdd}`) } try { @@ -571,7 +571,7 @@ export class Peer extends EventEmitter { protocolObj.protocol._handleMessage(msgCode, payload) } catch (err: any) { this.disconnect(DISCONNECT_REASONS.SUBPROTOCOL_ERROR) - debug(`Error on peer subprotocol message handling: ${err}`) + this._logger(`Error on peer subprotocol message handling: ${err}`) this.emit('error', err) } this._socketData.consume(bytesCount) @@ -603,7 +603,7 @@ export class Peer extends EventEmitter { } } catch (err: any) { this.disconnect(DISCONNECT_REASONS.SUBPROTOCOL_ERROR) - debug(`Error on peer socket data handling: ${err}`) + this._logger(`Error on peer socket data handling: ${err}`) this.emit('error', err) } } @@ -656,29 +656,6 @@ export class Peer extends EventEmitter { this._sendDisconnect(reason) } - private initMsgDebuggers() { - const MESSAGE_NAMES = Object.values(PREFIXES).filter( - (value) => typeof value === 'string' - ) as string[] - for (const name of MESSAGE_NAMES) { - this.msgDebuggers[name] = createDebugLogger(`${DEBUG_BASE_NAME}:${name}`) - } - - // Remote Peer IP logger - const ip = this._socket.remoteAddress - if (ip) { - this.msgDebuggers[ip] = createDebugLogger(`devp2p:${ip}`) - } - - // Special DISCONNECT per-reason logger - const DISCONNECT_NAMES = Object.values(DISCONNECT_REASONS).filter( - (value) => typeof value === 'string' - ) as string[] - for (const name of DISCONNECT_NAMES) { - this.msgDebuggers[name] = createDebugLogger(`${DEBUG_BASE_NAME}:DISCONNECT:${name}`) - } - } - /** * Called once from the subprotocol (e.g. `ETH`) on the peer * where a first successful `STATUS` msg exchange could be achieved. @@ -688,7 +665,7 @@ export class Peer extends EventEmitter { _addFirstPeerDebugger() { const ip = this._socket.remoteAddress if (ip) { - this.msgDebuggers[ip] = createDebugLogger(`devp2p:FIRST_PEER`) + this._logger = devp2pDebug.extend(ip).extend(`FIRST_PEER`).extend(DEBUG_BASE_NAME) } } @@ -697,20 +674,13 @@ export class Peer extends EventEmitter { * per-message debug logger * @param messageName Capitalized message name (e.g. `HELLO`) * @param msg Message text to debug + * @param disconnectReason Capitalized disconnect reason (e.g. 'TIMEOUT') */ private debug(messageName: string, msg: string, disconnectReason?: string) { - debug(msg) - if (this.msgDebuggers[messageName]) { - this.msgDebuggers[messageName](msg) - } - const ip = this._socket.remoteAddress - if (ip && this.msgDebuggers[ip]) { - this.msgDebuggers[ip](msg) - } - if (disconnectReason && messageName === 'DISCONNECT') { - if (this.msgDebuggers[disconnectReason]) { - this.msgDebuggers[disconnectReason](msg) - } + if (disconnectReason) { + this._logger.extend(messageName).extend(disconnectReason)(msg) + } else { + this._logger.extend(messageName)(msg) } } } diff --git a/packages/devp2p/src/rlpx/rlpx.ts b/packages/devp2p/src/rlpx/rlpx.ts index f52a1835b9..cecc9d2ba6 100644 --- a/packages/devp2p/src/rlpx/rlpx.ts +++ b/packages/devp2p/src/rlpx/rlpx.ts @@ -3,7 +3,8 @@ import * as os from 'os' import ms from 'ms' import { publicKeyCreate } from 'secp256k1' import { EventEmitter } from 'events' -import { debug as createDebugLogger } from 'debug' +import { debug as createDebugLogger, Debugger } from 'debug' +import { devp2pDebug } from '../util' import LRUCache from 'lru-cache' import Common from '@ethereumjs/common' // note: relative path only valid in .js file in dist @@ -12,7 +13,7 @@ import { pk2id, createDeferred, formatLogId, buffer2int } from '../util' import { Peer, DISCONNECT_REASONS, Capabilities } from './peer' import { DPT, PeerInfo } from '../dpt' -const debug = createDebugLogger('devp2p:rlpx') +const DEBUG_BASE_NAME = 'rlpx' const verbose = createDebugLogger('verbose').enabled export interface RLPxOptions { @@ -31,7 +32,7 @@ export interface RLPxOptions { export class RLPx extends EventEmitter { _privateKey: Buffer _id: Buffer - + _debug: Debugger _timeout: number _maxPeers: number _clientId: Buffer @@ -74,7 +75,7 @@ export class RLPx extends EventEmitter { this._dpt.on('peer:new', (peer: PeerInfo) => { if (!peer.tcpPort) { this._dpt!.banPeer(peer, ms('5m')) - debug(`banning peer with missing tcp port: ${peer.address}`) + this._debug(`banning peer with missing tcp port: ${peer.address}`) return } @@ -91,14 +92,15 @@ export class RLPx extends EventEmitter { ) }) } - // internal this._server = net.createServer() this._server.once('listening', () => this.emit('listening')) this._server.once('close', () => this.emit('close')) this._server.on('error', (err) => this.emit('error', err)) this._server.on('connection', (socket) => this._onConnect(socket, null)) - + this._debug = this._server.address() + ? devp2pDebug.extend(DEBUG_BASE_NAME).extend(this._server.address() as string) + : devp2pDebug.extend(DEBUG_BASE_NAME) this._peers = new Map() this._peersQueue = [] this._peersLRU = new LRUCache({ max: 25000 }) @@ -109,14 +111,14 @@ export class RLPx extends EventEmitter { listen(...args: any[]) { this._isAliveCheck() - debug('call .listen') + this._debug('call .listen') if (this._server) this._server.listen(...args) } destroy(...args: any[]) { this._isAliveCheck() - debug('call .destroy') + this._debug('call .destroy') clearInterval(this._refillIntervalId) @@ -136,7 +138,7 @@ export class RLPx extends EventEmitter { if (this._peers.has(peerKey)) throw new Error('Already connected') if (this._getOpenSlots() === 0) throw new Error('Too many peers already connected') - debug(`connect to ${peer.address}:${peer.tcpPort} (id: ${formatLogId(peerKey, verbose)})`) + this._debug(`connect to ${peer.address}:${peer.tcpPort} (id: ${formatLogId(peerKey, verbose)})`) const deferred = createDeferred() const socket = new net.Socket() @@ -185,7 +187,7 @@ export class RLPx extends EventEmitter { } _onConnect(socket: net.Socket, peerId: Buffer | null) { - debug(`connected to ${socket.remoteAddress}:${socket.remotePort}, handshake waiting..`) + this._debug(`connected to ${socket.remoteAddress}:${socket.remotePort}, handshake waiting..`) const peer: Peer = new Peer({ socket: socket, @@ -216,7 +218,7 @@ export class RLPx extends EventEmitter { if (peer._eciesSession._gotEIP8Ack === true) { msg += ` (peer eip8 ack)` } - debug(msg) + this._debug(msg) const id = peer.getId() if (id && id.equals(this._id)) { return peer.disconnect(DISCONNECT_REASONS.SAME_IDENTITY) @@ -234,12 +236,9 @@ export class RLPx extends EventEmitter { peer.once('close', (reason, disconnectWe) => { if (disconnectWe) { - debug( - `disconnect from ${socket.remoteAddress}:${socket.remotePort}, reason: ${DISCONNECT_REASONS[reason]}` - ) - } else { - debug( - `${socket.remoteAddress}:${socket.remotePort} disconnect, reason: ${DISCONNECT_REASONS[reason]}` + this._debug( + `disconnect from ${socket.remoteAddress}:${socket.remotePort}, reason: ${DISCONNECT_REASONS[reason]}`, + `disconnect` ) } @@ -267,7 +266,7 @@ export class RLPx extends EventEmitter { _refillConnections() { if (!this._isAlive()) return if (this._refillIntervalSelectionCounter === 0) { - debug( + this._debug( `Restart connection refill .. with selector ${ this._refillIntervalSelectionCounter } peers: ${this._peers.size}, queue size: ${ diff --git a/packages/devp2p/src/util.ts b/packages/devp2p/src/util.ts index f01c91cc90..f1c6c9feac 100644 --- a/packages/devp2p/src/util.ts +++ b/packages/devp2p/src/util.ts @@ -5,6 +5,9 @@ import createKeccakHash from 'keccak' import { rlp } from 'ethereumjs-util' import { ETH } from './eth' import { LES } from './les' +import { debug as createDebugLogger } from 'debug' + +export const devp2pDebug = createDebugLogger('devp2p') export function keccak256(...buffers: Buffer[]) { const buffer = Buffer.concat(buffers) diff --git a/packages/vm/package.json b/packages/vm/package.json index 7c895a0e7e..430d07076d 100644 --- a/packages/vm/package.json +++ b/packages/vm/package.json @@ -53,7 +53,7 @@ "merkle-patricia-tree": "^4.2.2", "async-eventemitter": "^0.2.4", "core-js-pure": "^3.0.1", - "debug": "^2.2.0", + "debug": "^4.3.3", "ethereumjs-util": "^7.1.3", "functional-red-black-tree": "^1.0.1", "mcl-wasm": "^0.7.1",