From 0e26ca5998ec2e472846d7670743dd3d41c0750d Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Tue, 25 Jan 2022 16:15:34 +0100 Subject: [PATCH 1/9] temp --- packages/validator/src/validator.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index b1e023e8054f..85c275a29df6 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -86,6 +86,9 @@ export class Validator { /** Waits for genesis and genesis time */ static async initializeFromBeaconNode(opts: ValidatorOptions, signal?: AbortSignal): Promise { const {config} = opts.dbOps; + // TODO [DA] is there any objection to renaming api to client? + // it then becomes obvious this is a client that makes a request to an api + // instead of this being an api that is being made available const api = typeof opts.api === "string" ? getClient(config, {baseUrl: opts.api, timeoutMs: 12000, getAbortSignal: () => signal}) From d100709cf674eae58a90cab36b4a7db2edfa5897 Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Wed, 26 Jan 2022 13:25:04 +0100 Subject: [PATCH 2/9] temp --- packages/cli/src/cmds/dev/handler.ts | 2 +- packages/lodestar/src/network/nodejs/util.ts | 9 ++++++--- packages/lodestar/src/network/peers/discover.ts | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/cmds/dev/handler.ts b/packages/cli/src/cmds/dev/handler.ts index dd79cdb8daae..29bda02ef496 100644 --- a/packages/cli/src/cmds/dev/handler.ts +++ b/packages/cli/src/cmds/dev/handler.ts @@ -66,7 +66,7 @@ export async function devHandler(args: IDevArgs & IGlobalArgs): Promise { if (args.logFormatGenesisTime === undefined) args.logFormatGenesisTime = genesisTime; // BeaconNode setup - const libp2p = await createNodeJsLibp2p(peerId, options.network); + const libp2p = await createNodeJsLibp2p(peerId, options.network, {peerStoreDir: beaconPaths.peerStoreDir}); const logger = getCliLogger(args, beaconPaths, config); logger.info("Lodestar", {version: getVersion(), network: args.network}); if (ACTIVE_PRESET === PresetName.minimal) logger.info("ACTIVE_PRESET == minimal preset"); diff --git a/packages/lodestar/src/network/nodejs/util.ts b/packages/lodestar/src/network/nodejs/util.ts index 3fa80374476d..47174bd01811 100644 --- a/packages/lodestar/src/network/nodejs/util.ts +++ b/packages/lodestar/src/network/nodejs/util.ts @@ -50,9 +50,12 @@ export async function createNodeJsLibp2p( // Append discv5.bootEnrs to bootMultiaddrs if requested if (networkOpts.connectToDiscv5Bootnodes) { - if (!networkOpts.bootMultiaddrs) networkOpts.bootMultiaddrs = []; - if (!networkOpts.discv5) networkOpts.discv5 = defaultDiscv5Options; - + if (!networkOpts.bootMultiaddrs) { + networkOpts.bootMultiaddrs = []; + } + if (!networkOpts.discv5) { + networkOpts.discv5 = defaultDiscv5Options; + } for (const enrOrStr of networkOpts.discv5.bootEnrs) { const enr = typeof enrOrStr === "string" ? ENR.decodeTxt(enrOrStr) : enrOrStr; const fullMultiAddr = await enr.getFullMultiaddr("tcp"); diff --git a/packages/lodestar/src/network/peers/discover.ts b/packages/lodestar/src/network/peers/discover.ts index f9a00a08a695..9a2d7c620460 100644 --- a/packages/lodestar/src/network/peers/discover.ts +++ b/packages/lodestar/src/network/peers/discover.ts @@ -237,6 +237,9 @@ export class PeerDiscovery { * Progressively called by discv5 as a result of any query. */ private onDiscovered = async (enr: ENR): Promise => { + console.log("peerId", (await enr.peerId()).toString()); + console.log("udp", (await enr.getFullMultiaddr("udp"))?.toString()); + console.log("tcp", (await enr.getFullMultiaddr("tcp"))?.toString()); const status = await this.handleDiscoveredPeer(enr); this.metrics?.discovery.discoveredStatus.inc({status}); }; From a8efda70abf2749ae864331191ca4c7f5dbb2243 Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Thu, 27 Jan 2022 16:15:33 +0100 Subject: [PATCH 3/9] Ensure the values passed via network.discv5.bootEnrs are connected to on startup. Updated the documentation of the dev command --- docs/usage/local.md | 88 +++++++++++-------- .../lodestar/src/network/peers/discover.ts | 5 +- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/docs/usage/local.md b/docs/usage/local.md index bd7a701434ca..d9c7f91eb693 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -1,58 +1,76 @@ # Local testnet -To quickly test and run Lodestar we recommend to start a local testnet. We recommend a simple configuration of two beacon nodes with multiple validators +To quickly test and run Lodestar we recommend starting a local testnet. We recommend a simple configuration of two beacon nodes with multiple validators **Terminal 1** -Run a beacon node with 8 validators and default settings. State will be written to .tmp/state.ssz +Run a beacon node, with 8 validators with the following command. ```bash -./lodestar dev --genesisValidators 8 --reset +./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --network prater --rootDir --reset ``` +`--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--rootDir` defines a path where lodestar should store the beacon state +while the `--reset` flag ensures the state is cleared on each restart - which is useful when testing locally. + +Once the node has started, make a request to `curl http://localhost:9596/eth/v1/node/identity` and copy the `enr` value. + +This would be used to connect from the second node. + +> enr stands for ethereum node records, which is a format for conveying p2p connectivity information for ethereum nodes. +> For more info see [eip-778](https://eips.ethereum.org/EIPS/eip-778) + **Terminal 2** -Connect to bootnode (node 1 default multiaddrs) but without starting validators. +Start the second node without starting any validators and connect to the first node by supplying the copied `enr` value: ```bash ./lodestar dev --startValidators 0:0 \ - --genesisStateFile ./dev/genesis.ssz \ - --network.localMultiaddrs /ip4/127.0.0.1/tcp/30607 \ - --sync.minPeers 1 + --genesisValidators 8 --genesisTime 1578787200 \ + --network prater \ + --rootDir /path/to/node2 \ + --port 9001 \ + --api.rest.port 9597 \ + --network.discv5.bootEnrs + --reset ``` ---- +By default, lodestar starts as many validators as the number supplied by `--genesisValidators`. In other to not start any validator, this is overridden by +the `--startValidators` option. Passing a value of `0:0` means no validators should be started. -Once both instances are running you should see an output similar to this +Also, take note that the values of `--genesisValidators` and `--genesisTime` must be the same as the ones passed to the first node in other for the two nodes +to have the same beacon chain. -**Terminal 1** +Finally `port` and `api.rest.port` are supplied since the default values will already be in use by the first node. + +Once the second node starts, you should see an output similar to the following in either of the terminals: -```bash -2020-06-21 16:42:40 [SYNC] warn: Current peerCount=0, required = 2 -2020-06-21 16:42:43 [SYNC] warn: Current peerCount=0, required = 2 -2020-06-21 16:42:43 [VALIDATOR 7] info: Validator is proposer at slot 9 -2020-06-21 16:42:43 [CHAIN] info: Processed new chain head newChainHeadRoot=0x3223a51b51fa4f42ea2281e8580806907a8e69f490cfe12a380b8e8b41b21d27, slot=9, epoch=1 -2020-06-21 16:42:43 [VALIDATOR 7] info: Proposed block with hash 0x3223a51b51fa4f42ea2281e8580806907a8e69f490cfe12a380b8e8b41b21d27 and slot 9 -2020-06-21 16:42:46 [SYNC] warn: Current peerCount=1, required = 2 -2020-06-21 16:42:49 [SYNC] warn: Current peerCount=1, required = 2 -2020-06-21 16:42:49 [VALIDATOR 6] info: Validator is proposer at slot 10 -2020-06-21 16:42:49 [CHAIN] info: Processed new chain head newChainHeadRoot=0x7c3a77ad892ca631b750b988277a6caca9cc011461e326537fb607c94359b95f, slot=10, epoch=1 -2020-06-21 16:42:49 [VALIDATOR 6] info: Proposed block with hash 0x7c3a77ad892ca631b750b988277a6caca9cc011461e326537fb607c94359b95f and slot 10 -2020-06-21 16:42:52 [SYNC] warn: Current peerCount=1, required = 2 -2020-06-21 16:42:55 [SYNC] warn: Current peerCount=1, required = 2 -2020-06-21 16:42:55 [VALIDATOR 2] info: Validator is proposer at slot 11 -2020-06-21 16:42:55 [CHAIN] info: Processed new chain head newChainHeadRoot=0x74e96be4058e0edec26028c2f727b30dbc05e12c3f29f364e487916e16777f4a, slot=11, epoch=1 -2020-06-21 16:42:55 [VALIDATOR 2] info: Proposed block with hash 0x74e96be4058e0edec26028c2f727b30dbc05e12c3f29f364e487916e16777f4a and slot 11 +``` +Eph 167991/6 6.007 [] info: Searching peers - peers: 1 - slot: 5375718 (skipped 5375718) - head: 0 0xcc67…3345 - finalized: 0x0000…0000:0 ``` -**Terminal 2** +For further confirmation that both nodes are connected as peers, make a request to the `/eth/v1/node/peers` endpoint. + +For example, making the request on the first node via the following command: + +`curl http://localhost:9596/eth/v1/node/peers | jq` + +will give a result similar to the following: -```bash -2020-06-21 16:42:49 [SYNC] info: Sync caught up to latest slot 9 -2020-06-21 16:42:49 [CHAIN] info: Processed new chain head newChainHeadRoot=0x7c3a77ad892ca631b750b988277a6caca9cc011461e326537fb607c94359b95f, slot=10, epoch=1 -2020-06-21 16:42:56 [CHAIN] info: Processed new chain head newChainHeadRoot=0x74e96be4058e0edec26028c2f727b30dbc05e12c3f29f364e487916e16777f4a, slot=11, epoch=1 -2020-06-21 16:43:01 [CHAIN] info: Processed new chain head newChainHeadRoot=0x75ff1e7143acead878913a516c87f620022e178298e7a7f4a9485fd731bc7128, slot=12, epoch=1 -2020-06-21 16:43:07 [CHAIN] info: Processed new chain head newChainHeadRoot=0x250da6f5ebad021894eb07824b535c3442fe0f7a67949f266d46ffa6b5a18b76, slot=13, epoch=1 -2020-06-21 16:43:13 [CHAIN] info: Processed new chain head newChainHeadRoot=0x2e90c4a2cea722cb8bfccdfe3b73a4211e7a21d07075d11307626d8b048b9074, slot=14, epoch=1 -2020-06-21 16:43:19 [CHAIN] info: Processed new chain head newChainHeadRoot=0x03fcff4f23de519c1e294f6b1256d194199c107c56b4466efed6bfab8d6e7e92, slot=15, epoch=1 ``` + +{ + "data": [ + { + "peer_id": "...", + "enr": "", + "last_seen_p2p_address": "....", + "direction": "inbound", + "state": "connected" + } + ], + "meta": { + "count": 1 + } +} +``` \ No newline at end of file diff --git a/packages/lodestar/src/network/peers/discover.ts b/packages/lodestar/src/network/peers/discover.ts index 9a2d7c620460..67ff673f8a87 100644 --- a/packages/lodestar/src/network/peers/discover.ts +++ b/packages/lodestar/src/network/peers/discover.ts @@ -127,6 +127,8 @@ export class PeerDiscovery { await this.discv5.start(); this.discv5StartMs = Date.now(); this.discv5.on("discovered", this.onDiscovered); + // on start, dial the discv5.bootEnrs that has been added to the routing table + this.discv5.kadValues().forEach((enr) => this.onDiscovered(enr)); } async stop(): Promise { @@ -237,9 +239,6 @@ export class PeerDiscovery { * Progressively called by discv5 as a result of any query. */ private onDiscovered = async (enr: ENR): Promise => { - console.log("peerId", (await enr.peerId()).toString()); - console.log("udp", (await enr.getFullMultiaddr("udp"))?.toString()); - console.log("tcp", (await enr.getFullMultiaddr("tcp"))?.toString()); const status = await this.handleDiscoveredPeer(enr); this.metrics?.discovery.discoveredStatus.inc({status}); }; From 6e79bc50c9732a4719bba8d67fa4392399558217 Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Wed, 2 Feb 2022 23:44:08 +0100 Subject: [PATCH 4/9] updated the documentation to include setting enr.ip for first node --- docs/usage/local.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/usage/local.md b/docs/usage/local.md index d9c7f91eb693..4ae4271b10fd 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -7,11 +7,11 @@ To quickly test and run Lodestar we recommend starting a local testnet. We recom Run a beacon node, with 8 validators with the following command. ```bash -./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --network prater --rootDir --reset +./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --network prater --enr.ip 127.0.0.1 --rootDir --reset ``` -`--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--rootDir` defines a path where lodestar should store the beacon state -while the `--reset` flag ensures the state is cleared on each restart - which is useful when testing locally. +`--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--rootDir` defines a path where +lodestar should store the beacon state, `--enr.ip` sets the enr ip entry for the node while the `--reset` flag ensures the state is cleared on each restart - which is useful when testing locally. Once the node has started, make a request to `curl http://localhost:9596/eth/v1/node/identity` and copy the `enr` value. From dcacf7f894d70186d3f0b625f5116fc9ec15102f Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Fri, 4 Feb 2022 09:26:02 +0100 Subject: [PATCH 5/9] Removed cli flags not needed when using dev command in the example --- docs/usage/local.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/usage/local.md b/docs/usage/local.md index 4ae4271b10fd..c8003bc38e51 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -7,7 +7,7 @@ To quickly test and run Lodestar we recommend starting a local testnet. We recom Run a beacon node, with 8 validators with the following command. ```bash -./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --network prater --enr.ip 127.0.0.1 --rootDir --reset +./lodestar dev --genesisValidators 8 --genesisTime 1578787200 --enr.ip 127.0.0.1 --rootDir --reset ``` `--genesisValidators` and `--genesisTime` define the genesis state of the beacon chain. `--rootDir` defines a path where @@ -27,7 +27,6 @@ Start the second node without starting any validators and connect to the first n ```bash ./lodestar dev --startValidators 0:0 \ --genesisValidators 8 --genesisTime 1578787200 \ - --network prater \ --rootDir /path/to/node2 \ --port 9001 \ --api.rest.port 9597 \ From 38c0b0298e4854f699ea2ea89063a952aae4bcee Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Fri, 4 Feb 2022 09:56:18 +0100 Subject: [PATCH 6/9] put connecting to disc5v behind a cli flag --- packages/lodestar/src/network/peers/discover.ts | 14 +++++++++++--- packages/lodestar/src/network/peers/peerManager.ts | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/lodestar/src/network/peers/discover.ts b/packages/lodestar/src/network/peers/discover.ts index 67ff673f8a87..823c6e8aad10 100644 --- a/packages/lodestar/src/network/peers/discover.ts +++ b/packages/lodestar/src/network/peers/discover.ts @@ -22,6 +22,7 @@ export type PeerDiscoveryOpts = { maxPeers: number; discv5FirstQueryDelayMs: number; discv5: Omit; + connectToDiscv5Bootnodes?: boolean; }; export type PeerDiscoveryModules = { @@ -91,6 +92,8 @@ export class PeerDiscovery { private discv5StartMs: number; private discv5FirstQueryDelayMs: number; + private connectToDiscv5BootnodesOnStart: boolean | undefined = false; + constructor(modules: PeerDiscoveryModules, opts: PeerDiscoveryOpts) { const {libp2p, peerRpcScores, metrics, logger, config} = modules; this.libp2p = libp2p; @@ -101,6 +104,7 @@ export class PeerDiscovery { this.maxPeers = opts.maxPeers; this.discv5StartMs = 0; this.discv5FirstQueryDelayMs = opts.discv5FirstQueryDelayMs; + this.connectToDiscv5BootnodesOnStart = opts.connectToDiscv5Bootnodes; this.discv5 = Discv5.create({ enr: opts.discv5.enr, @@ -112,7 +116,6 @@ export class PeerDiscovery { [K in keyof IMetrics["discv5"]]: IDiscv5Metrics[keyof IDiscv5Metrics]; }, }); - opts.discv5.bootEnrs.forEach((bootEnr) => this.discv5.addEnr(bootEnr)); if (metrics) { @@ -127,8 +130,13 @@ export class PeerDiscovery { await this.discv5.start(); this.discv5StartMs = Date.now(); this.discv5.on("discovered", this.onDiscovered); - // on start, dial the discv5.bootEnrs that has been added to the routing table - this.discv5.kadValues().forEach((enr) => this.onDiscovered(enr)); + if (this.connectToDiscv5BootnodesOnStart) { + // We are not supposed to connect to the Mainnet bootnodes + // so only connect to them on local devnet if the cli flag + // connectToDiscv5Bootnodes has been set to true. If so then + // on start, dial the discv5.bootEnrs that has been added to the routing table + this.discv5.kadValues().forEach((enr) => this.onDiscovered(enr)); + } } async stop(): Promise { diff --git a/packages/lodestar/src/network/peers/peerManager.ts b/packages/lodestar/src/network/peers/peerManager.ts index 4e03929dbb30..358b8456fd56 100644 --- a/packages/lodestar/src/network/peers/peerManager.ts +++ b/packages/lodestar/src/network/peers/peerManager.ts @@ -54,6 +54,10 @@ export type PeerManagerOpts = { * If null, Don't run discv5 queries, nor connect to cached peers in the peerStore */ discv5: IDiscv5DiscoveryInputOptions | null; + /** + * If set to true, connect to Discv5 bootnodes. If not set or false, do not connect + */ + connectToDiscv5Bootnodes?: boolean; }; export type PeerManagerModules = { @@ -137,6 +141,7 @@ export class PeerManager { maxPeers: opts.maxPeers, discv5FirstQueryDelayMs: opts.discv5FirstQueryDelayMs, discv5: opts.discv5, + connectToDiscv5Bootnodes: opts.connectToDiscv5Bootnodes, }); const {metrics} = modules; From 034b8ba04f32591c016a79d69f778df36e80956d Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Fri, 4 Feb 2022 09:57:32 +0100 Subject: [PATCH 7/9] Removed a TODO comment to be processed in another PR --- packages/validator/src/validator.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/validator/src/validator.ts b/packages/validator/src/validator.ts index 85c275a29df6..b1e023e8054f 100644 --- a/packages/validator/src/validator.ts +++ b/packages/validator/src/validator.ts @@ -86,9 +86,6 @@ export class Validator { /** Waits for genesis and genesis time */ static async initializeFromBeaconNode(opts: ValidatorOptions, signal?: AbortSignal): Promise { const {config} = opts.dbOps; - // TODO [DA] is there any objection to renaming api to client? - // it then becomes obvious this is a client that makes a request to an api - // instead of this being an api that is being made available const api = typeof opts.api === "string" ? getClient(config, {baseUrl: opts.api, timeoutMs: 12000, getAbortSignal: () => signal}) From ad28754cf3563cc19ee8fdc7174e81dc5a40d48e Mon Sep 17 00:00:00 2001 From: Dadepo Aderemi Date: Fri, 4 Feb 2022 12:21:36 +0100 Subject: [PATCH 8/9] Updated documentation to include the needed network.connectToDiscv5Bootnodes cli flag --- docs/usage/local.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/usage/local.md b/docs/usage/local.md index c8003bc38e51..c6755f11319d 100644 --- a/docs/usage/local.md +++ b/docs/usage/local.md @@ -30,6 +30,7 @@ Start the second node without starting any validators and connect to the first n --rootDir /path/to/node2 \ --port 9001 \ --api.rest.port 9597 \ + --network.connectToDiscv5Bootnodes true \ --network.discv5.bootEnrs --reset ``` @@ -40,7 +41,10 @@ the `--startValidators` option. Passing a value of `0:0` means no validators sho Also, take note that the values of `--genesisValidators` and `--genesisTime` must be the same as the ones passed to the first node in other for the two nodes to have the same beacon chain. -Finally `port` and `api.rest.port` are supplied since the default values will already be in use by the first node. +Finally `--port` and `--api.rest.port` are supplied since the default values will already be in use by the first node. + +The `--network.connectToDiscv5Bootnodes` flags needs to be set to true as this is needed to allow connection to boot enrs on local devnet. +The exact enr of node to connect to is then supplied via the `--network.discv5.bootEnrs` flag. Once the second node starts, you should see an output similar to the following in either of the terminals: From 9c5362ffed07e624681bb16f981d7b7655e146a4 Mon Sep 17 00:00:00 2001 From: Cayman Date: Fri, 4 Feb 2022 11:22:17 -0600 Subject: [PATCH 9/9] Update packages/lodestar/src/network/peers/discover.ts --- packages/lodestar/src/network/peers/discover.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/lodestar/src/network/peers/discover.ts b/packages/lodestar/src/network/peers/discover.ts index 823c6e8aad10..77a860661985 100644 --- a/packages/lodestar/src/network/peers/discover.ts +++ b/packages/lodestar/src/network/peers/discover.ts @@ -131,10 +131,9 @@ export class PeerDiscovery { this.discv5StartMs = Date.now(); this.discv5.on("discovered", this.onDiscovered); if (this.connectToDiscv5BootnodesOnStart) { - // We are not supposed to connect to the Mainnet bootnodes - // so only connect to them on local devnet if the cli flag - // connectToDiscv5Bootnodes has been set to true. If so then - // on start, dial the discv5.bootEnrs that has been added to the routing table + // In devnet scenarios, especially, we want more control over which peers we connect to. + // Only dial the discv5.bootEnrs if the option + // network.connectToDiscv5Bootnodes has been set to true. this.discv5.kadValues().forEach((enr) => this.onDiscovered(enr)); } }