v0.37.0 release

.benchrc.yaml

@@ -1,8 +1,8 @@
 # Mocha opts
+extension: ["ts"]
 colors: true
-require:
-  - ts-node/register
-  - packages/lodestar/test/setupBLS.ts
+node-option:
+  - "loader=ts-node/register"
 
 # benchmark opts
 threshold: 3

.eslintrc.js

@@ -13,8 +13,9 @@ module.exports = {
   parserOptions: {
     ecmaVersion: 10,
     project: "./tsconfig.json",
+    sourceType: "module",
   },
-  plugins: ["@typescript-eslint", "eslint-plugin-import", "eslint-plugin-node", "no-only-tests", "prettier"],
+  plugins: ["@typescript-eslint", "eslint-plugin-import", "@chainsafe/eslint-plugin-node", "no-only-tests", "prettier"],
   extends: [
     "eslint:recommended",
     "plugin:import/errors",
@@ -110,7 +111,7 @@ module.exports = {
     //if --fix is run it messes imports like /lib/presets/minimal & /lib/presets/mainnet
     "import/no-duplicates": "off",
     "import/no-relative-packages": "error",
-    "node/no-deprecated-api": "error",
+    "@chainsafe/node/no-deprecated-api": "error",
     "new-parens": "error",
     "no-caller": "error",
     "no-bitwise": "off",
@@ -147,6 +148,17 @@ module.exports = {
 
     // Prevents accidentally pushing a commit with .only in Mocha tests
     "no-only-tests/no-only-tests": "error",
+
+    // TEMP Disabled while eslint-plugin-import support ESM (Typescript does support it) https://github.com/import-js/eslint-plugin-import/issues/2170
+    "import/no-unresolved": "off",
+
+    "@chainsafe/node/file-extension-in-import": [
+      "error",
+      "always",
+      {
+        "esm": true
+      }
+    ],
   },
   settings: {
     "import/core-modules": [

.github/workflows/benchmark.yml

@@ -54,6 +54,9 @@ jobs:
       - name: Run benchmarks
         run: yarn benchmark
         env:
+          # To download content for tests
+          INFURA_ETH2_CREDENTIALS: ${{ secrets.INFURA_ETH2_CREDENTIALS }}
+          # To write to PRs and commits
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           # benchmark options
           BENCHMARK_S3: true

.github/workflows/release-nightly.yml

@@ -40,21 +40,39 @@ jobs:
         run: yarn build
         if: steps.cache-deps.outputs.cache-hit == 'true'
       # </common-build>
+      - name: Get version
+        id: version
+        run: |
+          PACKAGE_VERSION=$(node -p "require('./packages/lodestar/package.json').version")
+          export VERSION=${PACKAGE_VERSION}-dev.${GITHUB_SHA:0:10}
+          echo "::set-output name=version::$VERSION"
+          echo PACKAGE_VERSION $PACKAGE_VERSION GITHUB_SHA $GITHUB_SHA VERSION $VERSION
+
+      - name: Change and commit version
+        # Write version before publishing so it's picked up by `lerna publish from-package`.
+        # It must also be committed to ensure a clean git tree, otherwise `lerna publish` errors.
+        # This "temp" commit doesn't change the actually release commit which is captured above.
+        # git-data is also correct, since it's generated at build time, before `lerna version` run.
+        run: |
+          node_modules/.bin/lerna version ${{ steps.version.outputs.version }} \
+          --force-publish \
+          --exact \
+          --yes \
+          --no-git-tag-version
+
+          git config user.name 'temp'
+          git config user.email 'temp@github.com'
+          git commit -am "${{ steps.version.outputs.version }}"
 
       - name: Publish to npm registry
-        # Just use lerna publish with --canary option. Using 'from-package' ignore other options
-        # and only compares against the verison in package.json, and skips release if already
-        # published.
+        # Note: before https://github.com/ChainSafe/lodestar/commit/28e2c74cf0f1bede8b09c8c9fec26f54b367e3fd
+        # We used `lerna publish --canary` option. However, since we now publish must version on branches,
+        # i.e. v0.35.x branch, lerna fails to detect the latest version and publishes canary versions as
+        # `0.34.0-dev.173+28e2c74cf0` instead of `0.36.0-dev.4+28e2c74cf0`, which creates confusion.
         #
         # --no-git-reset:
         #   Do not delete code version artifacts so the next step can pick the version
         #
-        # --canary:
-        #   Format version with commit (1.1.0-alpha.0+81e3b443). Make sure the previous
-        #   released tags are not lightweight("commit" type), but proper annotated ("tag" type)
-        #   Otherwise the version canary will generate will be from last annotated tag type
-        #   Best way to create such a tag is by using 'git tag -a' or using lerna publish!
-        #
         # --dist-tag next:
         #   Make this nightly version installable with `@next`
         #
@@ -71,17 +89,15 @@ jobs:
         #
         # NOTE: Using --preid dev.$(git rev-parse --short=7 HEAD) results in `0.24.3-dev.3ddb91d.0+3ddb91d`
         run: |
-          node_modules/.bin/lerna publish --yes --no-verify-access \
-          --canary --dist-tag next --no-git-reset --force-publish \
-          --preid dev --exact
+          node_modules/.bin/lerna publish from-package \
+          --yes \
+          --no-verify-access \
+          --dist-tag next \
+          --no-git-reset \
+          --force-publish \
+          --exact
         env:
           NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
-      - name: Get version
-        id: version
-        run: |
-          VERSION=$(node -p "require('./packages/lodestar/package.json').version")
-          echo VERSION $VERSION
-          echo "::set-output name=version::$VERSION"
     outputs:
       version: ${{ steps.version.outputs.version }}
 

.github/workflows/test-spec.yml

@@ -43,31 +43,16 @@ jobs:
       - name: Download spec tests
         run: yarn download-spec-tests
         working-directory: packages/lodestar
-      - name: Check spec tests
-        run: yarn check-spec-tests
-        working-directory: packages/lodestar
 
       # Run them in different steps to quickly identifying which command failed
       # Otherwise just doing `yarn test:spec` you can't tell which specific suite failed
       # many of the suites have identical names for minimal and mainnet
       - name: Spec tests general
         run: yarn test:spec-general
         working-directory: packages/lodestar
-      - name: Spec tests phase0-minimal
-        run: yarn test:spec-phase0-minimal
-        working-directory: packages/lodestar
-      - name: Spec tests phase0-mainnet
-        run: yarn test:spec-phase0-mainnet
-        working-directory: packages/lodestar
-      - name: Spec tests altair-minimal
-        run: yarn test:spec-altair-minimal
-        working-directory: packages/lodestar
-      - name: Spec tests altair-mainnet
-        run: yarn test:spec-altair-mainnet
-        working-directory: packages/lodestar
-      - name: Spec tests bellatrix-minimal
-        run: yarn test:spec-bellatrix-minimal
+      - name: Spec tests minimal
+        run: yarn test:spec-minimal
         working-directory: packages/lodestar
-      - name: Spec tests bellatrix-mainnet
-        run: yarn test:spec-bellatrix-mainnet
+      - name: Spec tests mainnet
+        run: yarn test:spec-mainnet
         working-directory: packages/lodestar

.github/workflows/test.yml

@@ -53,10 +53,13 @@ jobs:
 
       - name: Check Types
         run: yarn run check-types
-      - name: README check
-        run: yarn run check-readme
+      # TODO: uncomment after https://github.com/bbc/typescript-docs-verifier/issues/17 is resolved
+      # - name: README check
+      #   run: yarn run check-readme
       - name: Lint
         run: yarn lint
+      - name: Check Build
+        run: yarn check-build
       - name: Unit tests
         run: yarn test:unit
       - name: Upload coverage data

.gitignore

@@ -46,7 +46,7 @@ docs/reference/cli.md
 .lodestar
 *.ssz
 .pyrmont
-packages/lodestar/.tmpdb/
+.tmpdb
 
 # Wallet CLI artifacts
 .pass

.mocharc.yaml

@@ -1,3 +1,4 @@
+extension: ["ts"]
 colors: true
-require:
-  - ts-node/register
+node-option:
+  - "loader=ts-node/esm"

RELEASE.md

@@ -11,21 +11,21 @@ In the same day:
 3. The selected commit is tagged as `v0.34.0-beta.0`, released, and published for testing as a Pre-Release
 
 ```
-export TAG=v0.34.0-beta.0 && git tag -a $TAG 9fceb02 -m "$TAG" && git push origin $TAG
+yarn release -t v0.34.0-beta.0
 ```
 
 4. The team creates a PR to bump `master` to the next version (in the example: `v0.35.0`) and continues releasing nightly builds.
 
 ```
-lerna version minor --no-git-tag-version --force-publish
+lerna version minor --no-git-tag-version --force-publish --yes
 ```
 
 After 3-5 days of testing:
 
 5. Tag final stable commit as `v0.34.0`, release and publish the stable release. This commit will be in `v0.34.x` branch and may note be on `master` if beta candidate required bug fixes.
 
 ```
-export TAG=v0.34.0 && git tag -a $TAG 9fceb02 -m "$TAG" && git push origin $TAG
+yarn release -t v0.34.0
 ```
 
 ## Pre-Releases

dashboards/lodestar_general.json

@@ -8632,6 +8632,18 @@
               "interval": "",
               "legendFormat": "till processed",
               "refId": "B"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "${DS_PROMETHEUS}"
+              },
+              "exemplar": false,
+              "expr": "delta(lodestar_gossip_block_elapsed_time_till_become_head_sum[$__rate_interval])\n/\ndelta(lodestar_gossip_block_elapsed_time_till_become_head_count[$__rate_interval])",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "till become head",
+              "refId": "C"
             }
           ],
           "title": "Gossip Block Received Delay",

docker/from_source.Dockerfile

@@ -30,6 +30,12 @@ RUN yarn install --non-interactive --frozen-lockfile --ignore-scripts
 COPY . .
 RUN yarn install --non-interactive --frozen-lockfile && yarn build
 
+# To have access to the specific branch and commit used to build this source,
+# a git-data.json file is created by persisting git data at build time. Then,
+# a version string like `v0.35.0-beta.0/HEAD/82219149 (git)` can be shown in
+# the terminal and in the logs; which is very useful to track tests better.
+RUN cd packages/cli && yarn write-git-data
+
 # Copy built src + node_modules to a new layer to prune unnecessary fs
 # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020)
 FROM node:16-alpine

docs/installation.md

@@ -10,7 +10,7 @@ Make sure to have [Yarn installed](https://classic.yarnpkg.com/en/docs/install).
 
 <!-- prettier-ignore-start -->
 !!! info
-    NodeJS versions < 12 are not supported by Lodestar.
+    NodeJS versions < 16.x are not supported by Lodestar. We currently recommend running NodeJS 16.x.
 <!-- prettier-ignore-end -->
 
 Clone the repo locally.

kiln/devnets/fixed.vars

@@ -0,0 +1,17 @@
+# Data dir will be mounted in /data/lodestar
+LODESTAR_FIXED_VARS="--rootDir /data/lodestar --eth1.providerUrls http://127.0.0.1:8545 --execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*' --network.connectToDiscv5Bootnodes --network.discv5.enabled true --eth1.enabled true --jwt-secret /data/jwtsecret"
+
+# Data dir will be mounted in /data/lodestar
+LODESTAR_VAL_FIXED_VARS="--rootDir /data/lodestar"
+
+# Data dir will be mounted in /data/nethermind
+NETHERMIND_FIXED_VARS="--datadir /data/nethermind --Network.DiscoveryPort=30303 --Network.P2PPort=30303 --Merge.Enabled=true --Init.DiagnosticMode=None --JsonRpc.Enabled=true --JsonRpc.Host=0.0.0.0 --JsonRpc.AdditionalRpcUrls \"http://localhost:8545|http;ws|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client\" --JsonRpc.JwtSecretFile /data/jwtsecret"
+
+# Data dir will be mounted in /data/geth
+GETH_FIXED_VARS="--datadir /data/geth --authrpc.jwtsecret /data/jwtsecret --http --http.api engine,net,eth,web3 --http.port 8545 --allow-insecure-unlock --http.addr 0.0.0.0 --http.corsdomain \"*\" --http.vhosts \"*\" --authrpc.port=8551"
+
+# Data dir will be mounted in /data/ethereumjs
+ETHEREUMJS_FIXED_VARS="--saveReceipts --rpc --rpcport 8545 --ws --rpcEngine --rpcEnginePort=8551 --rpcDebug --loglevel=debug"
+
+# Data dir will be mounted in /data/besu
+BESU_FIXED_VARS="--rpc-http-enabled=true --rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,TRACE --rpc-http-host=0.0.0.0 --rpc-http-port=8545 --Xmerge-support=true --engine-rpc-http-port=8551 --rpc-http-cors-origins=\"*\" --host-allowlist=\"*\" --engine-host-allowlist=\"*\" --p2p-enabled=true --engine-jwt-enabled=true"

kiln/devnets/kiln.vars

@@ -12,16 +12,16 @@ BESU_IMAGE=hyperledger/besu:develop
 
 LODESTAR_IMAGE=chainsafe/lodestar:next
 
-LODESTAR_EXTRA_ARGS="--execution.urls http://127.0.0.1:8551 --api.rest.enabled --api.rest.host 0.0.0.0 --api.rest.api '*'"
+LODESTAR_EXTRA_ARGS="--terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"
 
-LODESTAR_VALIDATOR_ARGS='--network kiln  --fromMnemonic "lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow" --mnemonicIndexes 0..5'
+LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"
 
-NETHERMIND_EXTRA_ARGS="--config kiln --Network.DiscoveryPort=30303 --Network.P2PPort=30303 --Merge.Enabled=true --Merge.TerminalTotalDifficulty=$MERGE_TTD  --Init.DiagnosticMode=None --JsonRpc.Enabled=true --JsonRpc.Host=0.0.0.0 --JsonRpc.AdditionalRpcUrls \"http://localhost:8545|http;ws|net;eth;subscribe;engine;web3;client|no-auth,http://localhost:8551|http;ws|net;eth;subscribe;engine;web3;client\""
+NETHERMIND_EXTRA_ARGS="--config kiln  --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"
 
-GETH_EXTRA_ARGS="--http --http.api engine,net,eth,web3 --http.port 8545 --allow-insecure-unlock --http.addr 0.0.0.0 --http.corsdomain \"*\" --http.vhosts \"*\" --authrpc.port=8551 --override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID"
+GETH_EXTRA_ARGS="--override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"
 
-ETHEREUMJS_EXTRA_ARGS="--saveReceipts --rpc --rpcport 8545 --ws --rpcEngine --rpcEnginePort=8551 --rpcDebug --loglevel=debug"
+ETHEREUMJS_EXTRA_ARGS="$ETHEREUMJS_FIXED_VARS"
 
-BESU_EXTRA_ARGS="--rpc-http-enabled=true --rpc-http-api=ADMIN,CLIQUE,MINER,ETH,NET,DEBUG,TXPOOL,TRACE --rpc-http-host=0.0.0.0 --rpc-http-port=8545 --Xmerge-support=true --engine-rpc-http-port=8551 --rpc-http-cors-origins=\"*\" --host-allowlist=\"*\" --engine-host-allowlist=\"*\" --p2p-enabled=true --engine-jwt-enabled=true --network-id=$NETWORK_ID"
+BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"
 
 EXTRA_BOOTNODES=""

kiln/devnets/mainnetshadow-5.vars

@@ -0,0 +1,27 @@
+DEVNET_NAME=mainnet-shadow-fork-5
+CONFIG_GIT_DIR=mainnet-shadow-fork-5
+NETWORK_ID=1
+MERGE_TTD=49407055721335634264064
+
+JWT_SECRET="0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"
+
+GETH_IMAGE=parithoshj/geth:master
+NETHERMIND_IMAGE=nethermindeth/nethermind:kiln_shadowfork
+ETHEREUMJS_IMAGE=g11tech/ethereumjs:kiln
+BESU_IMAGE=hyperledger/besu:develop
+
+LODESTAR_IMAGE=chainsafe/lodestar:next
+
+LODESTAR_EXTRA_ARGS="--terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"
+
+LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"
+
+NETHERMIND_EXTRA_ARGS="--config mainnet_shadowfork  --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"
+
+GETH_EXTRA_ARGS="--override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"
+
+ETHEREUMJS_EXTRA_ARGS="$ETHEREUMJS_FIXED_VARS"
+
+BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"
+
+EXTRA_BOOTNODES=""

kiln/devnets/ropsten.vars

@@ -0,0 +1,29 @@
+DEVNET_NAME=ropsten
+# Empty config git dir will be assumed to be clients having bakedin configs
+CONFIG_GIT_DIR=
+NETWORK_ID=3
+MERGE_TTD=43531756765713534
+
+# This will be available in /data/jwtsecret
+JWT_SECRET="0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"
+
+GETH_IMAGE=parithoshj/geth:master
+NETHERMIND_IMAGE=nethermindeth/nethermind:kiln_shadowfork
+ETHEREUMJS_IMAGE=g11tech/ethereumjs:kiln
+BESU_IMAGE=hyperledger/besu:develop
+
+LODESTAR_IMAGE=chainsafe/lodestar:next
+
+LODESTAR_EXTRA_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD $LODESTAR_FIXED_VARS"
+
+LODESTAR_VALIDATOR_ARGS="--network ropsten --terminal-total-difficulty-override $MERGE_TTD --fromMnemonic \"lens risk clerk foot verb planet drill roof boost aim salt omit celery tube list permit motor obvious flash demise churn hold wave hollow\" --mnemonicIndexes 0..5 $LODESTAR_VAL_FIXED_VARS"
+
+NETHERMIND_EXTRA_ARGS="--config ropsten  --Merge.TerminalTotalDifficulty=$MERGE_TTD $NETHERMIND_FIXED_VARS"
+
+GETH_EXTRA_ARGS="--ropsten --override.terminaltotaldifficulty=$MERGE_TTD --networkid $NETWORK_ID $GETH_FIXED_VARS"
+
+ETHEREUMJS_EXTRA_ARGS="--network ropsten $ETHEREUMJS_FIXED_VARS"
+
+BESU_EXTRA_ARGS="--network-id=$NETWORK_ID $BESU_FIXED_VARS"
+
+EXTRA_BOOTNODES=""