From 2e7a1876f0626aad13058f3c09ea53089bcf2123 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 00:09:36 +0300 Subject: [PATCH 01/27] Add specdocs static code analyzer --- BUILD.bazel | 1 + nogo_config.json | 11 +++++ tools/analyzers/specdocs/BUILD.bazel | 26 ++++++++++++ tools/analyzers/specdocs/analyzer.go | 63 ++++++++++++++++++++++++++++ 4 files changed, 101 insertions(+) create mode 100644 tools/analyzers/specdocs/BUILD.bazel create mode 100644 tools/analyzers/specdocs/analyzer.go diff --git a/BUILD.bazel b/BUILD.bazel index 36c24763a8f..e343cb30191 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -103,6 +103,7 @@ nogo( "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library", "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library", "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library", + "//tools/analyzers/specdocs:go_tool_library", "//tools/analyzers/maligned:go_tool_library", "//tools/analyzers/cryptorand:go_tool_library", "//tools/analyzers/errcheck:go_tool_library", diff --git a/nogo_config.json b/nogo_config.json index ad54ca3c9da..107cfbbeca4 100644 --- a/nogo_config.json +++ b/nogo_config.json @@ -146,5 +146,16 @@ ".*_test\\.go": "Tests are ok", "shared/fileutil/fileutil.go": "Package which defines the proper rules" } + }, + "specdocs": { + "only_files": { + "beacon-chain/.*": "", + "shared/.*": "", + "slasher/.*": "", + "validator/.*": "" + }, + "exclude_files": { + ".*/.*_test\\.go": "No spec comments are expected in tests" + } } } diff --git a/tools/analyzers/specdocs/BUILD.bazel b/tools/analyzers/specdocs/BUILD.bazel new file mode 100644 index 00000000000..8f30411d094 --- /dev/null +++ b/tools/analyzers/specdocs/BUILD.bazel @@ -0,0 +1,26 @@ +load("@prysm//tools/go:def.bzl", "go_library") +load("@io_bazel_rules_go//go:def.bzl", "go_tool_library") + +go_library( + name = "go_default_library", + srcs = ["analyzer.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs", + visibility = ["//visibility:public"], + deps = [ + "@org_golang_x_tools//go/analysis:go_default_library", + "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library", + "@org_golang_x_tools//go/ast/inspector:go_default_library", + ], +) + +go_tool_library( + name = "go_tool_library", + srcs = ["analyzer.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs", + visibility = ["//visibility:public"], + deps = [ + "@org_golang_x_tools//go/analysis:go_tool_library", + "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library", + "@org_golang_x_tools//go/ast/inspector:go_tool_library", + ], +) diff --git a/tools/analyzers/specdocs/analyzer.go b/tools/analyzers/specdocs/analyzer.go new file mode 100644 index 00000000000..130c071bb91 --- /dev/null +++ b/tools/analyzers/specdocs/analyzer.go @@ -0,0 +1,63 @@ +// Package specdocs implements a static analyzer to ensure that pseudo code we use in our comments, when implementing +// functions defined in specs, is up to date. Reference specs documentation is cached (so that we do not need to +// download it every time build is run). +package specdocs + +import ( + "errors" + "fmt" + "go/ast" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + "golang.org/x/tools/go/ast/inspector" +) + +// Doc explaining the tool. +const Doc = "Tool to enforce that specs pseudo code is up to date" + +var errWeakCrypto = errors.New("crypto-secure RNGs are required, use CSPRNG or PRNG defined in github.com/prysmaticlabs/prysm/shared/rand") + +// Analyzer runs static analysis. +var Analyzer = &analysis.Analyzer{ + Name: "specdocs", + Doc: Doc, + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +} + +func run(pass *analysis.Pass) (interface{}, error) { + inspection, ok := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) + if !ok { + return nil, errors.New("analyzer is not type *inspector.Inspector") + } + + nodeFilter := []ast.Node{ + (*ast.File)(nil), + (*ast.Comment)(nil), + } + + aliases := make(map[string]string) + inspection.Preorder(nodeFilter, func(node ast.Node) { + switch stmt := node.(type) { + case *ast.File: + fmt.Printf("node: %v", stmt) + // Reset aliases (per file). + aliases = make(map[string]string) + case *ast.Comment: + fmt.Printf("comment: %v", stmt) + } + }) + + return nil, nil +} + +func isPkgDot(expr ast.Expr, pkg, name string) bool { + sel, ok := expr.(*ast.SelectorExpr) + return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name) +} + +func isIdent(expr ast.Expr, ident string) bool { + id, ok := expr.(*ast.Ident) + return ok && id.Name == ident +} From 6f701792950fb853ee21ec371f87ba20c438a373 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 01:21:21 +0300 Subject: [PATCH 02/27] docs pulling script --- scripts/update-spec-docs.sh | 17 + .../specdocs/data/phase0/beacon-chain.md | 1879 +++++++++++++++++ .../specdocs/data/phase0/deposit-contract.md | 77 + .../specdocs/data/phase0/fork-choice.md | 406 ++++ .../specdocs/data/phase0/p2p-interface.md | 1526 +++++++++++++ .../specdocs/data/phase0/validator.md | 654 ++++++ .../specdocs/data/phase0/weak-subjectivity.md | 182 ++ 7 files changed, 4741 insertions(+) create mode 100755 scripts/update-spec-docs.sh create mode 100644 tools/analyzers/specdocs/data/phase0/beacon-chain.md create mode 100644 tools/analyzers/specdocs/data/phase0/deposit-contract.md create mode 100644 tools/analyzers/specdocs/data/phase0/fork-choice.md create mode 100644 tools/analyzers/specdocs/data/phase0/p2p-interface.md create mode 100644 tools/analyzers/specdocs/data/phase0/validator.md create mode 100644 tools/analyzers/specdocs/data/phase0/weak-subjectivity.md diff --git a/scripts/update-spec-docs.sh b/scripts/update-spec-docs.sh new file mode 100755 index 00000000000..c90da53cc6d --- /dev/null +++ b/scripts/update-spec-docs.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +declare -a files=("phase0/beacon-chain.md" + "phase0/deposit-contract.md" + "phase0/fork-choice.md" + "phase0/p2p-interface.md" + "phase0/validator.md" + "phase0/weak-subjectivity.md" +) + +BASE_URL="https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev/specs" +OUTPUT_DIR="tools/analyzers/specdocs/data" + +for file in "${files[@]}"; do + echo "downloading $file" + wget -q -O $OUTPUT_DIR/"$file" --no-check-certificate --content-disposition $BASE_URL/"$file" +done diff --git a/tools/analyzers/specdocs/data/phase0/beacon-chain.md b/tools/analyzers/specdocs/data/phase0/beacon-chain.md new file mode 100644 index 00000000000..cbd085bd3c8 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/beacon-chain.md @@ -0,0 +1,1879 @@ +# Ethereum 2.0 Phase 0 -- The Beacon Chain + +## Table of contents + + + + +- [Introduction](#introduction) +- [Notation](#notation) +- [Custom types](#custom-types) +- [Constants](#constants) +- [Configuration](#configuration) + - [Misc](#misc) + - [Gwei values](#gwei-values) + - [Initial values](#initial-values) + - [Withdrawal prefixes](#withdrawal-prefixes) + - [Time parameters](#time-parameters) + - [State list lengths](#state-list-lengths) + - [Rewards and penalties](#rewards-and-penalties) + - [Max operations per block](#max-operations-per-block) + - [Domain types](#domain-types) +- [Containers](#containers) + - [Misc dependencies](#misc-dependencies) + - [`Fork`](#fork) + - [`ForkData`](#forkdata) + - [`Checkpoint`](#checkpoint) + - [`Validator`](#validator) + - [`AttestationData`](#attestationdata) + - [`IndexedAttestation`](#indexedattestation) + - [`PendingAttestation`](#pendingattestation) + - [`Eth1Data`](#eth1data) + - [`HistoricalBatch`](#historicalbatch) + - [`DepositMessage`](#depositmessage) + - [`DepositData`](#depositdata) + - [`BeaconBlockHeader`](#beaconblockheader) + - [`SigningData`](#signingdata) + - [Beacon operations](#beacon-operations) + - [`ProposerSlashing`](#proposerslashing) + - [`AttesterSlashing`](#attesterslashing) + - [`Attestation`](#attestation) + - [`Deposit`](#deposit) + - [`VoluntaryExit`](#voluntaryexit) + - [Beacon blocks](#beacon-blocks) + - [`BeaconBlockBody`](#beaconblockbody) + - [`BeaconBlock`](#beaconblock) + - [Beacon state](#beacon-state) + - [`BeaconState`](#beaconstate) + - [Signed envelopes](#signed-envelopes) + - [`SignedVoluntaryExit`](#signedvoluntaryexit) + - [`SignedBeaconBlock`](#signedbeaconblock) + - [`SignedBeaconBlockHeader`](#signedbeaconblockheader) +- [Helper functions](#helper-functions) + - [Math](#math) + - [`integer_squareroot`](#integer_squareroot) + - [`xor`](#xor) + - [`uint_to_bytes`](#uint_to_bytes) + - [`bytes_to_uint64`](#bytes_to_uint64) + - [Crypto](#crypto) + - [`hash`](#hash) + - [`hash_tree_root`](#hash_tree_root) + - [BLS signatures](#bls-signatures) + - [Predicates](#predicates) + - [`is_active_validator`](#is_active_validator) + - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue) + - [`is_eligible_for_activation`](#is_eligible_for_activation) + - [`is_slashable_validator`](#is_slashable_validator) + - [`is_slashable_attestation_data`](#is_slashable_attestation_data) + - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) + - [`is_valid_merkle_branch`](#is_valid_merkle_branch) + - [Misc](#misc-1) + - [`compute_shuffled_index`](#compute_shuffled_index) + - [`compute_proposer_index`](#compute_proposer_index) + - [`compute_committee`](#compute_committee) + - [`compute_epoch_at_slot`](#compute_epoch_at_slot) + - [`compute_start_slot_at_epoch`](#compute_start_slot_at_epoch) + - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) + - [`compute_fork_data_root`](#compute_fork_data_root) + - [`compute_fork_digest`](#compute_fork_digest) + - [`compute_domain`](#compute_domain) + - [`compute_signing_root`](#compute_signing_root) + - [Beacon state accessors](#beacon-state-accessors) + - [`get_current_epoch`](#get_current_epoch) + - [`get_previous_epoch`](#get_previous_epoch) + - [`get_block_root`](#get_block_root) + - [`get_block_root_at_slot`](#get_block_root_at_slot) + - [`get_randao_mix`](#get_randao_mix) + - [`get_active_validator_indices`](#get_active_validator_indices) + - [`get_validator_churn_limit`](#get_validator_churn_limit) + - [`get_seed`](#get_seed) + - [`get_committee_count_per_slot`](#get_committee_count_per_slot) + - [`get_beacon_committee`](#get_beacon_committee) + - [`get_beacon_proposer_index`](#get_beacon_proposer_index) + - [`get_total_balance`](#get_total_balance) + - [`get_total_active_balance`](#get_total_active_balance) + - [`get_domain`](#get_domain) + - [`get_indexed_attestation`](#get_indexed_attestation) + - [`get_attesting_indices`](#get_attesting_indices) + - [Beacon state mutators](#beacon-state-mutators) + - [`increase_balance`](#increase_balance) + - [`decrease_balance`](#decrease_balance) + - [`initiate_validator_exit`](#initiate_validator_exit) + - [`slash_validator`](#slash_validator) +- [Genesis](#genesis) + - [Genesis state](#genesis-state) + - [Genesis block](#genesis-block) +- [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Epoch processing](#epoch-processing) + - [Helper functions](#helper-functions-1) + - [Justification and finalization](#justification-and-finalization) + - [Rewards and penalties](#rewards-and-penalties-1) + - [Helpers](#helpers) + - [Components of attestation deltas](#components-of-attestation-deltas) + - [`get_attestation_deltas`](#get_attestation_deltas) + - [`process_rewards_and_penalties`](#process_rewards_and_penalties) + - [Registry updates](#registry-updates) + - [Slashings](#slashings) + - [Eth1 data votes updates](#eth1-data-votes-updates) + - [Effective balances updates](#effective-balances-updates) + - [Slashings balances updates](#slashings-balances-updates) + - [Randao mixes updates](#randao-mixes-updates) + - [Historical roots updates](#historical-roots-updates) + - [Participation records rotation](#participation-records-rotation) + - [Block processing](#block-processing) + - [Block header](#block-header) + - [RANDAO](#randao) + - [Eth1 data](#eth1-data) + - [Operations](#operations) + - [Proposer slashings](#proposer-slashings) + - [Attester slashings](#attester-slashings) + - [Attestations](#attestations) + - [Deposits](#deposits) + - [Voluntary exits](#voluntary-exits) + + + + +## Introduction + +This document represents the specification for Phase 0 of Ethereum 2.0 -- The Beacon Chain. + +At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of validators. In the initial deployment phases of Ethereum 2.0, the only mechanism to become a validator is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a validator happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. +The primary source of load on the beacon chain is "attestations". Attestations are simultaneously availability votes for a shard block (in a later Eth2 upgrade) and proof-of-stake votes for a beacon block (Phase 0). + +## Notation + +Code snippets appearing in `this style` are to be interpreted as Python 3 code. + +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `Slot` | `uint64` | a slot number | +| `Epoch` | `uint64` | an epoch number | +| `CommitteeIndex` | `uint64` | a committee index at a slot | +| `ValidatorIndex` | `uint64` | a validator registry index | +| `Gwei` | `uint64` | an amount in Gwei | +| `Root` | `Bytes32` | a Merkle root | +| `Version` | `Bytes4` | a fork version number | +| `DomainType` | `Bytes4` | a domain type | +| `ForkDigest` | `Bytes4` | a digest of the current fork data | +| `Domain` | `Bytes32` | a signature domain | +| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | +| `BLSSignature` | `Bytes96` | a BLS12-381 signature | + +## Constants + +The following values are (non-configurable) constants used throughout the specification. + +| Name | Value | +| - | - | +| `GENESIS_SLOT` | `Slot(0)` | +| `GENESIS_EPOCH` | `Epoch(0)` | +| `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | +| `BASE_REWARDS_PER_EPOCH` | `uint64(4)` | +| `DEPOSIT_CONTRACT_TREE_DEPTH` | `uint64(2**5)` (= 32) | +| `JUSTIFICATION_BITS_LENGTH` | `uint64(4)` | +| `ENDIANNESS` | `'little'` | + +## Configuration + +*Note*: The default mainnet configuration values are included here for illustrative purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs) directory. + +### Misc + +| Name | Value | +| - | - | +| `ETH1_FOLLOW_DISTANCE` | `uint64(2**11)` (= 2,048) | +| `MAX_COMMITTEES_PER_SLOT` | `uint64(2**6)` (= 64) | +| `TARGET_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) | +| `MAX_VALIDATORS_PER_COMMITTEE` | `uint64(2**11)` (= 2,048) | +| `MIN_PER_EPOCH_CHURN_LIMIT` | `uint64(2**2)` (= 4) | +| `CHURN_LIMIT_QUOTIENT` | `uint64(2**16)` (= 65,536) | +| `SHUFFLE_ROUND_COUNT` | `uint64(90)` | +| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `uint64(2**14)` (= 16,384) | +| `MIN_GENESIS_TIME` | `uint64(1606824000)` (Dec 1, 2020, 12pm UTC) | +| `HYSTERESIS_QUOTIENT` | `uint64(4)` | +| `HYSTERESIS_DOWNWARD_MULTIPLIER` | `uint64(1)` | +| `HYSTERESIS_UPWARD_MULTIPLIER` | `uint64(5)` | + +- For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) + +### Gwei values + +| Name | Value | +| - | - | +| `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | +| `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | +| `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | +| `EFFECTIVE_BALANCE_INCREMENT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | + +### Initial values + +| Name | Value | +| - | - | +| `GENESIS_FORK_VERSION` | `Version('0x00000000')` | + +### Withdrawal prefixes + +| Name | Value | +| - | - | +| `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | +| `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `GENESIS_DELAY` | `uint64(604800)` | seconds | 7 days | +| `SECONDS_PER_SLOT` | `uint64(12)` | seconds | 12 seconds | +| `SECONDS_PER_ETH1_BLOCK` | `uint64(14)` | seconds | 14 seconds | +| `MIN_ATTESTATION_INCLUSION_DELAY` | `uint64(2**0)` (= 1) | slots | 12 seconds | +| `SLOTS_PER_EPOCH` | `uint64(2**5)` (= 32) | slots | 6.4 minutes | +| `MIN_SEED_LOOKAHEAD` | `uint64(2**0)` (= 1) | epochs | 6.4 minutes | +| `MAX_SEED_LOOKAHEAD` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes | +| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes | +| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `uint64(2**6)` (= 64) | epochs | ~6.8 hours | +| `SLOTS_PER_HISTORICAL_ROOT` | `uint64(2**13)` (= 8,192) | slots | ~27 hours | +| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `uint64(2**8)` (= 256) | epochs | ~27 hours | +| `SHARD_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours | + +### State list lengths + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `EPOCHS_PER_HISTORICAL_VECTOR` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 years | +| `EPOCHS_PER_SLASHINGS_VECTOR` | `uint64(2**13)` (= 8,192) | epochs | ~36 days | +| `HISTORICAL_ROOTS_LIMIT` | `uint64(2**24)` (= 16,777,216) | historical roots | ~52,262 years | +| `VALIDATOR_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | validators | + +### Rewards and penalties + +| Name | Value | +| - | - | +| `BASE_REWARD_FACTOR` | `uint64(2**6)` (= 64) | +| `WHISTLEBLOWER_REWARD_QUOTIENT` | `uint64(2**9)` (= 512) | +| `PROPOSER_REWARD_QUOTIENT` | `uint64(2**3)` (= 8) | +| `INACTIVITY_PENALTY_QUOTIENT` | `uint64(2**26)` (= 67,108,864) | +| `MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**7)` (= 128) | +| `PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(1)` | + +- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**13` epochs (about 36 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. Note this value will be upgraded to `2**24` after Phase 0 mainnet stabilizes to provide a faster recovery in the event of an inactivity leak. + +- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accountable safety margin. + +### Max operations per block + +| Name | Value | +| - | - | +| `MAX_PROPOSER_SLASHINGS` | `2**4` (= 16) | +| `MAX_ATTESTER_SLASHINGS` | `2**1` (= 2) | +| `MAX_ATTESTATIONS` | `2**7` (= 128) | +| `MAX_DEPOSITS` | `2**4` (= 16) | +| `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | + +### Domain types + +| Name | Value | +| - | - | +| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` | +| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` | +| `DOMAIN_RANDAO` | `DomainType('0x02000000')` | +| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` | +| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` | +| `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | +| `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | + +## Containers + +The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. + +*Note*: The definitions are ordered topologically to facilitate execution of the spec. + +*Note*: Fields missing in container instantiations default to their zero value. + +### Misc dependencies + +#### `Fork` + +```python +class Fork(Container): + previous_version: Version + current_version: Version + epoch: Epoch # Epoch of latest fork +``` + +#### `ForkData` + +```python +class ForkData(Container): + current_version: Version + genesis_validators_root: Root +``` + +#### `Checkpoint` + +```python +class Checkpoint(Container): + epoch: Epoch + root: Root +``` + +#### `Validator` + +```python +class Validator(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals + effective_balance: Gwei # Balance at stake + slashed: boolean + # Status epochs + activation_eligibility_epoch: Epoch # When criteria for activation were met + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw funds +``` + +#### `AttestationData` + +```python +class AttestationData(Container): + slot: Slot + index: CommitteeIndex + # LMD GHOST vote + beacon_block_root: Root + # FFG vote + source: Checkpoint + target: Checkpoint +``` + +#### `IndexedAttestation` + +```python +class IndexedAttestation(Container): + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature +``` + +#### `PendingAttestation` + +```python +class PendingAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + inclusion_delay: Slot + proposer_index: ValidatorIndex +``` + +#### `Eth1Data` + +```python +class Eth1Data(Container): + deposit_root: Root + deposit_count: uint64 + block_hash: Bytes32 +``` + +#### `HistoricalBatch` + +```python +class HistoricalBatch(Container): + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] +``` + +#### `DepositMessage` + +```python +class DepositMessage(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei +``` + +#### `DepositData` + +```python +class DepositData(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei + signature: BLSSignature # Signing over DepositMessage +``` + +#### `BeaconBlockHeader` + +```python +class BeaconBlockHeader(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body_root: Root +``` + +#### `SigningData` + +```python +class SigningData(Container): + object_root: Root + domain: Domain +``` + +### Beacon operations + +#### `ProposerSlashing` + +```python +class ProposerSlashing(Container): + signed_header_1: SignedBeaconBlockHeader + signed_header_2: SignedBeaconBlockHeader +``` + +#### `AttesterSlashing` + +```python +class AttesterSlashing(Container): + attestation_1: IndexedAttestation + attestation_2: IndexedAttestation +``` + +#### `Attestation` + +```python +class Attestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature +``` + +#### `Deposit` + +```python +class Deposit(Container): + proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root + data: DepositData +``` + +#### `VoluntaryExit` + +```python +class VoluntaryExit(Container): + epoch: Epoch # Earliest epoch when voluntary exit can be processed + validator_index: ValidatorIndex +``` + +### Beacon blocks + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(Container): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] +``` + +#### `BeaconBlock` + +```python +class BeaconBlock(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body: BeaconBlockBody +``` + +### Beacon state + +#### `BeaconState` + +```python +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Attestations + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint +``` + +### Signed envelopes + +#### `SignedVoluntaryExit` + +```python +class SignedVoluntaryExit(Container): + message: VoluntaryExit + signature: BLSSignature +``` + +#### `SignedBeaconBlock` + +```python +class SignedBeaconBlock(Container): + message: BeaconBlock + signature: BLSSignature +``` + +#### `SignedBeaconBlockHeader` + +```python +class SignedBeaconBlockHeader(Container): + message: BeaconBlockHeader + signature: BLSSignature +``` + +## Helper functions + +*Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. + +### Math + +#### `integer_squareroot` + +```python +def integer_squareroot(n: uint64) -> uint64: + """ + Return the largest integer ``x`` such that ``x**2 <= n``. + """ + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +``` + +#### `xor` + +```python +def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: + """ + Return the exclusive-or of two 32-byte strings. + """ + return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) +``` + +#### `uint_to_bytes` + +`def uint_to_bytes(n: uint) -> bytes` is a function for serializing the `uint` type object to bytes in ``ENDIANNESS``-endian. The expected length of the output is the byte-length of the `uint` type. + +#### `bytes_to_uint64` + +```python +def bytes_to_uint64(data: bytes) -> uint64: + """ + Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian. + """ + return uint64(int.from_bytes(data, ENDIANNESS)) +``` + +### Crypto + +#### `hash` + +`def hash(data: bytes) -> Bytes32` is SHA256. + +#### `hash_tree_root` + +`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization). + +#### BLS signatures + +The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: + +- `def Sign(privkey: int, message: Bytes) -> BLSSignature` +- `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` +- `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` +- `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` +- `def AggregateVerify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` + +The above functions are accessed through the `bls` module, e.g. `bls.Verify`. + +### Predicates + +#### `is_active_validator` + +```python +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +``` + +#### `is_eligible_for_activation_queue` + +```python +def is_eligible_for_activation_queue(validator: Validator) -> bool: + """ + Check if ``validator`` is eligible to be placed into the activation queue. + """ + return ( + validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH + and validator.effective_balance == MAX_EFFECTIVE_BALANCE + ) +``` + +#### `is_eligible_for_activation` + +```python +def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool: + """ + Check if ``validator`` is eligible for activation. + """ + return ( + # Placement in queue is finalized + validator.activation_eligibility_epoch <= state.finalized_checkpoint.epoch + # Has not yet been activated + and validator.activation_epoch == FAR_FUTURE_EPOCH + ) +``` + +#### `is_slashable_validator` + +```python +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) +``` + +#### `is_slashable_attestation_data` + +```python +def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: + """ + Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. + """ + return ( + # Double vote + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or + # Surround vote + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) + ) +``` + +#### `is_valid_indexed_attestation` + +```python +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. + """ + # Verify indices are sorted and unique + indices = indexed_attestation.attesting_indices + if len(indices) == 0 or not indices == sorted(set(indices)): + return False + # Verify aggregate signature + pubkeys = [state.validators[i].pubkey for i in indices] + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) + signing_root = compute_signing_root(indexed_attestation.data, domain) + return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) +``` + +#### `is_valid_merkle_branch` + +```python +def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint64, index: uint64, root: Root) -> bool: + """ + Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(branch[i] + value) + else: + value = hash(value + branch[i]) + return value == root +``` + +### Misc + +#### `compute_shuffled_index` + +```python +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: + """ + Return the shuffled index corresponding to ``seed`` (and ``index_count``). + """ + assert index < index_count + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_uint64(hash(seed + uint_to_bytes(uint8(current_round)))[0:8]) % index_count + flip = (pivot + index_count - index) % index_count + position = max(index, flip) + source = hash( + seed + + uint_to_bytes(uint8(current_round)) + + uint_to_bytes(uint32(position // 256)) + ) + byte = uint8(source[(position % 256) // 8]) + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index +``` + +#### `compute_proposer_index` + +```python +def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32) -> ValidatorIndex: + """ + Return from ``indices`` a random index sampled by effective balance. + """ + assert len(indices) > 0 + MAX_RANDOM_BYTE = 2**8 - 1 + i = uint64(0) + total = uint64(len(indices)) + while True: + candidate_index = indices[compute_shuffled_index(i % total, total, seed)] + random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return candidate_index + i += 1 +``` + +#### `compute_committee` + +```python +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Bytes32, + index: uint64, + count: uint64) -> Sequence[ValidatorIndex]: + """ + Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. + """ + start = (len(indices) * index) // count + end = (len(indices) * uint64(index + 1)) // count + return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)] +``` + +#### `compute_epoch_at_slot` + +```python +def compute_epoch_at_slot(slot: Slot) -> Epoch: + """ + Return the epoch number at ``slot``. + """ + return Epoch(slot // SLOTS_PER_EPOCH) +``` + +#### `compute_start_slot_at_epoch` + +```python +def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: + """ + Return the start slot of ``epoch``. + """ + return Slot(epoch * SLOTS_PER_EPOCH) +``` + +#### `compute_activation_exit_epoch` + +```python +def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. + """ + return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) +``` + +#### `compute_fork_data_root` + +```python +def compute_fork_data_root(current_version: Version, genesis_validators_root: Root) -> Root: + """ + Return the 32-byte fork data root for the ``current_version`` and ``genesis_validators_root``. + This is used primarily in signature domains to avoid collisions across forks/chains. + """ + return hash_tree_root(ForkData( + current_version=current_version, + genesis_validators_root=genesis_validators_root, + )) +``` + +#### `compute_fork_digest` + +```python +def compute_fork_digest(current_version: Version, genesis_validators_root: Root) -> ForkDigest: + """ + Return the 4-byte fork digest for the ``current_version`` and ``genesis_validators_root``. + This is a digest primarily used for domain separation on the p2p layer. + 4-bytes suffices for practical separation of forks/chains. + """ + return ForkDigest(compute_fork_data_root(current_version, genesis_validators_root)[:4]) +``` + +#### `compute_domain` + +```python +def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_validators_root: Root=None) -> Domain: + """ + Return the domain for the ``domain_type`` and ``fork_version``. + """ + if fork_version is None: + fork_version = GENESIS_FORK_VERSION + if genesis_validators_root is None: + genesis_validators_root = Root() # all bytes zero by default + fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) + return Domain(domain_type + fork_data_root[:28]) +``` + +#### `compute_signing_root` + +```python +def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: + """ + Return the signing root for the corresponding signing data. + """ + return hash_tree_root(SigningData( + object_root=hash_tree_root(ssz_object), + domain=domain, + )) +``` + +### Beacon state accessors + +#### `get_current_epoch` + +```python +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch. + """ + return compute_epoch_at_slot(state.slot) +``` + +#### `get_previous_epoch` + +```python +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). + """ + current_epoch = get_current_epoch(state) + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) +``` + +#### `get_block_root` + +```python +def get_block_root(state: BeaconState, epoch: Epoch) -> Root: + """ + Return the block root at the start of a recent ``epoch``. + """ + return get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) +``` + +#### `get_block_root_at_slot` + +```python +def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +``` + +#### `get_randao_mix` + +```python +def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] +``` + +#### `get_active_validator_indices` + +```python +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + """ + Return the sequence of active validator indices at ``epoch``. + """ + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] +``` + +#### `get_validator_churn_limit` + +```python +def get_validator_churn_limit(state: BeaconState) -> uint64: + """ + Return the validator churn limit for the current epoch. + """ + active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) + return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT) +``` + +#### `get_seed` + +```python +def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes32: + """ + Return the seed at ``epoch``. + """ + mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow + return hash(domain_type + uint_to_bytes(epoch) + mix) +``` + +#### `get_committee_count_per_slot` + +```python +def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: + """ + Return the number of committees in each slot for the given ``epoch``. + """ + return max(uint64(1), min( + MAX_COMMITTEES_PER_SLOT, + uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + )) +``` + +#### `get_beacon_committee` + +```python +def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]: + """ + Return the beacon committee at ``slot`` for ``index``. + """ + epoch = compute_epoch_at_slot(slot) + committees_per_slot = get_committee_count_per_slot(state, epoch) + return compute_committee( + indices=get_active_validator_indices(state, epoch), + seed=get_seed(state, epoch, DOMAIN_BEACON_ATTESTER), + index=(slot % SLOTS_PER_EPOCH) * committees_per_slot + index, + count=committees_per_slot * SLOTS_PER_EPOCH, + ) +``` + +#### `get_beacon_proposer_index` + +```python +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: + """ + Return the beacon proposer index at the current slot. + """ + epoch = get_current_epoch(state) + seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot)) + indices = get_active_validator_indices(state, epoch) + return compute_proposer_index(state, indices, seed) +``` + +#### `get_total_balance` + +```python +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of the ``indices``. + ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + Math safe up to ~10B ETH, afterwhich this overflows uint64. + """ + return Gwei(max(EFFECTIVE_BALANCE_INCREMENT, sum([state.validators[index].effective_balance for index in indices]))) +``` + +#### `get_total_active_balance` + +```python +def get_total_active_balance(state: BeaconState) -> Gwei: + """ + Return the combined effective balance of the active validators. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) +``` + +#### `get_domain` + +```python +def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: + """ + Return the signature domain (fork version concatenated with domain type) of a message. + """ + epoch = get_current_epoch(state) if epoch is None else epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return compute_domain(domain_type, fork_version, state.genesis_validators_root) +``` + +#### `get_indexed_attestation` + +```python +def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: + """ + Return the indexed attestation corresponding to ``attestation``. + """ + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + + return IndexedAttestation( + attesting_indices=sorted(attesting_indices), + data=attestation.data, + signature=attestation.signature, + ) +``` + +#### `get_attesting_indices` + +```python +def get_attesting_indices(state: BeaconState, + data: AttestationData, + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: + """ + Return the set of attesting indices corresponding to ``data`` and ``bits``. + """ + committee = get_beacon_committee(state, data.slot, data.index) + return set(index for i, index in enumerate(committee) if bits[i]) +``` + +### Beacon state mutators + +#### `increase_balance` + +```python +def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Increase the validator balance at index ``index`` by ``delta``. + """ + state.balances[index] += delta +``` + +#### `decrease_balance` + +```python +def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Decrease the validator balance at index ``index`` by ``delta``, with underflow protection. + """ + state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta +``` + +#### `initiate_validator_exit` + +```python +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the exit of the validator with index ``index``. + """ + # Return if validator already initiated exit + validator = state.validators[index] + if validator.exit_epoch != FAR_FUTURE_EPOCH: + return + + # Compute exit queue epoch + exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] + exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) + if exit_queue_churn >= get_validator_churn_limit(state): + exit_queue_epoch += Epoch(1) + + # Set validator exit epoch and withdrawable epoch + validator.exit_epoch = exit_queue_epoch + validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) +``` + +#### `slash_validator` + +```python +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: + """ + Slash the validator with index ``slashed_index``. + """ + epoch = get_current_epoch(state) + initiate_validator_exit(state, slashed_index) + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + + # Apply proposer and whistleblower rewards + proposer_index = get_beacon_proposer_index(state) + if whistleblower_index is None: + whistleblower_index = proposer_index + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) +``` + +## Genesis + +Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, let `candidate_state = initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: + +- `eth1_block_hash` is the hash of the Ethereum 1.0 block +- `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` +- `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` + +Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. + +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit]) -> BeaconState: + fork = Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + return state +``` + +*Note*: The ETH1 block with `eth1_timestamp` meeting the minimum genesis active validator count criteria can also occur before `MIN_GENESIS_TIME`. + +### Genesis state + +Let `genesis_state = candidate_state` whenever `is_valid_genesis_state(candidate_state) is True` for the first time. + +```python +def is_valid_genesis_state(state: BeaconState) -> bool: + if state.genesis_time < MIN_GENESIS_TIME: + return False + if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: + return False + return True +``` + +### Genesis block + +Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. + +## Beacon chain state transition function + +The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. State transitions that cause a `uint64` overflow or underflow are also considered invalid. + +```python +def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> None: + block = signed_block.message + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Verify signature + if validate_result: + assert verify_block_signature(state, signed_block) + # Process block + process_block(state, block) + # Verify state root + if validate_result: + assert block.state_root == hash_tree_root(state) +``` + +```python +def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: + proposer = state.validators[signed_block.message.proposer_index] + signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) +``` + +```python +def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot < slot + while state.slot < slot: + process_slot(state) + # Process epoch on the start slot of the next epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch(state) + state.slot = Slot(state.slot + 1) +``` + +```python +def process_slot(state: BeaconState) -> None: + # Cache state root + previous_state_root = hash_tree_root(state) + state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + # Cache latest block header state root + if state.latest_block_header.state_root == Bytes32(): + state.latest_block_header.state_root = previous_state_root + # Cache block root + previous_block_root = hash_tree_root(state.latest_block_header) + state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root +``` + +### Epoch processing + +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) + process_participation_record_updates(state) +``` + +#### Helper functions + +```python +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) + return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations +``` + +```python +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_source_attestations(state, epoch) + if a.data.target.root == get_block_root(state, epoch) + ] +``` + +```python +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_target_attestations(state, epoch) + if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) + ] +``` + +```python +def get_unslashed_attesting_indices(state: BeaconState, + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: + output = set() # type: Set[ValidatorIndex] + for a in attestations: + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) + return set(filter(lambda index: not state.validators[index].slashed, output)) +``` + +```python +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: + """ + Return the combined effective balance of the set of unslashed validators participating in ``attestations``. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +``` + +#### Justification and finalization + +```python +def process_justification_and_finalization(state: BeaconState) -> None: + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + previous_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + current_attestations = get_matching_target_attestations(state, get_current_epoch(state)) + total_active_balance = get_total_active_balance(state) + previous_target_balance = get_attesting_balance(state, previous_attestations) + current_target_balance = get_attesting_balance(state, current_attestations) + weigh_justification_and_finalization(state, total_active_balance, previous_target_balance, current_target_balance) +``` + +```python +def weigh_justification_and_finalization(state: BeaconState, + total_active_balance: Gwei, + previous_epoch_target_balance: Gwei, + current_epoch_target_balance: Gwei) -> None: + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 + if previous_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) + state.justification_bits[1] = 0b1 + if current_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) + state.justification_bits[0] = 0b1 + + # Process finalizations + bits = state.justification_bits + # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint +``` + +#### Rewards and penalties + +##### Helpers + +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) +``` + + +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` + + +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` + + +```python +def is_in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` + + +```python +def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: + previous_epoch = get_previous_epoch(state) + return [ + ValidatorIndex(index) for index, v in enumerate(state.validators) + if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) + ] +``` + +```python +def get_attestation_component_deltas(state: BeaconState, + attestations: Sequence[PendingAttestation] + ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Helper with shared logic for use by get source, target, and head deltas functions + """ + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + total_balance = get_total_active_balance(state) + unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) + attesting_balance = get_total_balance(state, unslashed_attesting_indices) + for index in get_eligible_validator_indices(state): + if index in unslashed_attesting_indices: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) + else: + penalties[index] += get_base_reward(state, index) + return rewards, penalties +``` + +##### Components of attestation deltas + +```python +def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for source-vote for each validator. + """ + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_source_attestations) +``` + +```python +def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for target-vote for each validator. + """ + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_target_attestations) +``` + +```python +def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for head-vote for each validator. + """ + matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_head_attestations) +``` + +```python +def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return proposer and inclusion delay micro-rewards/penalties for each validator. + """ + rewards = [Gwei(0) for _ in range(len(state.validators))] + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + for index in get_unslashed_attesting_indices(state, matching_source_attestations): + attestation = min([ + a for a in matching_source_attestations + if index in get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = Gwei(get_base_reward(state, index) - get_proposer_reward(state, index)) + rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) + + # No penalties associated with inclusion delay + penalties = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` + +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return inactivity reward/penalty deltas for each validator. + """ + penalties = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) + for index in get_eligible_validator_indices(state): + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) + if index not in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + # No rewards associated with inactivity penalties + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` + +##### `get_attestation_deltas` + +```python +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attestation reward/penalty deltas for each validator. + """ + source_rewards, source_penalties = get_source_deltas(state) + target_rewards, target_penalties = get_target_deltas(state) + head_rewards, head_penalties = get_head_deltas(state) + inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) + _, inactivity_penalties = get_inactivity_penalty_deltas(state) + + rewards = [ + source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] + for i in range(len(state.validators)) + ] + + penalties = [ + source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] + for i in range(len(state.validators)) + ] + + return rewards, penalties +``` + +##### `process_rewards_and_penalties` + +```python +def process_rewards_and_penalties(state: BeaconState) -> None: + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch + if get_current_epoch(state) == GENESIS_EPOCH: + return + + rewards, penalties = get_attestation_deltas(state) + for index in range(len(state.validators)): + increase_balance(state, ValidatorIndex(index), rewards[index]) + decrease_balance(state, ValidatorIndex(index), penalties[index]) +``` + +#### Registry updates + +```python +def process_registry_updates(state: BeaconState) -> None: + # Process activation eligibility and ejections + for index, validator in enumerate(state.validators): + if is_eligible_for_activation_queue(validator): + validator.activation_eligibility_epoch = get_current_epoch(state) + 1 + + if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: + initiate_validator_exit(state, ValidatorIndex(index)) + + # Queue validators eligible for activation and not yet dequeued for activation + activation_queue = sorted([ + index for index, validator in enumerate(state.validators) + if is_eligible_for_activation(state, validator) + # Order by the sequence of activation_eligibility_epoch setting and then index + ], key=lambda index: (state.validators[index].activation_eligibility_epoch, index)) + # Dequeued validators for activation up to churn limit + for index in activation_queue[:get_validator_churn_limit(state)]: + validator = state.validators[index] + validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) +``` + +#### Slashings + +```python +def process_slashings(state: BeaconState) -> None: + epoch = get_current_epoch(state) + total_balance = get_total_active_balance(state) + adjusted_total_slashing_balance = min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + for index, validator in enumerate(state.validators): + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance + penalty = penalty_numerator // total_balance * increment + decrease_balance(state, ValidatorIndex(index), penalty) +``` + +#### Eth1 data votes updates +```python +def process_eth1_data_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset eth1 data votes + if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] +``` + +#### Effective balances updates + +```python +def process_effective_balance_updates(state: BeaconState) -> None: + # Update effective balances with hysteresis + for index, validator in enumerate(state.validators): + balance = state.balances[index] + HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) + DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER + UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER + if ( + balance + DOWNWARD_THRESHOLD < validator.effective_balance + or validator.effective_balance + UPWARD_THRESHOLD < balance + ): + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) +``` + +#### Slashings balances updates + +```python +def process_slashings_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) +``` + +#### Randao mixes updates + +```python +def process_randao_mixes_reset(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) + # Set randao mix + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) +``` + +#### Historical roots updates +```python +def process_historical_roots_update(state: BeaconState) -> None: + # Set historical root accumulator + next_epoch = Epoch(get_current_epoch(state) + 1) + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) + state.historical_roots.append(hash_tree_root(historical_batch)) +``` + +#### Participation records rotation + +```python +def process_participation_record_updates(state: BeaconState) -> None: + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] +``` + +### Block processing + +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) +``` + +#### Block header + +```python +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + # Verify that proposer index is the correct index + assert block.proposer_index == get_beacon_proposer_index(state) + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), + ) + + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed +``` + +#### RANDAO + +```python +def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: + epoch = get_current_epoch(state) + # Verify RANDAO reveal + proposer = state.validators[get_beacon_proposer_index(state)] + signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) + assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal) + # Mix in RANDAO reveal + mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) + state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix +``` + +#### Eth1 data + +```python +def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: + state.eth1_data_votes.append(body.eth1_data) + if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.eth1_data = body.eth1_data +``` + +#### Operations + +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) +``` + +##### Proposer slashings + +```python +def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: + header_1 = proposer_slashing.signed_header_1.message + header_2 = proposer_slashing.signed_header_2.message + + # Verify header slots match + assert header_1.slot == header_2.slot + # Verify header proposer indices match + assert header_1.proposer_index == header_2.proposer_index + # Verify the headers are different + assert header_1 != header_2 + # Verify the proposer is slashable + proposer = state.validators[header_1.proposer_index] + assert is_slashable_validator(proposer, get_current_epoch(state)) + # Verify signatures + for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) + signing_root = compute_signing_root(signed_header.message, domain) + assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) + + slash_validator(state, header_1.proposer_index) +``` + +##### Attester slashings + +```python +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 + assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) + assert is_valid_indexed_attestation(state, attestation_1) + assert is_valid_indexed_attestation(state, attestation_2) + + slashed_any = False + indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices) + for index in sorted(indices): + if is_slashable_validator(state.validators[index], get_current_epoch(state)): + slash_validator(state, index) + slashed_any = True + assert slashed_any +``` + +##### Attestations + +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + assert data.index < get_committee_count_per_slot(state, data.target.epoch) + + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + + pending_attestation = PendingAttestation( + data=data, + aggregation_bits=attestation.aggregation_bits, + inclusion_delay=state.slot - data.slot, + proposer_index=get_beacon_proposer_index(state), + ) + + if data.target.epoch == get_current_epoch(state): + assert data.source == state.current_justified_checkpoint + state.current_epoch_attestations.append(pending_attestation) + else: + assert data.source == state.previous_justified_checkpoint + state.previous_epoch_attestations.append(pending_attestation) + + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) +``` + +##### Deposits + +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + ) +``` + +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` + +##### Voluntary exits + +```python +def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: + voluntary_exit = signed_voluntary_exit.message + validator = state.validators[voluntary_exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify exit has not been initiated + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= voluntary_exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD + # Verify signature + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) + # Initiate exit + initiate_validator_exit(state, voluntary_exit.validator_index) +``` diff --git a/tools/analyzers/specdocs/data/phase0/deposit-contract.md b/tools/analyzers/specdocs/data/phase0/deposit-contract.md new file mode 100644 index 00000000000..02e762daef2 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/deposit-contract.md @@ -0,0 +1,77 @@ +# Ethereum 2.0 Phase 0 -- Deposit Contract + +## Table of contents + + + + +- [Introduction](#introduction) +- [Constants](#constants) +- [Configuration](#configuration) +- [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) + - [`deposit` function](#deposit-function) + - [Deposit amount](#deposit-amount) + - [Withdrawal credentials](#withdrawal-credentials) + - [`DepositEvent` log](#depositevent-log) +- [Solidity code](#solidity-code) + + + + +## Introduction + +This document represents the specification for the beacon chain deposit contract, part of Ethereum 2.0 Phase 0. + +## Constants + +The following values are (non-configurable) constants used throughout the specification. + +| Name | Value | +| - | - | +| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | + +## Configuration + +*Note*: The default mainnet configuration values are included here for spec-design purposes. +The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs) directory. +These configurations are updated for releases and may be out of sync during `dev` changes. + +| Name | Value | +| - | - | +| `DEPOSIT_CHAIN_ID` | `1` | +| `DEPOSIT_NETWORK_ID` | `1` | +| `DEPOSIT_CONTRACT_ADDRESS` | `0x00000000219ab540356cBB839Cbe05303d7705Fa` | + +## Ethereum 1.0 deposit contract + +The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to the Ethereum 1.0 chain defined by the [chain-id](https://eips.ethereum.org/EIPS/eip-155) -- `DEPOSIT_CHAIN_ID` -- and the network-id -- `DEPOSIT_NETWORK_ID` -- for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2. + +_Note_: See [here](https://chainid.network/) for a comprehensive list of public Ethereum chain chain-id's and network-id's. + +### `deposit` function + +The deposit contract has a public `deposit` function to make deposits. It takes as arguments `bytes calldata pubkey, bytes calldata withdrawal_credentials, bytes calldata signature, bytes32 deposit_data_root`. The first three arguments populate a [`DepositData`](./beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata. + +#### Deposit amount + +The amount of ETH (rounded down to the closest Gwei) sent to the deposit contract is the deposit amount, which must be of size at least `MIN_DEPOSIT_AMOUNT` Gwei. Note that ETH consumed by the deposit contract is no longer usable on Ethereum 1.0. + +#### Withdrawal credentials + +One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. +The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. +Read more in the [validator guide](./validator.md#withdrawal-credentials). + +*Note*: The deposit contract does not validate the `withdrawal_credentials` field. +Support for new withdrawal prefixes can be added without modifying the deposit contract. + +#### `DepositEvent` log + +Every Ethereum 1.0 deposit emits a `DepositEvent` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. + +## Solidity code + +The deposit contract source code, written in Solidity, is available [here](../../solidity_deposit_contract/deposit_contract.sol). + +*Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. diff --git a/tools/analyzers/specdocs/data/phase0/fork-choice.md b/tools/analyzers/specdocs/data/phase0/fork-choice.md new file mode 100644 index 00000000000..b5689ecd239 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/fork-choice.md @@ -0,0 +1,406 @@ +# Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) +- [Fork choice](#fork-choice) + - [Configuration](#configuration) + - [Helpers](#helpers) + - [`LatestMessage`](#latestmessage) + - [`Store`](#store) + - [`get_forkchoice_store`](#get_forkchoice_store) + - [`get_slots_since_genesis`](#get_slots_since_genesis) + - [`get_current_slot`](#get_current_slot) + - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) + - [`get_ancestor`](#get_ancestor) + - [`get_latest_attesting_balance`](#get_latest_attesting_balance) + - [`filter_block_tree`](#filter_block_tree) + - [`get_filtered_block_tree`](#get_filtered_block_tree) + - [`get_head`](#get_head) + - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) + - [`on_attestation` helpers](#on_attestation-helpers) + - [`validate_on_attestation`](#validate_on_attestation) + - [`store_target_checkpoint_state`](#store_target_checkpoint_state) + - [`update_latest_messages`](#update_latest_messages) + - [Handlers](#handlers) + - [`on_tick`](#on_tick) + - [`on_block`](#on_block) + - [`on_attestation`](#on_attestation) + + + + +## Introduction + +This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./beacon-chain.md). + +## Fork choice + +The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_forkchoice_store(genesis_state)` and update `store` by running: + +- `on_tick(store, time)` whenever `time > store.time` where `time` is the current Unix time +- `on_block(store, block)` whenever a block `block: SignedBeaconBlock` is received +- `on_attestation(store, attestation)` whenever an attestation `attestation` is received + +Any of the above handlers that trigger an unhandled exception (e.g. a failed assert or an out-of-range list access) are considered invalid. Invalid calls to handlers must not modify `store`. + +*Notes*: + +1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). +2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. +3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](./validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. +4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. +5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). + +### Configuration + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `SAFE_SLOTS_TO_UPDATE_JUSTIFIED` | `2**3` (= 8) | slots | 96 seconds | + +### Helpers + +#### `LatestMessage` + +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root +``` + +#### `Store` + +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) +``` + +#### `get_forkchoice_store` + +The provided anchor-state will be regarded as a trusted state, to not roll back beyond. +This should be the genesis state for a full client. + +*Note* With regards to fork choice, block headers are interchangeable with blocks. The spec is likely to move to headers for reduced overhead in test vectors and better encapsulation. Full implementations store blocks as part of their database and will often use full blocks when dealing with production fork choice. + +```python +def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: + assert anchor_block.state_root == hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: copy(anchor_block)}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, + ) +``` + +#### `get_slots_since_genesis` + +```python +def get_slots_since_genesis(store: Store) -> int: + return (store.time - store.genesis_time) // SECONDS_PER_SLOT +``` + +#### `get_current_slot` + +```python +def get_current_slot(store: Store) -> Slot: + return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) +``` + +#### `compute_slots_since_epoch_start` + +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` + +#### `get_ancestor` + +```python +def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: + block = store.blocks[root] + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` + +#### `get_latest_attesting_balance` + +```python +def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + )) +``` + +#### `filter_block_tree` + +```python +def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: + block = store.blocks[block_root] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == block_root + ] + + # If any children branches contain expected finalized/justified checkpoints, + # add to filtered block-tree and signal viability to parent. + if any(children): + filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + # If leaf block, check finalized/justified checkpoints as matching latest. + head_state = store.block_states[block_root] + + correct_justified = ( + store.justified_checkpoint.epoch == GENESIS_EPOCH + or head_state.current_justified_checkpoint == store.justified_checkpoint + ) + correct_finalized = ( + store.finalized_checkpoint.epoch == GENESIS_EPOCH + or head_state.finalized_checkpoint == store.finalized_checkpoint + ) + # If expected finalized/justified, add to viable block-tree and signal viability to parent. + if correct_justified and correct_finalized: + blocks[block_root] = block + return True + + # Otherwise, branch not viable + return False +``` + +#### `get_filtered_block_tree` + +```python +def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: + """ + Retrieve a filtered block tree from ``store``, only returning branches + whose leaf state's justified/finalized info agrees with that in ``store``. + """ + base = store.justified_checkpoint.root + blocks: Dict[Root, BeaconBlock] = {} + filter_block_tree(store, base, blocks) + return blocks +``` + +#### `get_head` + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].parent_root == head + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) +``` + +#### `should_update_justified_checkpoint` + +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: + return False + + return True +``` + +#### `on_attestation` helpers + +##### `validate_on_attestation` + +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # LMD vote must be consistent with FFG vote target + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` + +##### `store_target_checkpoint_state` + +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = copy(store.block_states[target.root]) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` + +##### `update_latest_messages` + +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` + + +### Handlers + +#### `on_tick` + +```python +def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time + store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint +``` + +#### `on_block` + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint +``` + +#### `on_attestation` + +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) +``` diff --git a/tools/analyzers/specdocs/data/phase0/p2p-interface.md b/tools/analyzers/specdocs/data/phase0/p2p-interface.md new file mode 100644 index 00000000000..e9e8e092c47 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/p2p-interface.md @@ -0,0 +1,1526 @@ +# Ethereum 2.0 networking specification + +This document contains the networking specification for Ethereum 2.0 clients. + +It consists of four main sections: + +1. A specification of the network fundamentals. +2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. +3. The rationale and further explanation for the design choices made in the previous two sections. +4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. + +## Table of contents + + + + +- [Network fundamentals](#network-fundamentals) + - [Transport](#transport) + - [Encryption and identification](#encryption-and-identification) + - [Protocol Negotiation](#protocol-negotiation) + - [Multiplexing](#multiplexing) +- [Eth2 network interaction domains](#eth2-network-interaction-domains) + - [Configuration](#configuration) + - [MetaData](#metadata) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) + - [`beacon_aggregate_and_proof`](#beacon_aggregate_and_proof) + - [`voluntary_exit`](#voluntary_exit) + - [`proposer_slashing`](#proposer_slashing) + - [`attester_slashing`](#attester_slashing) + - [Attestation subnets](#attestation-subnets) + - [`beacon_attestation_{subnet_id}`](#beacon_attestation_subnet_id) + - [Attestations and Aggregation](#attestations-and-aggregation) + - [Encodings](#encodings) + - [The Req/Resp domain](#the-reqresp-domain) + - [Protocol identification](#protocol-identification) + - [Req/Resp interaction](#reqresp-interaction) + - [Requesting side](#requesting-side) + - [Responding side](#responding-side) + - [Encoding strategies](#encoding-strategies) + - [SSZ-snappy encoding strategy](#ssz-snappy-encoding-strategy) + - [Messages](#messages) + - [Status](#status) + - [Goodbye](#goodbye) + - [BeaconBlocksByRange](#beaconblocksbyrange) + - [BeaconBlocksByRoot](#beaconblocksbyroot) + - [Ping](#ping) + - [GetMetaData](#getmetadata) + - [The discovery domain: discv5](#the-discovery-domain-discv5) + - [Integration into libp2p stacks](#integration-into-libp2p-stacks) + - [ENR structure](#enr-structure) + - [Attestation subnet bitfield](#attestation-subnet-bitfield) + - [`eth2` field](#eth2-field) +- [Design decision rationale](#design-decision-rationale) + - [Transport](#transport-1) + - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) + - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) + - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) + - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) + - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) + - [Multiplexing](#multiplexing-1) + - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) + - [Protocol Negotiation](#protocol-negotiation-1) + - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) + - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) + - [Encryption](#encryption) + - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) + - [Why are we using Noise?](#why-are-we-using-noise) + - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) + - [Gossipsub](#gossipsub) + - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) + - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) + - [How do we upgrade gossip channels (e.g. changes in encoding, compression)?](#how-do-we-upgrade-gossip-channels-eg-changes-in-encoding-compression) + - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair) + - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes) + - [Why are we using the `StrictNoSign` signature policy?](#why-are-we-using-the-strictnosign-signature-policy) + - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id) + - [Why are these specific gossip parameters chosen?](#why-are-these-specific-gossip-parameters-chosen) + - [Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets?](#why-is-there-maximum_gossip_clock_disparity-when-validating-slot-ranges-of-messages-in-gossip-subnets) + - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets) + - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots) + - [Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s?](#why-are-aggregate-attestations-broadcast-to-the-global-topic-as-aggregateandproofs-rather-than-just-as-attestations) + - [Why are we sending entire objects in the pubsub and not just hashes?](#why-are-we-sending-entire-objects-in-the-pubsub-and-not-just-hashes) + - [Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc?](#should-clients-gossip-blocks-if-they-cannot-validate-the-proposer-signature-due-to-not-yet-being-synced-not-knowing-the-head-block-etc) + - [How are we going to discover peers in a gossipsub topic?](#how-are-we-going-to-discover-peers-in-a-gossipsub-topic) + - [How should fork version be used in practice?](#how-should-fork-version-be-used-in-practice) + - [Req/Resp](#reqresp) + - [Why segregate requests into dedicated protocol IDs?](#why-segregate-requests-into-dedicated-protocol-ids) + - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding) + - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver) + - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc) + - [Why do we allow empty responses in block requests?](#why-do-we-allow-empty-responses-in-block-requests) + - [Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from?](#why-does-beaconblocksbyrange-let-the-server-choose-which-branch-to-send-blocks-from) + - [What's the effect of empty slots on the sync algorithm?](#whats-the-effect-of-empty-slots-on-the-sync-algorithm) + - [Discovery](#discovery) + - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht) + - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs) + - [Why do we not form ENRs and find peers until genesis block/state is known?](#why-do-we-not-form-enrs-and-find-peers-until-genesis-blockstate-is-known) + - [Compression/Encoding](#compressionencoding) + - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding) + - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) + - [Why are we using Snappy for compression?](#why-are-we-using-snappy-for-compression) + - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) + - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) +- [libp2p implementations matrix](#libp2p-implementations-matrix) + + + + +# Network fundamentals + +This section outlines the specification for the networking stack in Ethereum 2.0 clients. + +## Transport + +Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), +we hereby define a profile for basic interoperability. + +All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). +The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). + +Clients must support listening on at least one of IPv4 or IPv6. +Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of +Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. + +All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. +(Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) + +Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), +MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. + +## Encryption and identification + +The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure +channel handshake with `secp256k1` identities will be used for encryption. + +As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. + +## Protocol Negotiation + +Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. + +Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) +and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. +Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. + +## Multiplexing + +During connection bootstrapping, libp2p dynamically negotiates a mutually supported multiplexing method to conduct parallel conversations. +This applies to transports that are natively incapable of multiplexing (e.g. TCP, WebSockets, WebRTC), +and is omitted for capable transports (e.g. QUIC). + +Two multiplexers are commonplace in libp2p implementations: +[mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). +Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. + +Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) +and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). +If both are supported by the client, yamux MUST take precedence during negotiation. +See the [Rationale](#design-decision-rationale) section below for tradeoffs. + +# Eth2 network interaction domains + +## Configuration + +This section outlines constants that are used in this spec. + +| Name | Value | Description | +|---|---|---| +| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | +| `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | +| `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | +| `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | +| `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | +| `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | +| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500ms` | The maximum milliseconds of clock disparity assumed between honest nodes. | +| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `0x00000000` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | +| `MESSAGE_DOMAIN_VALID_SNAPPY` | `0x01000000` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | + + +## MetaData + +Clients MUST locally store the following `MetaData`: + +``` +( + seq_number: uint64 + attnets: Bitvector[ATTESTATION_SUBNET_COUNT] +) +``` + +Where + +- `seq_number` is a `uint64` starting at `0` used to version the node's metadata. + If any other field in the local `MetaData` changes, the node MUST increment `seq_number` by 1. +- `attnets` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. + +*Note*: `MetaData.seq_number` is used for versioning of the node's metadata, +is entirely independent of the ENR sequence number, +and will in most cases be out of sync with the ENR sequence number. + +## The gossip domain: gossipsub + +Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md) libp2p Protocol +including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. + +**Protocol ID:** `/meshsub/1.1.0` + +**Gossipsub Parameters** + +The following gossipsub [parameters](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md#parameters) will be used: + +- `D` (topic stable mesh target count): 8 +- `D_low` (topic stable mesh low watermark): 6 +- `D_high` (topic stable mesh high watermark): 12 +- `D_lazy` (gossip target): 6 +- `heartbeat_interval` (frequency of heartbeat, seconds): 0.7 +- `fanout_ttl` (ttl for fanout maps for topics we are not subscribed to but have published to, seconds): 60 +- `mcache_len` (number of windows to retain full messages in cache for `IWANT` responses): 6 +- `mcache_gossip` (number of windows to gossip about): 3 +- `seen_ttl` (number of heartbeat intervals to retain message IDs): 550 + +*Note*: Gossipsub v1.1 introduces a number of +[additional parameters](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#overview-of-new-parameters) +for peer scoring and other attack mitigations. +These are currently under investigation and will be spec'd and released to mainnet when they are ready. + +### Topics and messages + +Topics are plain UTF-8 strings and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). +Topic strings have form: `/eth2/ForkDigestValue/Name/Encoding`. +This defines both the type of data being sent on the topic and how the data field of the message is encoded. + +- `ForkDigestValue` - the lowercase hex-encoded (no "0x" prefix) bytes of `compute_fork_digest(current_fork_version, genesis_validators_root)` where + - `current_fork_version` is the fork version of the epoch of the message to be sent on the topic + - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` +- `Name` - see table below +- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. + See the [Encodings](#Encodings) section for further details. + +*Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. +Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. + +Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. +Clients MUST reject (fail validation) messages that are over this size limit. +Likewise, clients MUST NOT emit or propagate messages larger than this limit. + +The optional `from` (1), `seqno` (3), `signature` (5) and `key` (6) protobuf fields are omitted from the message, +since messages are identified by content, anonymous, and signed where necessary in the application layer. +Starting from Gossipsub v1.1, clients MUST enforce this by applying the `StrictNoSign` +[signature policy](https://github.com/libp2p/specs/blob/master/pubsub/README.md#signature-policy-options). + +The `message-id` of a gossipsub message MUST be the following 20 byte value computed from the message data: +* If `message.data` has a valid snappy decompression, set `message-id` to the first 20 bytes of the `SHA256` hash of + the concatenation of `MESSAGE_DOMAIN_VALID_SNAPPY` with the snappy decompressed message data, + i.e. `SHA256(MESSAGE_DOMAIN_VALID_SNAPPY + snappy_decompress(message.data))[:20]`. +* Otherwise, set `message-id` to the first 20 bytes of the `SHA256` hash of + the concatenation of `MESSAGE_DOMAIN_INVALID_SNAPPY` with the raw message data, + i.e. `SHA256(MESSAGE_DOMAIN_INVALID_SNAPPY + message.data)[:20]`. + +*Note*: The above logic handles two exceptional cases: +(1) multiple snappy `data` can decompress to the same value, +and (2) some message `data` can fail to snappy decompress altogether. + +The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: + +| Name | Message Type | +|----------------------------------|---------------------------| +| `beacon_block` | `SignedBeaconBlock` | +| `beacon_aggregate_and_proof` | `SignedAggregateAndProof` | +| `beacon_attestation_{subnet_id}` | `Attestation` | +| `voluntary_exit` | `SignedVoluntaryExit` | +| `proposer_slashing` | `ProposerSlashing` | +| `attester_slashing` | `AttesterSlashing` | + +Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. + +When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. + +For any optional queueing, clients SHOULD maintain maximum queue sizes to avoid DoS vectors. + +Gossipsub v1.1 introduces [Extended Validators](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#extended-validators) +for the application to aid in the gossipsub peer-scoring scheme. +We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are application specific validations. +If all validations pass, return `ACCEPT`. +If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. + +#### Global topics + +There are two primary global topics used to propagate beacon blocks (`beacon_block`) +and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the network. + +There are three additional global topics that are used to propagate lower frequency validator messages +(`voluntary_exit`, `proposer_slashing`, and `attester_slashing`). + +##### `beacon_block` + +The `beacon_block` topic is used solely for propagating new signed beacon blocks to all nodes on the networks. +Signed blocks are sent in their entirety. + +The following validations MUST pass before forwarding the `signed_beacon_block` on the network. +- _[IGNORE]_ The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. validate that `signed_beacon_block.message.slot <= current_slot` + (a client MAY queue future blocks for processing at the appropriate slot). +- _[IGNORE]_ The block is from a slot greater than the latest finalized slot -- + i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` + (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). +- _[IGNORE]_ The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. +- _[REJECT]_ The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. +- _[IGNORE]_ The block's parent (defined by `block.parent_root`) has been seen + (via both gossip and non-gossip sources) + (a client MAY queue blocks for processing once the parent block is retrieved). +- _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. +- _[REJECT]_ The block is from a higher slot than its parent. +- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. + `get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + == store.finalized_checkpoint.root` +- _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot + in the context of the current shuffling (defined by `parent_root`/`slot`). + If the `proposer_index` cannot immediately be verified against the expected shuffling, + the block MAY be queued for later processing while proposers for the block's branch are calculated -- + in such a case _do not_ `REJECT`, instead `IGNORE` this message. + +##### `beacon_aggregate_and_proof` + +The `beacon_aggregate_and_proof` topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) +to subscribing nodes (typically validators) to be included in future blocks. + +The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. +(We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) +- _[IGNORE]_ `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` + (a client MAY queue future aggregates for processing at the appropriate slot). +- _[REJECT]_ The aggregate attestation's epoch matches its target -- i.e. `aggregate.data.target.epoch == + compute_epoch_at_slot(aggregate.data.slot)` +- _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator + with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. +- _[REJECT]_ The attestation has participants -- + that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. +- _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- + i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. +- _[REJECT]_ The aggregator's validator index is within the committee -- + i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. +- _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature + of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. +- _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. +- _[REJECT]_ The signature of `aggregate` is valid. +- _[IGNORE]_ The block being voted for (`aggregate.data.beacon_block_root`) has been seen + (via both gossip and non-gossip sources) + (a client MAY queue aggregates for processing once block is retrieved). +- _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. +- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `aggregate.data.beacon_block_root` -- i.e. + `get_ancestor(store, aggregate.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + == store.finalized_checkpoint.root` + + +##### `voluntary_exit` + +The `voluntary_exit` topic is used solely for propagating signed voluntary validator exits to proposers on the network. +Signed voluntary exits are sent in their entirety. + +The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network. +- _[IGNORE]_ The voluntary exit is the first valid voluntary exit received + for the validator with index `signed_voluntary_exit.message.validator_index`. +- _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. + +##### `proposer_slashing` + +The `proposer_slashing` topic is used solely for propagating proposer slashings to proposers on the network. +Proposer slashings are sent in their entirety. + +The following validations MUST pass before forwarding the `proposer_slashing` on to the network. +- _[IGNORE]_ The proposer slashing is the first valid proposer slashing received + for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. +- _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. + +##### `attester_slashing` + +The `attester_slashing` topic is used solely for propagating attester slashings to proposers on the network. +Attester slashings are sent in their entirety. + +Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. +- _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation + has not yet been seen in any prior `attester_slashing` + (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, + verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). +- _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. + +#### Attestation subnets + +Attestation subnets are used to propagate unaggregated attestations to subsections of the network. + +##### `beacon_attestation_{subnet_id}` + +The `beacon_attestation_{subnet_id}` topics are used to propagate unaggregated attestations +to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. + +The following validations MUST pass before forwarding the `attestation` on the subnet. +- _[REJECT]_ The committee index is within the expected range -- i.e. `data.index < get_committee_count_per_slot(state, data.target.epoch)`. +- _[REJECT]_ The attestation is for the correct subnet -- + i.e. `compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index) == subnet_id`, + where `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)`, + which may be pre-computed along with the committee information for the signature check. +- _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots + (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- + i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` + (a client MAY queue future attestations for processing at the appropriate slot). +- _[REJECT]_ The attestation's epoch matches its target -- i.e. `attestation.data.target.epoch == + compute_epoch_at_slot(attestation.data.slot)` +- _[REJECT]_ The attestation is unaggregated -- + that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit]) == 1`, i.e. exactly 1 bit is set). +- _[REJECT]_ The number of aggregation bits matches the committee size -- i.e. + `len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index))`. +- _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet + that has an identical `attestation.data.target.epoch` and participating validator index. +- _[REJECT]_ The signature of `attestation` is valid. +- _[IGNORE]_ The block being voted for (`attestation.data.beacon_block_root`) has been seen + (via both gossip and non-gossip sources) + (a client MAY queue attestations for processing once block is retrieved). +- _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. +- _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. + `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root` +- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `attestation.data.beacon_block_root` -- i.e. + `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + == store.finalized_checkpoint.root` + + + +#### Attestations and Aggregation + +Attestation broadcasting is grouped into subnets defined by a topic. +The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. +The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. +`beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees (future Eth2 upgrade). +The subnets are rotated through with `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)` subnets per slot. + +Unaggregated attestations are sent as `Attestation`s to the subnet topic, +`beacon_attestation_{compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index)}` as `Attestation`s. + +Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. + +### Encodings + +Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. + +- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. + Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, + the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` + that has been SSZ-encoded and then compressed with Snappy. + +Snappy has two formats: "block" and "frames" (streaming). +Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) +so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. + +Implementations MUST use a single encoding for gossip. +Changing an encoding will require coordination between participating implementations. + +## The Req/Resp domain + +### Protocol identification + +Each message type is segregated into its own libp2p protocol ID, which is a case-sensitive UTF-8 string of the form: + +``` +/ProtocolPrefix/MessageName/SchemaVersion/Encoding +``` + +With: + +- `ProtocolPrefix` - messages are grouped into families identified by a shared libp2p protocol name prefix. + In this case, we use `/eth2/beacon_chain/req`. +- `MessageName` - each request is identified by a name consisting of English alphabet, digits and underscores (`_`). +- `SchemaVersion` - an ordinal version number (e.g. 1, 2, 3…). + Each schema is versioned to facilitate backward and forward-compatibility when possible. +- `Encoding` - while the schema defines the data types in more abstract terms, + the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. + See the [Encodings](#Encoding-strategies) section for further details. + +This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` +to handle the request type, version, and encoding negotiation before establishing the underlying streams. + +### Req/Resp interaction + +We use ONE stream PER request/response interaction. +Streams are closed when the interaction finishes, whether in success or in error. + +Request/response messages MUST adhere to the encoding specified in the protocol name and follow this structure (relaxed BNF grammar): + +``` +request ::= | +response ::= * +response_chunk ::= | | +result ::= “0” | “1” | “2” | [“128” ... ”255”] +``` + +The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. +Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; +however, certain encodings like SSZ do, for added security. + +A `response` is formed by zero or more `response_chunk`s. +Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. +All other response types (non-Lists) send a single `response_chunk`. + +For both `request`s and `response`s, the `encoding-dependent-header` MUST be valid, +and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. +This includes type-specific bounds on payload size for some encoding strategies. +Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. + +Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. +Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. + +#### Requesting side + +Once a new stream with the protocol ID for the request type has been negotiated, the full request message SHOULD be sent immediately. +The request MUST be encoded according to the encoding strategy. + +The requester MUST close the write side of the stream once it finishes writing the request message. +At this point, the stream will be half-closed. + +The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). +On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. + +If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. + +A requester SHOULD read from the stream until either: +1. An error result is received in one of the chunks (the error payload MAY be read before stopping). +2. The responder closes the stream. +3. Any part of the `response_chunk` fails validation. +4. The maximum number of requested chunks are read. + +For requests consisting of a single valid `response_chunk`, +the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. + +#### Responding side + +Once a new stream with the protocol ID for the request type has been negotiated, +the responder SHOULD process the incoming request and MUST validate it before processing it. +Request processing and validation MUST be done according to the encoding strategy, until EOF (denoting stream half-closure by the requester). + +The responder MUST: + +1. Use the encoding strategy to read the optional header. +2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). + Should this not be the case, it should be treated as a failure. +3. Deserialize the expected type, and process the request. +4. Write the response which may consist of zero or more `response_chunk`s (result, optional header, payload). +5. Close their write side of the stream. At this point, the stream will be fully closed. + +If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. +Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. + +The entire request should be read in no more than `RESP_TIMEOUT`. +Upon a timeout, the responder SHOULD reset the stream. + +The responder SHOULD send a `response_chunk` promptly. +Chunks start with a **single-byte** response code which determines the contents of the `response_chunk` (`result` particle in the BNF grammar above). +For multiple chunks, only the last chunk is allowed to have a non-zero error code (i.e. The chunk stream is terminated once an error occurs). + +The response code can have one of the following values, encoded as a single unsigned byte: + +- 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. +- 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. + The response payload adheres to the `ErrorMessage` schema (described below). +- 2: **ServerError** -- the responder encountered an error while processing the request. + The response payload adheres to the `ErrorMessage` schema (described below). + +Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. + +The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognized expressly. + +The `ErrorMessage` schema is: + +``` +( + error_message: List[byte, 256] +) +``` + +*Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). +Clients MUST treat as valid any byte sequences. + +### Encoding strategies + +The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. +Only one value is possible at this time: + +- `ssz_snappy`: The contents are first [SSZ-encoded](../../ssz/simple-serialize.md) + and then compressed with [Snappy](https://github.com/google/snappy) frames compression. + For objects containing a single field, only the field is SSZ-encoded not a container with a single field. + For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. + This encoding type MUST be supported by all clients. + +#### SSZ-snappy encoding strategy + +The [SimpleSerialize (SSZ) specification](../../ssz/simple-serialize.md) outlines how objects are SSZ-encoded. + +To achieve snappy encoding on top of SSZ, we feed the serialized form of the object to the Snappy compressor on encoding. +The inverse happens on decoding. + +Snappy has two formats: "block" and "frames" (streaming). +To support large requests and response chunks, snappy-framing is used. + +Since snappy frame contents [have a maximum size of `65536` bytes](https://github.com/google/snappy/blob/master/framing_format.txt#L104) +and frame headers are just `identifier (1) + checksum (4)` bytes, the expected buffering of a single frame is acceptable. + +**Encoding-dependent header:** Req/Resp protocols using the `ssz_snappy` encoding strategy MUST encode the length of the raw SSZ bytes, +encoded as an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). + +*Writing*: By first computing and writing the SSZ byte length, the SSZ encoder can then directly write the chunk contents to the stream. +When Snappy is applied, it can be passed through a buffered Snappy writer to compress frame by frame. + +*Reading*: After reading the expected SSZ byte length, the SSZ decoder can directly read the contents from the stream. +When snappy is applied, it can be passed through a buffered Snappy reader to decompress frame by frame. + +Before reading the payload, the header MUST be validated: +- The unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes, which is sufficient for any `uint64`. +- The length-prefix is within the expected [size bounds derived from the payload SSZ type](#what-are-ssz-type-size-bounds). + +After reading a valid header, the payload MAY be read, while maintaining the size constraints from the header. + +A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length-prefix `n` from the header. +- For `ssz_snappy` this is: `32 + n + n // 6`. + This is considered the [worst-case compression result](https://github.com/google/snappy/blob/537f4ad6240e586970fe554614542e9717df7902/snappy.cc#L98) by Snappy. + +A reader SHOULD consider the following cases as invalid input: +- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected if more bytes are read than required. +- An early EOF, before fully reading the declared length-prefix worth of SSZ bytes. + +In case of an invalid input (header or payload), a reader MUST: +- From requests: send back an error message, response code `InvalidRequest`. The request itself is ignored. +- From responses: ignore the response, the response MUST be considered bad server behavior. + +All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. + +Responses that are SSZ-lists (for example `List[SignedBeaconBlock, ...]`) send their +constituents individually as `response_chunk`s. For example, the +`List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. +Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. + +### Messages + +#### Status + +**Protocol ID:** ``/eth2/beacon_chain/req/status/1/`` + +Request, Response Content: +``` +( + fork_digest: ForkDigest + finalized_root: Root + finalized_epoch: Epoch + head_root: Root + head_slot: Slot +) +``` +The fields are, as seen by the client at the time of sending the message: + +- `fork_digest`: The node's `ForkDigest` (`compute_fork_digest(current_fork_version, genesis_validators_root)`) where + - `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time + (not necessarily the epoch to which the node is sync) + - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` +- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block + (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). +- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. +- `head_root`: The `hash_tree_root` root of the current head block (`BeaconBlock`). +- `head_slot`: The slot of the block corresponding to the `head_root`. + +The dialing client MUST send a `Status` request upon connection. + +The request/response MUST be encoded as an SSZ-container. + +The response MUST consist of a single `response_chunk`. + +Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: + +1. If `fork_digest` does not match the node's local `fork_digest`, since the client’s chain is on another fork. +2. If the (`finalized_root`, `finalized_epoch`) shared by the peer is not in the client's chain at the expected epoch. + For example, if Peer 1 sends (root, epoch) of (A, 5) and Peer 2 sends (B, 3) but Peer 1 has root C at epoch 3, + then Peer 1 would disconnect because it knows that their chains are irreparably disjoint. + +Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) +SHOULD request beacon blocks from its counterparty via the `BeaconBlocksByRange` request. + +*Note*: Under abnormal network condition or after some rounds of `BeaconBlocksByRange` requests, +the client might need to send `Status` request again to learn if the peer has a higher head. +Implementers are free to implement such behavior in their own way. + +#### Goodbye + +**Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/`` + +Request, Response Content: +``` +( + uint64 +) +``` +Client MAY send goodbye messages upon disconnection. The reason field MAY be one of the following values: + +- 1: Client shut down. +- 2: Irrelevant network. +- 3: Fault/error. + +Clients MAY use reason codes above `128` to indicate alternative, erroneous request-specific responses. + +The range `[4, 127]` is RESERVED for future usage. + +The request/response MUST be encoded as a single SSZ-field. + +The response MUST consist of a single `response_chunk`. + +#### BeaconBlocksByRange + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/1/` + +Request Content: +``` +( + start_slot: Slot + count: uint64 + step: uint64 +) +``` + +Response Content: +``` +( + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] +) +``` + +Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. +`step` defines the slot increment between blocks. +For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. +In cases where a slot is empty for a given slot number, no block is returned. +For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. +A request MUST NOT have a 0 slot increment, i.e. `step >= 1`. + +`BeaconBlocksByRange` is primarily used to sync historical blocks. + +The request MUST be encoded as an SSZ-container. + +The response MUST consist of zero or more `response_chunk`. +Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. + +Clients MUST keep a record of signed blocks seen since the start of the weak subjectivity period +and MUST support serving requests of blocks up to their own `head_block_root`. + +Clients MUST respond with at least the first block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOCKS` blocks. + +The following blocks, where they exist, MUST be sent in consecutive order. + +Clients MAY limit the number of blocks in the response. + +The response MUST contain no more than `count` blocks. + +Clients MUST respond with blocks from their view of the current fork choice +-- that is, blocks from the single chain defined by the current head. +Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. + +Clients MUST respond with blocks that are consistent from a single chain within the context of the request. +This applies to any `step` value. +In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. + +After the initial block, clients MAY stop in the process of responding +if their fork choice changes the view of the chain in the context of the request. + +#### BeaconBlocksByRoot + +**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/1/` + +Request Content: + +``` +( + List[Root, MAX_REQUEST_BLOCKS] +) +``` + +Response Content: + +``` +( + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] +) +``` + +Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). +The response is a list of `SignedBeaconBlock` whose length is less than or equal to the number of requested blocks. +It may be less in the case that the responding peer is missing blocks. + +No more than `MAX_REQUEST_BLOCKS` may be requested at a time. + +`BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). + +The request MUST be encoded as an SSZ-field. + +The response MUST consist of zero or more `response_chunk`. +Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. + +Clients MUST support requesting blocks since the latest finalized epoch. + +Clients MUST respond with at least one block, if they have it. +Clients MAY limit the number of blocks in the response. + +#### Ping + +**Protocol ID:** `/eth2/beacon_chain/req/ping/1/` + +Request Content: + +``` +( + uint64 +) +``` + +Response Content: + +``` +( + uint64 +) +``` + +Sent intermittently, the `Ping` protocol checks liveness of connected peers. +Peers request and respond with their local metadata sequence number (`MetaData.seq_number`). + +If the peer does not respond to the `Ping` request, the client MAY disconnect from the peer. + +A client can then determine if their local record of a peer's MetaData is up to date +and MAY request an updated version via the `MetaData` RPC method if not. + +The request MUST be encoded as an SSZ-field. + +The response MUST consist of a single `response_chunk`. + +#### GetMetaData + +**Protocol ID:** `/eth2/beacon_chain/req/metadata/1/` + +No Request Content. + +Response Content: + +``` +( + MetaData +) +``` + +Requests the MetaData of a peer. +The request opens and negotiates the stream without sending any request content. +Once established the receiving peer responds with +it's local most up-to-date MetaData. + +The response MUST be encoded as an SSZ-container. + +The response MUST consist of a single `response_chunk`. + +## The discovery domain: discv5 + +Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) (Protocol version v5.1) is used for peer discovery. + +`discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. +`discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. + +### Integration into libp2p stacks + +`discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor +to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) +and [peer routing](https://github.com/libp2p/go-libp2p-core/blob/master/routing/routing.go#L36-L44) abstractions and interfaces (go-libp2p links provided). + +Inputs to operations include peer IDs (when locating a specific peer) or capabilities (when searching for peers with a specific capability), +and the outputs will be multiaddrs converted from the ENR records returned by the discv5 backend. + +This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. + +### ENR structure + +The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the following entries +(exclusive of the sequence number and signature, which MUST be present in an ENR): + +- The compressed secp256k1 publickey, 33 bytes (`secp256k1` field). + +The ENR MAY contain the following entries: + +- An IPv4 address (`ip` field) and/or IPv6 address (`ip6` field). +- A TCP port (`tcp` field) representing the local libp2p listening port. +- A UDP port (`udp` field) representing the local discv5 listening port. + +Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). + +#### Attestation subnet bitfield + +The ENR `attnets` entry signifies the attestation subnet bitfield with the following form +to more easily discover peers participating in particular attestation gossip subnets. + +| Key | Value | +|:-------------|:-------------------------------------------------| +| `attnets` | SSZ `Bitvector[ATTESTATION_SUBNET_COUNT]` | + +If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `attnets` entry with the same value as `MetaData.attnets`. + +If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. + +#### `eth2` field + +ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, +and next fork epoch to ensure connections are made with peers on the intended eth2 network. + +| Key | Value | +|:-------------|:--------------------| +| `eth2` | SSZ `ENRForkID` | + +Specifically, the value of the `eth2` key MUST be the following SSZ encoded object (`ENRForkID`) + +``` +( + fork_digest: ForkDigest + next_fork_version: Version + next_fork_epoch: Epoch +) +``` + +where the fields of `ENRForkID` are defined as + +* `fork_digest` is `compute_fork_digest(current_fork_version, genesis_validators_root)` where + * `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time + (not necessarily the epoch to which the node is sync) + * `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` +* `next_fork_version` is the fork version corresponding to the next planned hard fork at a future epoch. + If no future fork is planned, set `next_fork_version = current_fork_version` to signal this fact +* `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. + If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact + +*Note*: `fork_digest` is composed of values that are not known until the genesis block/state are available. +Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. +One notable exception to this rule is the distribution of bootnode ENRs prior to genesis. +In this case, bootnode ENRs SHOULD be initially distributed with `eth2` field set as +`ENRForkID(fork_digest=compute_fork_digest(GENESIS_FORK_VERSION, b'\x00'*32), next_fork_version=GENESIS_FORK_VERSION, next_fork_epoch=FAR_FUTURE_EPOCH)`. +After genesis values are known, the bootnodes SHOULD update ENRs to participate in normal discovery operations. + +Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `next_fork_epoch` that match local values. + +Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. +Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, +these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. + +# Design decision rationale + +## Transport + +### Why are we defining specific transports? + +libp2p peers can listen on multiple transports concurrently, and these can change over time. +Multiaddrs encode not only the address but also the transport to be used to dial. + +Due to this dynamic nature, agreeing on specific transports like TCP, QUIC, or WebSockets on paper becomes irrelevant. + +However, it is useful to define a minimum baseline for interoperability purposes. + +### Can clients support other transports/handshakes than the ones mandated by the spec? + +Clients may support other transports such as libp2p QUIC, WebSockets, and WebRTC transports, if available in the language of choice. +While interoperability shall not be harmed by lack of such support, the advantages are desirable: + +- Better latency, performance, and other QoS characteristics (QUIC). +- Paving the way for interfacing with future light clients (WebSockets, WebRTC). + +The libp2p QUIC transport inherently relies on TLS 1.3 per requirement in section 7 +of the [QUIC protocol specification](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7) +and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf-quic-tls-22). + +The usage of one handshake procedure or the other shall be transparent to the Eth2 application layer, +once the libp2p Host/Node object has been configured appropriately. + +### What are the advantages of using TCP/QUIC/Websockets? + +TCP is a reliable, ordered, full-duplex, congestion-controlled network protocol that powers much of the Internet as we know it today. +HTTP/1.1 and HTTP/2 run atop TCP. + +QUIC is a new protocol that’s in the final stages of specification by the IETF QUIC WG. +It emerged from Google’s SPDY experiment. The QUIC transport is undoubtedly promising. +It’s UDP-based yet reliable, ordered, multiplexed, natively secure (TLS 1.3), reduces latency vs. TCP, +and offers stream-level and connection-level congestion control (thus removing head-of-line blocking), +0-RTT connection establishment, and endpoint migration, amongst other features. +UDP also has better NAT traversal properties than TCP—something we desperately pursue in peer-to-peer networks. + +QUIC is being adopted as the underlying protocol for HTTP/3. +This has the potential to award us censorship resistance via deep packet inspection for free. +Provided that we use the same port numbers and encryption mechanisms as HTTP/3, our traffic may be indistinguishable from standard web traffic, +and we may only become subject to standard IP-based firewall filtering—something we can counteract via other mechanisms. + +WebSockets and/or WebRTC transports are necessary for interaction with browsers, +and will become increasingly important as we incorporate browser-based light clients to the Eth2 network. + +### Why do we not just support a single transport? + +Networks evolve. +Hardcoding design decisions leads to ossification, preventing the evolution of networks alongside the state of the art. +Introducing changes on an ossified protocol is very costly, and sometimes, downright impracticable without causing undesirable breakage. + +Modeling for upgradeability and dynamic transport selection from the get-go lays the foundation for a future-proof stack. + +Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments +(e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), +without the need for proxying or trust delegation to servers. + +### Why are we not using QUIC from the start? + +The QUIC standard is still not finalized (at working draft 22 at the time of writing), +and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). +One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). + +*Note*: [TLS 1.3 is a prerequisite of the QUIC transport](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), +although an experiment exists to integrate Noise as the QUIC crypto layer: [nQUIC](https://eprint.iacr.org/2019/028). + +On the other hand, TLS 1.3 is the newest, simplified iteration of TLS. +Old, insecure, obsolete ciphers and algorithms have been removed, adopting Ed25519 as the sole ECDH key agreement function. +Handshakes are faster, 1-RTT data is supported, and session resumption is a reality, amongst other features. + +## Multiplexing + +### Why are we using mplex/yamux? + +[Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) is a multiplexer invented by Hashicorp that supports stream-level congestion control. +Implementations exist in a limited set of languages, and it’s not a trivial piece to develop. + +Conscious of that, the libp2p community conceptualized [mplex](https://github.com/libp2p/specs/blob/master/mplex/README.md) +as a simple, minimal multiplexer for usage with libp2p. +It does not support stream-level congestion control and is subject to head-of-line blocking. + +Overlay multiplexers are not necessary with QUIC since the protocol provides native multiplexing, +but they need to be layered atop TCP, WebSockets, and other transports that lack such support. + +## Protocol Negotiation + +### When is multiselect 2.0 due and why do we plan to migrate to it? + +multiselect 2.0 is currently being conceptualized. +The debate started [on this issue](https://github.com/libp2p/specs/pull/95), +but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. + +At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, +in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. + +We plan to eventually migrate to multiselect 2.0 because it will: + +1. Reduce round trips during connection bootstrapping and stream protocol negotiation. +2. Enable efficient one-stream-per-request interaction patterns. +3. Leverage *push data* mechanisms of underlying protocols to expedite negotiation. +4. Provide the building blocks for enhanced censorship resistance. + +### What is the difference between connection-level and stream-level protocol negotiation? + +All libp2p connections must be authenticated, encrypted, and multiplexed. +Connections using network transports unsupportive of native authentication/encryption and multiplexing (e.g. TCP) need to undergo protocol negotiation to agree on a mutually supported: + +1. authentication/encryption mechanism (such as SecIO, TLS 1.3, Noise). +2. overlay multiplexer (such as mplex, Yamux, spdystream). + +In this specification, we refer to these two as *connection-level negotiations*. +Transports supporting those features natively (such as QUIC) omit those negotiations. + +After successfully selecting a multiplexer, all subsequent I/O happens over *streams*. +When opening streams, peers pin a protocol to that stream, by conducting *stream-level protocol negotiation*. + +At present, multistream-select 1.0 is used for both types of negotiation, +but multiselect 2.0 will use dedicated mechanisms for connection bootstrapping process and stream protocol negotiation. + +## Encryption + +### Why are we not supporting SecIO? + +SecIO has been the default encryption layer for libp2p for years. +It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. + +Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, +it requires several round trips to be sound, and doesn’t support early data (0-RTT data), +a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. + +SecIO is not considered secure for the purposes of this spec. + +### Why are we using Noise? + +Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): + +> Noise is a framework for building crypto protocols. +Noise protocols support mutual and optional authentication, identity hiding, forward secrecy, zero round-trip encryption, and other advanced features. + +Noise in itself does not specify a single handshake procedure, +but provides a framework to build secure handshakes based on Diffie-Hellman key agreement with a variety of tradeoffs and guarantees. + +Noise handshakes are lightweight and simple to understand, +and are used in major cryptographic-centric projects like WireGuard, I2P, and Lightning. +[Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) +have assessed the stated security goals of several Noise handshakes with positive results. + +### Why are we using encryption at all? + +Transport level encryption secures message exchange and provides properties that are useful for privacy, safety, and censorship resistance. +These properties are derived from the following security guarantees that apply to the entire communication between two peers: + +- Peer authentication: the peer I’m talking to is really who they claim to be and who I expect them to be. +- Confidentiality: no observer can eavesdrop on the content of our messages. +- Integrity: the data has not been tampered with by a third-party while in transit. +- Non-repudiation: the originating peer cannot dispute that they sent the message. +- Depending on the chosen algorithms and mechanisms (e.g. continuous HMAC), we may obtain additional guarantees, + such as non-replayability (this byte could’ve only been sent *now;* e.g. by using continuous HMACs), + or perfect forward secrecy (in the case that a peer key is compromised, the content of a past conversation will not be compromised). + +Note that transport-level encryption is not exclusive of application-level encryption or cryptography. +Transport-level encryption secures the communication itself, +while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). + +## Gossipsub + +### Why are we using a pub/sub algorithm for block and attestation propagation? + +Pubsub is a technique to broadcast/disseminate data across a network rapidly. +Such data is packaged in fire-and-forget messages that do not require a response from every recipient. +Peers subscribed to a topic participate in the propagation of messages in that topic. + +The alternative is to maintain a fully connected mesh (all peers connected to each other 1:1), which scales poorly (O(n^2)). + +### Why are we using topics to segregate encodings, yet only support one encoding? + +For future extensibility with almost zero overhead now (besides the extra bytes in the topic name). + +### How do we upgrade gossip channels (e.g. changes in encoding, compression)? + +Changing gossipsub/broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, during a hard fork. + +When a node is preparing for upcoming tasks (e.g. validator duty lookahead) on a gossipsub topic, +the node should join the topic of the future epoch in which the task is to occur in addition to listening to the topics for the current epoch. + +### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? + +Supporting multiple topics/encodings would require the presence of relayers to translate between encodings +and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, +making the protocol more complicated and fragile. + +Gossip protocols typically remember what messages they've seen for a finite period of time-based on message identity +-- if you publish the same message again after that time has passed, +it will be re-broadcast—adding a relay delay also makes this scenario more likely. + +One can imagine that in a complicated upgrade scenario, we might have peers publishing the same message on two topics/encodings, +but the price here is pretty high in terms of overhead -- both computational and networking -- so we'd rather avoid that. + +It is permitted for clients to publish data on alternative topics as long as they also publish on the network-wide mandatory topic. + +### Why are the topics strings and not hashes? + +Topic names have a hierarchical structure. +In the future, gossipsub may support wildcard subscriptions +(e.g. subscribe to all children topics under a root prefix) by way of prefix matching. +Enforcing hashes for topic names would preclude us from leveraging such features going forward. + +No security or privacy guarantees are lost as a result of choosing plaintext topic names, +since the domain is finite anyway, and calculating a digest's preimage would be trivial. + +Furthermore, the Eth2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), +so hashing topics would bloat messages unnecessarily. + +### Why are we using the `StrictNoSign` signature policy? + +The policy omits the `from` (1), `seqno` (3), `signature` (5) and `key` (6) fields. These fields would: +- Expose origin of sender (`from`), type of sender (based on `seqno`) +- Add extra unused data to the gossip, since message IDs are based on `data`, not on the `from` and `seqno`. +- Introduce more message validation than necessary, e.g. no `signature`. + +### Why are we overriding the default libp2p pubsub `message-id`? + +For our current purposes, there is no need to address messages based on source peer, or track a message `seqno`. +By overriding the default `message-id` to use content-addressing we can filter unnecessary duplicates before hitting the application layer. + +Some examples of where messages could be duplicated: + +* A validator client connected to multiple beacon nodes publishing duplicate gossip messages +* Attestation aggregation strategies where clients partially aggregate attestations and propagate them. + Partial aggregates could be duplicated +* Clients re-publishing seen messages + +### Why are these specific gossip parameters chosen? + +- `D`, `D_low`, `D_high`, `D_lazy`: recommended defaults. +- `heartbeat_interval`: 0.7 seconds, recommended for eth2 in the [GossipSub evaluation report by Protocol Labs](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4). +- `fanout_ttl`: 60 seconds, recommended default. + Fanout is primarily used by committees publishing attestations to subnets. + This happens once per epoch per validator and the subnet changes each epoch + so there is little to gain in having a `fanout_ttl` be increased from the recommended default. +- `mcache_len`: 6, increase by one to ensure that mcache is around for long + enough for `IWANT`s to respond to `IHAVE`s in the context of the shorter + `heartbeat_interval`. If `mcache_gossip` is increased, this param should be + increased to be at least `3` (~2 seconds) more than `mcache_gossip`. +- `mcache_gossip`: 3, recommended default. This can be increased to 5 or 6 + (~4 seconds) if gossip times are longer than expected and the current window + does not provide enough responsiveness during adverse conditions. +- `seen_ttl`: `SLOTS_PER_EPOCH * SECONDS_PER_SLOT / heartbeat_interval = approx. 550`. + Attestation gossip validity is bounded by an epoch, so this is the safe max bound. + + +### Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets? + +For some gossip channels (e.g. those for Attestations and BeaconBlocks), +there are designated ranges of slots during which particular messages can be sent, +limiting messages gossiped to those that can be reasonably used in the consensus at the current time/slot. +This is to reduce optionality in DoS attacks. + +`MAXIMUM_GOSSIP_CLOCK_DISPARITY` provides some leeway in validating slot ranges to prevent the gossip network +from becoming overly brittle with respect to clock disparity. +For minimum and maximum allowable slot broadcast times, +`MAXIMUM_GOSSIP_CLOCK_DISPARITY` MUST be subtracted and added respectively, marginally extending the valid range. +Although messages can at times be eagerly gossiped to the network, +the node's fork choice prevents integration of these messages into the actual consensus until the _actual local start_ of the designated slot. + +### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets? + +Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. +The exact grouping will be dependent on more involved network tests. +This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). +The value is currently set to be equal to `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. + +### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? + +Attestations can only be included on chain within an epoch's worth of slots so this is the natural cutoff. +There is no utility to the chain to broadcast attestations older than one epoch, +and because validators have a chance to make a new attestation each epoch, +there is minimal utility to the fork choice to relay old attestations as a new latest message can soon be created by each validator. + +In addition to this, relaying attestations requires validating the attestation in the context of the `state` during which it was created. +Thus, validating arbitrarily old attestations would put additional requirements on which states need to be readily available to the node. +This would result in a higher resource burden and could serve as a DoS vector. + +### Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s? + +The dominant strategy for an individual validator is to always broadcast an aggregate containing their own attestation +to the global channel to ensure that proposers see their attestation for inclusion. +Using a private selection criteria and providing this proof of selection alongside +the gossiped aggregate ensures that this dominant strategy will not flood the global channel. + +Also, an attacker can create any number of honest-looking aggregates and broadcast them to the global pubsub channel. +Thus without some sort of proof of selection as an aggregator, the global channel can trivially be spammed. + +### Why are we sending entire objects in the pubsub and not just hashes? + +Entire objects should be sent to get the greatest propagation speeds. +If only hashes are sent, then block and attestation propagation is dependent on recursive requests from each peer. +In a hash-only scenario, peers could receive hashes without knowing who to download the actual contents from. +Sending entire objects ensures that they get propagated through the entire network. + +### Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc? + +The prohibition of unverified-block-gossiping extends to nodes that cannot verify a signature +due to not being fully synced to ensure that such (amplified) DOS attacks are not possible. + +### How are we going to discover peers in a gossipsub topic? + +In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR. + +Although this method will be sufficient for early phases of Eth2, we aim to use the more appropriate discv5 topics for this and other similar tasks in the future. +ENRs should ultimately not be used for this purpose. +They are best suited to store identity, location, and capability information, rather than more volatile advertisements. + +### How should fork version be used in practice? + +Fork versions are to be manually updated (likely via incrementing) at each hard fork. +This is to provide native domain separation for signatures as well as to aid in usefulness for identitying peers (via ENRs) +and versioning network protocols (e.g. using fork version to naturally version gossipsub topics). + +`BeaconState.genesis_validators_root` is mixed into signature and ENR fork domains (`ForkDigest`) to aid in the ease of domain separation between chains. +This allows fork versions to safely be reused across chains except for the case of contentious forks using the same genesis. +In these cases, extra care should be taken to isolate fork versions (e.g. flip a high order bit in all future versions of one of the chains). + +A node locally stores all previous and future planned fork versions along with the each fork epoch. +This allows for handling sync and processing messages starting from past forks/epochs. + +## Req/Resp + +### Why segregate requests into dedicated protocol IDs? + +Requests are segregated by protocol ID to: + +1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. + This allows the handler function for each request type to be self-contained. + For an analogy, think about how you attach HTTP handlers to a REST API server. +2. Version requests independently. + In a coarser-grained umbrella protocol, the entire protocol would have to be versioned even if just one field in a single message changed. +3. Enable clients to select the individual requests/versions they support. + It would no longer be a strict requirement to support all requests, + and clients, in principle, could support a subset of requests and variety of versions. +4. Enable flexibility and agility for clients adopting spec changes that impact the request, by signalling to peers exactly which subset of new/old requests they support. +5. Enable clients to explicitly choose backwards compatibility at the request granularity. + Without this, clients would be forced to support entire versions of the coarser request protocol. +6. Parallelise RFCs (or Eth2 EIPs). + By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently + without relying on a synchronization point to version the general top-level protocol. + 1. This has the benefit that clients can explicitly choose which RFCs to deploy + without buying into all other RFCs that may be included in that top-level version. + 2. Affording this level of granularity with a top-level protocol would imply creating as many variants + (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). +7. Allow us to simplify the payload of requests. + Request-id’s and method-ids no longer need to be sent. + The encoding/request type and version can all be handled by the framework. + +**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. +It is somewhat naïve and introduces overhead on every request when negotiating streams, +although implementation-specific optimizations are possible to save this cost. +Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. +Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol +so the additional overhead is not expected to significantly hinder this domain. + +### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? + +We are using single-use streams where each stream is closed at the end of the message. +Thus, libp2p transparently handles message delimiting in the underlying stream. +libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). +We can therefore use stream closure to mark the end of the request and response independently. + +Nevertheless, in the case of `ssz_snappy`, messages are still length-prefixed with the length of the underlying data: +* A basic reader can prepare a correctly sized buffer before reading the message +* A more advanced reader can stream-decode SSZ given the length of the SSZ data. +* Alignment with protocols like gRPC over HTTP/2 that prefix with length +* Sanity checking of message length, and enabling much stricter message length limiting based on SSZ type information, + to provide even more DOS protection than the global message length already does. + E.g. a small `Status` message does not nearly require `MAX_CHUNK_SIZE` bytes. + +[Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length (unsigned here) ints. +Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. + +### Why do we version protocol strings with ordinals instead of semver? + +Using semver for network protocols is confusing. +It is never clear what a change in a field, even if backwards compatible on deserialization, actually implies. +Network protocol agreement should be explicit. Imagine two peers: + +- Peer A supporting v1.1.1 of protocol X. +- Peer B supporting v1.1.2 of protocol X. + +These two peers should never speak to each other because the results can be unpredictable. +This is an oversimplification: imagine the same problem with a set of 10 possible versions. +We now have 10^2 (100) possible outcomes that peers need to model for. The resulting complexity is unwieldy. + +For this reason, we rely on negotiation of explicit, verbatim protocols. +In the above case, peer B would provide backwards compatibility by supporting and advertising both v1.1.1 and v1.1.2 of the protocol. + +Therefore, semver would be relegated to convey expectations at the human level, and it wouldn't do a good job there either, +because it's unclear if "backwards compatibility" and "breaking change" apply only to wire schema level, to behavior, etc. + +For this reason, we remove and replace semver with ordinals that require explicit agreement and do not mandate a specific policy for changes. + +### Why is it called Req/Resp and not RPC? + +Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. + +### Why do we allow empty responses in block requests? + +When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. + +Thus, it may happen that we need to transmit an empty list - there are several ways to encode this: + +0) Close the stream without sending any data +1) Add a `null` option to the `success` response, for example by introducing an additional byte +2) Respond with an error result, using a specific error code for "No data" + +Semantically, it is not an error that a block is missing during a slot making option 2 unnatural. + +Option 1 allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. + +Under option 0, there is no way for a client to distinguish between a slot without a block and an incomplete response, +but given that it already must contain logic to handle the uncertainty of a malicious peer, option 0 was chosen. +Clients should mark any slots missing blocks as unknown until they can be verified as not containing a block by successive blocks. + +Assuming option 0 with no special `null` encoding, consider a request for slots `2, 3, 4` +-- if there was no block produced at slot 4, the response would be `2, 3, EOF`. +Now consider the same situation, but where only `4` is requested +-- closing the stream with only `EOF` (without any `response_chunk`) is consistent. + +Failing to provide blocks that nodes "should" have is reason to trust a peer less +-- for example, if a particular peer gossips a block, it should have access to its parent. +If a request for the parent fails, it's indicative of poor peer quality since peers should validate blocks before gossiping them. + +### Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from? + +When connecting, the `Status` message gives an idea about the sync status of a particular peer, but this changes over time. +By the time a subsequent `BeaconBlockByRange` request is processed, the information may be stale, +and the responding side might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. + +To avoid this race condition, we allow the responding side to choose which branch to send to the requesting client. +The requesting client then goes on to validate the blocks and incorporate them in their own database +-- because they follow the same rules, they should at this point arrive at the same canonical chain. + +### What's the effect of empty slots on the sync algorithm? + +When syncing one can only tell that a slot has been skipped on a particular branch +by examining subsequent blocks and analyzing the graph formed by the parent root. +Because the server side may choose to omit blocks in the response for any reason, clients must validate the graph and be prepared to fill in gaps. + +For example, if a peer responds with blocks [2, 3] when asked for [2, 3, 4], clients may not assume that block 4 doesn't exist +-- it merely means that the responding peer did not send it (they may not have it yet or may maliciously be trying to hide it) +and successive blocks will be needed to determine if there exists a block at slot 4 in this particular branch. + +## Discovery + +### Why are we using discv5 and not libp2p Kademlia DHT? + +discv5 is a standalone protocol, running on UDP on a dedicated port, meant for peer and service discovery only. +discv5 supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are, or will be, requirements in this context. + +On the other hand, libp2p Kademlia DHT is a fully-fledged DHT protocol/implementations +with content routing and storage capabilities, both of which are irrelevant in this context. + +Eth 1.0 nodes will evolve to support discv5. +By sharing the discovery network between Eth 1.0 and 2.0, +we benefit from the additive effect on network size that enhances resilience and resistance against certain attacks, +to which smaller networks are more vulnerable. +It should also help light clients of both networks find nodes with specific capabilities. + +discv5 is in the process of being audited. + +### What is the difference between an ENR and a multiaddr, and why are we using ENRs? + +Ethereum Node Records are self-certified node records. +Nodes craft and disseminate ENRs for themselves, proving authorship via a cryptographic signature. +ENRs are sequentially indexed, enabling conflicts to be resolved. + +ENRs are key-value records with string-indexed ASCII keys. +They can store arbitrary information, but EIP-778 specifies a pre-defined dictionary, including IPv4 and IPv6 addresses, secp256k1 public keys, etc. + +Comparing ENRs and multiaddrs is like comparing apples and oranges. +ENRs are self-certified containers of identity, addresses, and metadata about a node. +Multiaddrs are address strings with the peculiarity that they’re self-describing, composable and future-proof. +An ENR can contain multiaddrs, and multiaddrs can be derived securely from the fields of an authenticated ENR. + +discv5 uses ENRs and we will presumably need to: + +1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or – +2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR + (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Eth 1.0 nodes). + +### Why do we not form ENRs and find peers until genesis block/state is known? + +Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, +clients cannot form valid ENRs prior to this point. +ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains +so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. +Once genesis data is known, we can then form ENRs and safely find peers. + +When using an eth1 deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (7 days in mainnet configuration) before `genesis_time`, +providing ample time to find peers and form initial connections and gossip subnets prior to genesis. + +## Compression/Encoding + +### Why are we using SSZ for encoding? + +SSZ is used at the consensus layer, and all implementations should have support for SSZ-encoding/decoding, +requiring no further dependencies to be added to client implementations. +This is a natural choice for serializing objects to be sent across the wire. +The actual data in most protocols will be further compressed for efficiency. + +SSZ has well-defined schemas for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. +It also has defined all required types that are required for this network specification. + +### Why are we compressing, and at which layers? + +We compress on the wire to achieve smaller payloads per-message, which, in aggregate, +result in higher efficiency, better utilization of available bandwidth, and overall reduction in network-wide traffic overhead. + +At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated +and layered atop connections and streams, but it is [being considered](https://github.com/libp2p/libp2p/issues/81). + +This is a non-trivial feature because the behavior +of network IO loops, kernel buffers, chunking, and packet fragmentation, amongst others, need to be taken into account. +libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. + +Compression tends not to be a one-size-fits-all problem. +A lot of variables need careful evaluation, and generic approaches/choices lead to poor size shavings, +which may even be counterproductive when factoring in the CPU and memory tradeoff. + +For all these reasons, generically negotiating compression algorithms may be treated as a research problem at the libp2p community, +one we’re happy to tackle in the medium-term. + +At this stage, the wisest choice is to consider libp2p a messenger of bytes, +and to make application layer participate in compressing those bytes. +This looks different depending on the interaction layer: + +- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload + (when dictated by the encoding token in the topic name) prior to publishing the message via the API. + No length-prefixing is necessary because protobuf takes care of bounding the field in the serialized form. +- Req/Resp domain: since we define custom protocols that operate on byte streams, + implementers are encouraged to encapsulate the encoding and compression logic behind + MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. + +### Why are we using Snappy for compression? + +Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, +and can calculate the size of the uncompressed object without inflating it in memory. +This prevents DOS vectors where large uncompressed data is sent. + +### Can I get access to unencrypted bytes on the wire for debugging purposes? + +Yes, you can add loggers in your libp2p protocol handlers to log incoming and outgoing messages. +It is recommended to use programming design patterns to encapsulate the logging logic cleanly. + +If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node.js (javascript), +you can use logging facilities in those frameworks/runtimes to enable message tracing. + +For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) +(which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. + +### What are SSZ type size bounds? + +The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. +Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. +Other types are static, they have a fixed size: no dynamic-length content is involved, and the minimum and maximum bounds are the same. + +For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). +It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. + +# libp2p implementations matrix + +This section will soon contain a matrix showing the maturity/state of the libp2p features required +by this spec across the languages in which Eth2 clients are being developed. diff --git a/tools/analyzers/specdocs/data/phase0/validator.md b/tools/analyzers/specdocs/data/phase0/validator.md new file mode 100644 index 00000000000..a548003e1b3 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/validator.md @@ -0,0 +1,654 @@ +# Ethereum 2.0 Phase 0 -- Honest Validator + +This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Constants](#constants) + - [Misc](#misc) +- [Containers](#containers) + - [`Eth1Block`](#eth1block) + - [`AggregateAndProof`](#aggregateandproof) + - [`SignedAggregateAndProof`](#signedaggregateandproof) +- [Becoming a validator](#becoming-a-validator) + - [Initialization](#initialization) + - [BLS public key](#bls-public-key) + - [Withdrawal credentials](#withdrawal-credentials) + - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) + - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) + - [Submit deposit](#submit-deposit) + - [Process deposit](#process-deposit) + - [Validator index](#validator-index) + - [Activation](#activation) +- [Validator assignments](#validator-assignments) + - [Lookahead](#lookahead) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock) + - [Slot](#slot) + - [Proposer index](#proposer-index) + - [Parent root](#parent-root) + - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) + - [Randao reveal](#randao-reveal) + - [Eth1 Data](#eth1-data) + - [`get_eth1_data`](#get_eth1_data) + - [Proposer slashings](#proposer-slashings) + - [Attester slashings](#attester-slashings) + - [Attestations](#attestations) + - [Deposits](#deposits) + - [Voluntary exits](#voluntary-exits) + - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock) + - [State root](#state-root) + - [Signature](#signature) + - [Attesting](#attesting) + - [Attestation data](#attestation-data) + - [General](#general) + - [LMD GHOST vote](#lmd-ghost-vote) + - [FFG vote](#ffg-vote) + - [Construct attestation](#construct-attestation) + - [Data](#data) + - [Aggregation bits](#aggregation-bits) + - [Aggregate signature](#aggregate-signature) + - [Broadcast attestation](#broadcast-attestation) + - [Attestation aggregation](#attestation-aggregation) + - [Aggregation selection](#aggregation-selection) + - [Construct aggregate](#construct-aggregate) + - [Data](#data-1) + - [Aggregation bits](#aggregation-bits-1) + - [Aggregate signature](#aggregate-signature-1) + - [Broadcast aggregate](#broadcast-aggregate) +- [Phase 0 attestation subnet stability](#phase-0-attestation-subnet-stability) +- [How to avoid slashing](#how-to-avoid-slashing) + - [Proposer slashing](#proposer-slashing) + - [Attester slashing](#attester-slashing) +- [Protection best practices](#protection-best-practices) + + + + +## Introduction + +This document represents the expected behavior of an "honest validator" with respect to Phase 0 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. + +A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. + +## Prerequisites + +All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. + +## Constants + +### Misc + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | +| `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | +| `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | +| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | + +## Containers + +### `Eth1Block` + +```python +class Eth1Block(Container): + timestamp: uint64 + deposit_root: Root + deposit_count: uint64 + # All other eth1 block fields +``` + +### `AggregateAndProof` + +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` + +### `SignedAggregateAndProof` + +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` + +## Becoming a validator + +### Initialization + +A validator must initialize many parameters locally before submitting a deposit and joining the validator registry. + +#### BLS public key + +Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BLS12-381 curve](https://z.cash/blog/new-snark-curve). A private key, `privkey`, must be securely generated along with the resultant `pubkey`. This `privkey` must be "hot", that is, constantly available to sign data throughout the lifetime of the validator. + +#### Withdrawal credentials + +The `withdrawal_credentials` field constrains validator withdrawals. +The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. + +The following withdrawal prefixes are currently supported. + +##### `BLS_WITHDRAWAL_PREFIX` + +Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair +`(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. +The `withdrawal_credentials` field must be such that: + +* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` + +*Note*: The `bls_withdrawal_privkey` is not required for validating and can be kept in cold storage. + +##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` + +Withdrawal credentials with the Eth1 address withdrawal prefix specify +a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. +The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. + +The `withdrawal_credentials` field must be such that: + +* `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` +* `withdrawal_credentials[1:12] == b'\x00' * 11` +* `withdrawal_credentials[12:] == eth1_withdrawal_address` + +After the merge of the current Ethereum application layer (Eth1) into the Beacon Chain (Eth2), +withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) +triggered by a user transaction that will set the gas price and gas limit as well pay fees. +As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers, +the future withdrawal protocol is agnostic to all other implementation details. + +### Submit deposit + +In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 chain defined by `DEPOSIT_CHAIN_ID` and `DEPOSIT_NETWORK_ID`. Deposits are made to the [deposit contract](./deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. + +To submit a deposit: + +- Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](./beacon-chain.md#depositdata) SSZ object. +- Let `amount` be the amount in Gwei to be deposited by the validator where `amount >= MIN_DEPOSIT_AMOUNT`. +- Set `deposit_data.pubkey` to validator's `pubkey`. +- Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. +- Set `deposit_data.amount` to `amount`. +- Let `deposit_message` be a `DepositMessage` with all the `DepositData` contents except the `signature`. +- Let `signature` be the result of `bls.Sign` of the `compute_signing_root(deposit_message, domain)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (_Warning_: Deposits _must_ be signed with `GENESIS_FORK_VERSION`, calling `compute_domain` without a second argument defaults to the correct version). +- Let `deposit_data_root` be `hash_tree_root(deposit_data)`. +- Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. + +*Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. + +### Process deposit + +Deposits cannot be processed into the beacon chain until the Eth1 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth1 blocks (~8 hours) plus `EPOCHS_PER_ETH1_VOTING_PERIOD` epochs (~6.8 hours). Once the requisite Eth1 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validators` within an epoch or two. The validator is then in a queue to be activated. + +### Validator index + +Once a validator has been processed and added to the beacon state's `validators`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](./beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. + +### Activation + +In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `MAX_SEED_LOOKAHEAD` epochs (25.6 minutes). + +The function [`is_active_validator`](./beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: + +```python +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) +``` + +Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. + +*Note*: There is a maximum validator churn per finalized epoch, so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. + +## Validator assignments + +A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. + +```python +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex + ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the index to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. + """ + next_epoch = Epoch(get_current_epoch(state) + 1) + assert epoch <= next_epoch + + start_slot = compute_start_slot_at_epoch(epoch) + committee_count_per_slot = get_committee_count_per_slot(state, epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): + for index in range(committee_count_per_slot): + committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) + if validator_index in committee: + return committee, CommitteeIndex(index), Slot(slot) + return None +``` + +A validator can use the following function to see if they are supposed to propose during a slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. + +```python +def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` + +*Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. + +*Note*: `BeaconBlock` proposal is distinct from beacon committee assignment, and in a given epoch each responsibility might occur at a different slot. + +### Lookahead + +The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead +on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. +Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. + +`get_committee_assignment` should be called at the start of each epoch +to get the assignment for the next epoch (`current_epoch + 1`). +A validator should plan for future assignments by noting their assigned attestation +slot and joining the committee index attestation subnet related to their committee assignment. + +Specifically a validator should: +* Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. +* Calculate the committees per slot for the next epoch: `committees_per_slot = get_committee_count_per_slot(state, next_epoch)` +* Calculate the subnet index: `subnet_id = compute_subnet_for_attestation(committees_per_slot, slot, committee_index)` +* Find peers of the pubsub topic `beacon_attestation_{subnet_id}`. + * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][subnet_id] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. + * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. + +*Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. + +## Beacon chain responsibilities + +A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. + +### Block proposal + +A validator is expected to propose a [`SignedBeaconBlock`](./beacon-chain.md#signedbeaconblock) at +the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. +To propose, the validator selects the `BeaconBlock`, `parent`, +that in their view of the fork choice is the head of the chain during `slot - 1`. +The validator creates, signs, and broadcasts a `block` that is a child of `parent` +that satisfies a valid [beacon chain state transition](./beacon-chain.md#beacon-chain-state-transition-function). + +There is one proposer per slot, so if there are N active validators any individual validator +will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~6 weeks). + +*Note*: In this section, `state` is the state of the slot for the block proposal _without_ the block yet applied. +That is, `state` is the `previous_state` processed through any empty slots up to the assigned slot using `process_slots(previous_state, slot)`. + +#### Preparing for a `BeaconBlock` + +To construct a `BeaconBlockBody`, a `block` (`BeaconBlock`) is defined with the necessary context for a block proposal: + +##### Slot + +Set `block.slot = slot` where `slot` is the current slot at which the validator has been selected to propose. The `parent` selected must satisfy that `parent.slot < block.slot`. + +*Note*: There might be "skipped" slots between the `parent` and `block`. These skipped slots are processed in the state transition function without per-block processing. + +##### Proposer index + +Set `block.proposer_index = validator_index` where `validator_index` is the validator chosen to propose at this slot. The private key mapping to `state.validators[validator_index].pubkey` is used to sign the block. + +##### Parent root + +Set `block.parent_root = hash_tree_root(parent)`. + +#### Constructing the `BeaconBlockBody` + +##### Randao reveal + +Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obtained from: + +```python +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) +``` + +##### Eth1 Data + +The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 data. +This recent data contains an Eth1 block hash as well as the associated deposit root +(as calculated by the `get_deposit_root()` method of the deposit contract) and +deposit count after execution of the corresponding Eth1 block. +If over half of the block proposers in the current Eth1 voting period vote for the same +`eth1_data` then `state.eth1_data` updates immediately allowing new deposits to be processed. +Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. + +###### `get_eth1_data` + +Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` and depost contract data available. + +Let `get_eth1_data(block: Eth1Block) -> Eth1Data` be the function that returns the Eth1 data for a given Eth1 block. + +An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state, eth1_chain)` where: + +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) +``` + +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) + return compute_time_at_slot(state, eth1_voting_period_start_slot) +``` + +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start + ) +``` + +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] + + # Valid votes already cast during this period + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data + + return max( + valid_votes, + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance + default=default_vote + ) +``` + +##### Proposer slashings + +Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](./beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](./beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. + +##### Attester slashings + +Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](./beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](./beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. + +##### Attestations + +Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](./beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. + +##### Deposits + +If there are any unprocessed deposits for the existing `state.eth1_data` (i.e. `state.eth1_data.deposit_count > state.eth1_deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, eth1_data.deposit_count - state.eth1_deposit_index)`. These [`deposits`](./beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth1 deposit contract](./deposit-contract.md) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](./beacon-chain.md#deposits). + +The `proof` for each deposit must be constructed against the deposit root contained in `state.eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. + +##### Voluntary exits + +Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](./beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](./beacon-chain.md#voluntary-exits). + +*Note*: If a slashing for a validator is included in the same block as a +voluntary exit, the voluntary exit will fail and cause the block to be invalid +due to the slashing being processed first. Implementers must take heed of this +operation interaction when packing blocks. + +#### Packaging into a `SignedBeaconBlock` + +##### State root + +Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the `parent -> block` state transition. + +*Note*: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. +It is useful to be able to run a state transition function (working on a copy of the state) that does _not_ validate signatures or state root for this purpose: + +```python +def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) +``` + +##### Signature + +`signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: + +```python +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) + return bls.Sign(privkey, signing_root) +``` + +### Attesting + +A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `index`, and assigned `slot` for which the validator performs this role during an epoch are defined by `get_committee_assignment(state, epoch, validator_index)`. + +A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid block from the expected block proposer for the assigned `slot` or (b) one-third of the `slot` has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of `slot`) -- whichever comes _first_. + +*Note*: Although attestations during `GENESIS_EPOCH` do not count toward FFG finality, these initial attestations do give weight to the fork choice, are rewarded, and should be made. + +#### Attestation data + +First, the validator should construct `attestation_data`, an [`AttestationData`](./beacon-chain.md#attestationdata) object based upon the state at the assigned slot. + +- Let `head_block` be the result of running the fork choice during the assigned slot. +- Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. + +##### General + +* Set `attestation_data.slot = slot` where `slot` is the assigned slot. +* Set `attestation_data.index = index` where `index` is the index associated with the validator's committee. + +##### LMD GHOST vote + +Set `attestation_data.beacon_block_root = hash_tree_root(head_block)`. + +##### FFG vote + +- Set `attestation_data.source = head_state.current_justified_checkpoint`. +- Set `attestation_data.target = Checkpoint(epoch=get_current_epoch(head_state), root=epoch_boundary_block_root)` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. + +*Note*: `epoch_boundary_block_root` can be looked up in the state using: + +- Let `start_slot = compute_start_slot_at_epoch(get_current_epoch(head_state))`. +- Let `epoch_boundary_block_root = hash_tree_root(head_block) if start_slot == head_state.slot else get_block_root(state, get_current_epoch(head_state))`. + +#### Construct attestation + +Next, the validator creates `attestation`, an [`Attestation`](./beacon-chain.md#attestation) object. + +##### Data + +Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). + +##### Aggregation bits + +- Let `attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` of length `len(committee)`, where the bit of the index of the validator in the `committee` is set to `0b1`. + +*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. + +##### Aggregate signature + +Set `attestation.signature = attestation_signature` where `attestation_signature` is obtained from: + +```python +def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) +``` + +#### Broadcast attestation + +Finally, the validator broadcasts `attestation` to the associated attestation subnet, the `beacon_attestation_{subnet_id}` pubsub topic. + +The `subnet_id` for the `attestation` is calculated with: +- Let `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)`. +- Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.committee_index)`. + +```python +def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) + committees_since_epoch_start = committees_per_slot * slots_since_epoch_start + + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) +``` + +### Attestation aggregation + +Some validators are selected to locally aggregate attestations with a similar `attestation_data` to their constructed `attestation` for the assigned `slot`. + +#### Aggregation selection + +A validator is selected to aggregate based upon the return value of `is_aggregator()`. + +```python +def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` + +```python +def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: + committee = get_beacon_committee(state, slot, index) + modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) + return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 +``` + +#### Construct aggregate + +If the validator is selected to aggregate (`is_aggregator()`), they construct an aggregate attestation via the following. + +Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator. If `len(attestations) > 0`, create an `aggregate_attestation: Attestation` with the following fields. + +##### Data + +Set `aggregate_attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object that is the same for each individual attestation being aggregated. + +##### Aggregation bits + +Let `aggregate_attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` of length `len(committee)`, where each bit set from each individual attestation is set to `0b1`. + +##### Aggregate signature + +Set `aggregate_attestation.signature = aggregate_signature` where `aggregate_signature` is obtained from: + +```python +def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: + signatures = [attestation.signature for attestation in attestations] + return bls.Aggregate(signatures) +``` + +#### Broadcast aggregate + +If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. + +Selection proofs are provided in `AggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. + +`AggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. + +First, `aggregate_and_proof = get_aggregate_and_proof(state, validator_index, aggregate_attestation, privkey)` is constructed. + +```python +def get_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` + +Then `signed_aggregate_and_proof = SignedAggregateAndProof(message=aggregate_and_proof, signature=signature)` is constructed and broadcast. Where `signature` is obtained from: + +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` + +## Phase 0 attestation subnet stability + +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: + +* Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets +* Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets +* Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR + +*Note*: Short lived beacon committee assignments should not be added in into the ENR `attnets` entry. + +*Note*: When preparing for a hard fork, a validator must select and subscribe to random subnets of the future fork versioning at least `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. + +## How to avoid slashing + +"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. + +*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102, and vice versa. + +### Proposer slashing + +To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](./beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same slot. + +*In Phase 0, as long as the validator does not sign two different beacon blocks for the same slot, the validator is safe against proposer slashings.* + +Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: + +1. Save a record to hard disk that a beacon block has been signed for the `slot=block.slot`. +2. Generate and broadcast the block. + +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. + +### Attester slashing + +To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](./beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](./beacon-chain.md#is_slashable_attestation_data). + +Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: + +1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source.epoch`) and target (i.e. `attestation_data.target.epoch`). +2. Generate and broadcast attestation. + +If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. + +## Protection best practices + +A validator client should be considered standalone and should consider the beacon node as untrusted. This means that the validator client should protect: + +1) Private keys -- private keys should be protected from being exported accidentally or by an attacker. +2) Slashing -- before a validator client signs a message it should validate the data, check it against a local slashing database (do not sign a slashable attestation or block) and update its internal slashing database with the newly signed object. +3) Recovered validator -- Recovering a validator from a private key will result in an empty local slashing db. Best practice is to import (from a trusted source) that validator's attestation history. See [EIP 3076](https://github.com/ethereum/EIPs/pull/3076/files) for a standard slashing interchange format. +4) Far future signing requests -- A validator client can be requested to sign a far into the future attestation, resulting in a valid non-slashable request. If the validator client signs this message, it will result in it blocking itself from attesting any other attestation until the beacon-chain reaches that far into the future epoch. This will result in an inactivity penalty and potential ejection due to low balance. +A validator client should prevent itself from signing such requests by: a) keeping a local time clock if possible and following best practices to stop time server attacks and b) refusing to sign, by default, any message that has a large (>6h) gap from the current slashing protection database indicated a time "jump" or a long offline event. The administrator can manually override this protection to restart the validator after a genuine long offline event. diff --git a/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md b/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md new file mode 100644 index 00000000000..fd1b3cc2817 --- /dev/null +++ b/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md @@ -0,0 +1,182 @@ +# Ethereum 2.0 Phase 0 -- Weak Subjectivity Guide + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Custom Types](#custom-types) +- [Constants](#constants) +- [Configuration](#configuration) +- [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) +- [Weak Subjectivity Period](#weak-subjectivity-period) + - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) + - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) +- [Weak Subjectivity Sync](#weak-subjectivity-sync) + - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) + - [Checking for Stale Weak Subjectivity Checkpoint](#checking-for-stale-weak-subjectivity-checkpoint) + - [`is_within_weak_subjectivity_period`](#is_within_weak_subjectivity_period) +- [Distributing Weak Subjectivity Checkpoints](#distributing-weak-subjectivity-checkpoints) + + + + +## Introduction + +This document is a guide for implementing the Weak Subjectivity protections in Phase 0 of Ethereum 2.0. +This document is still a work-in-progress, and is subject to large changes. +For more information about weak subjectivity and why it is required, please refer to: + +- [Weak Subjectivity in Eth2.0](https://notes.ethereum.org/@adiasg/weak-subjectvity-eth2) +- [Proof of Stake: How I Learned to Love Weak Subjectivity](https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity/) + +## Prerequisites + +This document uses data structures, constants, functions, and terminology from +[Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Beacon Chain Fork Choice](./fork-choice.md). + +## Custom Types + +| Name | SSZ Equivalent | Description | +|---|---|---| +| `Ether` | `uint64` | an amount in Ether | + +## Constants + +| Name | Value | +|---|---| +| `ETH_TO_GWEI` | `uint64(10**9)` | + +## Configuration + +| Name | Value | +|---|---| +| `SAFETY_DECAY` | `uint64(10)` | + +## Weak Subjectivity Checkpoint + +Any `Checkpoint` object can be used as a Weak Subjectivity Checkpoint. +These Weak Subjectivity Checkpoints are distributed by providers, +downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. + +## Weak Subjectivity Period + +The Weak Subjectivity Period is the number of recent epochs within which there +must be a Weak Subjectivity Checkpoint to ensure that an attacker who takes control +of the validator set at the beginning of the period is slashed at least a minimum threshold +in the event that a conflicting `Checkpoint` is finalized. + +`SAFETY_DECAY` is defined as the maximum percentage tolerable loss in the one-third +safety margin of FFG finality. Thus, any attack exploiting the Weak Subjectivity Period has +a safety margin of at least `1/3 - SAFETY_DECAY/100`. + +### Calculating the Weak Subjectivity Period + +A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). + +*Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). + +*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magnitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magnitude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. + +#### `compute_weak_subjectivity_period` + +```python +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + if T * (200 + 3 * D) < t * (200 + 12 * D): + epochs_for_validator_set_churn = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + epochs_for_balance_top_ups = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) + else: + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) + + return ws_period +``` + +A brief reference for what these values look like in practice ([reference script](https://gist.github.com/adiasg/3aceab409b36aa9a9d9156c1baa3c248)): + +| Safety Decay | Avg. Val. Balance (ETH) | Val. Count | Weak Sub. Period (Epochs) | +| ---- | ---- | ---- | ---- | +| 10 | 28 | 32768 | 504 | +| 10 | 28 | 65536 | 752 | +| 10 | 28 | 131072 | 1248 | +| 10 | 28 | 262144 | 2241 | +| 10 | 28 | 524288 | 2241 | +| 10 | 28 | 1048576 | 2241 | +| 10 | 32 | 32768 | 665 | +| 10 | 32 | 65536 | 1075 | +| 10 | 32 | 131072 | 1894 | +| 10 | 32 | 262144 | 3532 | +| 10 | 32 | 524288 | 3532 | +| 10 | 32 | 1048576 | 3532 | + +## Weak Subjectivity Sync + +Clients should allow users to input a Weak Subjectivity Checkpoint at startup, and guarantee that any successful sync leads to the given Weak Subjectivity Checkpoint along the canonical chain. If such a sync is not possible, the client should treat this as a critical and irrecoverable failure. + +### Weak Subjectivity Sync Procedure + +1. Input a Weak Subjectivity Checkpoint as a CLI parameter in `block_root:epoch_number` format, + where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. + Example of the format: + + ``` + 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 + ``` + +2. Check the weak subjectivity requirements: + - *IF* `epoch_number > store.finalized_checkpoint.epoch`, + then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. + Emit descriptive critical error if this assert fails, then exit client process. + - *IF* `epoch_number <= store.finalized_checkpoint.epoch`, + then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. + Emit descriptive critical error if this assert fails, then exit client process. + +### Checking for Stale Weak Subjectivity Checkpoint + +Clients may choose to validate that the input Weak Subjectivity Checkpoint is not stale at the time of startup. +To support this mechanism, the client needs to take the state at the Weak Subjectivity Checkpoint as +a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). +The check can be implemented in the following way: + +#### `is_within_weak_subjectivity_period` + +```python +def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: + # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint + assert ws_state.latest_block_header.state_root == ws_checkpoint.root + assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch + + ws_period = compute_weak_subjectivity_period(ws_state) + ws_state_epoch = compute_epoch_at_slot(ws_state.slot) + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + return current_epoch <= ws_state_epoch + ws_period +``` + +## Distributing Weak Subjectivity Checkpoints + +This section will be updated soon. From 5c6ce1f6199b5de21b6d2077b2d0e6551786f5a7 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 01:48:16 +0300 Subject: [PATCH 03/27] update content pulling script --- scripts/update-spec-docs.sh | 16 +- .../phase0/{beacon-chain.md => all-defs.md} | 1229 ++++++------- .../specdocs/data/phase0/deposit-contract.md | 77 - .../specdocs/data/phase0/fork-choice.md | 406 ----- .../specdocs/data/phase0/p2p-interface.md | 1526 ----------------- .../specdocs/data/phase0/validator.md | 654 ------- .../specdocs/data/phase0/weak-subjectivity.md | 182 -- 7 files changed, 582 insertions(+), 3508 deletions(-) rename tools/analyzers/specdocs/data/phase0/{beacon-chain.md => all-defs.md} (69%) delete mode 100644 tools/analyzers/specdocs/data/phase0/deposit-contract.md delete mode 100644 tools/analyzers/specdocs/data/phase0/fork-choice.md delete mode 100644 tools/analyzers/specdocs/data/phase0/p2p-interface.md delete mode 100644 tools/analyzers/specdocs/data/phase0/validator.md delete mode 100644 tools/analyzers/specdocs/data/phase0/weak-subjectivity.md diff --git a/scripts/update-spec-docs.sh b/scripts/update-spec-docs.sh index c90da53cc6d..a0d1c0157ed 100755 --- a/scripts/update-spec-docs.sh +++ b/scripts/update-spec-docs.sh @@ -1,5 +1,8 @@ #!/bin/bash +# This script will pull the latest specs from https://github.com/ethereum/eth2.0-specs repo, extract code blocks +# and save them for reference in "all-defs.md" file (in separate directories for phase0, altair etc). + declare -a files=("phase0/beacon-chain.md" "phase0/deposit-contract.md" "phase0/fork-choice.md" @@ -11,7 +14,16 @@ declare -a files=("phase0/beacon-chain.md" BASE_URL="https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev/specs" OUTPUT_DIR="tools/analyzers/specdocs/data" +# Trunc all-defs files (they will contain extracted python code blocks). +echo -n >$OUTPUT_DIR/phase0/all-defs.md + for file in "${files[@]}"; do - echo "downloading $file" - wget -q -O $OUTPUT_DIR/"$file" --no-check-certificate --content-disposition $BASE_URL/"$file" + OUTPUT_PATH=$OUTPUT_DIR/$file + echo "$file" + echo "- downloading" + wget -q -O "$OUTPUT_PATH" --no-check-certificate --content-disposition $BASE_URL/"$file" + echo "- extracting all code blocks" + sed -n '/^```/,/^```/ p' <"$OUTPUT_PATH" >>"${OUTPUT_PATH%/*}"/all-defs.md + echo "- removing raw file" + rm "$OUTPUT_PATH" done diff --git a/tools/analyzers/specdocs/data/phase0/beacon-chain.md b/tools/analyzers/specdocs/data/phase0/all-defs.md similarity index 69% rename from tools/analyzers/specdocs/data/phase0/beacon-chain.md rename to tools/analyzers/specdocs/data/phase0/all-defs.md index cbd085bd3c8..6c1d3eb8205 100644 --- a/tools/analyzers/specdocs/data/phase0/beacon-chain.md +++ b/tools/analyzers/specdocs/data/phase0/all-defs.md @@ -1,328 +1,19 @@ -# Ethereum 2.0 Phase 0 -- The Beacon Chain - -## Table of contents - - - - -- [Introduction](#introduction) -- [Notation](#notation) -- [Custom types](#custom-types) -- [Constants](#constants) -- [Configuration](#configuration) - - [Misc](#misc) - - [Gwei values](#gwei-values) - - [Initial values](#initial-values) - - [Withdrawal prefixes](#withdrawal-prefixes) - - [Time parameters](#time-parameters) - - [State list lengths](#state-list-lengths) - - [Rewards and penalties](#rewards-and-penalties) - - [Max operations per block](#max-operations-per-block) - - [Domain types](#domain-types) -- [Containers](#containers) - - [Misc dependencies](#misc-dependencies) - - [`Fork`](#fork) - - [`ForkData`](#forkdata) - - [`Checkpoint`](#checkpoint) - - [`Validator`](#validator) - - [`AttestationData`](#attestationdata) - - [`IndexedAttestation`](#indexedattestation) - - [`PendingAttestation`](#pendingattestation) - - [`Eth1Data`](#eth1data) - - [`HistoricalBatch`](#historicalbatch) - - [`DepositMessage`](#depositmessage) - - [`DepositData`](#depositdata) - - [`BeaconBlockHeader`](#beaconblockheader) - - [`SigningData`](#signingdata) - - [Beacon operations](#beacon-operations) - - [`ProposerSlashing`](#proposerslashing) - - [`AttesterSlashing`](#attesterslashing) - - [`Attestation`](#attestation) - - [`Deposit`](#deposit) - - [`VoluntaryExit`](#voluntaryexit) - - [Beacon blocks](#beacon-blocks) - - [`BeaconBlockBody`](#beaconblockbody) - - [`BeaconBlock`](#beaconblock) - - [Beacon state](#beacon-state) - - [`BeaconState`](#beaconstate) - - [Signed envelopes](#signed-envelopes) - - [`SignedVoluntaryExit`](#signedvoluntaryexit) - - [`SignedBeaconBlock`](#signedbeaconblock) - - [`SignedBeaconBlockHeader`](#signedbeaconblockheader) -- [Helper functions](#helper-functions) - - [Math](#math) - - [`integer_squareroot`](#integer_squareroot) - - [`xor`](#xor) - - [`uint_to_bytes`](#uint_to_bytes) - - [`bytes_to_uint64`](#bytes_to_uint64) - - [Crypto](#crypto) - - [`hash`](#hash) - - [`hash_tree_root`](#hash_tree_root) - - [BLS signatures](#bls-signatures) - - [Predicates](#predicates) - - [`is_active_validator`](#is_active_validator) - - [`is_eligible_for_activation_queue`](#is_eligible_for_activation_queue) - - [`is_eligible_for_activation`](#is_eligible_for_activation) - - [`is_slashable_validator`](#is_slashable_validator) - - [`is_slashable_attestation_data`](#is_slashable_attestation_data) - - [`is_valid_indexed_attestation`](#is_valid_indexed_attestation) - - [`is_valid_merkle_branch`](#is_valid_merkle_branch) - - [Misc](#misc-1) - - [`compute_shuffled_index`](#compute_shuffled_index) - - [`compute_proposer_index`](#compute_proposer_index) - - [`compute_committee`](#compute_committee) - - [`compute_epoch_at_slot`](#compute_epoch_at_slot) - - [`compute_start_slot_at_epoch`](#compute_start_slot_at_epoch) - - [`compute_activation_exit_epoch`](#compute_activation_exit_epoch) - - [`compute_fork_data_root`](#compute_fork_data_root) - - [`compute_fork_digest`](#compute_fork_digest) - - [`compute_domain`](#compute_domain) - - [`compute_signing_root`](#compute_signing_root) - - [Beacon state accessors](#beacon-state-accessors) - - [`get_current_epoch`](#get_current_epoch) - - [`get_previous_epoch`](#get_previous_epoch) - - [`get_block_root`](#get_block_root) - - [`get_block_root_at_slot`](#get_block_root_at_slot) - - [`get_randao_mix`](#get_randao_mix) - - [`get_active_validator_indices`](#get_active_validator_indices) - - [`get_validator_churn_limit`](#get_validator_churn_limit) - - [`get_seed`](#get_seed) - - [`get_committee_count_per_slot`](#get_committee_count_per_slot) - - [`get_beacon_committee`](#get_beacon_committee) - - [`get_beacon_proposer_index`](#get_beacon_proposer_index) - - [`get_total_balance`](#get_total_balance) - - [`get_total_active_balance`](#get_total_active_balance) - - [`get_domain`](#get_domain) - - [`get_indexed_attestation`](#get_indexed_attestation) - - [`get_attesting_indices`](#get_attesting_indices) - - [Beacon state mutators](#beacon-state-mutators) - - [`increase_balance`](#increase_balance) - - [`decrease_balance`](#decrease_balance) - - [`initiate_validator_exit`](#initiate_validator_exit) - - [`slash_validator`](#slash_validator) -- [Genesis](#genesis) - - [Genesis state](#genesis-state) - - [Genesis block](#genesis-block) -- [Beacon chain state transition function](#beacon-chain-state-transition-function) - - [Epoch processing](#epoch-processing) - - [Helper functions](#helper-functions-1) - - [Justification and finalization](#justification-and-finalization) - - [Rewards and penalties](#rewards-and-penalties-1) - - [Helpers](#helpers) - - [Components of attestation deltas](#components-of-attestation-deltas) - - [`get_attestation_deltas`](#get_attestation_deltas) - - [`process_rewards_and_penalties`](#process_rewards_and_penalties) - - [Registry updates](#registry-updates) - - [Slashings](#slashings) - - [Eth1 data votes updates](#eth1-data-votes-updates) - - [Effective balances updates](#effective-balances-updates) - - [Slashings balances updates](#slashings-balances-updates) - - [Randao mixes updates](#randao-mixes-updates) - - [Historical roots updates](#historical-roots-updates) - - [Participation records rotation](#participation-records-rotation) - - [Block processing](#block-processing) - - [Block header](#block-header) - - [RANDAO](#randao) - - [Eth1 data](#eth1-data) - - [Operations](#operations) - - [Proposer slashings](#proposer-slashings) - - [Attester slashings](#attester-slashings) - - [Attestations](#attestations) - - [Deposits](#deposits) - - [Voluntary exits](#voluntary-exits) - - - - -## Introduction - -This document represents the specification for Phase 0 of Ethereum 2.0 -- The Beacon Chain. - -At the core of Ethereum 2.0 is a system chain called the "beacon chain". The beacon chain stores and manages the registry of validators. In the initial deployment phases of Ethereum 2.0, the only mechanism to become a validator is to make a one-way ETH transaction to a deposit contract on Ethereum 1.0. Activation as a validator happens when Ethereum 1.0 deposit receipts are processed by the beacon chain, the activation balance is reached, and a queuing process is completed. Exit is either voluntary or done forcibly as a penalty for misbehavior. -The primary source of load on the beacon chain is "attestations". Attestations are simultaneously availability votes for a shard block (in a later Eth2 upgrade) and proof-of-stake votes for a beacon block (Phase 0). - -## Notation - -Code snippets appearing in `this style` are to be interpreted as Python 3 code. - -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `Slot` | `uint64` | a slot number | -| `Epoch` | `uint64` | an epoch number | -| `CommitteeIndex` | `uint64` | a committee index at a slot | -| `ValidatorIndex` | `uint64` | a validator registry index | -| `Gwei` | `uint64` | an amount in Gwei | -| `Root` | `Bytes32` | a Merkle root | -| `Version` | `Bytes4` | a fork version number | -| `DomainType` | `Bytes4` | a domain type | -| `ForkDigest` | `Bytes4` | a digest of the current fork data | -| `Domain` | `Bytes32` | a signature domain | -| `BLSPubkey` | `Bytes48` | a BLS12-381 public key | -| `BLSSignature` | `Bytes96` | a BLS12-381 signature | - -## Constants - -The following values are (non-configurable) constants used throughout the specification. - -| Name | Value | -| - | - | -| `GENESIS_SLOT` | `Slot(0)` | -| `GENESIS_EPOCH` | `Epoch(0)` | -| `FAR_FUTURE_EPOCH` | `Epoch(2**64 - 1)` | -| `BASE_REWARDS_PER_EPOCH` | `uint64(4)` | -| `DEPOSIT_CONTRACT_TREE_DEPTH` | `uint64(2**5)` (= 32) | -| `JUSTIFICATION_BITS_LENGTH` | `uint64(4)` | -| `ENDIANNESS` | `'little'` | - -## Configuration - -*Note*: The default mainnet configuration values are included here for illustrative purposes. The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs) directory. - -### Misc - -| Name | Value | -| - | - | -| `ETH1_FOLLOW_DISTANCE` | `uint64(2**11)` (= 2,048) | -| `MAX_COMMITTEES_PER_SLOT` | `uint64(2**6)` (= 64) | -| `TARGET_COMMITTEE_SIZE` | `uint64(2**7)` (= 128) | -| `MAX_VALIDATORS_PER_COMMITTEE` | `uint64(2**11)` (= 2,048) | -| `MIN_PER_EPOCH_CHURN_LIMIT` | `uint64(2**2)` (= 4) | -| `CHURN_LIMIT_QUOTIENT` | `uint64(2**16)` (= 65,536) | -| `SHUFFLE_ROUND_COUNT` | `uint64(90)` | -| `MIN_GENESIS_ACTIVE_VALIDATOR_COUNT` | `uint64(2**14)` (= 16,384) | -| `MIN_GENESIS_TIME` | `uint64(1606824000)` (Dec 1, 2020, 12pm UTC) | -| `HYSTERESIS_QUOTIENT` | `uint64(4)` | -| `HYSTERESIS_DOWNWARD_MULTIPLIER` | `uint64(1)` | -| `HYSTERESIS_UPWARD_MULTIPLIER` | `uint64(5)` | - -- For the safety of committees, `TARGET_COMMITTEE_SIZE` exceeds [the recommended minimum committee size of 111](http://web.archive.org/web/20190504131341/https://vitalik.ca/files/Ithaca201807_Sharding.pdf); with sufficient active validators (at least `SLOTS_PER_EPOCH * TARGET_COMMITTEE_SIZE`), the shuffling algorithm ensures committee sizes of at least `TARGET_COMMITTEE_SIZE`. (Unbiasable randomness with a Verifiable Delay Function (VDF) will improve committee robustness and lower the safe minimum committee size.) - -### Gwei values - -| Name | Value | -| - | - | -| `MIN_DEPOSIT_AMOUNT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | -| `MAX_EFFECTIVE_BALANCE` | `Gwei(2**5 * 10**9)` (= 32,000,000,000) | -| `EJECTION_BALANCE` | `Gwei(2**4 * 10**9)` (= 16,000,000,000) | -| `EFFECTIVE_BALANCE_INCREMENT` | `Gwei(2**0 * 10**9)` (= 1,000,000,000) | - -### Initial values - -| Name | Value | -| - | - | -| `GENESIS_FORK_VERSION` | `Version('0x00000000')` | - -### Withdrawal prefixes - -| Name | Value | -| - | - | -| `BLS_WITHDRAWAL_PREFIX` | `Bytes1('0x00')` | -| `ETH1_ADDRESS_WITHDRAWAL_PREFIX` | `Bytes1('0x01')` | - -### Time parameters - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `GENESIS_DELAY` | `uint64(604800)` | seconds | 7 days | -| `SECONDS_PER_SLOT` | `uint64(12)` | seconds | 12 seconds | -| `SECONDS_PER_ETH1_BLOCK` | `uint64(14)` | seconds | 14 seconds | -| `MIN_ATTESTATION_INCLUSION_DELAY` | `uint64(2**0)` (= 1) | slots | 12 seconds | -| `SLOTS_PER_EPOCH` | `uint64(2**5)` (= 32) | slots | 6.4 minutes | -| `MIN_SEED_LOOKAHEAD` | `uint64(2**0)` (= 1) | epochs | 6.4 minutes | -| `MAX_SEED_LOOKAHEAD` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes | -| `MIN_EPOCHS_TO_INACTIVITY_PENALTY` | `uint64(2**2)` (= 4) | epochs | 25.6 minutes | -| `EPOCHS_PER_ETH1_VOTING_PERIOD` | `uint64(2**6)` (= 64) | epochs | ~6.8 hours | -| `SLOTS_PER_HISTORICAL_ROOT` | `uint64(2**13)` (= 8,192) | slots | ~27 hours | -| `MIN_VALIDATOR_WITHDRAWABILITY_DELAY` | `uint64(2**8)` (= 256) | epochs | ~27 hours | -| `SHARD_COMMITTEE_PERIOD` | `uint64(2**8)` (= 256) | epochs | ~27 hours | - -### State list lengths - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `EPOCHS_PER_HISTORICAL_VECTOR` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 years | -| `EPOCHS_PER_SLASHINGS_VECTOR` | `uint64(2**13)` (= 8,192) | epochs | ~36 days | -| `HISTORICAL_ROOTS_LIMIT` | `uint64(2**24)` (= 16,777,216) | historical roots | ~52,262 years | -| `VALIDATOR_REGISTRY_LIMIT` | `uint64(2**40)` (= 1,099,511,627,776) | validators | - -### Rewards and penalties - -| Name | Value | -| - | - | -| `BASE_REWARD_FACTOR` | `uint64(2**6)` (= 64) | -| `WHISTLEBLOWER_REWARD_QUOTIENT` | `uint64(2**9)` (= 512) | -| `PROPOSER_REWARD_QUOTIENT` | `uint64(2**3)` (= 8) | -| `INACTIVITY_PENALTY_QUOTIENT` | `uint64(2**26)` (= 67,108,864) | -| `MIN_SLASHING_PENALTY_QUOTIENT` | `uint64(2**7)` (= 128) | -| `PROPORTIONAL_SLASHING_MULTIPLIER` | `uint64(1)` | - -- The `INACTIVITY_PENALTY_QUOTIENT` equals `INVERSE_SQRT_E_DROP_TIME**2` where `INVERSE_SQRT_E_DROP_TIME := 2**13` epochs (about 36 days) is the time it takes the inactivity penalty to reduce the balance of non-participating validators to about `1/sqrt(e) ~= 60.6%`. Indeed, the balance retained by offline validators after `n` epochs is about `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(n**2/2)`; so after `INVERSE_SQRT_E_DROP_TIME` epochs, it is roughly `(1 - 1/INACTIVITY_PENALTY_QUOTIENT)**(INACTIVITY_PENALTY_QUOTIENT/2) ~= 1/sqrt(e)`. Note this value will be upgraded to `2**24` after Phase 0 mainnet stabilizes to provide a faster recovery in the event of an inactivity leak. - -- The `PROPORTIONAL_SLASHING_MULTIPLIER` is set to `1` at initial mainnet launch, resulting in one-third of the minimum accountable safety margin in the event of a finality attack. After Phase 0 mainnet stablizes, this value will be upgraded to `3` to provide the maximal minimum accountable safety margin. - -### Max operations per block - -| Name | Value | -| - | - | -| `MAX_PROPOSER_SLASHINGS` | `2**4` (= 16) | -| `MAX_ATTESTER_SLASHINGS` | `2**1` (= 2) | -| `MAX_ATTESTATIONS` | `2**7` (= 128) | -| `MAX_DEPOSITS` | `2**4` (= 16) | -| `MAX_VOLUNTARY_EXITS` | `2**4` (= 16) | - -### Domain types - -| Name | Value | -| - | - | -| `DOMAIN_BEACON_PROPOSER` | `DomainType('0x00000000')` | -| `DOMAIN_BEACON_ATTESTER` | `DomainType('0x01000000')` | -| `DOMAIN_RANDAO` | `DomainType('0x02000000')` | -| `DOMAIN_DEPOSIT` | `DomainType('0x03000000')` | -| `DOMAIN_VOLUNTARY_EXIT` | `DomainType('0x04000000')` | -| `DOMAIN_SELECTION_PROOF` | `DomainType('0x05000000')` | -| `DOMAIN_AGGREGATE_AND_PROOF` | `DomainType('0x06000000')` | - -## Containers - -The following types are [SimpleSerialize (SSZ)](../../ssz/simple-serialize.md) containers. - -*Note*: The definitions are ordered topologically to facilitate execution of the spec. - -*Note*: Fields missing in container instantiations default to their zero value. - -### Misc dependencies - -#### `Fork` - ```python class Fork(Container): previous_version: Version current_version: Version epoch: Epoch # Epoch of latest fork ``` - -#### `ForkData` - ```python class ForkData(Container): current_version: Version genesis_validators_root: Root ``` - -#### `Checkpoint` - ```python class Checkpoint(Container): epoch: Epoch root: Root ``` - -#### `Validator` - ```python class Validator(Container): pubkey: BLSPubkey @@ -335,9 +26,6 @@ class Validator(Container): exit_epoch: Epoch withdrawable_epoch: Epoch # When validator can withdraw funds ``` - -#### `AttestationData` - ```python class AttestationData(Container): slot: Slot @@ -348,18 +36,12 @@ class AttestationData(Container): source: Checkpoint target: Checkpoint ``` - -#### `IndexedAttestation` - ```python class IndexedAttestation(Container): attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData signature: BLSSignature ``` - -#### `PendingAttestation` - ```python class PendingAttestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] @@ -367,35 +49,23 @@ class PendingAttestation(Container): inclusion_delay: Slot proposer_index: ValidatorIndex ``` - -#### `Eth1Data` - ```python class Eth1Data(Container): deposit_root: Root deposit_count: uint64 block_hash: Bytes32 ``` - -#### `HistoricalBatch` - ```python class HistoricalBatch(Container): block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] ``` - -#### `DepositMessage` - ```python class DepositMessage(Container): pubkey: BLSPubkey withdrawal_credentials: Bytes32 amount: Gwei ``` - -#### `DepositData` - ```python class DepositData(Container): pubkey: BLSPubkey @@ -403,9 +73,6 @@ class DepositData(Container): amount: Gwei signature: BLSSignature # Signing over DepositMessage ``` - -#### `BeaconBlockHeader` - ```python class BeaconBlockHeader(Container): slot: Slot @@ -414,62 +81,37 @@ class BeaconBlockHeader(Container): state_root: Root body_root: Root ``` - -#### `SigningData` - ```python class SigningData(Container): object_root: Root domain: Domain ``` - -### Beacon operations - -#### `ProposerSlashing` - ```python class ProposerSlashing(Container): signed_header_1: SignedBeaconBlockHeader signed_header_2: SignedBeaconBlockHeader ``` - -#### `AttesterSlashing` - ```python class AttesterSlashing(Container): attestation_1: IndexedAttestation attestation_2: IndexedAttestation ``` - -#### `Attestation` - ```python class Attestation(Container): aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] data: AttestationData signature: BLSSignature ``` - -#### `Deposit` - ```python class Deposit(Container): proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root data: DepositData ``` - -#### `VoluntaryExit` - ```python class VoluntaryExit(Container): epoch: Epoch # Earliest epoch when voluntary exit can be processed validator_index: ValidatorIndex ``` - -### Beacon blocks - -#### `BeaconBlockBody` - ```python class BeaconBlockBody(Container): randao_reveal: BLSSignature @@ -482,9 +124,6 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] ``` - -#### `BeaconBlock` - ```python class BeaconBlock(Container): slot: Slot @@ -493,11 +132,6 @@ class BeaconBlock(Container): state_root: Root body: BeaconBlockBody ``` - -### Beacon state - -#### `BeaconState` - ```python class BeaconState(Container): # Versioning @@ -530,41 +164,21 @@ class BeaconState(Container): current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint ``` - -### Signed envelopes - -#### `SignedVoluntaryExit` - ```python class SignedVoluntaryExit(Container): message: VoluntaryExit signature: BLSSignature ``` - -#### `SignedBeaconBlock` - ```python class SignedBeaconBlock(Container): message: BeaconBlock signature: BLSSignature ``` - -#### `SignedBeaconBlockHeader` - ```python class SignedBeaconBlockHeader(Container): message: BeaconBlockHeader signature: BLSSignature ``` - -## Helper functions - -*Note*: The definitions below are for specification purposes and are not necessarily optimal implementations. - -### Math - -#### `integer_squareroot` - ```python def integer_squareroot(n: uint64) -> uint64: """ @@ -577,9 +191,6 @@ def integer_squareroot(n: uint64) -> uint64: y = (x + n // x) // 2 return x ``` - -#### `xor` - ```python def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: """ @@ -587,13 +198,6 @@ def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: """ return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) ``` - -#### `uint_to_bytes` - -`def uint_to_bytes(n: uint) -> bytes` is a function for serializing the `uint` type object to bytes in ``ENDIANNESS``-endian. The expected length of the output is the byte-length of the `uint` type. - -#### `bytes_to_uint64` - ```python def bytes_to_uint64(data: bytes) -> uint64: """ @@ -601,33 +205,6 @@ def bytes_to_uint64(data: bytes) -> uint64: """ return uint64(int.from_bytes(data, ENDIANNESS)) ``` - -### Crypto - -#### `hash` - -`def hash(data: bytes) -> Bytes32` is SHA256. - -#### `hash_tree_root` - -`def hash_tree_root(object: SSZSerializable) -> Root` is a function for hashing objects into a single root by utilizing a hash tree structure, as defined in the [SSZ spec](../../ssz/simple-serialize.md#merkleization). - -#### BLS signatures - -The [IETF BLS signature draft standard v4](https://tools.ietf.org/html/draft-irtf-cfrg-bls-signature-04) with ciphersuite `BLS_SIG_BLS12381G2_XMD:SHA-256_SSWU_RO_POP_` defines the following functions: - -- `def Sign(privkey: int, message: Bytes) -> BLSSignature` -- `def Verify(pubkey: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool` -- `def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature` -- `def FastAggregateVerify(pubkeys: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool` -- `def AggregateVerify(pubkeys: Sequence[BLSPubkey], messages: Sequence[Bytes], signature: BLSSignature) -> bool` - -The above functions are accessed through the `bls` module, e.g. `bls.Verify`. - -### Predicates - -#### `is_active_validator` - ```python def is_active_validator(validator: Validator, epoch: Epoch) -> bool: """ @@ -635,9 +212,6 @@ def is_active_validator(validator: Validator, epoch: Epoch) -> bool: """ return validator.activation_epoch <= epoch < validator.exit_epoch ``` - -#### `is_eligible_for_activation_queue` - ```python def is_eligible_for_activation_queue(validator: Validator) -> bool: """ @@ -648,9 +222,6 @@ def is_eligible_for_activation_queue(validator: Validator) -> bool: and validator.effective_balance == MAX_EFFECTIVE_BALANCE ) ``` - -#### `is_eligible_for_activation` - ```python def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool: """ @@ -663,9 +234,6 @@ def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool and validator.activation_epoch == FAR_FUTURE_EPOCH ) ``` - -#### `is_slashable_validator` - ```python def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ @@ -673,9 +241,6 @@ def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: """ return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) ``` - -#### `is_slashable_attestation_data` - ```python def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: """ @@ -688,9 +253,6 @@ def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationDa (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) ) ``` - -#### `is_valid_indexed_attestation` - ```python def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: """ @@ -706,9 +268,6 @@ def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: Indexe signing_root = compute_signing_root(indexed_attestation.data, domain) return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) ``` - -#### `is_valid_merkle_branch` - ```python def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint64, index: uint64, root: Root) -> bool: """ @@ -722,11 +281,6 @@ def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint value = hash(value + branch[i]) return value == root ``` - -### Misc - -#### `compute_shuffled_index` - ```python def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: """ @@ -751,9 +305,6 @@ def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> return index ``` - -#### `compute_proposer_index` - ```python def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32) -> ValidatorIndex: """ @@ -771,9 +322,6 @@ def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex] return candidate_index i += 1 ``` - -#### `compute_committee` - ```python def compute_committee(indices: Sequence[ValidatorIndex], seed: Bytes32, @@ -786,9 +334,6 @@ def compute_committee(indices: Sequence[ValidatorIndex], end = (len(indices) * uint64(index + 1)) // count return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)] ``` - -#### `compute_epoch_at_slot` - ```python def compute_epoch_at_slot(slot: Slot) -> Epoch: """ @@ -796,9 +341,6 @@ def compute_epoch_at_slot(slot: Slot) -> Epoch: """ return Epoch(slot // SLOTS_PER_EPOCH) ``` - -#### `compute_start_slot_at_epoch` - ```python def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: """ @@ -806,9 +348,6 @@ def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: """ return Slot(epoch * SLOTS_PER_EPOCH) ``` - -#### `compute_activation_exit_epoch` - ```python def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: """ @@ -816,9 +355,6 @@ def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: """ return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) ``` - -#### `compute_fork_data_root` - ```python def compute_fork_data_root(current_version: Version, genesis_validators_root: Root) -> Root: """ @@ -830,9 +366,6 @@ def compute_fork_data_root(current_version: Version, genesis_validators_root: Ro genesis_validators_root=genesis_validators_root, )) ``` - -#### `compute_fork_digest` - ```python def compute_fork_digest(current_version: Version, genesis_validators_root: Root) -> ForkDigest: """ @@ -842,9 +375,6 @@ def compute_fork_digest(current_version: Version, genesis_validators_root: Root) """ return ForkDigest(compute_fork_data_root(current_version, genesis_validators_root)[:4]) ``` - -#### `compute_domain` - ```python def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_validators_root: Root=None) -> Domain: """ @@ -857,9 +387,6 @@ def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_ fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) return Domain(domain_type + fork_data_root[:28]) ``` - -#### `compute_signing_root` - ```python def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: """ @@ -870,11 +397,6 @@ def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: domain=domain, )) ``` - -### Beacon state accessors - -#### `get_current_epoch` - ```python def get_current_epoch(state: BeaconState) -> Epoch: """ @@ -882,9 +404,6 @@ def get_current_epoch(state: BeaconState) -> Epoch: """ return compute_epoch_at_slot(state.slot) ``` - -#### `get_previous_epoch` - ```python def get_previous_epoch(state: BeaconState) -> Epoch: """` @@ -893,9 +412,6 @@ def get_previous_epoch(state: BeaconState) -> Epoch: current_epoch = get_current_epoch(state) return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) ``` - -#### `get_block_root` - ```python def get_block_root(state: BeaconState, epoch: Epoch) -> Root: """ @@ -903,9 +419,6 @@ def get_block_root(state: BeaconState, epoch: Epoch) -> Root: """ return get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) ``` - -#### `get_block_root_at_slot` - ```python def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: """ @@ -914,9 +427,6 @@ def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] ``` - -#### `get_randao_mix` - ```python def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: """ @@ -924,9 +434,6 @@ def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: """ return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] ``` - -#### `get_active_validator_indices` - ```python def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: """ @@ -934,9 +441,6 @@ def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[V """ return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] ``` - -#### `get_validator_churn_limit` - ```python def get_validator_churn_limit(state: BeaconState) -> uint64: """ @@ -945,9 +449,6 @@ def get_validator_churn_limit(state: BeaconState) -> uint64: active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT) ``` - -#### `get_seed` - ```python def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes32: """ @@ -956,9 +457,6 @@ def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow return hash(domain_type + uint_to_bytes(epoch) + mix) ``` - -#### `get_committee_count_per_slot` - ```python def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: """ @@ -969,9 +467,6 @@ def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, )) ``` - -#### `get_beacon_committee` - ```python def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]: """ @@ -986,9 +481,6 @@ def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) count=committees_per_slot * SLOTS_PER_EPOCH, ) ``` - -#### `get_beacon_proposer_index` - ```python def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: """ @@ -999,9 +491,6 @@ def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: indices = get_active_validator_indices(state, epoch) return compute_proposer_index(state, indices, seed) ``` - -#### `get_total_balance` - ```python def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ @@ -1011,9 +500,6 @@ def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: """ return Gwei(max(EFFECTIVE_BALANCE_INCREMENT, sum([state.validators[index].effective_balance for index in indices]))) ``` - -#### `get_total_active_balance` - ```python def get_total_active_balance(state: BeaconState) -> Gwei: """ @@ -1022,9 +508,6 @@ def get_total_active_balance(state: BeaconState) -> Gwei: """ return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) ``` - -#### `get_domain` - ```python def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: """ @@ -1034,9 +517,6 @@ def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) - fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version return compute_domain(domain_type, fork_version, state.genesis_validators_root) ``` - -#### `get_indexed_attestation` - ```python def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: """ @@ -1050,9 +530,6 @@ def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> Ind signature=attestation.signature, ) ``` - -#### `get_attesting_indices` - ```python def get_attesting_indices(state: BeaconState, data: AttestationData, @@ -1063,11 +540,6 @@ def get_attesting_indices(state: BeaconState, committee = get_beacon_committee(state, data.slot, data.index) return set(index for i, index in enumerate(committee) if bits[i]) ``` - -### Beacon state mutators - -#### `increase_balance` - ```python def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ @@ -1075,9 +547,6 @@ def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> """ state.balances[index] += delta ``` - -#### `decrease_balance` - ```python def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: """ @@ -1085,9 +554,6 @@ def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> """ state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta ``` - -#### `initiate_validator_exit` - ```python def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: """ @@ -1109,9 +575,6 @@ def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: validator.exit_epoch = exit_queue_epoch validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) ``` - -#### `slash_validator` - ```python def slash_validator(state: BeaconState, slashed_index: ValidatorIndex, @@ -1136,17 +599,6 @@ def slash_validator(state: BeaconState, increase_balance(state, proposer_index, proposer_reward) increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) ``` - -## Genesis - -Before the Ethereum 2.0 genesis has been triggered, and for every Ethereum 1.0 block, let `candidate_state = initialize_beacon_state_from_eth1(eth1_block_hash, eth1_timestamp, deposits)` where: - -- `eth1_block_hash` is the hash of the Ethereum 1.0 block -- `eth1_timestamp` is the Unix timestamp corresponding to `eth1_block_hash` -- `deposits` is the sequence of all deposits, ordered chronologically, up to (and including) the block with hash `eth1_block_hash` - -Eth1 blocks must only be considered once they are at least `SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE` seconds old (i.e. `eth1_timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= current_unix_time`). Due to this constraint, if `GENESIS_DELAY < SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE`, then the `genesis_time` can happen before the time/state is first known. Values should be configured to avoid this case. - ```python def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, eth1_timestamp: uint64, @@ -1184,13 +636,6 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, return state ``` - -*Note*: The ETH1 block with `eth1_timestamp` meeting the minimum genesis active validator count criteria can also occur before `MIN_GENESIS_TIME`. - -### Genesis state - -Let `genesis_state = candidate_state` whenever `is_valid_genesis_state(candidate_state) is True` for the first time. - ```python def is_valid_genesis_state(state: BeaconState) -> bool: if state.genesis_time < MIN_GENESIS_TIME: @@ -1199,15 +644,6 @@ def is_valid_genesis_state(state: BeaconState) -> bool: return False return True ``` - -### Genesis block - -Let `genesis_block = BeaconBlock(state_root=hash_tree_root(genesis_state))`. - -## Beacon chain state transition function - -The post-state corresponding to a pre-state `state` and a signed block `signed_block` is defined as `state_transition(state, signed_block)`. State transitions that trigger an unhandled exception (e.g. a failed `assert` or an out-of-range list access) are considered invalid. State transitions that cause a `uint64` overflow or underflow are also considered invalid. - ```python def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> None: block = signed_block.message @@ -1222,14 +658,12 @@ def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, valida if validate_result: assert block.state_root == hash_tree_root(state) ``` - ```python def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: proposer = state.validators[signed_block.message.proposer_index] signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) ``` - ```python def process_slots(state: BeaconState, slot: Slot) -> None: assert state.slot < slot @@ -1240,7 +674,6 @@ def process_slots(state: BeaconState, slot: Slot) -> None: process_epoch(state) state.slot = Slot(state.slot + 1) ``` - ```python def process_slot(state: BeaconState) -> None: # Cache state root @@ -1253,9 +686,6 @@ def process_slot(state: BeaconState) -> None: previous_block_root = hash_tree_root(state.latest_block_header) state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root ``` - -### Epoch processing - ```python def process_epoch(state: BeaconState) -> None: process_justification_and_finalization(state) @@ -1269,15 +699,11 @@ def process_epoch(state: BeaconState) -> None: process_historical_roots_update(state) process_participation_record_updates(state) ``` - -#### Helper functions - ```python def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: assert epoch in (get_previous_epoch(state), get_current_epoch(state)) return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations ``` - ```python def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ @@ -1285,7 +711,6 @@ def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequen if a.data.target.root == get_block_root(state, epoch) ] ``` - ```python def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: return [ @@ -1293,7 +718,6 @@ def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) ] ``` - ```python def get_unslashed_attesting_indices(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: @@ -1302,7 +726,6 @@ def get_unslashed_attesting_indices(state: BeaconState, output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) return set(filter(lambda index: not state.validators[index].slashed, output)) ``` - ```python def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: """ @@ -1311,9 +734,6 @@ def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAtte """ return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) ``` - -#### Justification and finalization - ```python def process_justification_and_finalization(state: BeaconState) -> None: # Initial FFG checkpoint values have a `0x00` stub for `root`. @@ -1327,7 +747,6 @@ def process_justification_and_finalization(state: BeaconState) -> None: current_target_balance = get_attesting_balance(state, current_attestations) weigh_justification_and_finalization(state, total_active_balance, previous_target_balance, current_target_balance) ``` - ```python def weigh_justification_and_finalization(state: BeaconState, total_active_balance: Gwei, @@ -1366,37 +785,24 @@ def weigh_justification_and_finalization(state: BeaconState, if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: state.finalized_checkpoint = old_current_justified_checkpoint ``` - -#### Rewards and penalties - -##### Helpers - ```python def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: total_balance = get_total_active_balance(state) effective_balance = state.validators[index].effective_balance return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) ``` - - ```python def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) ``` - - ```python def get_finality_delay(state: BeaconState) -> uint64: return get_previous_epoch(state) - state.finalized_checkpoint.epoch ``` - - ```python def is_in_inactivity_leak(state: BeaconState) -> bool: return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY ``` - - ```python def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: previous_epoch = get_previous_epoch(state) @@ -1405,7 +811,6 @@ def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorInde if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) ] ``` - ```python def get_attestation_component_deltas(state: BeaconState, attestations: Sequence[PendingAttestation] @@ -1432,9 +837,6 @@ def get_attestation_component_deltas(state: BeaconState, penalties[index] += get_base_reward(state, index) return rewards, penalties ``` - -##### Components of attestation deltas - ```python def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1443,7 +845,6 @@ def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) return get_attestation_component_deltas(state, matching_source_attestations) ``` - ```python def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1452,7 +853,6 @@ def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) return get_attestation_component_deltas(state, matching_target_attestations) ``` - ```python def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1461,7 +861,6 @@ def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]] matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) return get_attestation_component_deltas(state, matching_head_attestations) ``` - ```python def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1482,7 +881,6 @@ def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequ penalties = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` - ```python def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1504,9 +902,6 @@ def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], S rewards = [Gwei(0) for _ in range(len(state.validators))] return rewards, penalties ``` - -##### `get_attestation_deltas` - ```python def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: """ @@ -1530,9 +925,6 @@ def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence return rewards, penalties ``` - -##### `process_rewards_and_penalties` - ```python def process_rewards_and_penalties(state: BeaconState) -> None: # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch @@ -1544,9 +936,6 @@ def process_rewards_and_penalties(state: BeaconState) -> None: increase_balance(state, ValidatorIndex(index), rewards[index]) decrease_balance(state, ValidatorIndex(index), penalties[index]) ``` - -#### Registry updates - ```python def process_registry_updates(state: BeaconState) -> None: # Process activation eligibility and ejections @@ -1568,9 +957,6 @@ def process_registry_updates(state: BeaconState) -> None: validator = state.validators[index] validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) ``` - -#### Slashings - ```python def process_slashings(state: BeaconState) -> None: epoch = get_current_epoch(state) @@ -1583,8 +969,6 @@ def process_slashings(state: BeaconState) -> None: penalty = penalty_numerator // total_balance * increment decrease_balance(state, ValidatorIndex(index), penalty) ``` - -#### Eth1 data votes updates ```python def process_eth1_data_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) @@ -1592,9 +976,6 @@ def process_eth1_data_reset(state: BeaconState) -> None: if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: state.eth1_data_votes = [] ``` - -#### Effective balances updates - ```python def process_effective_balance_updates(state: BeaconState) -> None: # Update effective balances with hysteresis @@ -1609,18 +990,12 @@ def process_effective_balance_updates(state: BeaconState) -> None: ): validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) ``` - -#### Slashings balances updates - ```python def process_slashings_reset(state: BeaconState) -> None: next_epoch = Epoch(get_current_epoch(state) + 1) # Reset slashings state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) ``` - -#### Randao mixes updates - ```python def process_randao_mixes_reset(state: BeaconState) -> None: current_epoch = get_current_epoch(state) @@ -1628,8 +1003,6 @@ def process_randao_mixes_reset(state: BeaconState) -> None: # Set randao mix state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) ``` - -#### Historical roots updates ```python def process_historical_roots_update(state: BeaconState) -> None: # Set historical root accumulator @@ -1638,18 +1011,12 @@ def process_historical_roots_update(state: BeaconState) -> None: historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) state.historical_roots.append(hash_tree_root(historical_batch)) ``` - -#### Participation records rotation - ```python def process_participation_record_updates(state: BeaconState) -> None: # Rotate current/previous epoch attestations state.previous_epoch_attestations = state.current_epoch_attestations state.current_epoch_attestations = [] ``` - -### Block processing - ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) @@ -1657,9 +1024,6 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_operations(state, block.body) ``` - -#### Block header - ```python def process_block_header(state: BeaconState, block: BeaconBlock) -> None: # Verify that the slots match @@ -1683,9 +1047,6 @@ def process_block_header(state: BeaconState, block: BeaconBlock) -> None: proposer = state.validators[block.proposer_index] assert not proposer.slashed ``` - -#### RANDAO - ```python def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: epoch = get_current_epoch(state) @@ -1697,18 +1058,12 @@ def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix ``` - -#### Eth1 data - ```python def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: state.eth1_data_votes.append(body.eth1_data) if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: state.eth1_data = body.eth1_data ``` - -#### Operations - ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: # Verify that outstanding deposits are processed up to the maximum number of deposits @@ -1724,9 +1079,6 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) ``` - -##### Proposer slashings - ```python def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: header_1 = proposer_slashing.signed_header_1.message @@ -1749,9 +1101,6 @@ def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSla slash_validator(state, header_1.proposer_index) ``` - -##### Attester slashings - ```python def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: attestation_1 = attester_slashing.attestation_1 @@ -1768,9 +1117,6 @@ def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSla slashed_any = True assert slashed_any ``` - -##### Attestations - ```python def process_attestation(state: BeaconState, attestation: Attestation) -> None: data = attestation.data @@ -1799,9 +1145,6 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None: # Verify signature assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) ``` - -##### Deposits - ```python def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: amount = deposit.data.amount @@ -1817,7 +1160,6 @@ def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validato effective_balance=effective_balance, ) ``` - ```python def process_deposit(state: BeaconState, deposit: Deposit) -> None: # Verify the Merkle branch @@ -1855,9 +1197,6 @@ def process_deposit(state: BeaconState, deposit: Deposit) -> None: index = ValidatorIndex(validator_pubkeys.index(pubkey)) increase_balance(state, index, amount) ``` - -##### Voluntary exits - ```python def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: voluntary_exit = signed_voluntary_exit.message @@ -1877,3 +1216,571 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) ``` +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root +``` +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) +``` +```python +def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: + assert anchor_block.state_root == hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: copy(anchor_block)}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, + ) +``` +```python +def get_slots_since_genesis(store: Store) -> int: + return (store.time - store.genesis_time) // SECONDS_PER_SLOT +``` +```python +def get_current_slot(store: Store) -> Slot: + return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) +``` +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` +```python +def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: + block = store.blocks[root] + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` +```python +def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + )) +``` +```python +def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: + block = store.blocks[block_root] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == block_root + ] + + # If any children branches contain expected finalized/justified checkpoints, + # add to filtered block-tree and signal viability to parent. + if any(children): + filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + # If leaf block, check finalized/justified checkpoints as matching latest. + head_state = store.block_states[block_root] + + correct_justified = ( + store.justified_checkpoint.epoch == GENESIS_EPOCH + or head_state.current_justified_checkpoint == store.justified_checkpoint + ) + correct_finalized = ( + store.finalized_checkpoint.epoch == GENESIS_EPOCH + or head_state.finalized_checkpoint == store.finalized_checkpoint + ) + # If expected finalized/justified, add to viable block-tree and signal viability to parent. + if correct_justified and correct_finalized: + blocks[block_root] = block + return True + + # Otherwise, branch not viable + return False +``` +```python +def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: + """ + Retrieve a filtered block tree from ``store``, only returning branches + whose leaf state's justified/finalized info agrees with that in ``store``. + """ + base = store.justified_checkpoint.root + blocks: Dict[Root, BeaconBlock] = {} + filter_block_tree(store, base, blocks) + return blocks +``` +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].parent_root == head + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) +``` +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: + return False + + return True +``` +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # LMD vote must be consistent with FFG vote target + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = copy(store.block_states[target.root]) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` +```python +def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time + store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint +``` +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint +``` +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) +``` +``` +( + seq_number: uint64 + attnets: Bitvector[ATTESTATION_SUBNET_COUNT] +) +``` +``` +/ProtocolPrefix/MessageName/SchemaVersion/Encoding +``` +``` +request ::= | +response ::= * +response_chunk ::= | | +result ::= “0” | “1” | “2” | [“128” ... ”255”] +``` +``` +( + error_message: List[byte, 256] +) +``` +``` +( + fork_digest: ForkDigest + finalized_root: Root + finalized_epoch: Epoch + head_root: Root + head_slot: Slot +) +``` +``` +( + uint64 +) +``` +``` +( + start_slot: Slot + count: uint64 + step: uint64 +) +``` +``` +( + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] +) +``` +``` +( + List[Root, MAX_REQUEST_BLOCKS] +) +``` +``` +( + List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] +) +``` +``` +( + uint64 +) +``` +``` +( + uint64 +) +``` +``` +( + MetaData +) +``` +``` +( + fork_digest: ForkDigest + next_fork_version: Version + next_fork_epoch: Epoch +) +``` +```python +class Eth1Block(Container): + timestamp: uint64 + deposit_root: Root + deposit_count: uint64 + # All other eth1 block fields +``` +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` +```python +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) +``` +```python +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex + ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the index to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. + """ + next_epoch = Epoch(get_current_epoch(state) + 1) + assert epoch <= next_epoch + + start_slot = compute_start_slot_at_epoch(epoch) + committee_count_per_slot = get_committee_count_per_slot(state, epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): + for index in range(committee_count_per_slot): + committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) + if validator_index in committee: + return committee, CommitteeIndex(index), Slot(slot) + return None +``` +```python +def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` +```python +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) +``` +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) + return compute_time_at_slot(state, eth1_voting_period_start_slot) +``` +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start + ) +``` +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] + + # Valid votes already cast during this period + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data + + return max( + valid_votes, + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance + default=default_vote + ) +``` +```python +def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) +``` +```python +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) + committees_since_epoch_start = committees_per_slot * slots_since_epoch_start + + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) +``` +```python +def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: + committee = get_beacon_committee(state, slot, index) + modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) + return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 +``` +```python +def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: + signatures = [attestation.signature for attestation in attestations] + return bls.Aggregate(signatures) +``` +```python +def get_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + if T * (200 + 3 * D) < t * (200 + 12 * D): + epochs_for_validator_set_churn = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + epochs_for_balance_top_ups = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) + else: + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) + + return ws_period +``` +```python +def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: + # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint + assert ws_state.latest_block_header.state_root == ws_checkpoint.root + assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch + + ws_period = compute_weak_subjectivity_period(ws_state) + ws_state_epoch = compute_epoch_at_slot(ws_state.slot) + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + return current_epoch <= ws_state_epoch + ws_period +``` diff --git a/tools/analyzers/specdocs/data/phase0/deposit-contract.md b/tools/analyzers/specdocs/data/phase0/deposit-contract.md deleted file mode 100644 index 02e762daef2..00000000000 --- a/tools/analyzers/specdocs/data/phase0/deposit-contract.md +++ /dev/null @@ -1,77 +0,0 @@ -# Ethereum 2.0 Phase 0 -- Deposit Contract - -## Table of contents - - - - -- [Introduction](#introduction) -- [Constants](#constants) -- [Configuration](#configuration) -- [Ethereum 1.0 deposit contract](#ethereum-10-deposit-contract) - - [`deposit` function](#deposit-function) - - [Deposit amount](#deposit-amount) - - [Withdrawal credentials](#withdrawal-credentials) - - [`DepositEvent` log](#depositevent-log) -- [Solidity code](#solidity-code) - - - - -## Introduction - -This document represents the specification for the beacon chain deposit contract, part of Ethereum 2.0 Phase 0. - -## Constants - -The following values are (non-configurable) constants used throughout the specification. - -| Name | Value | -| - | - | -| `DEPOSIT_CONTRACT_TREE_DEPTH` | `2**5` (= 32) | - -## Configuration - -*Note*: The default mainnet configuration values are included here for spec-design purposes. -The different configurations for mainnet, testnets, and YAML-based testing can be found in the [`configs/constant_presets`](../../configs) directory. -These configurations are updated for releases and may be out of sync during `dev` changes. - -| Name | Value | -| - | - | -| `DEPOSIT_CHAIN_ID` | `1` | -| `DEPOSIT_NETWORK_ID` | `1` | -| `DEPOSIT_CONTRACT_ADDRESS` | `0x00000000219ab540356cBB839Cbe05303d7705Fa` | - -## Ethereum 1.0 deposit contract - -The initial deployment phases of Ethereum 2.0 are implemented without consensus changes to Ethereum 1.0. A deposit contract at address `DEPOSIT_CONTRACT_ADDRESS` is added to the Ethereum 1.0 chain defined by the [chain-id](https://eips.ethereum.org/EIPS/eip-155) -- `DEPOSIT_CHAIN_ID` -- and the network-id -- `DEPOSIT_NETWORK_ID` -- for deposits of ETH to the beacon chain. Validator balances will be withdrawable to the shards in Phase 2. - -_Note_: See [here](https://chainid.network/) for a comprehensive list of public Ethereum chain chain-id's and network-id's. - -### `deposit` function - -The deposit contract has a public `deposit` function to make deposits. It takes as arguments `bytes calldata pubkey, bytes calldata withdrawal_credentials, bytes calldata signature, bytes32 deposit_data_root`. The first three arguments populate a [`DepositData`](./beacon-chain.md#depositdata) object, and `deposit_data_root` is the expected `DepositData` root as a protection against malformatted calldata. - -#### Deposit amount - -The amount of ETH (rounded down to the closest Gwei) sent to the deposit contract is the deposit amount, which must be of size at least `MIN_DEPOSIT_AMOUNT` Gwei. Note that ETH consumed by the deposit contract is no longer usable on Ethereum 1.0. - -#### Withdrawal credentials - -One of the `DepositData` fields is `withdrawal_credentials` which constrains validator withdrawals. -The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. -The withdrawal prefixes currently supported are `BLS_WITHDRAWAL_PREFIX` and `ETH1_ADDRESS_WITHDRAWAL_PREFIX`. -Read more in the [validator guide](./validator.md#withdrawal-credentials). - -*Note*: The deposit contract does not validate the `withdrawal_credentials` field. -Support for new withdrawal prefixes can be added without modifying the deposit contract. - -#### `DepositEvent` log - -Every Ethereum 1.0 deposit emits a `DepositEvent` log for consumption by the beacon chain. The deposit contract does little validation, pushing most of the validator onboarding logic to the beacon chain. In particular, the proof of possession (a BLS12-381 signature) is not verified by the deposit contract. - -## Solidity code - -The deposit contract source code, written in Solidity, is available [here](../../solidity_deposit_contract/deposit_contract.sol). - -*Note*: To save on gas, the deposit contract uses a progressive Merkle root calculation algorithm that requires only O(log(n)) storage. See [here](https://github.com/ethereum/research/blob/master/beacon_chain_impl/progressive_merkle_tree.py) for a Python implementation, and [here](https://github.com/runtimeverification/verified-smart-contracts/blob/master/deposit/formal-incremental-merkle-tree-algorithm.pdf) for a formal correctness proof. diff --git a/tools/analyzers/specdocs/data/phase0/fork-choice.md b/tools/analyzers/specdocs/data/phase0/fork-choice.md deleted file mode 100644 index b5689ecd239..00000000000 --- a/tools/analyzers/specdocs/data/phase0/fork-choice.md +++ /dev/null @@ -1,406 +0,0 @@ -# Ethereum 2.0 Phase 0 -- Beacon Chain Fork Choice - -## Table of contents - - - - -- [Introduction](#introduction) -- [Fork choice](#fork-choice) - - [Configuration](#configuration) - - [Helpers](#helpers) - - [`LatestMessage`](#latestmessage) - - [`Store`](#store) - - [`get_forkchoice_store`](#get_forkchoice_store) - - [`get_slots_since_genesis`](#get_slots_since_genesis) - - [`get_current_slot`](#get_current_slot) - - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) - - [`get_ancestor`](#get_ancestor) - - [`get_latest_attesting_balance`](#get_latest_attesting_balance) - - [`filter_block_tree`](#filter_block_tree) - - [`get_filtered_block_tree`](#get_filtered_block_tree) - - [`get_head`](#get_head) - - [`should_update_justified_checkpoint`](#should_update_justified_checkpoint) - - [`on_attestation` helpers](#on_attestation-helpers) - - [`validate_on_attestation`](#validate_on_attestation) - - [`store_target_checkpoint_state`](#store_target_checkpoint_state) - - [`update_latest_messages`](#update_latest_messages) - - [Handlers](#handlers) - - [`on_tick`](#on_tick) - - [`on_block`](#on_block) - - [`on_attestation`](#on_attestation) - - - - -## Introduction - -This document is the beacon chain fork choice spec, part of Ethereum 2.0 Phase 0. It assumes the [beacon chain state transition function spec](./beacon-chain.md). - -## Fork choice - -The head block root associated with a `store` is defined as `get_head(store)`. At genesis, let `store = get_forkchoice_store(genesis_state)` and update `store` by running: - -- `on_tick(store, time)` whenever `time > store.time` where `time` is the current Unix time -- `on_block(store, block)` whenever a block `block: SignedBeaconBlock` is received -- `on_attestation(store, attestation)` whenever an attestation `attestation` is received - -Any of the above handlers that trigger an unhandled exception (e.g. a failed assert or an out-of-range list access) are considered invalid. Invalid calls to handlers must not modify `store`. - -*Notes*: - -1) **Leap seconds**: Slots will last `SECONDS_PER_SLOT + 1` or `SECONDS_PER_SLOT - 1` seconds around leap seconds. This is automatically handled by [UNIX time](https://en.wikipedia.org/wiki/Unix_time). -2) **Honest clocks**: Honest nodes are assumed to have clocks synchronized within `SECONDS_PER_SLOT` seconds of each other. -3) **Eth1 data**: The large `ETH1_FOLLOW_DISTANCE` specified in the [honest validator document](./validator.md) should ensure that `state.latest_eth1_data` of the canonical Ethereum 2.0 chain remains consistent with the canonical Ethereum 1.0 chain. If not, emergency manual intervention will be required. -4) **Manual forks**: Manual forks may arbitrarily change the fork choice rule but are expected to be enacted at epoch transitions, with the fork details reflected in `state.fork`. -5) **Implementation**: The implementation found in this specification is constructed for ease of understanding rather than for optimization in computation, space, or any other resource. A number of optimized alternatives can be found [here](https://github.com/protolambda/lmd-ghost). - -### Configuration - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `SAFE_SLOTS_TO_UPDATE_JUSTIFIED` | `2**3` (= 8) | slots | 96 seconds | - -### Helpers - -#### `LatestMessage` - -```python -@dataclass(eq=True, frozen=True) -class LatestMessage(object): - epoch: Epoch - root: Root -``` - -#### `Store` - -```python -@dataclass -class Store(object): - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) -``` - -#### `get_forkchoice_store` - -The provided anchor-state will be regarded as a trusted state, to not roll back beyond. -This should be the genesis state for a full client. - -*Note* With regards to fork choice, block headers are interchangeable with blocks. The spec is likely to move to headers for reduced overhead in test vectors and better encapsulation. Full implementations store blocks as part of their database and will often use full blocks when dealing with production fork choice. - -```python -def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: - assert anchor_block.state_root == hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - return Store( - time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: copy(anchor_block)}, - block_states={anchor_root: copy(anchor_state)}, - checkpoint_states={justified_checkpoint: copy(anchor_state)}, - ) -``` - -#### `get_slots_since_genesis` - -```python -def get_slots_since_genesis(store: Store) -> int: - return (store.time - store.genesis_time) // SECONDS_PER_SLOT -``` - -#### `get_current_slot` - -```python -def get_current_slot(store: Store) -> Slot: - return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) -``` - -#### `compute_slots_since_epoch_start` - -```python -def compute_slots_since_epoch_start(slot: Slot) -> int: - return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) -``` - -#### `get_ancestor` - -```python -def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: - block = store.blocks[root] - if block.slot > slot: - return get_ancestor(store, block.parent_root, slot) - elif block.slot == slot: - return root - else: - # root is older than queried slot, thus a skip slot. Return most recent root prior to slot - return root -``` - -#### `get_latest_attesting_balance` - -```python -def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: - state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state, get_current_epoch(state)) - return Gwei(sum( - state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages - and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) - )) -``` - -#### `filter_block_tree` - -```python -def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: - block = store.blocks[block_root] - children = [ - root for root in store.blocks.keys() - if store.blocks[root].parent_root == block_root - ] - - # If any children branches contain expected finalized/justified checkpoints, - # add to filtered block-tree and signal viability to parent. - if any(children): - filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - # If leaf block, check finalized/justified checkpoints as matching latest. - head_state = store.block_states[block_root] - - correct_justified = ( - store.justified_checkpoint.epoch == GENESIS_EPOCH - or head_state.current_justified_checkpoint == store.justified_checkpoint - ) - correct_finalized = ( - store.finalized_checkpoint.epoch == GENESIS_EPOCH - or head_state.finalized_checkpoint == store.finalized_checkpoint - ) - # If expected finalized/justified, add to viable block-tree and signal viability to parent. - if correct_justified and correct_finalized: - blocks[block_root] = block - return True - - # Otherwise, branch not viable - return False -``` - -#### `get_filtered_block_tree` - -```python -def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: - """ - Retrieve a filtered block tree from ``store``, only returning branches - whose leaf state's justified/finalized info agrees with that in ``store``. - """ - base = store.justified_checkpoint.root - blocks: Dict[Root, BeaconBlock] = {} - filter_block_tree(store, base, blocks) - return blocks -``` - -#### `get_head` - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - children = [ - root for root in blocks.keys() - if blocks[root].parent_root == head - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) -``` - -#### `should_update_justified_checkpoint` - -```python -def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: - """ - To address the bouncing attack, only update conflicting justified - checkpoints in the fork choice if in the early slots of the epoch. - Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. - - See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. - """ - if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: - return True - - justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) - if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: - return False - - return True -``` - -#### `on_attestation` helpers - -##### `validate_on_attestation` - -```python -def validate_on_attestation(store: Store, attestation: Attestation) -> None: - target = attestation.data.target - - # Attestations must be from the current or previous epoch - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - # Use GENESIS_EPOCH for previous when genesis to avoid underflow - previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH - # If attestation target is from a future epoch, delay consideration until the epoch arrives - assert target.epoch in [current_epoch, previous_epoch] - assert target.epoch == compute_epoch_at_slot(attestation.data.slot) - - # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found - assert target.root in store.blocks - - # Attestations must be for a known block. If block is unknown, delay consideration until the block is found - assert attestation.data.beacon_block_root in store.blocks - # Attestations must not be for blocks in the future. If not, the attestation should not be considered - assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - - # LMD vote must be consistent with FFG vote target - target_slot = compute_start_slot_at_epoch(target.epoch) - assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) - - # Attestations can only affect the fork choice of subsequent slots. - # Delay consideration in the fork choice until their slot is in the past. - assert get_current_slot(store) >= attestation.data.slot + 1 -``` - -##### `store_target_checkpoint_state` - -```python -def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: - # Store target checkpoint state if not yet seen - if target not in store.checkpoint_states: - base_state = copy(store.block_states[target.root]) - if base_state.slot < compute_start_slot_at_epoch(target.epoch): - process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) - store.checkpoint_states[target] = base_state -``` - -##### `update_latest_messages` - -```python -def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: - target = attestation.data.target - beacon_block_root = attestation.data.beacon_block_root - for i in attesting_indices: - if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: - store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) -``` - - -### Handlers - -#### `on_tick` - -```python -def on_tick(store: Store, time: uint64) -> None: - previous_slot = get_current_slot(store) - - # update store time - store.time = time - - current_slot = get_current_slot(store) - # Not a new epoch, return - if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): - return - # Update store.justified_checkpoint if a better checkpoint is known - if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = store.best_justified_checkpoint -``` - -#### `on_block` - -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - pre_state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root - - # Check the block is valid and compute the post-state - state = pre_state.copy() - state_transition(state, signed_block, True) - # Add new block to the store - store.blocks[hash_tree_root(block)] = block - # Add new state for this block to the store - store.block_states[hash_tree_root(block)] = state - - # Update justified checkpoint - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: - store.best_justified_checkpoint = state.current_justified_checkpoint - if should_update_justified_checkpoint(store, state.current_justified_checkpoint): - store.justified_checkpoint = state.current_justified_checkpoint - - # Update finalized checkpoint - if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: - store.finalized_checkpoint = state.finalized_checkpoint - - # Potentially update justified if different from store - if store.justified_checkpoint != state.current_justified_checkpoint: - # Update justified if new justified is later than store justified - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = state.current_justified_checkpoint - return - - # Update justified if store justified is not in chain with finalized checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) - if ancestor_at_finalized_slot != store.finalized_checkpoint.root: - store.justified_checkpoint = state.current_justified_checkpoint -``` - -#### `on_attestation` - -```python -def on_attestation(store: Store, attestation: Attestation) -> None: - """ - Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. - - An ``attestation`` that is asserted as invalid may be valid at a later time, - consider scheduling it for later processing in such case. - """ - validate_on_attestation(store, attestation) - store_target_checkpoint_state(store, attestation.data.target) - - # Get state at the `target` to fully validate attestation - target_state = store.checkpoint_states[attestation.data.target] - indexed_attestation = get_indexed_attestation(target_state, attestation) - assert is_valid_indexed_attestation(target_state, indexed_attestation) - - # Update latest messages for attesting indices - update_latest_messages(store, indexed_attestation.attesting_indices, attestation) -``` diff --git a/tools/analyzers/specdocs/data/phase0/p2p-interface.md b/tools/analyzers/specdocs/data/phase0/p2p-interface.md deleted file mode 100644 index e9e8e092c47..00000000000 --- a/tools/analyzers/specdocs/data/phase0/p2p-interface.md +++ /dev/null @@ -1,1526 +0,0 @@ -# Ethereum 2.0 networking specification - -This document contains the networking specification for Ethereum 2.0 clients. - -It consists of four main sections: - -1. A specification of the network fundamentals. -2. A specification of the three network interaction *domains* of Eth2: (a) the gossip domain, (b) the discovery domain, and (c) the Req/Resp domain. -3. The rationale and further explanation for the design choices made in the previous two sections. -4. An analysis of the maturity/state of the libp2p features required by this spec across the languages in which Eth2 clients are being developed. - -## Table of contents - - - - -- [Network fundamentals](#network-fundamentals) - - [Transport](#transport) - - [Encryption and identification](#encryption-and-identification) - - [Protocol Negotiation](#protocol-negotiation) - - [Multiplexing](#multiplexing) -- [Eth2 network interaction domains](#eth2-network-interaction-domains) - - [Configuration](#configuration) - - [MetaData](#metadata) - - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - - [Topics and messages](#topics-and-messages) - - [Global topics](#global-topics) - - [`beacon_block`](#beacon_block) - - [`beacon_aggregate_and_proof`](#beacon_aggregate_and_proof) - - [`voluntary_exit`](#voluntary_exit) - - [`proposer_slashing`](#proposer_slashing) - - [`attester_slashing`](#attester_slashing) - - [Attestation subnets](#attestation-subnets) - - [`beacon_attestation_{subnet_id}`](#beacon_attestation_subnet_id) - - [Attestations and Aggregation](#attestations-and-aggregation) - - [Encodings](#encodings) - - [The Req/Resp domain](#the-reqresp-domain) - - [Protocol identification](#protocol-identification) - - [Req/Resp interaction](#reqresp-interaction) - - [Requesting side](#requesting-side) - - [Responding side](#responding-side) - - [Encoding strategies](#encoding-strategies) - - [SSZ-snappy encoding strategy](#ssz-snappy-encoding-strategy) - - [Messages](#messages) - - [Status](#status) - - [Goodbye](#goodbye) - - [BeaconBlocksByRange](#beaconblocksbyrange) - - [BeaconBlocksByRoot](#beaconblocksbyroot) - - [Ping](#ping) - - [GetMetaData](#getmetadata) - - [The discovery domain: discv5](#the-discovery-domain-discv5) - - [Integration into libp2p stacks](#integration-into-libp2p-stacks) - - [ENR structure](#enr-structure) - - [Attestation subnet bitfield](#attestation-subnet-bitfield) - - [`eth2` field](#eth2-field) -- [Design decision rationale](#design-decision-rationale) - - [Transport](#transport-1) - - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) - - [Can clients support other transports/handshakes than the ones mandated by the spec?](#can-clients-support-other-transportshandshakes-than-the-ones-mandated-by-the-spec) - - [What are the advantages of using TCP/QUIC/Websockets?](#what-are-the-advantages-of-using-tcpquicwebsockets) - - [Why do we not just support a single transport?](#why-do-we-not-just-support-a-single-transport) - - [Why are we not using QUIC from the start?](#why-are-we-not-using-quic-from-the-start) - - [Multiplexing](#multiplexing-1) - - [Why are we using mplex/yamux?](#why-are-we-using-mplexyamux) - - [Protocol Negotiation](#protocol-negotiation-1) - - [When is multiselect 2.0 due and why do we plan to migrate to it?](#when-is-multiselect-20-due-and-why-do-we-plan-to-migrate-to-it) - - [What is the difference between connection-level and stream-level protocol negotiation?](#what-is-the-difference-between-connection-level-and-stream-level-protocol-negotiation) - - [Encryption](#encryption) - - [Why are we not supporting SecIO?](#why-are-we-not-supporting-secio) - - [Why are we using Noise?](#why-are-we-using-noise) - - [Why are we using encryption at all?](#why-are-we-using-encryption-at-all) - - [Gossipsub](#gossipsub) - - [Why are we using a pub/sub algorithm for block and attestation propagation?](#why-are-we-using-a-pubsub-algorithm-for-block-and-attestation-propagation) - - [Why are we using topics to segregate encodings, yet only support one encoding?](#why-are-we-using-topics-to-segregate-encodings-yet-only-support-one-encoding) - - [How do we upgrade gossip channels (e.g. changes in encoding, compression)?](#how-do-we-upgrade-gossip-channels-eg-changes-in-encoding-compression) - - [Why must all clients use the same gossip topic instead of one negotiated between each peer pair?](#why-must-all-clients-use-the-same-gossip-topic-instead-of-one-negotiated-between-each-peer-pair) - - [Why are the topics strings and not hashes?](#why-are-the-topics-strings-and-not-hashes) - - [Why are we using the `StrictNoSign` signature policy?](#why-are-we-using-the-strictnosign-signature-policy) - - [Why are we overriding the default libp2p pubsub `message-id`?](#why-are-we-overriding-the-default-libp2p-pubsub-message-id) - - [Why are these specific gossip parameters chosen?](#why-are-these-specific-gossip-parameters-chosen) - - [Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets?](#why-is-there-maximum_gossip_clock_disparity-when-validating-slot-ranges-of-messages-in-gossip-subnets) - - [Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets?](#why-are-there-attestation_subnet_count-attestation-subnets) - - [Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots?](#why-are-attestations-limited-to-be-broadcast-on-gossip-channels-within-slots_per_epoch-slots) - - [Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s?](#why-are-aggregate-attestations-broadcast-to-the-global-topic-as-aggregateandproofs-rather-than-just-as-attestations) - - [Why are we sending entire objects in the pubsub and not just hashes?](#why-are-we-sending-entire-objects-in-the-pubsub-and-not-just-hashes) - - [Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc?](#should-clients-gossip-blocks-if-they-cannot-validate-the-proposer-signature-due-to-not-yet-being-synced-not-knowing-the-head-block-etc) - - [How are we going to discover peers in a gossipsub topic?](#how-are-we-going-to-discover-peers-in-a-gossipsub-topic) - - [How should fork version be used in practice?](#how-should-fork-version-be-used-in-practice) - - [Req/Resp](#reqresp) - - [Why segregate requests into dedicated protocol IDs?](#why-segregate-requests-into-dedicated-protocol-ids) - - [Why are messages length-prefixed with a protobuf varint in the SSZ-encoding?](#why-are-messages-length-prefixed-with-a-protobuf-varint-in-the-ssz-encoding) - - [Why do we version protocol strings with ordinals instead of semver?](#why-do-we-version-protocol-strings-with-ordinals-instead-of-semver) - - [Why is it called Req/Resp and not RPC?](#why-is-it-called-reqresp-and-not-rpc) - - [Why do we allow empty responses in block requests?](#why-do-we-allow-empty-responses-in-block-requests) - - [Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from?](#why-does-beaconblocksbyrange-let-the-server-choose-which-branch-to-send-blocks-from) - - [What's the effect of empty slots on the sync algorithm?](#whats-the-effect-of-empty-slots-on-the-sync-algorithm) - - [Discovery](#discovery) - - [Why are we using discv5 and not libp2p Kademlia DHT?](#why-are-we-using-discv5-and-not-libp2p-kademlia-dht) - - [What is the difference between an ENR and a multiaddr, and why are we using ENRs?](#what-is-the-difference-between-an-enr-and-a-multiaddr-and-why-are-we-using-enrs) - - [Why do we not form ENRs and find peers until genesis block/state is known?](#why-do-we-not-form-enrs-and-find-peers-until-genesis-blockstate-is-known) - - [Compression/Encoding](#compressionencoding) - - [Why are we using SSZ for encoding?](#why-are-we-using-ssz-for-encoding) - - [Why are we compressing, and at which layers?](#why-are-we-compressing-and-at-which-layers) - - [Why are we using Snappy for compression?](#why-are-we-using-snappy-for-compression) - - [Can I get access to unencrypted bytes on the wire for debugging purposes?](#can-i-get-access-to-unencrypted-bytes-on-the-wire-for-debugging-purposes) - - [What are SSZ type size bounds?](#what-are-ssz-type-size-bounds) -- [libp2p implementations matrix](#libp2p-implementations-matrix) - - - - -# Network fundamentals - -This section outlines the specification for the networking stack in Ethereum 2.0 clients. - -## Transport - -Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), -we hereby define a profile for basic interoperability. - -All implementations MUST support the TCP libp2p transport, and it MUST be enabled for both dialing and listening (i.e. outbound and inbound connections). -The libp2p TCP transport supports listening on IPv4 and IPv6 addresses (and on multiple simultaneously). - -Clients must support listening on at least one of IPv4 or IPv6. -Clients that do _not_ have support for listening on IPv4 SHOULD be cognizant of the potential disadvantages in terms of -Internet-wide routability/support. Clients MAY choose to listen only on IPv6, but MUST be capable of dialing both IPv4 and IPv6 addresses. - -All listening endpoints must be publicly dialable, and thus not rely on libp2p circuit relay, AutoNAT, or AutoRelay facilities. -(Usage of circuit relay, AutoNAT, or AutoRelay will be specifically re-examined soon.) - -Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), -MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. - -## Encryption and identification - -The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure -channel handshake with `secp256k1` identities will be used for encryption. - -As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. - -## Protocol Negotiation - -Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. - -Clients MUST support [multistream-select 1.0](https://github.com/multiformats/multistream-select/) -and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. -Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. - -## Multiplexing - -During connection bootstrapping, libp2p dynamically negotiates a mutually supported multiplexing method to conduct parallel conversations. -This applies to transports that are natively incapable of multiplexing (e.g. TCP, WebSockets, WebRTC), -and is omitted for capable transports (e.g. QUIC). - -Two multiplexers are commonplace in libp2p implementations: -[mplex](https://github.com/libp2p/specs/tree/master/mplex) and [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). -Their protocol IDs are, respectively: `/mplex/6.7.0` and `/yamux/1.0.0`. - -Clients MUST support [mplex](https://github.com/libp2p/specs/tree/master/mplex) -and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). -If both are supported by the client, yamux MUST take precedence during negotiation. -See the [Rationale](#design-decision-rationale) section below for tradeoffs. - -# Eth2 network interaction domains - -## Configuration - -This section outlines constants that are used in this spec. - -| Name | Value | Description | -|---|---|---| -| `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | -| `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | -| `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | -| `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | -| `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | -| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500ms` | The maximum milliseconds of clock disparity assumed between honest nodes. | -| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `0x00000000` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | -| `MESSAGE_DOMAIN_VALID_SNAPPY` | `0x01000000` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | - - -## MetaData - -Clients MUST locally store the following `MetaData`: - -``` -( - seq_number: uint64 - attnets: Bitvector[ATTESTATION_SUBNET_COUNT] -) -``` - -Where - -- `seq_number` is a `uint64` starting at `0` used to version the node's metadata. - If any other field in the local `MetaData` changes, the node MUST increment `seq_number` by 1. -- `attnets` is a `Bitvector` representing the node's persistent attestation subnet subscriptions. - -*Note*: `MetaData.seq_number` is used for versioning of the node's metadata, -is entirely independent of the ENR sequence number, -and will in most cases be out of sync with the ENR sequence number. - -## The gossip domain: gossipsub - -Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md) libp2p Protocol -including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. - -**Protocol ID:** `/meshsub/1.1.0` - -**Gossipsub Parameters** - -The following gossipsub [parameters](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md#parameters) will be used: - -- `D` (topic stable mesh target count): 8 -- `D_low` (topic stable mesh low watermark): 6 -- `D_high` (topic stable mesh high watermark): 12 -- `D_lazy` (gossip target): 6 -- `heartbeat_interval` (frequency of heartbeat, seconds): 0.7 -- `fanout_ttl` (ttl for fanout maps for topics we are not subscribed to but have published to, seconds): 60 -- `mcache_len` (number of windows to retain full messages in cache for `IWANT` responses): 6 -- `mcache_gossip` (number of windows to gossip about): 3 -- `seen_ttl` (number of heartbeat intervals to retain message IDs): 550 - -*Note*: Gossipsub v1.1 introduces a number of -[additional parameters](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#overview-of-new-parameters) -for peer scoring and other attack mitigations. -These are currently under investigation and will be spec'd and released to mainnet when they are ready. - -### Topics and messages - -Topics are plain UTF-8 strings and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). -Topic strings have form: `/eth2/ForkDigestValue/Name/Encoding`. -This defines both the type of data being sent on the topic and how the data field of the message is encoded. - -- `ForkDigestValue` - the lowercase hex-encoded (no "0x" prefix) bytes of `compute_fork_digest(current_fork_version, genesis_validators_root)` where - - `current_fork_version` is the fork version of the epoch of the message to be sent on the topic - - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -- `Name` - see table below -- `Encoding` - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. - See the [Encodings](#Encodings) section for further details. - -*Note*: `ForkDigestValue` is composed of values that are not known until the genesis block/state are available. -Due to this, clients SHOULD NOT subscribe to gossipsub topics until these genesis values are known. - -Each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24) has a maximum size of `GOSSIP_MAX_SIZE`. -Clients MUST reject (fail validation) messages that are over this size limit. -Likewise, clients MUST NOT emit or propagate messages larger than this limit. - -The optional `from` (1), `seqno` (3), `signature` (5) and `key` (6) protobuf fields are omitted from the message, -since messages are identified by content, anonymous, and signed where necessary in the application layer. -Starting from Gossipsub v1.1, clients MUST enforce this by applying the `StrictNoSign` -[signature policy](https://github.com/libp2p/specs/blob/master/pubsub/README.md#signature-policy-options). - -The `message-id` of a gossipsub message MUST be the following 20 byte value computed from the message data: -* If `message.data` has a valid snappy decompression, set `message-id` to the first 20 bytes of the `SHA256` hash of - the concatenation of `MESSAGE_DOMAIN_VALID_SNAPPY` with the snappy decompressed message data, - i.e. `SHA256(MESSAGE_DOMAIN_VALID_SNAPPY + snappy_decompress(message.data))[:20]`. -* Otherwise, set `message-id` to the first 20 bytes of the `SHA256` hash of - the concatenation of `MESSAGE_DOMAIN_INVALID_SNAPPY` with the raw message data, - i.e. `SHA256(MESSAGE_DOMAIN_INVALID_SNAPPY + message.data)[:20]`. - -*Note*: The above logic handles two exceptional cases: -(1) multiple snappy `data` can decompress to the same value, -and (2) some message `data` can fail to snappy decompress altogether. - -The payload is carried in the `data` field of a gossipsub message, and varies depending on the topic: - -| Name | Message Type | -|----------------------------------|---------------------------| -| `beacon_block` | `SignedBeaconBlock` | -| `beacon_aggregate_and_proof` | `SignedAggregateAndProof` | -| `beacon_attestation_{subnet_id}` | `Attestation` | -| `voluntary_exit` | `SignedVoluntaryExit` | -| `proposer_slashing` | `ProposerSlashing` | -| `attester_slashing` | `AttesterSlashing` | - -Clients MUST reject (fail validation) messages containing an incorrect type, or invalid payload. - -When processing incoming gossip, clients MAY descore or disconnect peers who fail to observe these constraints. - -For any optional queueing, clients SHOULD maintain maximum queue sizes to avoid DoS vectors. - -Gossipsub v1.1 introduces [Extended Validators](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#extended-validators) -for the application to aid in the gossipsub peer-scoring scheme. -We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are application specific validations. -If all validations pass, return `ACCEPT`. -If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. - -#### Global topics - -There are two primary global topics used to propagate beacon blocks (`beacon_block`) -and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the network. - -There are three additional global topics that are used to propagate lower frequency validator messages -(`voluntary_exit`, `proposer_slashing`, and `attester_slashing`). - -##### `beacon_block` - -The `beacon_block` topic is used solely for propagating new signed beacon blocks to all nodes on the networks. -Signed blocks are sent in their entirety. - -The following validations MUST pass before forwarding the `signed_beacon_block` on the network. -- _[IGNORE]_ The block is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. validate that `signed_beacon_block.message.slot <= current_slot` - (a client MAY queue future blocks for processing at the appropriate slot). -- _[IGNORE]_ The block is from a slot greater than the latest finalized slot -- - i.e. validate that `signed_beacon_block.message.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` - (a client MAY choose to validate and store such blocks for additional purposes -- e.g. slashing detection, archive nodes, etc). -- _[IGNORE]_ The block is the first block with valid signature received for the proposer for the slot, `signed_beacon_block.message.slot`. -- _[REJECT]_ The proposer signature, `signed_beacon_block.signature`, is valid with respect to the `proposer_index` pubkey. -- _[IGNORE]_ The block's parent (defined by `block.parent_root`) has been seen - (via both gossip and non-gossip sources) - (a client MAY queue blocks for processing once the parent block is retrieved). -- _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. -- _[REJECT]_ The block is from a higher slot than its parent. -- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. - `get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) - == store.finalized_checkpoint.root` -- _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot - in the context of the current shuffling (defined by `parent_root`/`slot`). - If the `proposer_index` cannot immediately be verified against the expected shuffling, - the block MAY be queued for later processing while proposers for the block's branch are calculated -- - in such a case _do not_ `REJECT`, instead `IGNORE` this message. - -##### `beacon_aggregate_and_proof` - -The `beacon_aggregate_and_proof` topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) -to subscribing nodes (typically validators) to be included in future blocks. - -The following validations MUST pass before forwarding the `signed_aggregate_and_proof` on the network. -(We define the following for convenience -- `aggregate_and_proof = signed_aggregate_and_proof.message` and `aggregate = aggregate_and_proof.aggregate`) -- _[IGNORE]_ `aggregate.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. `aggregate.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= aggregate.data.slot` - (a client MAY queue future aggregates for processing at the appropriate slot). -- _[REJECT]_ The aggregate attestation's epoch matches its target -- i.e. `aggregate.data.target.epoch == - compute_epoch_at_slot(aggregate.data.slot)` -- _[IGNORE]_ The `aggregate` is the first valid aggregate received for the aggregator - with index `aggregate_and_proof.aggregator_index` for the epoch `aggregate.data.target.epoch`. -- _[REJECT]_ The attestation has participants -- - that is, `len(get_attesting_indices(state, aggregate.data, aggregate.aggregation_bits)) >= 1`. -- _[REJECT]_ `aggregate_and_proof.selection_proof` selects the validator as an aggregator for the slot -- - i.e. `is_aggregator(state, aggregate.data.slot, aggregate.data.index, aggregate_and_proof.selection_proof)` returns `True`. -- _[REJECT]_ The aggregator's validator index is within the committee -- - i.e. `aggregate_and_proof.aggregator_index in get_beacon_committee(state, aggregate.data.slot, aggregate.data.index)`. -- _[REJECT]_ The `aggregate_and_proof.selection_proof` is a valid signature - of the `aggregate.data.slot` by the validator with index `aggregate_and_proof.aggregator_index`. -- _[REJECT]_ The aggregator signature, `signed_aggregate_and_proof.signature`, is valid. -- _[REJECT]_ The signature of `aggregate` is valid. -- _[IGNORE]_ The block being voted for (`aggregate.data.beacon_block_root`) has been seen - (via both gossip and non-gossip sources) - (a client MAY queue aggregates for processing once block is retrieved). -- _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. -- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `aggregate.data.beacon_block_root` -- i.e. - `get_ancestor(store, aggregate.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) - == store.finalized_checkpoint.root` - - -##### `voluntary_exit` - -The `voluntary_exit` topic is used solely for propagating signed voluntary validator exits to proposers on the network. -Signed voluntary exits are sent in their entirety. - -The following validations MUST pass before forwarding the `signed_voluntary_exit` on to the network. -- _[IGNORE]_ The voluntary exit is the first valid voluntary exit received - for the validator with index `signed_voluntary_exit.message.validator_index`. -- _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. - -##### `proposer_slashing` - -The `proposer_slashing` topic is used solely for propagating proposer slashings to proposers on the network. -Proposer slashings are sent in their entirety. - -The following validations MUST pass before forwarding the `proposer_slashing` on to the network. -- _[IGNORE]_ The proposer slashing is the first valid proposer slashing received - for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. -- _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. - -##### `attester_slashing` - -The `attester_slashing` topic is used solely for propagating attester slashings to proposers on the network. -Attester slashings are sent in their entirety. - -Clients who receive an attester slashing on this topic MUST validate the conditions within `process_attester_slashing` before forwarding it across the network. -- _[IGNORE]_ At least one index in the intersection of the attesting indices of each attestation - has not yet been seen in any prior `attester_slashing` - (i.e. `attester_slashed_indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices)`, - verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). -- _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. - -#### Attestation subnets - -Attestation subnets are used to propagate unaggregated attestations to subsections of the network. - -##### `beacon_attestation_{subnet_id}` - -The `beacon_attestation_{subnet_id}` topics are used to propagate unaggregated attestations -to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. - -The following validations MUST pass before forwarding the `attestation` on the subnet. -- _[REJECT]_ The committee index is within the expected range -- i.e. `data.index < get_committee_count_per_slot(state, data.target.epoch)`. -- _[REJECT]_ The attestation is for the correct subnet -- - i.e. `compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index) == subnet_id`, - where `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)`, - which may be pre-computed along with the committee information for the signature check. -- _[IGNORE]_ `attestation.data.slot` is within the last `ATTESTATION_PROPAGATION_SLOT_RANGE` slots - (within a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- - i.e. `attestation.data.slot + ATTESTATION_PROPAGATION_SLOT_RANGE >= current_slot >= attestation.data.slot` - (a client MAY queue future attestations for processing at the appropriate slot). -- _[REJECT]_ The attestation's epoch matches its target -- i.e. `attestation.data.target.epoch == - compute_epoch_at_slot(attestation.data.slot)` -- _[REJECT]_ The attestation is unaggregated -- - that is, it has exactly one participating validator (`len([bit for bit in attestation.aggregation_bits if bit]) == 1`, i.e. exactly 1 bit is set). -- _[REJECT]_ The number of aggregation bits matches the committee size -- i.e. - `len(attestation.aggregation_bits) == len(get_beacon_committee(state, data.slot, data.index))`. -- _[IGNORE]_ There has been no other valid attestation seen on an attestation subnet - that has an identical `attestation.data.target.epoch` and participating validator index. -- _[REJECT]_ The signature of `attestation` is valid. -- _[IGNORE]_ The block being voted for (`attestation.data.beacon_block_root`) has been seen - (via both gossip and non-gossip sources) - (a client MAY queue attestations for processing once block is retrieved). -- _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. -- _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. - `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root` -- _[REJECT]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `attestation.data.beacon_block_root` -- i.e. - `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) - == store.finalized_checkpoint.root` - - - -#### Attestations and Aggregation - -Attestation broadcasting is grouped into subnets defined by a topic. -The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. -The correct subnet for an attestation can be calculated with `compute_subnet_for_attestation`. -`beacon_attestation_{subnet_id}` topics, are rotated through throughout the epoch in a similar fashion to rotating through shards in committees (future Eth2 upgrade). -The subnets are rotated through with `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)` subnets per slot. - -Unaggregated attestations are sent as `Attestation`s to the subnet topic, -`beacon_attestation_{compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index)}` as `Attestation`s. - -Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. - -### Encodings - -Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. - -- `ssz_snappy` - All objects are SSZ-encoded and then compressed with [Snappy](https://github.com/google/snappy) block compression. - Example: The beacon aggregate attestation topic string is `/eth2/446a7232/beacon_aggregate_and_proof/ssz_snappy`, - the fork digest is `446a7232` and the data field of a gossipsub message is an `AggregateAndProof` - that has been SSZ-encoded and then compressed with Snappy. - -Snappy has two formats: "block" and "frames" (streaming). -Gossip messages remain relatively small (100s of bytes to 100s of kilobytes) -so [basic snappy block compression](https://github.com/google/snappy/blob/master/format_description.txt) is used to avoid the additional overhead associated with snappy frames. - -Implementations MUST use a single encoding for gossip. -Changing an encoding will require coordination between participating implementations. - -## The Req/Resp domain - -### Protocol identification - -Each message type is segregated into its own libp2p protocol ID, which is a case-sensitive UTF-8 string of the form: - -``` -/ProtocolPrefix/MessageName/SchemaVersion/Encoding -``` - -With: - -- `ProtocolPrefix` - messages are grouped into families identified by a shared libp2p protocol name prefix. - In this case, we use `/eth2/beacon_chain/req`. -- `MessageName` - each request is identified by a name consisting of English alphabet, digits and underscores (`_`). -- `SchemaVersion` - an ordinal version number (e.g. 1, 2, 3…). - Each schema is versioned to facilitate backward and forward-compatibility when possible. -- `Encoding` - while the schema defines the data types in more abstract terms, - the encoding strategy describes a specific representation of bytes that will be transmitted over the wire. - See the [Encodings](#Encoding-strategies) section for further details. - -This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` -to handle the request type, version, and encoding negotiation before establishing the underlying streams. - -### Req/Resp interaction - -We use ONE stream PER request/response interaction. -Streams are closed when the interaction finishes, whether in success or in error. - -Request/response messages MUST adhere to the encoding specified in the protocol name and follow this structure (relaxed BNF grammar): - -``` -request ::= | -response ::= * -response_chunk ::= | | -result ::= “0” | “1” | “2” | [“128” ... ”255”] -``` - -The encoding-dependent header may carry metadata or assertions such as the encoded payload length, for integrity and attack proofing purposes. -Because req/resp streams are single-use and stream closures implicitly delimit the boundaries, it is not strictly necessary to length-prefix payloads; -however, certain encodings like SSZ do, for added security. - -A `response` is formed by zero or more `response_chunk`s. -Responses that consist of a single SSZ-list (such as `BlocksByRange` and `BlocksByRoot`) send each list item as a `response_chunk`. -All other response types (non-Lists) send a single `response_chunk`. - -For both `request`s and `response`s, the `encoding-dependent-header` MUST be valid, -and the `encoded-payload` must be valid within the constraints of the `encoding-dependent-header`. -This includes type-specific bounds on payload size for some encoding strategies. -Regardless of these type specific bounds, a global maximum uncompressed byte size of `MAX_CHUNK_SIZE` MUST be applied to all method response chunks. - -Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. -Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. - -#### Requesting side - -Once a new stream with the protocol ID for the request type has been negotiated, the full request message SHOULD be sent immediately. -The request MUST be encoded according to the encoding strategy. - -The requester MUST close the write side of the stream once it finishes writing the request message. -At this point, the stream will be half-closed. - -The requester MUST wait a maximum of `TTFB_TIMEOUT` for the first response byte to arrive (time to first byte—or TTFB—timeout). -On that happening, the requester allows a further `RESP_TIMEOUT` for each subsequent `response_chunk` received. - -If any of these timeouts fire, the requester SHOULD reset the stream and deem the req/resp operation to have failed. - -A requester SHOULD read from the stream until either: -1. An error result is received in one of the chunks (the error payload MAY be read before stopping). -2. The responder closes the stream. -3. Any part of the `response_chunk` fails validation. -4. The maximum number of requested chunks are read. - -For requests consisting of a single valid `response_chunk`, -the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. - -#### Responding side - -Once a new stream with the protocol ID for the request type has been negotiated, -the responder SHOULD process the incoming request and MUST validate it before processing it. -Request processing and validation MUST be done according to the encoding strategy, until EOF (denoting stream half-closure by the requester). - -The responder MUST: - -1. Use the encoding strategy to read the optional header. -2. If there are any length assertions for length `N`, it should read exactly `N` bytes from the stream, at which point an EOF should arise (no more bytes). - Should this not be the case, it should be treated as a failure. -3. Deserialize the expected type, and process the request. -4. Write the response which may consist of zero or more `response_chunk`s (result, optional header, payload). -5. Close their write side of the stream. At this point, the stream will be fully closed. - -If steps (1), (2), or (3) fail due to invalid, malformed, or inconsistent data, the responder MUST respond in error. -Clients tracking peer reputation MAY record such failures, as well as unexpected events, e.g. early stream resets. - -The entire request should be read in no more than `RESP_TIMEOUT`. -Upon a timeout, the responder SHOULD reset the stream. - -The responder SHOULD send a `response_chunk` promptly. -Chunks start with a **single-byte** response code which determines the contents of the `response_chunk` (`result` particle in the BNF grammar above). -For multiple chunks, only the last chunk is allowed to have a non-zero error code (i.e. The chunk stream is terminated once an error occurs). - -The response code can have one of the following values, encoded as a single unsigned byte: - -- 0: **Success** -- a normal response follows, with contents matching the expected message schema and encoding specified in the request. -- 1: **InvalidRequest** -- the contents of the request are semantically invalid, or the payload is malformed, or could not be understood. - The response payload adheres to the `ErrorMessage` schema (described below). -- 2: **ServerError** -- the responder encountered an error while processing the request. - The response payload adheres to the `ErrorMessage` schema (described below). - -Clients MAY use response codes above `128` to indicate alternative, erroneous request-specific responses. - -The range `[3, 127]` is RESERVED for future usages, and should be treated as error if not recognized expressly. - -The `ErrorMessage` schema is: - -``` -( - error_message: List[byte, 256] -) -``` - -*Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). -Clients MUST treat as valid any byte sequences. - -### Encoding strategies - -The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. -Only one value is possible at this time: - -- `ssz_snappy`: The contents are first [SSZ-encoded](../../ssz/simple-serialize.md) - and then compressed with [Snappy](https://github.com/google/snappy) frames compression. - For objects containing a single field, only the field is SSZ-encoded not a container with a single field. - For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. - This encoding type MUST be supported by all clients. - -#### SSZ-snappy encoding strategy - -The [SimpleSerialize (SSZ) specification](../../ssz/simple-serialize.md) outlines how objects are SSZ-encoded. - -To achieve snappy encoding on top of SSZ, we feed the serialized form of the object to the Snappy compressor on encoding. -The inverse happens on decoding. - -Snappy has two formats: "block" and "frames" (streaming). -To support large requests and response chunks, snappy-framing is used. - -Since snappy frame contents [have a maximum size of `65536` bytes](https://github.com/google/snappy/blob/master/framing_format.txt#L104) -and frame headers are just `identifier (1) + checksum (4)` bytes, the expected buffering of a single frame is acceptable. - -**Encoding-dependent header:** Req/Resp protocols using the `ssz_snappy` encoding strategy MUST encode the length of the raw SSZ bytes, -encoded as an unsigned [protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints). - -*Writing*: By first computing and writing the SSZ byte length, the SSZ encoder can then directly write the chunk contents to the stream. -When Snappy is applied, it can be passed through a buffered Snappy writer to compress frame by frame. - -*Reading*: After reading the expected SSZ byte length, the SSZ decoder can directly read the contents from the stream. -When snappy is applied, it can be passed through a buffered Snappy reader to decompress frame by frame. - -Before reading the payload, the header MUST be validated: -- The unsigned protobuf varint used for the length-prefix MUST not be longer than 10 bytes, which is sufficient for any `uint64`. -- The length-prefix is within the expected [size bounds derived from the payload SSZ type](#what-are-ssz-type-size-bounds). - -After reading a valid header, the payload MAY be read, while maintaining the size constraints from the header. - -A reader SHOULD NOT read more than `max_encoded_len(n)` bytes after reading the SSZ length-prefix `n` from the header. -- For `ssz_snappy` this is: `32 + n + n // 6`. - This is considered the [worst-case compression result](https://github.com/google/snappy/blob/537f4ad6240e586970fe554614542e9717df7902/snappy.cc#L98) by Snappy. - -A reader SHOULD consider the following cases as invalid input: -- Any remaining bytes, after having read the `n` SSZ bytes. An EOF is expected if more bytes are read than required. -- An early EOF, before fully reading the declared length-prefix worth of SSZ bytes. - -In case of an invalid input (header or payload), a reader MUST: -- From requests: send back an error message, response code `InvalidRequest`. The request itself is ignored. -- From responses: ignore the response, the response MUST be considered bad server behavior. - -All messages that contain only a single field MUST be encoded directly as the type of that field and MUST NOT be encoded as an SSZ container. - -Responses that are SSZ-lists (for example `List[SignedBeaconBlock, ...]`) send their -constituents individually as `response_chunk`s. For example, the -`List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. -Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. - -### Messages - -#### Status - -**Protocol ID:** ``/eth2/beacon_chain/req/status/1/`` - -Request, Response Content: -``` -( - fork_digest: ForkDigest - finalized_root: Root - finalized_epoch: Epoch - head_root: Root - head_slot: Slot -) -``` -The fields are, as seen by the client at the time of sending the message: - -- `fork_digest`: The node's `ForkDigest` (`compute_fork_digest(current_fork_version, genesis_validators_root)`) where - - `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time - (not necessarily the epoch to which the node is sync) - - `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -- `finalized_root`: `state.finalized_checkpoint.root` for the state corresponding to the head block - (Note this defaults to `Root(b'\x00' * 32)` for the genesis finalized checkpoint). -- `finalized_epoch`: `state.finalized_checkpoint.epoch` for the state corresponding to the head block. -- `head_root`: The `hash_tree_root` root of the current head block (`BeaconBlock`). -- `head_slot`: The slot of the block corresponding to the `head_root`. - -The dialing client MUST send a `Status` request upon connection. - -The request/response MUST be encoded as an SSZ-container. - -The response MUST consist of a single `response_chunk`. - -Clients SHOULD immediately disconnect from one another following the handshake above under the following conditions: - -1. If `fork_digest` does not match the node's local `fork_digest`, since the client’s chain is on another fork. -2. If the (`finalized_root`, `finalized_epoch`) shared by the peer is not in the client's chain at the expected epoch. - For example, if Peer 1 sends (root, epoch) of (A, 5) and Peer 2 sends (B, 3) but Peer 1 has root C at epoch 3, - then Peer 1 would disconnect because it knows that their chains are irreparably disjoint. - -Once the handshake completes, the client with the lower `finalized_epoch` or `head_slot` (if the clients have equal `finalized_epoch`s) -SHOULD request beacon blocks from its counterparty via the `BeaconBlocksByRange` request. - -*Note*: Under abnormal network condition or after some rounds of `BeaconBlocksByRange` requests, -the client might need to send `Status` request again to learn if the peer has a higher head. -Implementers are free to implement such behavior in their own way. - -#### Goodbye - -**Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/`` - -Request, Response Content: -``` -( - uint64 -) -``` -Client MAY send goodbye messages upon disconnection. The reason field MAY be one of the following values: - -- 1: Client shut down. -- 2: Irrelevant network. -- 3: Fault/error. - -Clients MAY use reason codes above `128` to indicate alternative, erroneous request-specific responses. - -The range `[4, 127]` is RESERVED for future usage. - -The request/response MUST be encoded as a single SSZ-field. - -The response MUST consist of a single `response_chunk`. - -#### BeaconBlocksByRange - -**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/1/` - -Request Content: -``` -( - start_slot: Slot - count: uint64 - step: uint64 -) -``` - -Response Content: -``` -( - List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] -) -``` - -Requests beacon blocks in the slot range `[start_slot, start_slot + count * step)`, leading up to the current head block as selected by fork choice. -`step` defines the slot increment between blocks. -For example, requesting blocks starting at `start_slot` 2 with a step value of 2 would return the blocks at slots [2, 4, 6, …]. -In cases where a slot is empty for a given slot number, no block is returned. -For example, if slot 4 were empty in the previous example, the returned array would contain [2, 6, …]. -A request MUST NOT have a 0 slot increment, i.e. `step >= 1`. - -`BeaconBlocksByRange` is primarily used to sync historical blocks. - -The request MUST be encoded as an SSZ-container. - -The response MUST consist of zero or more `response_chunk`. -Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. - -Clients MUST keep a record of signed blocks seen since the start of the weak subjectivity period -and MUST support serving requests of blocks up to their own `head_block_root`. - -Clients MUST respond with at least the first block that exists in the range, if they have it, and no more than `MAX_REQUEST_BLOCKS` blocks. - -The following blocks, where they exist, MUST be sent in consecutive order. - -Clients MAY limit the number of blocks in the response. - -The response MUST contain no more than `count` blocks. - -Clients MUST respond with blocks from their view of the current fork choice --- that is, blocks from the single chain defined by the current head. -Of note, blocks from slots before the finalization MUST lead to the finalized block reported in the `Status` handshake. - -Clients MUST respond with blocks that are consistent from a single chain within the context of the request. -This applies to any `step` value. -In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_root` of the preceding block. - -After the initial block, clients MAY stop in the process of responding -if their fork choice changes the view of the chain in the context of the request. - -#### BeaconBlocksByRoot - -**Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/1/` - -Request Content: - -``` -( - List[Root, MAX_REQUEST_BLOCKS] -) -``` - -Response Content: - -``` -( - List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] -) -``` - -Requests blocks by block root (= `hash_tree_root(SignedBeaconBlock.message)`). -The response is a list of `SignedBeaconBlock` whose length is less than or equal to the number of requested blocks. -It may be less in the case that the responding peer is missing blocks. - -No more than `MAX_REQUEST_BLOCKS` may be requested at a time. - -`BeaconBlocksByRoot` is primarily used to recover recent blocks (e.g. when receiving a block or attestation whose parent is unknown). - -The request MUST be encoded as an SSZ-field. - -The response MUST consist of zero or more `response_chunk`. -Each _successful_ `response_chunk` MUST contain a single `SignedBeaconBlock` payload. - -Clients MUST support requesting blocks since the latest finalized epoch. - -Clients MUST respond with at least one block, if they have it. -Clients MAY limit the number of blocks in the response. - -#### Ping - -**Protocol ID:** `/eth2/beacon_chain/req/ping/1/` - -Request Content: - -``` -( - uint64 -) -``` - -Response Content: - -``` -( - uint64 -) -``` - -Sent intermittently, the `Ping` protocol checks liveness of connected peers. -Peers request and respond with their local metadata sequence number (`MetaData.seq_number`). - -If the peer does not respond to the `Ping` request, the client MAY disconnect from the peer. - -A client can then determine if their local record of a peer's MetaData is up to date -and MAY request an updated version via the `MetaData` RPC method if not. - -The request MUST be encoded as an SSZ-field. - -The response MUST consist of a single `response_chunk`. - -#### GetMetaData - -**Protocol ID:** `/eth2/beacon_chain/req/metadata/1/` - -No Request Content. - -Response Content: - -``` -( - MetaData -) -``` - -Requests the MetaData of a peer. -The request opens and negotiates the stream without sending any request content. -Once established the receiving peer responds with -it's local most up-to-date MetaData. - -The response MUST be encoded as an SSZ-container. - -The response MUST consist of a single `response_chunk`. - -## The discovery domain: discv5 - -Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) (Protocol version v5.1) is used for peer discovery. - -`discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. -`discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. - -### Integration into libp2p stacks - -`discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor -to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) -and [peer routing](https://github.com/libp2p/go-libp2p-core/blob/master/routing/routing.go#L36-L44) abstractions and interfaces (go-libp2p links provided). - -Inputs to operations include peer IDs (when locating a specific peer) or capabilities (when searching for peers with a specific capability), -and the outputs will be multiaddrs converted from the ENR records returned by the discv5 backend. - -This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. - -### ENR structure - -The Ethereum Node Record (ENR) for an Ethereum 2.0 client MUST contain the following entries -(exclusive of the sequence number and signature, which MUST be present in an ENR): - -- The compressed secp256k1 publickey, 33 bytes (`secp256k1` field). - -The ENR MAY contain the following entries: - -- An IPv4 address (`ip` field) and/or IPv6 address (`ip6` field). -- A TCP port (`tcp` field) representing the local libp2p listening port. -- A UDP port (`udp` field) representing the local discv5 listening port. - -Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). - -#### Attestation subnet bitfield - -The ENR `attnets` entry signifies the attestation subnet bitfield with the following form -to more easily discover peers participating in particular attestation gossip subnets. - -| Key | Value | -|:-------------|:-------------------------------------------------| -| `attnets` | SSZ `Bitvector[ATTESTATION_SUBNET_COUNT]` | - -If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `attnets` entry with the same value as `MetaData.attnets`. - -If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. - -#### `eth2` field - -ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, -and next fork epoch to ensure connections are made with peers on the intended eth2 network. - -| Key | Value | -|:-------------|:--------------------| -| `eth2` | SSZ `ENRForkID` | - -Specifically, the value of the `eth2` key MUST be the following SSZ encoded object (`ENRForkID`) - -``` -( - fork_digest: ForkDigest - next_fork_version: Version - next_fork_epoch: Epoch -) -``` - -where the fields of `ENRForkID` are defined as - -* `fork_digest` is `compute_fork_digest(current_fork_version, genesis_validators_root)` where - * `current_fork_version` is the fork version at the node's current epoch defined by the wall-clock time - (not necessarily the epoch to which the node is sync) - * `genesis_validators_root` is the static `Root` found in `state.genesis_validators_root` -* `next_fork_version` is the fork version corresponding to the next planned hard fork at a future epoch. - If no future fork is planned, set `next_fork_version = current_fork_version` to signal this fact -* `next_fork_epoch` is the epoch at which the next fork is planned and the `current_fork_version` will be updated. - If no future fork is planned, set `next_fork_epoch = FAR_FUTURE_EPOCH` to signal this fact - -*Note*: `fork_digest` is composed of values that are not known until the genesis block/state are available. -Due to this, clients SHOULD NOT form ENRs and begin peer discovery until genesis values are known. -One notable exception to this rule is the distribution of bootnode ENRs prior to genesis. -In this case, bootnode ENRs SHOULD be initially distributed with `eth2` field set as -`ENRForkID(fork_digest=compute_fork_digest(GENESIS_FORK_VERSION, b'\x00'*32), next_fork_version=GENESIS_FORK_VERSION, next_fork_epoch=FAR_FUTURE_EPOCH)`. -After genesis values are known, the bootnodes SHOULD update ENRs to participate in normal discovery operations. - -Clients SHOULD connect to peers with `fork_digest`, `next_fork_version`, and `next_fork_epoch` that match local values. - -Clients MAY connect to peers with the same `fork_digest` but a different `next_fork_version`/`next_fork_epoch`. -Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, -these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. - -# Design decision rationale - -## Transport - -### Why are we defining specific transports? - -libp2p peers can listen on multiple transports concurrently, and these can change over time. -Multiaddrs encode not only the address but also the transport to be used to dial. - -Due to this dynamic nature, agreeing on specific transports like TCP, QUIC, or WebSockets on paper becomes irrelevant. - -However, it is useful to define a minimum baseline for interoperability purposes. - -### Can clients support other transports/handshakes than the ones mandated by the spec? - -Clients may support other transports such as libp2p QUIC, WebSockets, and WebRTC transports, if available in the language of choice. -While interoperability shall not be harmed by lack of such support, the advantages are desirable: - -- Better latency, performance, and other QoS characteristics (QUIC). -- Paving the way for interfacing with future light clients (WebSockets, WebRTC). - -The libp2p QUIC transport inherently relies on TLS 1.3 per requirement in section 7 -of the [QUIC protocol specification](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7) -and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf-quic-tls-22). - -The usage of one handshake procedure or the other shall be transparent to the Eth2 application layer, -once the libp2p Host/Node object has been configured appropriately. - -### What are the advantages of using TCP/QUIC/Websockets? - -TCP is a reliable, ordered, full-duplex, congestion-controlled network protocol that powers much of the Internet as we know it today. -HTTP/1.1 and HTTP/2 run atop TCP. - -QUIC is a new protocol that’s in the final stages of specification by the IETF QUIC WG. -It emerged from Google’s SPDY experiment. The QUIC transport is undoubtedly promising. -It’s UDP-based yet reliable, ordered, multiplexed, natively secure (TLS 1.3), reduces latency vs. TCP, -and offers stream-level and connection-level congestion control (thus removing head-of-line blocking), -0-RTT connection establishment, and endpoint migration, amongst other features. -UDP also has better NAT traversal properties than TCP—something we desperately pursue in peer-to-peer networks. - -QUIC is being adopted as the underlying protocol for HTTP/3. -This has the potential to award us censorship resistance via deep packet inspection for free. -Provided that we use the same port numbers and encryption mechanisms as HTTP/3, our traffic may be indistinguishable from standard web traffic, -and we may only become subject to standard IP-based firewall filtering—something we can counteract via other mechanisms. - -WebSockets and/or WebRTC transports are necessary for interaction with browsers, -and will become increasingly important as we incorporate browser-based light clients to the Eth2 network. - -### Why do we not just support a single transport? - -Networks evolve. -Hardcoding design decisions leads to ossification, preventing the evolution of networks alongside the state of the art. -Introducing changes on an ossified protocol is very costly, and sometimes, downright impracticable without causing undesirable breakage. - -Modeling for upgradeability and dynamic transport selection from the get-go lays the foundation for a future-proof stack. - -Clients can adopt new transports without breaking old ones, and the multi-transport ability enables constrained and sandboxed environments -(e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), -without the need for proxying or trust delegation to servers. - -### Why are we not using QUIC from the start? - -The QUIC standard is still not finalized (at working draft 22 at the time of writing), -and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). -One remarkable example is node.js, where the QUIC implementation is [in early development](https://github.com/nodejs/quic). - -*Note*: [TLS 1.3 is a prerequisite of the QUIC transport](https://tools.ietf.org/html/draft-ietf-quic-transport-22#section-7), -although an experiment exists to integrate Noise as the QUIC crypto layer: [nQUIC](https://eprint.iacr.org/2019/028). - -On the other hand, TLS 1.3 is the newest, simplified iteration of TLS. -Old, insecure, obsolete ciphers and algorithms have been removed, adopting Ed25519 as the sole ECDH key agreement function. -Handshakes are faster, 1-RTT data is supported, and session resumption is a reality, amongst other features. - -## Multiplexing - -### Why are we using mplex/yamux? - -[Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) is a multiplexer invented by Hashicorp that supports stream-level congestion control. -Implementations exist in a limited set of languages, and it’s not a trivial piece to develop. - -Conscious of that, the libp2p community conceptualized [mplex](https://github.com/libp2p/specs/blob/master/mplex/README.md) -as a simple, minimal multiplexer for usage with libp2p. -It does not support stream-level congestion control and is subject to head-of-line blocking. - -Overlay multiplexers are not necessary with QUIC since the protocol provides native multiplexing, -but they need to be layered atop TCP, WebSockets, and other transports that lack such support. - -## Protocol Negotiation - -### When is multiselect 2.0 due and why do we plan to migrate to it? - -multiselect 2.0 is currently being conceptualized. -The debate started [on this issue](https://github.com/libp2p/specs/pull/95), -but it got overloaded—as it tends to happen with large conceptual OSS discussions that touch the heart and core of a system. - -At some point in 2020, we expect a renewed initiative to first define the requirements, constraints, assumptions, and features, -in order to lock in basic consensus upfront and subsequently build on that consensus by submitting a specification for implementation. - -We plan to eventually migrate to multiselect 2.0 because it will: - -1. Reduce round trips during connection bootstrapping and stream protocol negotiation. -2. Enable efficient one-stream-per-request interaction patterns. -3. Leverage *push data* mechanisms of underlying protocols to expedite negotiation. -4. Provide the building blocks for enhanced censorship resistance. - -### What is the difference between connection-level and stream-level protocol negotiation? - -All libp2p connections must be authenticated, encrypted, and multiplexed. -Connections using network transports unsupportive of native authentication/encryption and multiplexing (e.g. TCP) need to undergo protocol negotiation to agree on a mutually supported: - -1. authentication/encryption mechanism (such as SecIO, TLS 1.3, Noise). -2. overlay multiplexer (such as mplex, Yamux, spdystream). - -In this specification, we refer to these two as *connection-level negotiations*. -Transports supporting those features natively (such as QUIC) omit those negotiations. - -After successfully selecting a multiplexer, all subsequent I/O happens over *streams*. -When opening streams, peers pin a protocol to that stream, by conducting *stream-level protocol negotiation*. - -At present, multistream-select 1.0 is used for both types of negotiation, -but multiselect 2.0 will use dedicated mechanisms for connection bootstrapping process and stream protocol negotiation. - -## Encryption - -### Why are we not supporting SecIO? - -SecIO has been the default encryption layer for libp2p for years. -It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. - -Although SecIO has wide language support, we won’t be using it for mainnet because, amongst other things, -it requires several round trips to be sound, and doesn’t support early data (0-RTT data), -a mechanism that multiselect 2.0 will leverage to reduce round trips during connection bootstrapping. - -SecIO is not considered secure for the purposes of this spec. - -### Why are we using Noise? - -Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): - -> Noise is a framework for building crypto protocols. -Noise protocols support mutual and optional authentication, identity hiding, forward secrecy, zero round-trip encryption, and other advanced features. - -Noise in itself does not specify a single handshake procedure, -but provides a framework to build secure handshakes based on Diffie-Hellman key agreement with a variety of tradeoffs and guarantees. - -Noise handshakes are lightweight and simple to understand, -and are used in major cryptographic-centric projects like WireGuard, I2P, and Lightning. -[Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) -have assessed the stated security goals of several Noise handshakes with positive results. - -### Why are we using encryption at all? - -Transport level encryption secures message exchange and provides properties that are useful for privacy, safety, and censorship resistance. -These properties are derived from the following security guarantees that apply to the entire communication between two peers: - -- Peer authentication: the peer I’m talking to is really who they claim to be and who I expect them to be. -- Confidentiality: no observer can eavesdrop on the content of our messages. -- Integrity: the data has not been tampered with by a third-party while in transit. -- Non-repudiation: the originating peer cannot dispute that they sent the message. -- Depending on the chosen algorithms and mechanisms (e.g. continuous HMAC), we may obtain additional guarantees, - such as non-replayability (this byte could’ve only been sent *now;* e.g. by using continuous HMACs), - or perfect forward secrecy (in the case that a peer key is compromised, the content of a past conversation will not be compromised). - -Note that transport-level encryption is not exclusive of application-level encryption or cryptography. -Transport-level encryption secures the communication itself, -while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). - -## Gossipsub - -### Why are we using a pub/sub algorithm for block and attestation propagation? - -Pubsub is a technique to broadcast/disseminate data across a network rapidly. -Such data is packaged in fire-and-forget messages that do not require a response from every recipient. -Peers subscribed to a topic participate in the propagation of messages in that topic. - -The alternative is to maintain a fully connected mesh (all peers connected to each other 1:1), which scales poorly (O(n^2)). - -### Why are we using topics to segregate encodings, yet only support one encoding? - -For future extensibility with almost zero overhead now (besides the extra bytes in the topic name). - -### How do we upgrade gossip channels (e.g. changes in encoding, compression)? - -Changing gossipsub/broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, during a hard fork. - -When a node is preparing for upcoming tasks (e.g. validator duty lookahead) on a gossipsub topic, -the node should join the topic of the future epoch in which the task is to occur in addition to listening to the topics for the current epoch. - -### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? - -Supporting multiple topics/encodings would require the presence of relayers to translate between encodings -and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, -making the protocol more complicated and fragile. - -Gossip protocols typically remember what messages they've seen for a finite period of time-based on message identity --- if you publish the same message again after that time has passed, -it will be re-broadcast—adding a relay delay also makes this scenario more likely. - -One can imagine that in a complicated upgrade scenario, we might have peers publishing the same message on two topics/encodings, -but the price here is pretty high in terms of overhead -- both computational and networking -- so we'd rather avoid that. - -It is permitted for clients to publish data on alternative topics as long as they also publish on the network-wide mandatory topic. - -### Why are the topics strings and not hashes? - -Topic names have a hierarchical structure. -In the future, gossipsub may support wildcard subscriptions -(e.g. subscribe to all children topics under a root prefix) by way of prefix matching. -Enforcing hashes for topic names would preclude us from leveraging such features going forward. - -No security or privacy guarantees are lost as a result of choosing plaintext topic names, -since the domain is finite anyway, and calculating a digest's preimage would be trivial. - -Furthermore, the Eth2 topic names are shorter than their digest equivalents (assuming SHA-256 hash), -so hashing topics would bloat messages unnecessarily. - -### Why are we using the `StrictNoSign` signature policy? - -The policy omits the `from` (1), `seqno` (3), `signature` (5) and `key` (6) fields. These fields would: -- Expose origin of sender (`from`), type of sender (based on `seqno`) -- Add extra unused data to the gossip, since message IDs are based on `data`, not on the `from` and `seqno`. -- Introduce more message validation than necessary, e.g. no `signature`. - -### Why are we overriding the default libp2p pubsub `message-id`? - -For our current purposes, there is no need to address messages based on source peer, or track a message `seqno`. -By overriding the default `message-id` to use content-addressing we can filter unnecessary duplicates before hitting the application layer. - -Some examples of where messages could be duplicated: - -* A validator client connected to multiple beacon nodes publishing duplicate gossip messages -* Attestation aggregation strategies where clients partially aggregate attestations and propagate them. - Partial aggregates could be duplicated -* Clients re-publishing seen messages - -### Why are these specific gossip parameters chosen? - -- `D`, `D_low`, `D_high`, `D_lazy`: recommended defaults. -- `heartbeat_interval`: 0.7 seconds, recommended for eth2 in the [GossipSub evaluation report by Protocol Labs](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4). -- `fanout_ttl`: 60 seconds, recommended default. - Fanout is primarily used by committees publishing attestations to subnets. - This happens once per epoch per validator and the subnet changes each epoch - so there is little to gain in having a `fanout_ttl` be increased from the recommended default. -- `mcache_len`: 6, increase by one to ensure that mcache is around for long - enough for `IWANT`s to respond to `IHAVE`s in the context of the shorter - `heartbeat_interval`. If `mcache_gossip` is increased, this param should be - increased to be at least `3` (~2 seconds) more than `mcache_gossip`. -- `mcache_gossip`: 3, recommended default. This can be increased to 5 or 6 - (~4 seconds) if gossip times are longer than expected and the current window - does not provide enough responsiveness during adverse conditions. -- `seen_ttl`: `SLOTS_PER_EPOCH * SECONDS_PER_SLOT / heartbeat_interval = approx. 550`. - Attestation gossip validity is bounded by an epoch, so this is the safe max bound. - - -### Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets? - -For some gossip channels (e.g. those for Attestations and BeaconBlocks), -there are designated ranges of slots during which particular messages can be sent, -limiting messages gossiped to those that can be reasonably used in the consensus at the current time/slot. -This is to reduce optionality in DoS attacks. - -`MAXIMUM_GOSSIP_CLOCK_DISPARITY` provides some leeway in validating slot ranges to prevent the gossip network -from becoming overly brittle with respect to clock disparity. -For minimum and maximum allowable slot broadcast times, -`MAXIMUM_GOSSIP_CLOCK_DISPARITY` MUST be subtracted and added respectively, marginally extending the valid range. -Although messages can at times be eagerly gossiped to the network, -the node's fork choice prevents integration of these messages into the actual consensus until the _actual local start_ of the designated slot. - -### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets? - -Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. -The exact grouping will be dependent on more involved network tests. -This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). -The value is currently set to be equal to `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. - -### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? - -Attestations can only be included on chain within an epoch's worth of slots so this is the natural cutoff. -There is no utility to the chain to broadcast attestations older than one epoch, -and because validators have a chance to make a new attestation each epoch, -there is minimal utility to the fork choice to relay old attestations as a new latest message can soon be created by each validator. - -In addition to this, relaying attestations requires validating the attestation in the context of the `state` during which it was created. -Thus, validating arbitrarily old attestations would put additional requirements on which states need to be readily available to the node. -This would result in a higher resource burden and could serve as a DoS vector. - -### Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s? - -The dominant strategy for an individual validator is to always broadcast an aggregate containing their own attestation -to the global channel to ensure that proposers see their attestation for inclusion. -Using a private selection criteria and providing this proof of selection alongside -the gossiped aggregate ensures that this dominant strategy will not flood the global channel. - -Also, an attacker can create any number of honest-looking aggregates and broadcast them to the global pubsub channel. -Thus without some sort of proof of selection as an aggregator, the global channel can trivially be spammed. - -### Why are we sending entire objects in the pubsub and not just hashes? - -Entire objects should be sent to get the greatest propagation speeds. -If only hashes are sent, then block and attestation propagation is dependent on recursive requests from each peer. -In a hash-only scenario, peers could receive hashes without knowing who to download the actual contents from. -Sending entire objects ensures that they get propagated through the entire network. - -### Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc? - -The prohibition of unverified-block-gossiping extends to nodes that cannot verify a signature -due to not being fully synced to ensure that such (amplified) DOS attacks are not possible. - -### How are we going to discover peers in a gossipsub topic? - -In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR. - -Although this method will be sufficient for early phases of Eth2, we aim to use the more appropriate discv5 topics for this and other similar tasks in the future. -ENRs should ultimately not be used for this purpose. -They are best suited to store identity, location, and capability information, rather than more volatile advertisements. - -### How should fork version be used in practice? - -Fork versions are to be manually updated (likely via incrementing) at each hard fork. -This is to provide native domain separation for signatures as well as to aid in usefulness for identitying peers (via ENRs) -and versioning network protocols (e.g. using fork version to naturally version gossipsub topics). - -`BeaconState.genesis_validators_root` is mixed into signature and ENR fork domains (`ForkDigest`) to aid in the ease of domain separation between chains. -This allows fork versions to safely be reused across chains except for the case of contentious forks using the same genesis. -In these cases, extra care should be taken to isolate fork versions (e.g. flip a high order bit in all future versions of one of the chains). - -A node locally stores all previous and future planned fork versions along with the each fork epoch. -This allows for handling sync and processing messages starting from past forks/epochs. - -## Req/Resp - -### Why segregate requests into dedicated protocol IDs? - -Requests are segregated by protocol ID to: - -1. Leverage protocol routing in libp2p, such that the libp2p stack will route the incoming stream to the appropriate handler. - This allows the handler function for each request type to be self-contained. - For an analogy, think about how you attach HTTP handlers to a REST API server. -2. Version requests independently. - In a coarser-grained umbrella protocol, the entire protocol would have to be versioned even if just one field in a single message changed. -3. Enable clients to select the individual requests/versions they support. - It would no longer be a strict requirement to support all requests, - and clients, in principle, could support a subset of requests and variety of versions. -4. Enable flexibility and agility for clients adopting spec changes that impact the request, by signalling to peers exactly which subset of new/old requests they support. -5. Enable clients to explicitly choose backwards compatibility at the request granularity. - Without this, clients would be forced to support entire versions of the coarser request protocol. -6. Parallelise RFCs (or Eth2 EIPs). - By decoupling requests from one another, each RFC that affects the request protocol can be deployed/tested/debated independently - without relying on a synchronization point to version the general top-level protocol. - 1. This has the benefit that clients can explicitly choose which RFCs to deploy - without buying into all other RFCs that may be included in that top-level version. - 2. Affording this level of granularity with a top-level protocol would imply creating as many variants - (e.g. /protocol/43-{a,b,c,d,...}) as the cartesian product of RFCs inflight, O(n^2). -7. Allow us to simplify the payload of requests. - Request-id’s and method-ids no longer need to be sent. - The encoding/request type and version can all be handled by the framework. - -**Caveat**: The protocol negotiation component in the current version of libp2p is called multistream-select 1.0. -It is somewhat naïve and introduces overhead on every request when negotiating streams, -although implementation-specific optimizations are possible to save this cost. -Multiselect 2.0 will eventually remove this overhead by memoizing previously selected protocols, and modeling shared protocol tables. -Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol -so the additional overhead is not expected to significantly hinder this domain. - -### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? - -We are using single-use streams where each stream is closed at the end of the message. -Thus, libp2p transparently handles message delimiting in the underlying stream. -libp2p streams are full-duplex, and each party is responsible for closing their write side (like in TCP). -We can therefore use stream closure to mark the end of the request and response independently. - -Nevertheless, in the case of `ssz_snappy`, messages are still length-prefixed with the length of the underlying data: -* A basic reader can prepare a correctly sized buffer before reading the message -* A more advanced reader can stream-decode SSZ given the length of the SSZ data. -* Alignment with protocols like gRPC over HTTP/2 that prefix with length -* Sanity checking of message length, and enabling much stricter message length limiting based on SSZ type information, - to provide even more DOS protection than the global message length already does. - E.g. a small `Status` message does not nearly require `MAX_CHUNK_SIZE` bytes. - -[Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length (unsigned here) ints. -Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. - -### Why do we version protocol strings with ordinals instead of semver? - -Using semver for network protocols is confusing. -It is never clear what a change in a field, even if backwards compatible on deserialization, actually implies. -Network protocol agreement should be explicit. Imagine two peers: - -- Peer A supporting v1.1.1 of protocol X. -- Peer B supporting v1.1.2 of protocol X. - -These two peers should never speak to each other because the results can be unpredictable. -This is an oversimplification: imagine the same problem with a set of 10 possible versions. -We now have 10^2 (100) possible outcomes that peers need to model for. The resulting complexity is unwieldy. - -For this reason, we rely on negotiation of explicit, verbatim protocols. -In the above case, peer B would provide backwards compatibility by supporting and advertising both v1.1.1 and v1.1.2 of the protocol. - -Therefore, semver would be relegated to convey expectations at the human level, and it wouldn't do a good job there either, -because it's unclear if "backwards compatibility" and "breaking change" apply only to wire schema level, to behavior, etc. - -For this reason, we remove and replace semver with ordinals that require explicit agreement and do not mandate a specific policy for changes. - -### Why is it called Req/Resp and not RPC? - -Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. - -### Why do we allow empty responses in block requests? - -When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. - -Thus, it may happen that we need to transmit an empty list - there are several ways to encode this: - -0) Close the stream without sending any data -1) Add a `null` option to the `success` response, for example by introducing an additional byte -2) Respond with an error result, using a specific error code for "No data" - -Semantically, it is not an error that a block is missing during a slot making option 2 unnatural. - -Option 1 allows the responder to signal "no block", but this information may be wrong - for example in the case of a malicious node. - -Under option 0, there is no way for a client to distinguish between a slot without a block and an incomplete response, -but given that it already must contain logic to handle the uncertainty of a malicious peer, option 0 was chosen. -Clients should mark any slots missing blocks as unknown until they can be verified as not containing a block by successive blocks. - -Assuming option 0 with no special `null` encoding, consider a request for slots `2, 3, 4` --- if there was no block produced at slot 4, the response would be `2, 3, EOF`. -Now consider the same situation, but where only `4` is requested --- closing the stream with only `EOF` (without any `response_chunk`) is consistent. - -Failing to provide blocks that nodes "should" have is reason to trust a peer less --- for example, if a particular peer gossips a block, it should have access to its parent. -If a request for the parent fails, it's indicative of poor peer quality since peers should validate blocks before gossiping them. - -### Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from? - -When connecting, the `Status` message gives an idea about the sync status of a particular peer, but this changes over time. -By the time a subsequent `BeaconBlockByRange` request is processed, the information may be stale, -and the responding side might have moved on to a new finalization point and pruned blocks around the previous head and finalized blocks. - -To avoid this race condition, we allow the responding side to choose which branch to send to the requesting client. -The requesting client then goes on to validate the blocks and incorporate them in their own database --- because they follow the same rules, they should at this point arrive at the same canonical chain. - -### What's the effect of empty slots on the sync algorithm? - -When syncing one can only tell that a slot has been skipped on a particular branch -by examining subsequent blocks and analyzing the graph formed by the parent root. -Because the server side may choose to omit blocks in the response for any reason, clients must validate the graph and be prepared to fill in gaps. - -For example, if a peer responds with blocks [2, 3] when asked for [2, 3, 4], clients may not assume that block 4 doesn't exist --- it merely means that the responding peer did not send it (they may not have it yet or may maliciously be trying to hide it) -and successive blocks will be needed to determine if there exists a block at slot 4 in this particular branch. - -## Discovery - -### Why are we using discv5 and not libp2p Kademlia DHT? - -discv5 is a standalone protocol, running on UDP on a dedicated port, meant for peer and service discovery only. -discv5 supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are, or will be, requirements in this context. - -On the other hand, libp2p Kademlia DHT is a fully-fledged DHT protocol/implementations -with content routing and storage capabilities, both of which are irrelevant in this context. - -Eth 1.0 nodes will evolve to support discv5. -By sharing the discovery network between Eth 1.0 and 2.0, -we benefit from the additive effect on network size that enhances resilience and resistance against certain attacks, -to which smaller networks are more vulnerable. -It should also help light clients of both networks find nodes with specific capabilities. - -discv5 is in the process of being audited. - -### What is the difference between an ENR and a multiaddr, and why are we using ENRs? - -Ethereum Node Records are self-certified node records. -Nodes craft and disseminate ENRs for themselves, proving authorship via a cryptographic signature. -ENRs are sequentially indexed, enabling conflicts to be resolved. - -ENRs are key-value records with string-indexed ASCII keys. -They can store arbitrary information, but EIP-778 specifies a pre-defined dictionary, including IPv4 and IPv6 addresses, secp256k1 public keys, etc. - -Comparing ENRs and multiaddrs is like comparing apples and oranges. -ENRs are self-certified containers of identity, addresses, and metadata about a node. -Multiaddrs are address strings with the peculiarity that they’re self-describing, composable and future-proof. -An ENR can contain multiaddrs, and multiaddrs can be derived securely from the fields of an authenticated ENR. - -discv5 uses ENRs and we will presumably need to: - -1. Add `multiaddr` to the dictionary, so that nodes can advertise their multiaddr under a reserved namespace in ENRs. – and/or – -2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR - (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Eth 1.0 nodes). - -### Why do we not form ENRs and find peers until genesis block/state is known? - -Although client software might very well be running locally prior to the solidification of the eth2 genesis state and block, -clients cannot form valid ENRs prior to this point. -ENRs contain `fork_digest` which utilizes the `genesis_validators_root` for a cleaner separation between chains -so prior to knowing genesis, we cannot use `fork_digest` to cleanly find peers on our intended chain. -Once genesis data is known, we can then form ENRs and safely find peers. - -When using an eth1 deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (7 days in mainnet configuration) before `genesis_time`, -providing ample time to find peers and form initial connections and gossip subnets prior to genesis. - -## Compression/Encoding - -### Why are we using SSZ for encoding? - -SSZ is used at the consensus layer, and all implementations should have support for SSZ-encoding/decoding, -requiring no further dependencies to be added to client implementations. -This is a natural choice for serializing objects to be sent across the wire. -The actual data in most protocols will be further compressed for efficiency. - -SSZ has well-defined schemas for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. -It also has defined all required types that are required for this network specification. - -### Why are we compressing, and at which layers? - -We compress on the wire to achieve smaller payloads per-message, which, in aggregate, -result in higher efficiency, better utilization of available bandwidth, and overall reduction in network-wide traffic overhead. - -At this time, libp2p does not have an out-of-the-box compression feature that can be dynamically negotiated -and layered atop connections and streams, but it is [being considered](https://github.com/libp2p/libp2p/issues/81). - -This is a non-trivial feature because the behavior -of network IO loops, kernel buffers, chunking, and packet fragmentation, amongst others, need to be taken into account. -libp2p streams are unbounded streams, whereas compression algorithms work best on bounded byte streams of which we have some prior knowledge. - -Compression tends not to be a one-size-fits-all problem. -A lot of variables need careful evaluation, and generic approaches/choices lead to poor size shavings, -which may even be counterproductive when factoring in the CPU and memory tradeoff. - -For all these reasons, generically negotiating compression algorithms may be treated as a research problem at the libp2p community, -one we’re happy to tackle in the medium-term. - -At this stage, the wisest choice is to consider libp2p a messenger of bytes, -and to make application layer participate in compressing those bytes. -This looks different depending on the interaction layer: - -- Gossip domain: since gossipsub has a framing protocol and exposes an API, we compress the payload - (when dictated by the encoding token in the topic name) prior to publishing the message via the API. - No length-prefixing is necessary because protobuf takes care of bounding the field in the serialized form. -- Req/Resp domain: since we define custom protocols that operate on byte streams, - implementers are encouraged to encapsulate the encoding and compression logic behind - MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. - -### Why are we using Snappy for compression? - -Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, -and can calculate the size of the uncompressed object without inflating it in memory. -This prevents DOS vectors where large uncompressed data is sent. - -### Can I get access to unencrypted bytes on the wire for debugging purposes? - -Yes, you can add loggers in your libp2p protocol handlers to log incoming and outgoing messages. -It is recommended to use programming design patterns to encapsulate the logging logic cleanly. - -If your libp2p library relies on frameworks/runtimes such as Netty (jvm) or Node.js (javascript), -you can use logging facilities in those frameworks/runtimes to enable message tracing. - -For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) -(which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. - -### What are SSZ type size bounds? - -The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. -Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. -Other types are static, they have a fixed size: no dynamic-length content is involved, and the minimum and maximum bounds are the same. - -For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). -It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. - -# libp2p implementations matrix - -This section will soon contain a matrix showing the maturity/state of the libp2p features required -by this spec across the languages in which Eth2 clients are being developed. diff --git a/tools/analyzers/specdocs/data/phase0/validator.md b/tools/analyzers/specdocs/data/phase0/validator.md deleted file mode 100644 index a548003e1b3..00000000000 --- a/tools/analyzers/specdocs/data/phase0/validator.md +++ /dev/null @@ -1,654 +0,0 @@ -# Ethereum 2.0 Phase 0 -- Honest Validator - -This is an accompanying document to [Ethereum 2.0 Phase 0 -- The Beacon Chain](./beacon-chain.md), which describes the expected actions of a "validator" participating in the Ethereum 2.0 protocol. - -## Table of contents - - - - - -- [Introduction](#introduction) -- [Prerequisites](#prerequisites) -- [Constants](#constants) - - [Misc](#misc) -- [Containers](#containers) - - [`Eth1Block`](#eth1block) - - [`AggregateAndProof`](#aggregateandproof) - - [`SignedAggregateAndProof`](#signedaggregateandproof) -- [Becoming a validator](#becoming-a-validator) - - [Initialization](#initialization) - - [BLS public key](#bls-public-key) - - [Withdrawal credentials](#withdrawal-credentials) - - [`BLS_WITHDRAWAL_PREFIX`](#bls_withdrawal_prefix) - - [`ETH1_ADDRESS_WITHDRAWAL_PREFIX`](#eth1_address_withdrawal_prefix) - - [Submit deposit](#submit-deposit) - - [Process deposit](#process-deposit) - - [Validator index](#validator-index) - - [Activation](#activation) -- [Validator assignments](#validator-assignments) - - [Lookahead](#lookahead) -- [Beacon chain responsibilities](#beacon-chain-responsibilities) - - [Block proposal](#block-proposal) - - [Preparing for a `BeaconBlock`](#preparing-for-a-beaconblock) - - [Slot](#slot) - - [Proposer index](#proposer-index) - - [Parent root](#parent-root) - - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) - - [Randao reveal](#randao-reveal) - - [Eth1 Data](#eth1-data) - - [`get_eth1_data`](#get_eth1_data) - - [Proposer slashings](#proposer-slashings) - - [Attester slashings](#attester-slashings) - - [Attestations](#attestations) - - [Deposits](#deposits) - - [Voluntary exits](#voluntary-exits) - - [Packaging into a `SignedBeaconBlock`](#packaging-into-a-signedbeaconblock) - - [State root](#state-root) - - [Signature](#signature) - - [Attesting](#attesting) - - [Attestation data](#attestation-data) - - [General](#general) - - [LMD GHOST vote](#lmd-ghost-vote) - - [FFG vote](#ffg-vote) - - [Construct attestation](#construct-attestation) - - [Data](#data) - - [Aggregation bits](#aggregation-bits) - - [Aggregate signature](#aggregate-signature) - - [Broadcast attestation](#broadcast-attestation) - - [Attestation aggregation](#attestation-aggregation) - - [Aggregation selection](#aggregation-selection) - - [Construct aggregate](#construct-aggregate) - - [Data](#data-1) - - [Aggregation bits](#aggregation-bits-1) - - [Aggregate signature](#aggregate-signature-1) - - [Broadcast aggregate](#broadcast-aggregate) -- [Phase 0 attestation subnet stability](#phase-0-attestation-subnet-stability) -- [How to avoid slashing](#how-to-avoid-slashing) - - [Proposer slashing](#proposer-slashing) - - [Attester slashing](#attester-slashing) -- [Protection best practices](#protection-best-practices) - - - - -## Introduction - -This document represents the expected behavior of an "honest validator" with respect to Phase 0 of the Ethereum 2.0 protocol. This document does not distinguish between a "node" (i.e. the functionality of following and reading the beacon chain) and a "validator client" (i.e. the functionality of actively participating in consensus). The separation of concerns between these (potentially) two pieces of software is left as a design decision that is out of scope. - -A validator is an entity that participates in the consensus of the Ethereum 2.0 protocol. This is an optional role for users in which they can post ETH as collateral and verify and attest to the validity of blocks to seek financial returns in exchange for building and securing the protocol. This is similar to proof-of-work networks in which miners provide collateral in the form of hardware/hash-power to seek returns in exchange for building and securing the protocol. - -## Prerequisites - -All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. - -## Constants - -### Misc - -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | -| `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | -| `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | - -## Containers - -### `Eth1Block` - -```python -class Eth1Block(Container): - timestamp: uint64 - deposit_root: Root - deposit_count: uint64 - # All other eth1 block fields -``` - -### `AggregateAndProof` - -```python -class AggregateAndProof(Container): - aggregator_index: ValidatorIndex - aggregate: Attestation - selection_proof: BLSSignature -``` - -### `SignedAggregateAndProof` - -```python -class SignedAggregateAndProof(Container): - message: AggregateAndProof - signature: BLSSignature -``` - -## Becoming a validator - -### Initialization - -A validator must initialize many parameters locally before submitting a deposit and joining the validator registry. - -#### BLS public key - -Validator public keys are [G1 points](beacon-chain.md#bls-signatures) on the [BLS12-381 curve](https://z.cash/blog/new-snark-curve). A private key, `privkey`, must be securely generated along with the resultant `pubkey`. This `privkey` must be "hot", that is, constantly available to sign data throughout the lifetime of the validator. - -#### Withdrawal credentials - -The `withdrawal_credentials` field constrains validator withdrawals. -The first byte of this 32-byte field is a withdrawal prefix which defines the semantics of the remaining 31 bytes. - -The following withdrawal prefixes are currently supported. - -##### `BLS_WITHDRAWAL_PREFIX` - -Withdrawal credentials with the BLS withdrawal prefix allow a BLS key pair -`(bls_withdrawal_privkey, bls_withdrawal_pubkey)` to trigger withdrawals. -The `withdrawal_credentials` field must be such that: - -* `withdrawal_credentials[:1] == BLS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:] == hash(bls_withdrawal_pubkey)[1:]` - -*Note*: The `bls_withdrawal_privkey` is not required for validating and can be kept in cold storage. - -##### `ETH1_ADDRESS_WITHDRAWAL_PREFIX` - -Withdrawal credentials with the Eth1 address withdrawal prefix specify -a 20-byte Eth1 address `eth1_withdrawal_address` as the recipient for all withdrawals. -The `eth1_withdrawal_address` can be the address of either an externally owned account or of a contract. - -The `withdrawal_credentials` field must be such that: - -* `withdrawal_credentials[:1] == ETH1_ADDRESS_WITHDRAWAL_PREFIX` -* `withdrawal_credentials[1:12] == b'\x00' * 11` -* `withdrawal_credentials[12:] == eth1_withdrawal_address` - -After the merge of the current Ethereum application layer (Eth1) into the Beacon Chain (Eth2), -withdrawals to `eth1_withdrawal_address` will be normal ETH transfers (with no payload other than the validator's ETH) -triggered by a user transaction that will set the gas price and gas limit as well pay fees. -As long as the account or contract with address `eth1_withdrawal_address` can receive ETH transfers, -the future withdrawal protocol is agnostic to all other implementation details. - -### Submit deposit - -In Phase 0, all incoming validator deposits originate from the Ethereum 1.0 chain defined by `DEPOSIT_CHAIN_ID` and `DEPOSIT_NETWORK_ID`. Deposits are made to the [deposit contract](./deposit-contract.md) located at `DEPOSIT_CONTRACT_ADDRESS`. - -To submit a deposit: - -- Pack the validator's [initialization parameters](#initialization) into `deposit_data`, a [`DepositData`](./beacon-chain.md#depositdata) SSZ object. -- Let `amount` be the amount in Gwei to be deposited by the validator where `amount >= MIN_DEPOSIT_AMOUNT`. -- Set `deposit_data.pubkey` to validator's `pubkey`. -- Set `deposit_data.withdrawal_credentials` to `withdrawal_credentials`. -- Set `deposit_data.amount` to `amount`. -- Let `deposit_message` be a `DepositMessage` with all the `DepositData` contents except the `signature`. -- Let `signature` be the result of `bls.Sign` of the `compute_signing_root(deposit_message, domain)` with `domain=compute_domain(DOMAIN_DEPOSIT)`. (_Warning_: Deposits _must_ be signed with `GENESIS_FORK_VERSION`, calling `compute_domain` without a second argument defaults to the correct version). -- Let `deposit_data_root` be `hash_tree_root(deposit_data)`. -- Send a transaction on the Ethereum 1.0 chain to `DEPOSIT_CONTRACT_ADDRESS` executing `def deposit(pubkey: bytes[48], withdrawal_credentials: bytes[32], signature: bytes[96], deposit_data_root: bytes32)` along with a deposit of `amount` Gwei. - -*Note*: Deposits made for the same `pubkey` are treated as for the same validator. A singular `Validator` will be added to `state.validators` with each additional deposit amount added to the validator's balance. A validator can only be activated when total deposits for the validator pubkey meet or exceed `MAX_EFFECTIVE_BALANCE`. - -### Process deposit - -Deposits cannot be processed into the beacon chain until the Eth1 block in which they were deposited or any of its descendants is added to the beacon chain `state.eth1_data`. This takes _a minimum_ of `ETH1_FOLLOW_DISTANCE` Eth1 blocks (~8 hours) plus `EPOCHS_PER_ETH1_VOTING_PERIOD` epochs (~6.8 hours). Once the requisite Eth1 data is added, the deposit will normally be added to a beacon chain block and processed into the `state.validators` within an epoch or two. The validator is then in a queue to be activated. - -### Validator index - -Once a validator has been processed and added to the beacon state's `validators`, the validator's `validator_index` is defined by the index into the registry at which the [`ValidatorRecord`](./beacon-chain.md#validator) contains the `pubkey` specified in the validator's deposit. A validator's `validator_index` is guaranteed to not change from the time of initial deposit until the validator exits and fully withdraws. This `validator_index` is used throughout the specification to dictate validator roles and responsibilities at any point and should be stored locally. - -### Activation - -In normal operation, the validator is quickly activated, at which point the validator is added to the shuffling and begins validation after an additional `MAX_SEED_LOOKAHEAD` epochs (25.6 minutes). - -The function [`is_active_validator`](./beacon-chain.md#is_active_validator) can be used to check if a validator is active during a given epoch. Usage is as follows: - -```python -def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: - validator = state.validators[validator_index] - return is_active_validator(validator, get_current_epoch(state)) -``` - -Once a validator is activated, the validator is assigned [responsibilities](#beacon-chain-responsibilities) until exited. - -*Note*: There is a maximum validator churn per finalized epoch, so the delay until activation is variable depending upon finality, total active validator balance, and the number of validators in the queue to be activated. - -## Validator assignments - -A validator can get committee assignments for a given epoch using the following helper via `get_committee_assignment(state, epoch, validator_index)` where `epoch <= next_epoch`. - -```python -def get_committee_assignment(state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex - ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the index to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - Return None if no assignment. - """ - next_epoch = Epoch(get_current_epoch(state) + 1) - assert epoch <= next_epoch - - start_slot = compute_start_slot_at_epoch(epoch) - committee_count_per_slot = get_committee_count_per_slot(state, epoch) - for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): - for index in range(committee_count_per_slot): - committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) - if validator_index in committee: - return committee, CommitteeIndex(index), Slot(slot) - return None -``` - -A validator can use the following function to see if they are supposed to propose during a slot. This function can only be run with a `state` of the slot in question. Proposer selection is only stable within the context of the current epoch. - -```python -def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: - return get_beacon_proposer_index(state) == validator_index -``` - -*Note*: To see if a validator is assigned to propose during the slot, the beacon state must be in the epoch in question. At the epoch boundaries, the validator must run an epoch transition into the epoch to successfully check the proposal assignment of the first slot. - -*Note*: `BeaconBlock` proposal is distinct from beacon committee assignment, and in a given epoch each responsibility might occur at a different slot. - -### Lookahead - -The beacon chain shufflings are designed to provide a minimum of 1 epoch lookahead -on the validator's upcoming committee assignments for attesting dictated by the shuffling and slot. -Note that this lookahead does not apply to proposing, which must be checked during the epoch in question. - -`get_committee_assignment` should be called at the start of each epoch -to get the assignment for the next epoch (`current_epoch + 1`). -A validator should plan for future assignments by noting their assigned attestation -slot and joining the committee index attestation subnet related to their committee assignment. - -Specifically a validator should: -* Call `get_committee_assignment(state, next_epoch, validator_index)` when checking for next epoch assignments. -* Calculate the committees per slot for the next epoch: `committees_per_slot = get_committee_count_per_slot(state, next_epoch)` -* Calculate the subnet index: `subnet_id = compute_subnet_for_attestation(committees_per_slot, slot, committee_index)` -* Find peers of the pubsub topic `beacon_attestation_{subnet_id}`. - * If an _insufficient_ number of current peers are subscribed to the topic, the validator must discover new peers on this topic. Via the discovery protocol, find peers with an ENR containing the `attnets` entry such that `ENR["attnets"][subnet_id] == True`. Then validate that the peers are still persisted on the desired topic by requesting `GetMetaData` and checking the resulting `attnets` field. - * If the validator is assigned to be an aggregator for the slot (see `is_aggregator()`), then subscribe to the topic. - -*Note*: If the validator is _not_ assigned to be an aggregator, the validator only needs sufficient number of peers on the topic to be able to publish messages. The validator does not need to _subscribe_ and listen to all messages on the topic. - -## Beacon chain responsibilities - -A validator has two primary responsibilities to the beacon chain: [proposing blocks](#block-proposal) and [creating attestations](#attestations-1). Proposals happen infrequently, whereas attestations should be created once per epoch. - -### Block proposal - -A validator is expected to propose a [`SignedBeaconBlock`](./beacon-chain.md#signedbeaconblock) at -the beginning of any slot during which `is_proposer(state, validator_index)` returns `True`. -To propose, the validator selects the `BeaconBlock`, `parent`, -that in their view of the fork choice is the head of the chain during `slot - 1`. -The validator creates, signs, and broadcasts a `block` that is a child of `parent` -that satisfies a valid [beacon chain state transition](./beacon-chain.md#beacon-chain-state-transition-function). - -There is one proposer per slot, so if there are N active validators any individual validator -will on average be assigned to propose once per N slots (e.g. at 312,500 validators = 10 million ETH, that's once per ~6 weeks). - -*Note*: In this section, `state` is the state of the slot for the block proposal _without_ the block yet applied. -That is, `state` is the `previous_state` processed through any empty slots up to the assigned slot using `process_slots(previous_state, slot)`. - -#### Preparing for a `BeaconBlock` - -To construct a `BeaconBlockBody`, a `block` (`BeaconBlock`) is defined with the necessary context for a block proposal: - -##### Slot - -Set `block.slot = slot` where `slot` is the current slot at which the validator has been selected to propose. The `parent` selected must satisfy that `parent.slot < block.slot`. - -*Note*: There might be "skipped" slots between the `parent` and `block`. These skipped slots are processed in the state transition function without per-block processing. - -##### Proposer index - -Set `block.proposer_index = validator_index` where `validator_index` is the validator chosen to propose at this slot. The private key mapping to `state.validators[validator_index].pubkey` is used to sign the block. - -##### Parent root - -Set `block.parent_root = hash_tree_root(parent)`. - -#### Constructing the `BeaconBlockBody` - -##### Randao reveal - -Set `block.body.randao_reveal = epoch_signature` where `epoch_signature` is obtained from: - -```python -def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) - return bls.Sign(privkey, signing_root) -``` - -##### Eth1 Data - -The `block.body.eth1_data` field is for block proposers to vote on recent Eth1 data. -This recent data contains an Eth1 block hash as well as the associated deposit root -(as calculated by the `get_deposit_root()` method of the deposit contract) and -deposit count after execution of the corresponding Eth1 block. -If over half of the block proposers in the current Eth1 voting period vote for the same -`eth1_data` then `state.eth1_data` updates immediately allowing new deposits to be processed. -Each deposit in `block.body.deposits` must verify against `state.eth1_data.eth1_deposit_root`. - -###### `get_eth1_data` - -Let `Eth1Block` be an abstract object representing Eth1 blocks with the `timestamp` and depost contract data available. - -Let `get_eth1_data(block: Eth1Block) -> Eth1Data` be the function that returns the Eth1 data for a given Eth1 block. - -An honest block proposer sets `block.body.eth1_data = get_eth1_vote(state, eth1_chain)` where: - -```python -def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: - return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) -``` - -```python -def voting_period_start_time(state: BeaconState) -> uint64: - eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) - return compute_time_at_slot(state, eth1_voting_period_start_slot) -``` - -```python -def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: - return ( - block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start - and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start - ) -``` - -```python -def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: - period_start = voting_period_start_time(state) - # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [ - get_eth1_data(block) for block in eth1_chain - if ( - is_candidate_block(block, period_start) - # Ensure cannot move back to earlier deposit contract states - and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count - ) - ] - - # Valid votes already cast during this period - valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] - - # Default vote on latest eth1 block data in the period range unless eth1 chain is not live - # Non-substantive casting for linter - state_eth1_data: Eth1Data = state.eth1_data - default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data - - return max( - valid_votes, - key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance - default=default_vote - ) -``` - -##### Proposer slashings - -Up to `MAX_PROPOSER_SLASHINGS`, [`ProposerSlashing`](./beacon-chain.md#proposerslashing) objects can be included in the `block`. The proposer slashings must satisfy the verification conditions found in [proposer slashings processing](./beacon-chain.md#proposer-slashings). The validator receives a small "whistleblower" reward for each proposer slashing found and included. - -##### Attester slashings - -Up to `MAX_ATTESTER_SLASHINGS`, [`AttesterSlashing`](./beacon-chain.md#attesterslashing) objects can be included in the `block`. The attester slashings must satisfy the verification conditions found in [attester slashings processing](./beacon-chain.md#attester-slashings). The validator receives a small "whistleblower" reward for each attester slashing found and included. - -##### Attestations - -Up to `MAX_ATTESTATIONS`, aggregate attestations can be included in the `block`. The attestations added must satisfy the verification conditions found in [attestation processing](./beacon-chain.md#attestations). To maximize profit, the validator should attempt to gather aggregate attestations that include singular attestations from the largest number of validators whose signatures from the same epoch have not previously been added on chain. - -##### Deposits - -If there are any unprocessed deposits for the existing `state.eth1_data` (i.e. `state.eth1_data.deposit_count > state.eth1_deposit_index`), then pending deposits _must_ be added to the block. The expected number of deposits is exactly `min(MAX_DEPOSITS, eth1_data.deposit_count - state.eth1_deposit_index)`. These [`deposits`](./beacon-chain.md#deposit) are constructed from the `Deposit` logs from the [Eth1 deposit contract](./deposit-contract.md) and must be processed in sequential order. The deposits included in the `block` must satisfy the verification conditions found in [deposits processing](./beacon-chain.md#deposits). - -The `proof` for each deposit must be constructed against the deposit root contained in `state.eth1_data` rather than the deposit root at the time the deposit was initially logged from the 1.0 chain. This entails storing a full deposit merkle tree locally and computing updated proofs against the `eth1_data.deposit_root` as needed. See [`minimal_merkle.py`](https://github.com/ethereum/research/blob/master/spec_pythonizer/utils/merkle_minimal.py) for a sample implementation. - -##### Voluntary exits - -Up to `MAX_VOLUNTARY_EXITS`, [`VoluntaryExit`](./beacon-chain.md#voluntaryexit) objects can be included in the `block`. The exits must satisfy the verification conditions found in [exits processing](./beacon-chain.md#voluntary-exits). - -*Note*: If a slashing for a validator is included in the same block as a -voluntary exit, the voluntary exit will fail and cause the block to be invalid -due to the slashing being processed first. Implementers must take heed of this -operation interaction when packing blocks. - -#### Packaging into a `SignedBeaconBlock` - -##### State root - -Set `block.state_root = hash_tree_root(state)` of the resulting `state` of the `parent -> block` state transition. - -*Note*: To calculate `state_root`, the validator should first run the state transition function on an unsigned `block` containing a stub for the `state_root`. -It is useful to be able to run a state transition function (working on a copy of the state) that does _not_ validate signatures or state root for this purpose: - -```python -def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - temp_state: BeaconState = state.copy() - signed_block = SignedBeaconBlock(message=block) - state_transition(temp_state, signed_block, validate_result=False) - return hash_tree_root(temp_state) -``` - -##### Signature - -`signed_block = SignedBeaconBlock(message=block, signature=block_signature)`, where `block_signature` is obtained from: - -```python -def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(block, domain) - return bls.Sign(privkey, signing_root) -``` - -### Attesting - -A validator is expected to create, sign, and broadcast an attestation during each epoch. The `committee`, assigned `index`, and assigned `slot` for which the validator performs this role during an epoch are defined by `get_committee_assignment(state, epoch, validator_index)`. - -A validator should create and broadcast the `attestation` to the associated attestation subnet when either (a) the validator has received a valid block from the expected block proposer for the assigned `slot` or (b) one-third of the `slot` has transpired (`SECONDS_PER_SLOT / 3` seconds after the start of `slot`) -- whichever comes _first_. - -*Note*: Although attestations during `GENESIS_EPOCH` do not count toward FFG finality, these initial attestations do give weight to the fork choice, are rewarded, and should be made. - -#### Attestation data - -First, the validator should construct `attestation_data`, an [`AttestationData`](./beacon-chain.md#attestationdata) object based upon the state at the assigned slot. - -- Let `head_block` be the result of running the fork choice during the assigned slot. -- Let `head_state` be the state of `head_block` processed through any empty slots up to the assigned slot using `process_slots(state, slot)`. - -##### General - -* Set `attestation_data.slot = slot` where `slot` is the assigned slot. -* Set `attestation_data.index = index` where `index` is the index associated with the validator's committee. - -##### LMD GHOST vote - -Set `attestation_data.beacon_block_root = hash_tree_root(head_block)`. - -##### FFG vote - -- Set `attestation_data.source = head_state.current_justified_checkpoint`. -- Set `attestation_data.target = Checkpoint(epoch=get_current_epoch(head_state), root=epoch_boundary_block_root)` where `epoch_boundary_block_root` is the root of block at the most recent epoch boundary. - -*Note*: `epoch_boundary_block_root` can be looked up in the state using: - -- Let `start_slot = compute_start_slot_at_epoch(get_current_epoch(head_state))`. -- Let `epoch_boundary_block_root = hash_tree_root(head_block) if start_slot == head_state.slot else get_block_root(state, get_current_epoch(head_state))`. - -#### Construct attestation - -Next, the validator creates `attestation`, an [`Attestation`](./beacon-chain.md#attestation) object. - -##### Data - -Set `attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object defined in the previous section, [attestation data](#attestation-data). - -##### Aggregation bits - -- Let `attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` of length `len(committee)`, where the bit of the index of the validator in the `committee` is set to `0b1`. - -*Note*: Calling `get_attesting_indices(state, attestation.data, attestation.aggregation_bits)` should return a list of length equal to 1, containing `validator_index`. - -##### Aggregate signature - -Set `attestation.signature = attestation_signature` where `attestation_signature` is obtained from: - -```python -def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = compute_signing_root(attestation_data, domain) - return bls.Sign(privkey, signing_root) -``` - -#### Broadcast attestation - -Finally, the validator broadcasts `attestation` to the associated attestation subnet, the `beacon_attestation_{subnet_id}` pubsub topic. - -The `subnet_id` for the `attestation` is calculated with: -- Let `committees_per_slot = get_committee_count_per_slot(state, attestation.data.target.epoch)`. -- Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.committee_index)`. - -```python -def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: - """ - Compute the correct subnet for an attestation for Phase 0. - Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. - """ - slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) - committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - - return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) -``` - -### Attestation aggregation - -Some validators are selected to locally aggregate attestations with a similar `attestation_data` to their constructed `attestation` for the assigned `slot`. - -#### Aggregation selection - -A validator is selected to aggregate based upon the return value of `is_aggregator()`. - -```python -def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) - signing_root = compute_signing_root(slot, domain) - return bls.Sign(privkey, signing_root) -``` - -```python -def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: - committee = get_beacon_committee(state, slot, index) - modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) - return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 -``` - -#### Construct aggregate - -If the validator is selected to aggregate (`is_aggregator()`), they construct an aggregate attestation via the following. - -Collect `attestations` seen via gossip during the `slot` that have an equivalent `attestation_data` to that constructed by the validator. If `len(attestations) > 0`, create an `aggregate_attestation: Attestation` with the following fields. - -##### Data - -Set `aggregate_attestation.data = attestation_data` where `attestation_data` is the `AttestationData` object that is the same for each individual attestation being aggregated. - -##### Aggregation bits - -Let `aggregate_attestation.aggregation_bits` be a `Bitlist[MAX_VALIDATORS_PER_COMMITTEE]` of length `len(committee)`, where each bit set from each individual attestation is set to `0b1`. - -##### Aggregate signature - -Set `aggregate_attestation.signature = aggregate_signature` where `aggregate_signature` is obtained from: - -```python -def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: - signatures = [attestation.signature for attestation in attestations] - return bls.Aggregate(signatures) -``` - -#### Broadcast aggregate - -If the validator is selected to aggregate (`is_aggregator`), then they broadcast their best aggregate as a `SignedAggregateAndProof` to the global aggregate channel (`beacon_aggregate_and_proof`) two-thirds of the way through the `slot`-that is, `SECONDS_PER_SLOT * 2 / 3` seconds after the start of `slot`. - -Selection proofs are provided in `AggregateAndProof` to prove to the gossip channel that the validator has been selected as an aggregator. - -`AggregateAndProof` messages are signed by the aggregator and broadcast inside of `SignedAggregateAndProof` objects to prevent a class of DoS attacks and message forgeries. - -First, `aggregate_and_proof = get_aggregate_and_proof(state, validator_index, aggregate_attestation, privkey)` is constructed. - -```python -def get_aggregate_and_proof(state: BeaconState, - aggregator_index: ValidatorIndex, - aggregate: Attestation, - privkey: int) -> AggregateAndProof: - return AggregateAndProof( - aggregator_index=aggregator_index, - aggregate=aggregate, - selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), - ) -``` - -Then `signed_aggregate_and_proof = SignedAggregateAndProof(message=aggregate_and_proof, signature=signature)` is constructed and broadcast. Where `signature` is obtained from: - -```python -def get_aggregate_and_proof_signature(state: BeaconState, - aggregate_and_proof: AggregateAndProof, - privkey: int) -> BLSSignature: - aggregate = aggregate_and_proof.aggregate - domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) - signing_root = compute_signing_root(aggregate_and_proof, domain) - return bls.Sign(privkey, signing_root) -``` - -## Phase 0 attestation subnet stability - -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: - -* Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets -* Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets -* Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR - -*Note*: Short lived beacon committee assignments should not be added in into the ENR `attnets` entry. - -*Note*: When preparing for a hard fork, a validator must select and subscribe to random subnets of the future fork versioning at least `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. - -## How to avoid slashing - -"Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. - -*Note*: Signed data must be within a sequential `Fork` context to conflict. Messages cannot be slashed across diverging forks. If the previous fork version is 1 and the chain splits into fork 2 and 102, messages from 1 can slashable against messages in forks 1, 2, and 102. Messages in 2 cannot be slashable against messages in 102, and vice versa. - -### Proposer slashing - -To avoid "proposer slashings", a validator must not sign two conflicting [`BeaconBlock`](./beacon-chain.md#beaconblock) where conflicting is defined as two distinct blocks within the same slot. - -*In Phase 0, as long as the validator does not sign two different beacon blocks for the same slot, the validator is safe against proposer slashings.* - -Specifically, when signing a `BeaconBlock`, a validator should perform the following steps in the following order: - -1. Save a record to hard disk that a beacon block has been signed for the `slot=block.slot`. -2. Generate and broadcast the block. - -If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast block and can effectively avoid slashing. - -### Attester slashing - -To avoid "attester slashings", a validator must not sign two conflicting [`AttestationData`](./beacon-chain.md#attestationdata) objects, i.e. two attestations that satisfy [`is_slashable_attestation_data`](./beacon-chain.md#is_slashable_attestation_data). - -Specifically, when signing an `Attestation`, a validator should perform the following steps in the following order: - -1. Save a record to hard disk that an attestation has been signed for source (i.e. `attestation_data.source.epoch`) and target (i.e. `attestation_data.target.epoch`). -2. Generate and broadcast attestation. - -If the software crashes at some point within this routine, then when the validator comes back online, the hard disk has the record of the *potentially* signed/broadcast attestation and can effectively avoid slashing. - -## Protection best practices - -A validator client should be considered standalone and should consider the beacon node as untrusted. This means that the validator client should protect: - -1) Private keys -- private keys should be protected from being exported accidentally or by an attacker. -2) Slashing -- before a validator client signs a message it should validate the data, check it against a local slashing database (do not sign a slashable attestation or block) and update its internal slashing database with the newly signed object. -3) Recovered validator -- Recovering a validator from a private key will result in an empty local slashing db. Best practice is to import (from a trusted source) that validator's attestation history. See [EIP 3076](https://github.com/ethereum/EIPs/pull/3076/files) for a standard slashing interchange format. -4) Far future signing requests -- A validator client can be requested to sign a far into the future attestation, resulting in a valid non-slashable request. If the validator client signs this message, it will result in it blocking itself from attesting any other attestation until the beacon-chain reaches that far into the future epoch. This will result in an inactivity penalty and potential ejection due to low balance. -A validator client should prevent itself from signing such requests by: a) keeping a local time clock if possible and following best practices to stop time server attacks and b) refusing to sign, by default, any message that has a large (>6h) gap from the current slashing protection database indicated a time "jump" or a long offline event. The administrator can manually override this protection to restart the validator after a genuine long offline event. diff --git a/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md b/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md deleted file mode 100644 index fd1b3cc2817..00000000000 --- a/tools/analyzers/specdocs/data/phase0/weak-subjectivity.md +++ /dev/null @@ -1,182 +0,0 @@ -# Ethereum 2.0 Phase 0 -- Weak Subjectivity Guide - -## Table of contents - - - - - -- [Introduction](#introduction) -- [Prerequisites](#prerequisites) -- [Custom Types](#custom-types) -- [Constants](#constants) -- [Configuration](#configuration) -- [Weak Subjectivity Checkpoint](#weak-subjectivity-checkpoint) -- [Weak Subjectivity Period](#weak-subjectivity-period) - - [Calculating the Weak Subjectivity Period](#calculating-the-weak-subjectivity-period) - - [`compute_weak_subjectivity_period`](#compute_weak_subjectivity_period) -- [Weak Subjectivity Sync](#weak-subjectivity-sync) - - [Weak Subjectivity Sync Procedure](#weak-subjectivity-sync-procedure) - - [Checking for Stale Weak Subjectivity Checkpoint](#checking-for-stale-weak-subjectivity-checkpoint) - - [`is_within_weak_subjectivity_period`](#is_within_weak_subjectivity_period) -- [Distributing Weak Subjectivity Checkpoints](#distributing-weak-subjectivity-checkpoints) - - - - -## Introduction - -This document is a guide for implementing the Weak Subjectivity protections in Phase 0 of Ethereum 2.0. -This document is still a work-in-progress, and is subject to large changes. -For more information about weak subjectivity and why it is required, please refer to: - -- [Weak Subjectivity in Eth2.0](https://notes.ethereum.org/@adiasg/weak-subjectvity-eth2) -- [Proof of Stake: How I Learned to Love Weak Subjectivity](https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity/) - -## Prerequisites - -This document uses data structures, constants, functions, and terminology from -[Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Beacon Chain Fork Choice](./fork-choice.md). - -## Custom Types - -| Name | SSZ Equivalent | Description | -|---|---|---| -| `Ether` | `uint64` | an amount in Ether | - -## Constants - -| Name | Value | -|---|---| -| `ETH_TO_GWEI` | `uint64(10**9)` | - -## Configuration - -| Name | Value | -|---|---| -| `SAFETY_DECAY` | `uint64(10)` | - -## Weak Subjectivity Checkpoint - -Any `Checkpoint` object can be used as a Weak Subjectivity Checkpoint. -These Weak Subjectivity Checkpoints are distributed by providers, -downloaded by users and/or distributed as a part of clients, and used as input while syncing a client. - -## Weak Subjectivity Period - -The Weak Subjectivity Period is the number of recent epochs within which there -must be a Weak Subjectivity Checkpoint to ensure that an attacker who takes control -of the validator set at the beginning of the period is slashed at least a minimum threshold -in the event that a conflicting `Checkpoint` is finalized. - -`SAFETY_DECAY` is defined as the maximum percentage tolerable loss in the one-third -safety margin of FFG finality. Thus, any attack exploiting the Weak Subjectivity Period has -a safety margin of at least `1/3 - SAFETY_DECAY/100`. - -### Calculating the Weak Subjectivity Period - -A detailed analysis of the calculation of the weak subjectivity period is made in [this report](https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf). - -*Note*: The expressions in the report use fractions, whereas eth2.0-specs uses only `uint64` arithmetic. The expressions have been simplified to avoid computing fractions, and more details can be found [here](https://www.overleaf.com/read/wgjzjdjpvpsd). - -*Note*: The calculations here use `Ether` instead of `Gwei`, because the large magnitude of balances in `Gwei` can cause an overflow while computing using `uint64` arithmetic operations. Using `Ether` reduces the magnitude of the multiplicative factors by an order of `ETH_TO_GWEI` (`= 10**9`) and avoid the scope for overflows in `uint64`. - -#### `compute_weak_subjectivity_period` - -```python -def compute_weak_subjectivity_period(state: BeaconState) -> uint64: - """ - Returns the weak subjectivity period for the current ``state``. - This computation takes into account the effect of: - - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and - - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). - A detailed calculation can be found at: - https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf - """ - ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - N = len(get_active_validator_indices(state, get_current_epoch(state))) - t = get_total_active_balance(state) // N // ETH_TO_GWEI - T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI - delta = get_validator_churn_limit(state) - Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH - D = SAFETY_DECAY - - if T * (200 + 3 * D) < t * (200 + 12 * D): - epochs_for_validator_set_churn = ( - N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) - ) - epochs_for_balance_top_ups = ( - N * (200 + 3 * D) // (600 * Delta) - ) - ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) - else: - ws_period += ( - 3 * N * D * t // (200 * Delta * (T - t)) - ) - - return ws_period -``` - -A brief reference for what these values look like in practice ([reference script](https://gist.github.com/adiasg/3aceab409b36aa9a9d9156c1baa3c248)): - -| Safety Decay | Avg. Val. Balance (ETH) | Val. Count | Weak Sub. Period (Epochs) | -| ---- | ---- | ---- | ---- | -| 10 | 28 | 32768 | 504 | -| 10 | 28 | 65536 | 752 | -| 10 | 28 | 131072 | 1248 | -| 10 | 28 | 262144 | 2241 | -| 10 | 28 | 524288 | 2241 | -| 10 | 28 | 1048576 | 2241 | -| 10 | 32 | 32768 | 665 | -| 10 | 32 | 65536 | 1075 | -| 10 | 32 | 131072 | 1894 | -| 10 | 32 | 262144 | 3532 | -| 10 | 32 | 524288 | 3532 | -| 10 | 32 | 1048576 | 3532 | - -## Weak Subjectivity Sync - -Clients should allow users to input a Weak Subjectivity Checkpoint at startup, and guarantee that any successful sync leads to the given Weak Subjectivity Checkpoint along the canonical chain. If such a sync is not possible, the client should treat this as a critical and irrecoverable failure. - -### Weak Subjectivity Sync Procedure - -1. Input a Weak Subjectivity Checkpoint as a CLI parameter in `block_root:epoch_number` format, - where `block_root` (an "0x" prefixed 32-byte hex string) and `epoch_number` (an integer) represent a valid `Checkpoint`. - Example of the format: - - ``` - 0x8584188b86a9296932785cc2827b925f9deebacce6d72ad8d53171fa046b43d9:9544 - ``` - -2. Check the weak subjectivity requirements: - - *IF* `epoch_number > store.finalized_checkpoint.epoch`, - then *ASSERT* during block sync that block with root `block_root` is in the sync path at epoch `epoch_number`. - Emit descriptive critical error if this assert fails, then exit client process. - - *IF* `epoch_number <= store.finalized_checkpoint.epoch`, - then *ASSERT* that the block in the canonical chain at epoch `epoch_number` has root `block_root`. - Emit descriptive critical error if this assert fails, then exit client process. - -### Checking for Stale Weak Subjectivity Checkpoint - -Clients may choose to validate that the input Weak Subjectivity Checkpoint is not stale at the time of startup. -To support this mechanism, the client needs to take the state at the Weak Subjectivity Checkpoint as -a CLI parameter input (or fetch the state associated with the input Weak Subjectivity Checkpoint from some source). -The check can be implemented in the following way: - -#### `is_within_weak_subjectivity_period` - -```python -def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: - # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint - assert ws_state.latest_block_header.state_root == ws_checkpoint.root - assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch - - ws_period = compute_weak_subjectivity_period(ws_state) - ws_state_epoch = compute_epoch_at_slot(ws_state.slot) - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - return current_epoch <= ws_state_epoch + ws_period -``` - -## Distributing Weak Subjectivity Checkpoints - -This section will be updated soon. From 8d58f93d2dace4b3aa2ce8a16c3111a75da40e5a Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 15:49:29 +0300 Subject: [PATCH 04/27] add test --- tools/analyzers/specdocs/BUILD.bazel | 2 + tools/analyzers/specdocs/analyzer_test.go | 11 +++ tools/analyzers/specdocs/testdata/BUILD.bazel | 8 ++ .../specdocs/testdata/weak_subjectivity.go | 87 +++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 tools/analyzers/specdocs/analyzer_test.go create mode 100644 tools/analyzers/specdocs/testdata/BUILD.bazel create mode 100644 tools/analyzers/specdocs/testdata/weak_subjectivity.go diff --git a/tools/analyzers/specdocs/BUILD.bazel b/tools/analyzers/specdocs/BUILD.bazel index 8f30411d094..c5f431cd580 100644 --- a/tools/analyzers/specdocs/BUILD.bazel +++ b/tools/analyzers/specdocs/BUILD.bazel @@ -24,3 +24,5 @@ go_tool_library( "@org_golang_x_tools//go/ast/inspector:go_tool_library", ], ) + +# gazelle:exclude analyzer_test.go diff --git a/tools/analyzers/specdocs/analyzer_test.go b/tools/analyzers/specdocs/analyzer_test.go new file mode 100644 index 00000000000..5ce7e7545ca --- /dev/null +++ b/tools/analyzers/specdocs/analyzer_test.go @@ -0,0 +1,11 @@ +package specdocs + +import ( + "testing" + + "golang.org/x/tools/go/analysis/analysistest" +) + +func TestAnalyzer(t *testing.T) { + analysistest.Run(t, analysistest.TestData(), Analyzer) +} diff --git a/tools/analyzers/specdocs/testdata/BUILD.bazel b/tools/analyzers/specdocs/testdata/BUILD.bazel new file mode 100644 index 00000000000..4256b585a40 --- /dev/null +++ b/tools/analyzers/specdocs/testdata/BUILD.bazel @@ -0,0 +1,8 @@ +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["weak_subjectivity.go"], + importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs/testdata", + visibility = ["//visibility:public"], +) diff --git a/tools/analyzers/specdocs/testdata/weak_subjectivity.go b/tools/analyzers/specdocs/testdata/weak_subjectivity.go new file mode 100644 index 00000000000..7c46c2cee1e --- /dev/null +++ b/tools/analyzers/specdocs/testdata/weak_subjectivity.go @@ -0,0 +1,87 @@ +package testdata + +// ComputeWeakSubjectivityPeriod returns weak subjectivity period for the active validator count and finalized epoch. +// +// Reference spec implementation: +// https://github.com/ethereum/eth2.0-specs/blob/master/specs/phase0/weak-subjectivity.md#calculating-the-weak-subjectivity-period +// +// def compute_weak_subjectivity_period(state: BeaconState) -> uint64: +// """ +// Returns the weak subjectivity period for the current ``state``. +// This computation takes into account the effect of: +// - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and +// - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). +// A detailed calculation can be found at: +// https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf +// """ +// ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY +// N = len(get_active_validator_indices(state, get_current_epoch(state))) +// t = get_total_active_balance(state) // N // ETH_TO_GWEI +// T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI +// delta = get_validator_churn_limit(state) +// Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH +// D = SAFETY_DECAY +// +// if T * (200 + 3 * D) < t * (200 + 12 * D): +// epochs_for_validator_set_churn = ( +// N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) +// ) +// epochs_for_balance_top_ups = ( +// N * (200 + 3 * D) // (600 * Delta) +// ) +// ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) +// else: +// ws_period += ( +// 3 * N * D * t // (200 * Delta * (T - t)) +// ) +// +// return ws_period +func ComputeWeakSubjectivityPeriod(st string) (uint64, error) { + return 0, nil +} + +// IsWithinWeakSubjectivityPeriod verifies if a given weak subjectivity checkpoint is not stale i.e. +// the current node is so far beyond, that a given state and checkpoint are not for the latest weak +// subjectivity point. Provided checkpoint still can be used to double-check that node's block root +// at a given epoch matches that of the checkpoint. +// +// Reference implementation: +// https://github.com/ethereum/eth2.0-specs/blob/master/specs/phase0/weak-subjectivity.md#checking-for-stale-weak-subjectivity-checkpoint + +// def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: +// # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint +// assert ws_state.latest_block_header.state_root == ws_checkpoint.root +// assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch +// +// ws_period = compute_weak_subjectivity_period(ws_state) +// ws_state_epoch = compute_epoch_at_slot(ws_state.slot) +// current_epoch = compute_epoch_at_slot(get_current_slot(store)) +// return current_epoch <= ws_state_epoch + ws_period +func IsWithinWeakSubjectivityPeriod(st string) (bool, error) { + return false, nil +} + +// SlotToEpoch returns the epoch number of the input slot. +// +// Spec pseudocode definition: +// def compute_epoch_of_slot(slot: Slot) -> Epoch: +// """ +// Return the epoch number of ``slot``. +// """ +// return Epoch(slot // SLOTS_PER_EPOCH) +func SlotToEpoch(slot uint64) uint64 { + return slot / 32 +} + +// CurrentEpoch returns the current epoch number calculated from +// the slot number stored in beacon state. +// +// Spec pseudocode definition: +// def get_current_epoch(state: BeaconState) -> Epoch: +// """ +// Return the current epoch. +// """ +// return compute_epoch_of_slot(state.slot) +func CurrentEpoch(state string) uint64 { + return 42 +} From 49cb9e249dbd1ec15b055a0538226e302c9e82fd Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 19:02:30 +0300 Subject: [PATCH 05/27] better parsing of incoming docs --- scripts/update-spec-docs.sh | 2 +- .../specdocs/data/phase0/all-defs.md | 78 ------------------- 2 files changed, 1 insertion(+), 79 deletions(-) diff --git a/scripts/update-spec-docs.sh b/scripts/update-spec-docs.sh index a0d1c0157ed..20f75702e74 100755 --- a/scripts/update-spec-docs.sh +++ b/scripts/update-spec-docs.sh @@ -23,7 +23,7 @@ for file in "${files[@]}"; do echo "- downloading" wget -q -O "$OUTPUT_PATH" --no-check-certificate --content-disposition $BASE_URL/"$file" echo "- extracting all code blocks" - sed -n '/^```/,/^```/ p' <"$OUTPUT_PATH" >>"${OUTPUT_PATH%/*}"/all-defs.md + sed -n '/^```python/,/^```/ p' <"$OUTPUT_PATH" >>"${OUTPUT_PATH%/*}"/all-defs.md echo "- removing raw file" rm "$OUTPUT_PATH" done diff --git a/tools/analyzers/specdocs/data/phase0/all-defs.md b/tools/analyzers/specdocs/data/phase0/all-defs.md index 6c1d3eb8205..9485b68f7e3 100644 --- a/tools/analyzers/specdocs/data/phase0/all-defs.md +++ b/tools/analyzers/specdocs/data/phase0/all-defs.md @@ -1494,84 +1494,6 @@ def on_attestation(store: Store, attestation: Attestation) -> None: # Update latest messages for attesting indices update_latest_messages(store, indexed_attestation.attesting_indices, attestation) ``` -``` -( - seq_number: uint64 - attnets: Bitvector[ATTESTATION_SUBNET_COUNT] -) -``` -``` -/ProtocolPrefix/MessageName/SchemaVersion/Encoding -``` -``` -request ::= | -response ::= * -response_chunk ::= | | -result ::= “0” | “1” | “2” | [“128” ... ”255”] -``` -``` -( - error_message: List[byte, 256] -) -``` -``` -( - fork_digest: ForkDigest - finalized_root: Root - finalized_epoch: Epoch - head_root: Root - head_slot: Slot -) -``` -``` -( - uint64 -) -``` -``` -( - start_slot: Slot - count: uint64 - step: uint64 -) -``` -``` -( - List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] -) -``` -``` -( - List[Root, MAX_REQUEST_BLOCKS] -) -``` -``` -( - List[SignedBeaconBlock, MAX_REQUEST_BLOCKS] -) -``` -``` -( - uint64 -) -``` -``` -( - uint64 -) -``` -``` -( - MetaData -) -``` -``` -( - fork_digest: ForkDigest - next_fork_version: Version - next_fork_epoch: Epoch -) -``` ```python class Eth1Block(Container): timestamp: uint64 From 4195460565127566e85d9d4b697214ee40d18c0f Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 22:56:47 +0300 Subject: [PATCH 06/27] update test --- .../specdocs/testdata/weak_subjectivity.go | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/analyzers/specdocs/testdata/weak_subjectivity.go b/tools/analyzers/specdocs/testdata/weak_subjectivity.go index 7c46c2cee1e..c4f2f6c580d 100644 --- a/tools/analyzers/specdocs/testdata/weak_subjectivity.go +++ b/tools/analyzers/specdocs/testdata/weak_subjectivity.go @@ -64,7 +64,7 @@ func IsWithinWeakSubjectivityPeriod(st string) (bool, error) { // SlotToEpoch returns the epoch number of the input slot. // // Spec pseudocode definition: -// def compute_epoch_of_slot(slot: Slot) -> Epoch: +// def compute_epoch_at_slot(slot: Slot) -> Epoch: // """ // Return the epoch number of ``slot``. // """ @@ -82,6 +82,17 @@ func SlotToEpoch(slot uint64) uint64 { // Return the current epoch. // """ // return compute_epoch_of_slot(state.slot) +// We might have further comments, they shouldn't trigger analyzer. func CurrentEpoch(state string) uint64 { return 42 } + +func FuncWithoutComment() { + +} + +// FuncWithNoSpecComment is just a function that has comments, but none of those is from specs. +// So, parser should ignore it. +func FuncWithNoSpecComment() { + +} From 50d6eeb202ec03b70765984c99c1417a5a5522b3 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Wed, 7 Apr 2021 23:23:23 +0300 Subject: [PATCH 07/27] implements analyzer --- tools/analyzers/specdocs/analyzer.go | 111 ++++++++++++++++++++++----- 1 file changed, 93 insertions(+), 18 deletions(-) diff --git a/tools/analyzers/specdocs/analyzer.go b/tools/analyzers/specdocs/analyzer.go index 130c071bb91..8e351f232ec 100644 --- a/tools/analyzers/specdocs/analyzer.go +++ b/tools/analyzers/specdocs/analyzer.go @@ -4,20 +4,26 @@ package specdocs import ( + _ "embed" "errors" - "fmt" "go/ast" + "regexp" + "strings" "golang.org/x/tools/go/analysis" "golang.org/x/tools/go/analysis/passes/inspect" "golang.org/x/tools/go/ast/inspector" ) +//go:embed data +var specText string + +// Regex to find Python's "def". +var m = regexp.MustCompile("def\\s(.*)\\(.*") + // Doc explaining the tool. const Doc = "Tool to enforce that specs pseudo code is up to date" -var errWeakCrypto = errors.New("crypto-secure RNGs are required, use CSPRNG or PRNG defined in github.com/prysmaticlabs/prysm/shared/rand") - // Analyzer runs static analysis. var Analyzer = &analysis.Analyzer{ Name: "specdocs", @@ -33,31 +39,100 @@ func run(pass *analysis.Pass) (interface{}, error) { } nodeFilter := []ast.Node{ - (*ast.File)(nil), - (*ast.Comment)(nil), + (*ast.CommentGroup)(nil), + } + + // Obtain reference snippets. + defs, err := parseSpecs(specText) + if err != nil { + return nil, err } - aliases := make(map[string]string) inspection.Preorder(nodeFilter, func(node ast.Node) { switch stmt := node.(type) { - case *ast.File: - fmt.Printf("node: %v", stmt) - // Reset aliases (per file). - aliases = make(map[string]string) - case *ast.Comment: - fmt.Printf("comment: %v", stmt) + case *ast.CommentGroup: + // Ignore comment groups that do not have python pseudo-code. + chunk := stmt.Text() + if !m.MatchString(chunk) { + return + } + + // Trim the chunk, so that it starts from Python's "def". + loc := m.FindStringIndex(chunk) + chunk = chunk[loc[0]:] + + // Find out Python function name. + defName, defBody := parseDefChunk(chunk) + if defName == "" { + pass.Reportf(node.Pos(), "cannot parse comment pseudo code") + return + } + + // Calculate differences with reference implementation. + refDefs, ok := defs[defName] + if !ok { + pass.Reportf(node.Pos(), "%q is not found in spec docs", defName) + return + } + if !matchesRefImplementation(defName, refDefs, defBody) { + pass.Reportf(node.Pos(), "%q code doesn not match reference implementation in specs", defName) + return + } } }) return nil, nil } -func isPkgDot(expr ast.Expr, pkg, name string) bool { - sel, ok := expr.(*ast.SelectorExpr) - return ok && isIdent(sel.X, pkg) && isIdent(sel.Sel, name) +// parseSpecs parses input spec docs into map of function name -> array of function bodies +// (single entity may have several definitions). +func parseSpecs(input string) (map[string][]string, error) { + chunks := strings.Split(strings.ReplaceAll(input, "```python", ""), "```") + defs := make(map[string][]string, len(chunks)) + for _, chunk := range chunks { + defName, defBody := parseDefChunk(chunk) + if defName == "" { + continue + } + defs[defName] = append(defs[defName], defBody) + } + return defs, nil } -func isIdent(expr ast.Expr, ident string) bool { - id, ok := expr.(*ast.Ident) - return ok && id.Name == ident +func parseDefChunk(chunk string) (string, string) { + chunk = strings.TrimLeft(chunk, "\n") + if chunk == "" { + return "", "" + } + chunkLines := strings.Split(chunk, "\n") + // Ignore all snippets, that do not define functions. + if chunkLines[0][:4] != "def " { + return "", "" + } + defMatches := m.FindStringSubmatch(chunkLines[0]) + if len(defMatches) < 2 { + return "", "" + } + return strings.Trim(defMatches[1], " "), chunk +} + +// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). +func matchesRefImplementation(defName string, refDefs []string, input string) bool { + for _, refDef := range refDefs { + refDefLines := strings.Split(refDef, "\n") + inputLines := strings.Split(input, "\n") + + matchesPerfectly := true + for i := 0; i < len(refDefs); i++ { + a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") + if a != b { + matchesPerfectly = false + break + } + } + if matchesPerfectly { + return true + } + } + return false } From b58d04b585eff1a914c05ecf27bb89551c3befc8 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 03:25:56 +0300 Subject: [PATCH 08/27] separate tool --- tools/analyzers/specdocs/BUILD.bazel | 28 --- tools/specs-checker/BUILD.bazel | 20 ++ .../data/phase0/all-defs.md | 0 tools/specs-checker/main.go | 192 ++++++++++++++++++ .../testdata/BUILD.bazel | 2 +- .../testdata/weak_subjectivity.go | 0 6 files changed, 213 insertions(+), 29 deletions(-) delete mode 100644 tools/analyzers/specdocs/BUILD.bazel create mode 100644 tools/specs-checker/BUILD.bazel rename tools/{analyzers/specdocs => specs-checker}/data/phase0/all-defs.md (100%) create mode 100644 tools/specs-checker/main.go rename tools/{analyzers/specdocs => specs-checker}/testdata/BUILD.bazel (67%) rename tools/{analyzers/specdocs => specs-checker}/testdata/weak_subjectivity.go (100%) diff --git a/tools/analyzers/specdocs/BUILD.bazel b/tools/analyzers/specdocs/BUILD.bazel deleted file mode 100644 index c5f431cd580..00000000000 --- a/tools/analyzers/specdocs/BUILD.bazel +++ /dev/null @@ -1,28 +0,0 @@ -load("@prysm//tools/go:def.bzl", "go_library") -load("@io_bazel_rules_go//go:def.bzl", "go_tool_library") - -go_library( - name = "go_default_library", - srcs = ["analyzer.go"], - importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs", - visibility = ["//visibility:public"], - deps = [ - "@org_golang_x_tools//go/analysis:go_default_library", - "@org_golang_x_tools//go/analysis/passes/inspect:go_default_library", - "@org_golang_x_tools//go/ast/inspector:go_default_library", - ], -) - -go_tool_library( - name = "go_tool_library", - srcs = ["analyzer.go"], - importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs", - visibility = ["//visibility:public"], - deps = [ - "@org_golang_x_tools//go/analysis:go_tool_library", - "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library", - "@org_golang_x_tools//go/ast/inspector:go_tool_library", - ], -) - -# gazelle:exclude analyzer_test.go diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel new file mode 100644 index 00000000000..df3310c283d --- /dev/null +++ b/tools/specs-checker/BUILD.bazel @@ -0,0 +1,20 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + embedsrcs = ["data/phase0/all-defs.md"], + importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", + visibility = ["//visibility:public"], + deps = [ + "@com_github_logrusorgru_aurora//:go_default_library", + "@com_github_urfave_cli_v2//:go_default_library", + ], +) + +go_binary( + name = "specs-checker", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/analyzers/specdocs/data/phase0/all-defs.md b/tools/specs-checker/data/phase0/all-defs.md similarity index 100% rename from tools/analyzers/specdocs/data/phase0/all-defs.md rename to tools/specs-checker/data/phase0/all-defs.md diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go new file mode 100644 index 00000000000..cc77ec12f95 --- /dev/null +++ b/tools/specs-checker/main.go @@ -0,0 +1,192 @@ +package main + +import ( + _ "embed" + "fmt" + "go/ast" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "regexp" + "strings" + + "github.com/logrusorgru/aurora" + "github.com/urfave/cli/v2" +) + +//go:embed data +var specText string + +// Regex to find Python's "def". +var m = regexp.MustCompile("def\\s(.*)\\(.*") + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "Path to a directory containing Golang files to check", + Required: true, + } + au = aurora.NewAurora(true /* enable colors */) +) + +func main() { + app := &cli.App{ + Name: "Specs checker utility", + Description: "Checks that specs pseudo code used in comments is up to date", + Usage: "helps keeping specs pseudo code up to date!", + Commands: []*cli.Command{ + { + Name: "check", + Usage: "Checks that all doc strings", + Flags: []cli.Flag{ + dirFlag, + }, + Action: check, + }, + { + Name: "download", + Usage: "Downloads the latest specs docs", + Action: download, + }, + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} + +func check(cliCtx *cli.Context) error { + // Obtain reference snippets. + defs, err := parseSpecs(specText) + if err != nil { + return err + } + + // Walk the path, and process all contained Golang files. + fileWalker := func(path string, info os.FileInfo, err error) error { + if info == nil { + return fmt.Errorf("invalid input dir %q", path) + } + if !strings.HasSuffix(info.Name(), ".go") { + return nil + } + if err := inspectFile(path, defs); err != nil { + return err + } + return nil + } + if err := filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker); err != nil { + return err + } + + return nil +} + +func inspectFile(path string, defs map[string][]string) error { + // Parse source files, and check the pseudo code. + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + + ast.Inspect(file, func(node ast.Node) bool { + switch stmt := node.(type) { + case *ast.CommentGroup: + // Ignore comment groups that do not have python pseudo-code. + chunk := stmt.Text() + if !m.MatchString(chunk) { + return true + } + + pos := fset.Position(node.Pos()) + + // Trim the chunk, so that it starts from Python's "def". + loc := m.FindStringIndex(chunk) + chunk = chunk[loc[0]:] + + // Find out Python function name. + defName, defBody := parseDefChunk(chunk) + if defName == "" { + fmt.Printf("%s: cannot parse comment pseudo code\n", pos) + return false + } + + // Calculate differences with reference implementation. + refDefs, ok := defs[defName] + if !ok { + fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) + return false + } + if !matchesRefImplementation(defName, refDefs, defBody) { + fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) + return false + } + } + return true + }) + + return nil +} + +func download(cliCtx *cli.Context) error { + return nil +} + +// parseSpecs parses input spec docs into map of function name -> array of function bodies +// (single entity may have several definitions). +func parseSpecs(input string) (map[string][]string, error) { + chunks := strings.Split(strings.ReplaceAll(input, "```python", ""), "```") + defs := make(map[string][]string, len(chunks)) + for _, chunk := range chunks { + defName, defBody := parseDefChunk(chunk) + if defName == "" { + continue + } + defs[defName] = append(defs[defName], defBody) + } + return defs, nil +} + +func parseDefChunk(chunk string) (string, string) { + chunk = strings.TrimLeft(chunk, "\n") + if chunk == "" { + return "", "" + } + chunkLines := strings.Split(chunk, "\n") + // Ignore all snippets, that do not define functions. + if chunkLines[0][:4] != "def " { + return "", "" + } + defMatches := m.FindStringSubmatch(chunkLines[0]) + if len(defMatches) < 2 { + return "", "" + } + return strings.Trim(defMatches[1], " "), chunk +} + +// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). +func matchesRefImplementation(defName string, refDefs []string, input string) bool { + for _, refDef := range refDefs { + refDefLines := strings.Split(refDef, "\n") + inputLines := strings.Split(input, "\n") + + matchesPerfectly := true + for i := 0; i < len(refDefs); i++ { + a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") + if a != b { + matchesPerfectly = false + break + } + } + if matchesPerfectly { + return true + } + } + return false +} diff --git a/tools/analyzers/specdocs/testdata/BUILD.bazel b/tools/specs-checker/testdata/BUILD.bazel similarity index 67% rename from tools/analyzers/specdocs/testdata/BUILD.bazel rename to tools/specs-checker/testdata/BUILD.bazel index 4256b585a40..d77791fe7ee 100644 --- a/tools/analyzers/specdocs/testdata/BUILD.bazel +++ b/tools/specs-checker/testdata/BUILD.bazel @@ -3,6 +3,6 @@ load("@prysm//tools/go:def.bzl", "go_library") go_library( name = "go_default_library", srcs = ["weak_subjectivity.go"], - importpath = "github.com/prysmaticlabs/prysm/tools/analyzers/specdocs/testdata", + importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker/testdata", visibility = ["//visibility:public"], ) diff --git a/tools/analyzers/specdocs/testdata/weak_subjectivity.go b/tools/specs-checker/testdata/weak_subjectivity.go similarity index 100% rename from tools/analyzers/specdocs/testdata/weak_subjectivity.go rename to tools/specs-checker/testdata/weak_subjectivity.go From 49159c7f5f1c0e62c1eac631917073070a693139 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 03:27:12 +0300 Subject: [PATCH 09/27] remove analyzer code --- BUILD.bazel | 1 - tools/analyzers/specdocs/analyzer.go | 138 ---------------------- tools/analyzers/specdocs/analyzer_test.go | 11 -- 3 files changed, 150 deletions(-) delete mode 100644 tools/analyzers/specdocs/analyzer.go delete mode 100644 tools/analyzers/specdocs/analyzer_test.go diff --git a/BUILD.bazel b/BUILD.bazel index e343cb30191..36c24763a8f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -103,7 +103,6 @@ nogo( "@org_golang_x_tools//go/analysis/passes/assign:go_tool_library", "@org_golang_x_tools//go/analysis/passes/inspect:go_tool_library", "@org_golang_x_tools//go/analysis/passes/asmdecl:go_tool_library", - "//tools/analyzers/specdocs:go_tool_library", "//tools/analyzers/maligned:go_tool_library", "//tools/analyzers/cryptorand:go_tool_library", "//tools/analyzers/errcheck:go_tool_library", diff --git a/tools/analyzers/specdocs/analyzer.go b/tools/analyzers/specdocs/analyzer.go deleted file mode 100644 index 8e351f232ec..00000000000 --- a/tools/analyzers/specdocs/analyzer.go +++ /dev/null @@ -1,138 +0,0 @@ -// Package specdocs implements a static analyzer to ensure that pseudo code we use in our comments, when implementing -// functions defined in specs, is up to date. Reference specs documentation is cached (so that we do not need to -// download it every time build is run). -package specdocs - -import ( - _ "embed" - "errors" - "go/ast" - "regexp" - "strings" - - "golang.org/x/tools/go/analysis" - "golang.org/x/tools/go/analysis/passes/inspect" - "golang.org/x/tools/go/ast/inspector" -) - -//go:embed data -var specText string - -// Regex to find Python's "def". -var m = regexp.MustCompile("def\\s(.*)\\(.*") - -// Doc explaining the tool. -const Doc = "Tool to enforce that specs pseudo code is up to date" - -// Analyzer runs static analysis. -var Analyzer = &analysis.Analyzer{ - Name: "specdocs", - Doc: Doc, - Requires: []*analysis.Analyzer{inspect.Analyzer}, - Run: run, -} - -func run(pass *analysis.Pass) (interface{}, error) { - inspection, ok := pass.ResultOf[inspect.Analyzer].(*inspector.Inspector) - if !ok { - return nil, errors.New("analyzer is not type *inspector.Inspector") - } - - nodeFilter := []ast.Node{ - (*ast.CommentGroup)(nil), - } - - // Obtain reference snippets. - defs, err := parseSpecs(specText) - if err != nil { - return nil, err - } - - inspection.Preorder(nodeFilter, func(node ast.Node) { - switch stmt := node.(type) { - case *ast.CommentGroup: - // Ignore comment groups that do not have python pseudo-code. - chunk := stmt.Text() - if !m.MatchString(chunk) { - return - } - - // Trim the chunk, so that it starts from Python's "def". - loc := m.FindStringIndex(chunk) - chunk = chunk[loc[0]:] - - // Find out Python function name. - defName, defBody := parseDefChunk(chunk) - if defName == "" { - pass.Reportf(node.Pos(), "cannot parse comment pseudo code") - return - } - - // Calculate differences with reference implementation. - refDefs, ok := defs[defName] - if !ok { - pass.Reportf(node.Pos(), "%q is not found in spec docs", defName) - return - } - if !matchesRefImplementation(defName, refDefs, defBody) { - pass.Reportf(node.Pos(), "%q code doesn not match reference implementation in specs", defName) - return - } - } - }) - - return nil, nil -} - -// parseSpecs parses input spec docs into map of function name -> array of function bodies -// (single entity may have several definitions). -func parseSpecs(input string) (map[string][]string, error) { - chunks := strings.Split(strings.ReplaceAll(input, "```python", ""), "```") - defs := make(map[string][]string, len(chunks)) - for _, chunk := range chunks { - defName, defBody := parseDefChunk(chunk) - if defName == "" { - continue - } - defs[defName] = append(defs[defName], defBody) - } - return defs, nil -} - -func parseDefChunk(chunk string) (string, string) { - chunk = strings.TrimLeft(chunk, "\n") - if chunk == "" { - return "", "" - } - chunkLines := strings.Split(chunk, "\n") - // Ignore all snippets, that do not define functions. - if chunkLines[0][:4] != "def " { - return "", "" - } - defMatches := m.FindStringSubmatch(chunkLines[0]) - if len(defMatches) < 2 { - return "", "" - } - return strings.Trim(defMatches[1], " "), chunk -} - -// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). -func matchesRefImplementation(defName string, refDefs []string, input string) bool { - for _, refDef := range refDefs { - refDefLines := strings.Split(refDef, "\n") - inputLines := strings.Split(input, "\n") - - matchesPerfectly := true - for i := 0; i < len(refDefs); i++ { - a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") - if a != b { - matchesPerfectly = false - break - } - } - if matchesPerfectly { - return true - } - } - return false -} diff --git a/tools/analyzers/specdocs/analyzer_test.go b/tools/analyzers/specdocs/analyzer_test.go deleted file mode 100644 index 5ce7e7545ca..00000000000 --- a/tools/analyzers/specdocs/analyzer_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package specdocs - -import ( - "testing" - - "golang.org/x/tools/go/analysis/analysistest" -) - -func TestAnalyzer(t *testing.T) { - analysistest.Run(t, analysistest.TestData(), Analyzer) -} From 4691d966c412790136fcfa9c0ba1489b5db8c862 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 03:29:52 +0300 Subject: [PATCH 10/27] cleanup --- nogo_config.json | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/nogo_config.json b/nogo_config.json index 107cfbbeca4..ad54ca3c9da 100644 --- a/nogo_config.json +++ b/nogo_config.json @@ -146,16 +146,5 @@ ".*_test\\.go": "Tests are ok", "shared/fileutil/fileutil.go": "Package which defines the proper rules" } - }, - "specdocs": { - "only_files": { - "beacon-chain/.*": "", - "shared/.*": "", - "slasher/.*": "", - "validator/.*": "" - }, - "exclude_files": { - ".*/.*_test\\.go": "No spec comments are expected in tests" - } } } From 0d504538a6b21a2d48d7f92438b1571bb79be7d4 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 03:38:25 +0300 Subject: [PATCH 11/27] deep source fixes --- tools/specs-checker/BUILD.bazel | 5 +-- tools/specs-checker/main.go | 71 +++++++++++++++------------------ 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel index df3310c283d..4de9e4a005a 100644 --- a/tools/specs-checker/BUILD.bazel +++ b/tools/specs-checker/BUILD.bazel @@ -7,10 +7,7 @@ go_library( embedsrcs = ["data/phase0/all-defs.md"], importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", visibility = ["//visibility:public"], - deps = [ - "@com_github_logrusorgru_aurora//:go_default_library", - "@com_github_urfave_cli_v2//:go_default_library", - ], + deps = ["@com_github_urfave_cli_v2//:go_default_library"], ) go_binary( diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go index cc77ec12f95..1305e63afe7 100644 --- a/tools/specs-checker/main.go +++ b/tools/specs-checker/main.go @@ -12,7 +12,6 @@ import ( "regexp" "strings" - "github.com/logrusorgru/aurora" "github.com/urfave/cli/v2" ) @@ -20,7 +19,7 @@ import ( var specText string // Regex to find Python's "def". -var m = regexp.MustCompile("def\\s(.*)\\(.*") +var m = regexp.MustCompile(`def\s(.*)\(.*`) var ( dirFlag = &cli.StringFlag{ @@ -29,7 +28,6 @@ var ( Usage: "Path to a directory containing Golang files to check", Required: true, } - au = aurora.NewAurora(true /* enable colors */) ) func main() { @@ -75,16 +73,9 @@ func check(cliCtx *cli.Context) error { if !strings.HasSuffix(info.Name(), ".go") { return nil } - if err := inspectFile(path, defs); err != nil { - return err - } - return nil - } - if err := filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker); err != nil { - return err + return inspectFile(path, defs) } - - return nil + return filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker) } func inspectFile(path string, defs map[string][]string) error { @@ -96,38 +87,40 @@ func inspectFile(path string, defs map[string][]string) error { } ast.Inspect(file, func(node ast.Node) bool { - switch stmt := node.(type) { - case *ast.CommentGroup: - // Ignore comment groups that do not have python pseudo-code. - chunk := stmt.Text() - if !m.MatchString(chunk) { - return true - } + stmt, ok := node.(*ast.CommentGroup) + if !ok { + return true + } + // Ignore comment groups that do not have python pseudo-code. + chunk := stmt.Text() + if !m.MatchString(chunk) { + return true + } - pos := fset.Position(node.Pos()) + pos := fset.Position(node.Pos()) - // Trim the chunk, so that it starts from Python's "def". - loc := m.FindStringIndex(chunk) - chunk = chunk[loc[0]:] + // Trim the chunk, so that it starts from Python's "def". + loc := m.FindStringIndex(chunk) + chunk = chunk[loc[0]:] - // Find out Python function name. - defName, defBody := parseDefChunk(chunk) - if defName == "" { - fmt.Printf("%s: cannot parse comment pseudo code\n", pos) - return false - } + // Find out Python function name. + defName, defBody := parseDefChunk(chunk) + if defName == "" { + fmt.Printf("%s: cannot parse comment pseudo code\n", pos) + return false + } - // Calculate differences with reference implementation. - refDefs, ok := defs[defName] - if !ok { - fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) - return false - } - if !matchesRefImplementation(defName, refDefs, defBody) { - fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) - return false - } + // Calculate differences with reference implementation. + refDefs, ok := defs[defName] + if !ok { + fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) + return false } + if !matchesRefImplementation(defName, refDefs, defBody) { + fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) + return false + } + return true }) From a97896e8a3fabc59a8254c8884b5fa64e51f6c27 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 03:42:46 +0300 Subject: [PATCH 12/27] untrack raw specs files --- tools/specs-checker/data/phase0/all-defs.md | 1708 ------------------- 1 file changed, 1708 deletions(-) delete mode 100644 tools/specs-checker/data/phase0/all-defs.md diff --git a/tools/specs-checker/data/phase0/all-defs.md b/tools/specs-checker/data/phase0/all-defs.md deleted file mode 100644 index 9485b68f7e3..00000000000 --- a/tools/specs-checker/data/phase0/all-defs.md +++ /dev/null @@ -1,1708 +0,0 @@ -```python -class Fork(Container): - previous_version: Version - current_version: Version - epoch: Epoch # Epoch of latest fork -``` -```python -class ForkData(Container): - current_version: Version - genesis_validators_root: Root -``` -```python -class Checkpoint(Container): - epoch: Epoch - root: Root -``` -```python -class Validator(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals - effective_balance: Gwei # Balance at stake - slashed: boolean - # Status epochs - activation_eligibility_epoch: Epoch # When criteria for activation were met - activation_epoch: Epoch - exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw funds -``` -```python -class AttestationData(Container): - slot: Slot - index: CommitteeIndex - # LMD GHOST vote - beacon_block_root: Root - # FFG vote - source: Checkpoint - target: Checkpoint -``` -```python -class IndexedAttestation(Container): - attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - signature: BLSSignature -``` -```python -class PendingAttestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - inclusion_delay: Slot - proposer_index: ValidatorIndex -``` -```python -class Eth1Data(Container): - deposit_root: Root - deposit_count: uint64 - block_hash: Bytes32 -``` -```python -class HistoricalBatch(Container): - block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] -``` -```python -class DepositMessage(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei -``` -```python -class DepositData(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei - signature: BLSSignature # Signing over DepositMessage -``` -```python -class BeaconBlockHeader(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body_root: Root -``` -```python -class SigningData(Container): - object_root: Root - domain: Domain -``` -```python -class ProposerSlashing(Container): - signed_header_1: SignedBeaconBlockHeader - signed_header_2: SignedBeaconBlockHeader -``` -```python -class AttesterSlashing(Container): - attestation_1: IndexedAttestation - attestation_2: IndexedAttestation -``` -```python -class Attestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - signature: BLSSignature -``` -```python -class Deposit(Container): - proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root - data: DepositData -``` -```python -class VoluntaryExit(Container): - epoch: Epoch # Earliest epoch when voluntary exit can be processed - validator_index: ValidatorIndex -``` -```python -class BeaconBlockBody(Container): - randao_reveal: BLSSignature - eth1_data: Eth1Data # Eth1 data vote - graffiti: Bytes32 # Arbitrary data - # Operations - proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] - attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] - attestations: List[Attestation, MAX_ATTESTATIONS] - deposits: List[Deposit, MAX_DEPOSITS] - voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] -``` -```python -class BeaconBlock(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body: BeaconBlockBody -``` -```python -class BeaconState(Container): - # Versioning - genesis_time: uint64 - genesis_validators_root: Root - slot: Slot - fork: Fork - # History - latest_block_header: BeaconBlockHeader - block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] - # Eth1 - eth1_data: Eth1Data - eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] - eth1_deposit_index: uint64 - # Registry - validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] - balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] - # Randomness - randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] - # Slashings - slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances - # Attestations - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - # Finality - justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch - previous_justified_checkpoint: Checkpoint # Previous epoch snapshot - current_justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint -``` -```python -class SignedVoluntaryExit(Container): - message: VoluntaryExit - signature: BLSSignature -``` -```python -class SignedBeaconBlock(Container): - message: BeaconBlock - signature: BLSSignature -``` -```python -class SignedBeaconBlockHeader(Container): - message: BeaconBlockHeader - signature: BLSSignature -``` -```python -def integer_squareroot(n: uint64) -> uint64: - """ - Return the largest integer ``x`` such that ``x**2 <= n``. - """ - x = n - y = (x + 1) // 2 - while y < x: - x = y - y = (x + n // x) // 2 - return x -``` -```python -def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: - """ - Return the exclusive-or of two 32-byte strings. - """ - return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) -``` -```python -def bytes_to_uint64(data: bytes) -> uint64: - """ - Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian. - """ - return uint64(int.from_bytes(data, ENDIANNESS)) -``` -```python -def is_active_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is active. - """ - return validator.activation_epoch <= epoch < validator.exit_epoch -``` -```python -def is_eligible_for_activation_queue(validator: Validator) -> bool: - """ - Check if ``validator`` is eligible to be placed into the activation queue. - """ - return ( - validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH - and validator.effective_balance == MAX_EFFECTIVE_BALANCE - ) -``` -```python -def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool: - """ - Check if ``validator`` is eligible for activation. - """ - return ( - # Placement in queue is finalized - validator.activation_eligibility_epoch <= state.finalized_checkpoint.epoch - # Has not yet been activated - and validator.activation_epoch == FAR_FUTURE_EPOCH - ) -``` -```python -def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: - """ - Check if ``validator`` is slashable. - """ - return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) -``` -```python -def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: - """ - Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. - """ - return ( - # Double vote - (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or - # Surround vote - (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) - ) -``` -```python -def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: - """ - Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. - """ - # Verify indices are sorted and unique - indices = indexed_attestation.attesting_indices - if len(indices) == 0 or not indices == sorted(set(indices)): - return False - # Verify aggregate signature - pubkeys = [state.validators[i].pubkey for i in indices] - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) - signing_root = compute_signing_root(indexed_attestation.data, domain) - return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) -``` -```python -def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint64, index: uint64, root: Root) -> bool: - """ - Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. - """ - value = leaf - for i in range(depth): - if index // (2**i) % 2: - value = hash(branch[i] + value) - else: - value = hash(value + branch[i]) - return value == root -``` -```python -def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: - """ - Return the shuffled index corresponding to ``seed`` (and ``index_count``). - """ - assert index < index_count - - # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) - # See the 'generalized domain' algorithm on page 3 - for current_round in range(SHUFFLE_ROUND_COUNT): - pivot = bytes_to_uint64(hash(seed + uint_to_bytes(uint8(current_round)))[0:8]) % index_count - flip = (pivot + index_count - index) % index_count - position = max(index, flip) - source = hash( - seed - + uint_to_bytes(uint8(current_round)) - + uint_to_bytes(uint32(position // 256)) - ) - byte = uint8(source[(position % 256) // 8]) - bit = (byte >> (position % 8)) % 2 - index = flip if bit else index - - return index -``` -```python -def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32) -> ValidatorIndex: - """ - Return from ``indices`` a random index sampled by effective balance. - """ - assert len(indices) > 0 - MAX_RANDOM_BYTE = 2**8 - 1 - i = uint64(0) - total = uint64(len(indices)) - while True: - candidate_index = indices[compute_shuffled_index(i % total, total, seed)] - random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] - effective_balance = state.validators[candidate_index].effective_balance - if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: - return candidate_index - i += 1 -``` -```python -def compute_committee(indices: Sequence[ValidatorIndex], - seed: Bytes32, - index: uint64, - count: uint64) -> Sequence[ValidatorIndex]: - """ - Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. - """ - start = (len(indices) * index) // count - end = (len(indices) * uint64(index + 1)) // count - return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)] -``` -```python -def compute_epoch_at_slot(slot: Slot) -> Epoch: - """ - Return the epoch number at ``slot``. - """ - return Epoch(slot // SLOTS_PER_EPOCH) -``` -```python -def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: - """ - Return the start slot of ``epoch``. - """ - return Slot(epoch * SLOTS_PER_EPOCH) -``` -```python -def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: - """ - Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. - """ - return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) -``` -```python -def compute_fork_data_root(current_version: Version, genesis_validators_root: Root) -> Root: - """ - Return the 32-byte fork data root for the ``current_version`` and ``genesis_validators_root``. - This is used primarily in signature domains to avoid collisions across forks/chains. - """ - return hash_tree_root(ForkData( - current_version=current_version, - genesis_validators_root=genesis_validators_root, - )) -``` -```python -def compute_fork_digest(current_version: Version, genesis_validators_root: Root) -> ForkDigest: - """ - Return the 4-byte fork digest for the ``current_version`` and ``genesis_validators_root``. - This is a digest primarily used for domain separation on the p2p layer. - 4-bytes suffices for practical separation of forks/chains. - """ - return ForkDigest(compute_fork_data_root(current_version, genesis_validators_root)[:4]) -``` -```python -def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_validators_root: Root=None) -> Domain: - """ - Return the domain for the ``domain_type`` and ``fork_version``. - """ - if fork_version is None: - fork_version = GENESIS_FORK_VERSION - if genesis_validators_root is None: - genesis_validators_root = Root() # all bytes zero by default - fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) - return Domain(domain_type + fork_data_root[:28]) -``` -```python -def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: - """ - Return the signing root for the corresponding signing data. - """ - return hash_tree_root(SigningData( - object_root=hash_tree_root(ssz_object), - domain=domain, - )) -``` -```python -def get_current_epoch(state: BeaconState) -> Epoch: - """ - Return the current epoch. - """ - return compute_epoch_at_slot(state.slot) -``` -```python -def get_previous_epoch(state: BeaconState) -> Epoch: - """` - Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). - """ - current_epoch = get_current_epoch(state) - return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) -``` -```python -def get_block_root(state: BeaconState, epoch: Epoch) -> Root: - """ - Return the block root at the start of a recent ``epoch``. - """ - return get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) -``` -```python -def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: - """ - Return the block root at a recent ``slot``. - """ - assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT - return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] -``` -```python -def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: - """ - Return the randao mix at a recent ``epoch``. - """ - return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] -``` -```python -def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: - """ - Return the sequence of active validator indices at ``epoch``. - """ - return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] -``` -```python -def get_validator_churn_limit(state: BeaconState) -> uint64: - """ - Return the validator churn limit for the current epoch. - """ - active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) - return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT) -``` -```python -def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes32: - """ - Return the seed at ``epoch``. - """ - mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow - return hash(domain_type + uint_to_bytes(epoch) + mix) -``` -```python -def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: - """ - Return the number of committees in each slot for the given ``epoch``. - """ - return max(uint64(1), min( - MAX_COMMITTEES_PER_SLOT, - uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, - )) -``` -```python -def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]: - """ - Return the beacon committee at ``slot`` for ``index``. - """ - epoch = compute_epoch_at_slot(slot) - committees_per_slot = get_committee_count_per_slot(state, epoch) - return compute_committee( - indices=get_active_validator_indices(state, epoch), - seed=get_seed(state, epoch, DOMAIN_BEACON_ATTESTER), - index=(slot % SLOTS_PER_EPOCH) * committees_per_slot + index, - count=committees_per_slot * SLOTS_PER_EPOCH, - ) -``` -```python -def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: - """ - Return the beacon proposer index at the current slot. - """ - epoch = get_current_epoch(state) - seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot)) - indices = get_active_validator_indices(state, epoch) - return compute_proposer_index(state, indices, seed) -``` -```python -def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: - """ - Return the combined effective balance of the ``indices``. - ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. - Math safe up to ~10B ETH, afterwhich this overflows uint64. - """ - return Gwei(max(EFFECTIVE_BALANCE_INCREMENT, sum([state.validators[index].effective_balance for index in indices]))) -``` -```python -def get_total_active_balance(state: BeaconState) -> Gwei: - """ - Return the combined effective balance of the active validators. - Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. - """ - return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) -``` -```python -def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: - """ - Return the signature domain (fork version concatenated with domain type) of a message. - """ - epoch = get_current_epoch(state) if epoch is None else epoch - fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version - return compute_domain(domain_type, fork_version, state.genesis_validators_root) -``` -```python -def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: - """ - Return the indexed attestation corresponding to ``attestation``. - """ - attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) - - return IndexedAttestation( - attesting_indices=sorted(attesting_indices), - data=attestation.data, - signature=attestation.signature, - ) -``` -```python -def get_attesting_indices(state: BeaconState, - data: AttestationData, - bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: - """ - Return the set of attesting indices corresponding to ``data`` and ``bits``. - """ - committee = get_beacon_committee(state, data.slot, data.index) - return set(index for i, index in enumerate(committee) if bits[i]) -``` -```python -def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: - """ - Increase the validator balance at index ``index`` by ``delta``. - """ - state.balances[index] += delta -``` -```python -def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: - """ - Decrease the validator balance at index ``index`` by ``delta``, with underflow protection. - """ - state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta -``` -```python -def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: - """ - Initiate the exit of the validator with index ``index``. - """ - # Return if validator already initiated exit - validator = state.validators[index] - if validator.exit_epoch != FAR_FUTURE_EPOCH: - return - - # Compute exit queue epoch - exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] - exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) - exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) - if exit_queue_churn >= get_validator_churn_limit(state): - exit_queue_epoch += Epoch(1) - - # Set validator exit epoch and withdrawable epoch - validator.exit_epoch = exit_queue_epoch - validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) -``` -```python -def slash_validator(state: BeaconState, - slashed_index: ValidatorIndex, - whistleblower_index: ValidatorIndex=None) -> None: - """ - Slash the validator with index ``slashed_index``. - """ - epoch = get_current_epoch(state) - initiate_validator_exit(state, slashed_index) - validator = state.validators[slashed_index] - validator.slashed = True - validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) - state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance - decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) - - # Apply proposer and whistleblower rewards - proposer_index = get_beacon_proposer_index(state) - if whistleblower_index is None: - whistleblower_index = proposer_index - whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) - proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) - increase_balance(state, proposer_index, proposer_reward) - increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) -``` -```python -def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, - eth1_timestamp: uint64, - deposits: Sequence[Deposit]) -> BeaconState: - fork = Fork( - previous_version=GENESIS_FORK_VERSION, - current_version=GENESIS_FORK_VERSION, - epoch=GENESIS_EPOCH, - ) - state = BeaconState( - genesis_time=eth1_timestamp + GENESIS_DELAY, - fork=fork, - eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), - latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), - randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy - ) - - # Process deposits - leaves = list(map(lambda deposit: deposit.data, deposits)) - for index, deposit in enumerate(deposits): - deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) - state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) - process_deposit(state, deposit) - - # Process activations - for index, validator in enumerate(state.validators): - balance = state.balances[index] - validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) - if validator.effective_balance == MAX_EFFECTIVE_BALANCE: - validator.activation_eligibility_epoch = GENESIS_EPOCH - validator.activation_epoch = GENESIS_EPOCH - - # Set genesis validators root for domain separation and chain versioning - state.genesis_validators_root = hash_tree_root(state.validators) - - return state -``` -```python -def is_valid_genesis_state(state: BeaconState) -> bool: - if state.genesis_time < MIN_GENESIS_TIME: - return False - if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: - return False - return True -``` -```python -def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> None: - block = signed_block.message - # Process slots (including those with no blocks) since block - process_slots(state, block.slot) - # Verify signature - if validate_result: - assert verify_block_signature(state, signed_block) - # Process block - process_block(state, block) - # Verify state root - if validate_result: - assert block.state_root == hash_tree_root(state) -``` -```python -def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: - proposer = state.validators[signed_block.message.proposer_index] - signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) - return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) -``` -```python -def process_slots(state: BeaconState, slot: Slot) -> None: - assert state.slot < slot - while state.slot < slot: - process_slot(state) - # Process epoch on the start slot of the next epoch - if (state.slot + 1) % SLOTS_PER_EPOCH == 0: - process_epoch(state) - state.slot = Slot(state.slot + 1) -``` -```python -def process_slot(state: BeaconState) -> None: - # Cache state root - previous_state_root = hash_tree_root(state) - state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root - # Cache latest block header state root - if state.latest_block_header.state_root == Bytes32(): - state.latest_block_header.state_root = previous_state_root - # Cache block root - previous_block_root = hash_tree_root(state.latest_block_header) - state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root -``` -```python -def process_epoch(state: BeaconState) -> None: - process_justification_and_finalization(state) - process_rewards_and_penalties(state) - process_registry_updates(state) - process_slashings(state) - process_eth1_data_reset(state) - process_effective_balance_updates(state) - process_slashings_reset(state) - process_randao_mixes_reset(state) - process_historical_roots_update(state) - process_participation_record_updates(state) -``` -```python -def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: - assert epoch in (get_previous_epoch(state), get_current_epoch(state)) - return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations -``` -```python -def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: - return [ - a for a in get_matching_source_attestations(state, epoch) - if a.data.target.root == get_block_root(state, epoch) - ] -``` -```python -def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: - return [ - a for a in get_matching_target_attestations(state, epoch) - if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) - ] -``` -```python -def get_unslashed_attesting_indices(state: BeaconState, - attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: - output = set() # type: Set[ValidatorIndex] - for a in attestations: - output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) - return set(filter(lambda index: not state.validators[index].slashed, output)) -``` -```python -def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: - """ - Return the combined effective balance of the set of unslashed validators participating in ``attestations``. - Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. - """ - return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) -``` -```python -def process_justification_and_finalization(state: BeaconState) -> None: - # Initial FFG checkpoint values have a `0x00` stub for `root`. - # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. - if get_current_epoch(state) <= GENESIS_EPOCH + 1: - return - previous_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - current_attestations = get_matching_target_attestations(state, get_current_epoch(state)) - total_active_balance = get_total_active_balance(state) - previous_target_balance = get_attesting_balance(state, previous_attestations) - current_target_balance = get_attesting_balance(state, current_attestations) - weigh_justification_and_finalization(state, total_active_balance, previous_target_balance, current_target_balance) -``` -```python -def weigh_justification_and_finalization(state: BeaconState, - total_active_balance: Gwei, - previous_epoch_target_balance: Gwei, - current_epoch_target_balance: Gwei) -> None: - previous_epoch = get_previous_epoch(state) - current_epoch = get_current_epoch(state) - old_previous_justified_checkpoint = state.previous_justified_checkpoint - old_current_justified_checkpoint = state.current_justified_checkpoint - - # Process justifications - state.previous_justified_checkpoint = state.current_justified_checkpoint - state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] - state.justification_bits[0] = 0b0 - if previous_epoch_target_balance * 3 >= total_active_balance * 2: - state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, - root=get_block_root(state, previous_epoch)) - state.justification_bits[1] = 0b1 - if current_epoch_target_balance * 3 >= total_active_balance * 2: - state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, - root=get_block_root(state, current_epoch)) - state.justification_bits[0] = 0b1 - - # Process finalizations - bits = state.justification_bits - # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source - if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: - state.finalized_checkpoint = old_previous_justified_checkpoint - # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source - if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: - state.finalized_checkpoint = old_previous_justified_checkpoint - # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source - if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: - state.finalized_checkpoint = old_current_justified_checkpoint - # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source - if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: - state.finalized_checkpoint = old_current_justified_checkpoint -``` -```python -def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: - total_balance = get_total_active_balance(state) - effective_balance = state.validators[index].effective_balance - return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) -``` -```python -def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: - return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) -``` -```python -def get_finality_delay(state: BeaconState) -> uint64: - return get_previous_epoch(state) - state.finalized_checkpoint.epoch -``` -```python -def is_in_inactivity_leak(state: BeaconState) -> bool: - return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY -``` -```python -def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: - previous_epoch = get_previous_epoch(state) - return [ - ValidatorIndex(index) for index, v in enumerate(state.validators) - if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) - ] -``` -```python -def get_attestation_component_deltas(state: BeaconState, - attestations: Sequence[PendingAttestation] - ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Helper with shared logic for use by get source, target, and head deltas functions - """ - rewards = [Gwei(0)] * len(state.validators) - penalties = [Gwei(0)] * len(state.validators) - total_balance = get_total_active_balance(state) - unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) - attesting_balance = get_total_balance(state, unslashed_attesting_indices) - for index in get_eligible_validator_indices(state): - if index in unslashed_attesting_indices: - increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow - if is_in_inactivity_leak(state): - # Since full base reward will be canceled out by inactivity penalty deltas, - # optimal participation receives full base reward compensation here. - rewards[index] += get_base_reward(state, index) - else: - reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) - rewards[index] += reward_numerator // (total_balance // increment) - else: - penalties[index] += get_base_reward(state, index) - return rewards, penalties -``` -```python -def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return attester micro-rewards/penalties for source-vote for each validator. - """ - matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) - return get_attestation_component_deltas(state, matching_source_attestations) -``` -```python -def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return attester micro-rewards/penalties for target-vote for each validator. - """ - matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - return get_attestation_component_deltas(state, matching_target_attestations) -``` -```python -def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return attester micro-rewards/penalties for head-vote for each validator. - """ - matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) - return get_attestation_component_deltas(state, matching_head_attestations) -``` -```python -def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return proposer and inclusion delay micro-rewards/penalties for each validator. - """ - rewards = [Gwei(0) for _ in range(len(state.validators))] - matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) - for index in get_unslashed_attesting_indices(state, matching_source_attestations): - attestation = min([ - a for a in matching_source_attestations - if index in get_attesting_indices(state, a.data, a.aggregation_bits) - ], key=lambda a: a.inclusion_delay) - rewards[attestation.proposer_index] += get_proposer_reward(state, index) - max_attester_reward = Gwei(get_base_reward(state, index) - get_proposer_reward(state, index)) - rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) - - # No penalties associated with inclusion delay - penalties = [Gwei(0) for _ in range(len(state.validators))] - return rewards, penalties -``` -```python -def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return inactivity reward/penalty deltas for each validator. - """ - penalties = [Gwei(0) for _ in range(len(state.validators))] - if is_in_inactivity_leak(state): - matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) - matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) - for index in get_eligible_validator_indices(state): - # If validator is performing optimally this cancels all rewards for a neutral balance - base_reward = get_base_reward(state, index) - penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) - if index not in matching_target_attesting_indices: - effective_balance = state.validators[index].effective_balance - penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) - - # No rewards associated with inactivity penalties - rewards = [Gwei(0) for _ in range(len(state.validators))] - return rewards, penalties -``` -```python -def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: - """ - Return attestation reward/penalty deltas for each validator. - """ - source_rewards, source_penalties = get_source_deltas(state) - target_rewards, target_penalties = get_target_deltas(state) - head_rewards, head_penalties = get_head_deltas(state) - inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) - _, inactivity_penalties = get_inactivity_penalty_deltas(state) - - rewards = [ - source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] - for i in range(len(state.validators)) - ] - - penalties = [ - source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] - for i in range(len(state.validators)) - ] - - return rewards, penalties -``` -```python -def process_rewards_and_penalties(state: BeaconState) -> None: - # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch - if get_current_epoch(state) == GENESIS_EPOCH: - return - - rewards, penalties = get_attestation_deltas(state) - for index in range(len(state.validators)): - increase_balance(state, ValidatorIndex(index), rewards[index]) - decrease_balance(state, ValidatorIndex(index), penalties[index]) -``` -```python -def process_registry_updates(state: BeaconState) -> None: - # Process activation eligibility and ejections - for index, validator in enumerate(state.validators): - if is_eligible_for_activation_queue(validator): - validator.activation_eligibility_epoch = get_current_epoch(state) + 1 - - if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: - initiate_validator_exit(state, ValidatorIndex(index)) - - # Queue validators eligible for activation and not yet dequeued for activation - activation_queue = sorted([ - index for index, validator in enumerate(state.validators) - if is_eligible_for_activation(state, validator) - # Order by the sequence of activation_eligibility_epoch setting and then index - ], key=lambda index: (state.validators[index].activation_eligibility_epoch, index)) - # Dequeued validators for activation up to churn limit - for index in activation_queue[:get_validator_churn_limit(state)]: - validator = state.validators[index] - validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) -``` -```python -def process_slashings(state: BeaconState) -> None: - epoch = get_current_epoch(state) - total_balance = get_total_active_balance(state) - adjusted_total_slashing_balance = min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) - for index, validator in enumerate(state.validators): - if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: - increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow - penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance - penalty = penalty_numerator // total_balance * increment - decrease_balance(state, ValidatorIndex(index), penalty) -``` -```python -def process_eth1_data_reset(state: BeaconState) -> None: - next_epoch = Epoch(get_current_epoch(state) + 1) - # Reset eth1 data votes - if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: - state.eth1_data_votes = [] -``` -```python -def process_effective_balance_updates(state: BeaconState) -> None: - # Update effective balances with hysteresis - for index, validator in enumerate(state.validators): - balance = state.balances[index] - HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) - DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER - UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER - if ( - balance + DOWNWARD_THRESHOLD < validator.effective_balance - or validator.effective_balance + UPWARD_THRESHOLD < balance - ): - validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) -``` -```python -def process_slashings_reset(state: BeaconState) -> None: - next_epoch = Epoch(get_current_epoch(state) + 1) - # Reset slashings - state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) -``` -```python -def process_randao_mixes_reset(state: BeaconState) -> None: - current_epoch = get_current_epoch(state) - next_epoch = Epoch(current_epoch + 1) - # Set randao mix - state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) -``` -```python -def process_historical_roots_update(state: BeaconState) -> None: - # Set historical root accumulator - next_epoch = Epoch(get_current_epoch(state) + 1) - if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: - historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) - state.historical_roots.append(hash_tree_root(historical_batch)) -``` -```python -def process_participation_record_updates(state: BeaconState) -> None: - # Rotate current/previous epoch attestations - state.previous_epoch_attestations = state.current_epoch_attestations - state.current_epoch_attestations = [] -``` -```python -def process_block(state: BeaconState, block: BeaconBlock) -> None: - process_block_header(state, block) - process_randao(state, block.body) - process_eth1_data(state, block.body) - process_operations(state, block.body) -``` -```python -def process_block_header(state: BeaconState, block: BeaconBlock) -> None: - # Verify that the slots match - assert block.slot == state.slot - # Verify that the block is newer than latest block header - assert block.slot > state.latest_block_header.slot - # Verify that proposer index is the correct index - assert block.proposer_index == get_beacon_proposer_index(state) - # Verify that the parent matches - assert block.parent_root == hash_tree_root(state.latest_block_header) - # Cache current block as the new latest block - state.latest_block_header = BeaconBlockHeader( - slot=block.slot, - proposer_index=block.proposer_index, - parent_root=block.parent_root, - state_root=Bytes32(), # Overwritten in the next process_slot call - body_root=hash_tree_root(block.body), - ) - - # Verify proposer is not slashed - proposer = state.validators[block.proposer_index] - assert not proposer.slashed -``` -```python -def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: - epoch = get_current_epoch(state) - # Verify RANDAO reveal - proposer = state.validators[get_beacon_proposer_index(state)] - signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) - assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal) - # Mix in RANDAO reveal - mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) - state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix -``` -```python -def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: - state.eth1_data_votes.append(body.eth1_data) - if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: - state.eth1_data = body.eth1_data -``` -```python -def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: - # Verify that outstanding deposits are processed up to the maximum number of deposits - assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - - def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: - for operation in operations: - fn(state, operation) - - for_ops(body.proposer_slashings, process_proposer_slashing) - for_ops(body.attester_slashings, process_attester_slashing) - for_ops(body.attestations, process_attestation) - for_ops(body.deposits, process_deposit) - for_ops(body.voluntary_exits, process_voluntary_exit) -``` -```python -def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: - header_1 = proposer_slashing.signed_header_1.message - header_2 = proposer_slashing.signed_header_2.message - - # Verify header slots match - assert header_1.slot == header_2.slot - # Verify header proposer indices match - assert header_1.proposer_index == header_2.proposer_index - # Verify the headers are different - assert header_1 != header_2 - # Verify the proposer is slashable - proposer = state.validators[header_1.proposer_index] - assert is_slashable_validator(proposer, get_current_epoch(state)) - # Verify signatures - for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) - signing_root = compute_signing_root(signed_header.message, domain) - assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) - - slash_validator(state, header_1.proposer_index) -``` -```python -def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: - attestation_1 = attester_slashing.attestation_1 - attestation_2 = attester_slashing.attestation_2 - assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) - assert is_valid_indexed_attestation(state, attestation_1) - assert is_valid_indexed_attestation(state, attestation_2) - - slashed_any = False - indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices) - for index in sorted(indices): - if is_slashable_validator(state.validators[index], get_current_epoch(state)): - slash_validator(state, index) - slashed_any = True - assert slashed_any -``` -```python -def process_attestation(state: BeaconState, attestation: Attestation) -> None: - data = attestation.data - assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) - assert data.target.epoch == compute_epoch_at_slot(data.slot) - assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH - assert data.index < get_committee_count_per_slot(state, data.target.epoch) - - committee = get_beacon_committee(state, data.slot, data.index) - assert len(attestation.aggregation_bits) == len(committee) - - pending_attestation = PendingAttestation( - data=data, - aggregation_bits=attestation.aggregation_bits, - inclusion_delay=state.slot - data.slot, - proposer_index=get_beacon_proposer_index(state), - ) - - if data.target.epoch == get_current_epoch(state): - assert data.source == state.current_justified_checkpoint - state.current_epoch_attestations.append(pending_attestation) - else: - assert data.source == state.previous_justified_checkpoint - state.previous_epoch_attestations.append(pending_attestation) - - # Verify signature - assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) -``` -```python -def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: - amount = deposit.data.amount - effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) - - return Validator( - pubkey=deposit.data.pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - activation_eligibility_epoch=FAR_FUTURE_EPOCH, - activation_epoch=FAR_FUTURE_EPOCH, - exit_epoch=FAR_FUTURE_EPOCH, - withdrawable_epoch=FAR_FUTURE_EPOCH, - effective_balance=effective_balance, - ) -``` -```python -def process_deposit(state: BeaconState, deposit: Deposit) -> None: - # Verify the Merkle branch - assert is_valid_merkle_branch( - leaf=hash_tree_root(deposit.data), - branch=deposit.proof, - depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in - index=state.eth1_deposit_index, - root=state.eth1_data.deposit_root, - ) - - # Deposits must be processed in order - state.eth1_deposit_index += 1 - - pubkey = deposit.data.pubkey - amount = deposit.data.amount - validator_pubkeys = [v.pubkey for v in state.validators] - if pubkey not in validator_pubkeys: - # Verify the deposit signature (proof of possession) which is not checked by the deposit contract - deposit_message = DepositMessage( - pubkey=deposit.data.pubkey, - withdrawal_credentials=deposit.data.withdrawal_credentials, - amount=deposit.data.amount, - ) - domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks - signing_root = compute_signing_root(deposit_message, domain) - if not bls.Verify(pubkey, signing_root, deposit.data.signature): - return - - # Add validator and balance entries - state.validators.append(get_validator_from_deposit(state, deposit)) - state.balances.append(amount) - else: - # Increase balance by deposit amount - index = ValidatorIndex(validator_pubkeys.index(pubkey)) - increase_balance(state, index, amount) -``` -```python -def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: - voluntary_exit = signed_voluntary_exit.message - validator = state.validators[voluntary_exit.validator_index] - # Verify the validator is active - assert is_active_validator(validator, get_current_epoch(state)) - # Verify exit has not been initiated - assert validator.exit_epoch == FAR_FUTURE_EPOCH - # Exits must specify an epoch when they become valid; they are not valid before then - assert get_current_epoch(state) >= voluntary_exit.epoch - # Verify the validator has been active long enough - assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD - # Verify signature - domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) - signing_root = compute_signing_root(voluntary_exit, domain) - assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) - # Initiate exit - initiate_validator_exit(state, voluntary_exit.validator_index) -``` -```python -@dataclass(eq=True, frozen=True) -class LatestMessage(object): - epoch: Epoch - root: Root -``` -```python -@dataclass -class Store(object): - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) -``` -```python -def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: - assert anchor_block.state_root == hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - return Store( - time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: copy(anchor_block)}, - block_states={anchor_root: copy(anchor_state)}, - checkpoint_states={justified_checkpoint: copy(anchor_state)}, - ) -``` -```python -def get_slots_since_genesis(store: Store) -> int: - return (store.time - store.genesis_time) // SECONDS_PER_SLOT -``` -```python -def get_current_slot(store: Store) -> Slot: - return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) -``` -```python -def compute_slots_since_epoch_start(slot: Slot) -> int: - return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) -``` -```python -def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: - block = store.blocks[root] - if block.slot > slot: - return get_ancestor(store, block.parent_root, slot) - elif block.slot == slot: - return root - else: - # root is older than queried slot, thus a skip slot. Return most recent root prior to slot - return root -``` -```python -def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: - state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state, get_current_epoch(state)) - return Gwei(sum( - state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages - and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) - )) -``` -```python -def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: - block = store.blocks[block_root] - children = [ - root for root in store.blocks.keys() - if store.blocks[root].parent_root == block_root - ] - - # If any children branches contain expected finalized/justified checkpoints, - # add to filtered block-tree and signal viability to parent. - if any(children): - filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - # If leaf block, check finalized/justified checkpoints as matching latest. - head_state = store.block_states[block_root] - - correct_justified = ( - store.justified_checkpoint.epoch == GENESIS_EPOCH - or head_state.current_justified_checkpoint == store.justified_checkpoint - ) - correct_finalized = ( - store.finalized_checkpoint.epoch == GENESIS_EPOCH - or head_state.finalized_checkpoint == store.finalized_checkpoint - ) - # If expected finalized/justified, add to viable block-tree and signal viability to parent. - if correct_justified and correct_finalized: - blocks[block_root] = block - return True - - # Otherwise, branch not viable - return False -``` -```python -def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: - """ - Retrieve a filtered block tree from ``store``, only returning branches - whose leaf state's justified/finalized info agrees with that in ``store``. - """ - base = store.justified_checkpoint.root - blocks: Dict[Root, BeaconBlock] = {} - filter_block_tree(store, base, blocks) - return blocks -``` -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - children = [ - root for root in blocks.keys() - if blocks[root].parent_root == head - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) -``` -```python -def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: - """ - To address the bouncing attack, only update conflicting justified - checkpoints in the fork choice if in the early slots of the epoch. - Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. - - See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. - """ - if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: - return True - - justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) - if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: - return False - - return True -``` -```python -def validate_on_attestation(store: Store, attestation: Attestation) -> None: - target = attestation.data.target - - # Attestations must be from the current or previous epoch - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - # Use GENESIS_EPOCH for previous when genesis to avoid underflow - previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH - # If attestation target is from a future epoch, delay consideration until the epoch arrives - assert target.epoch in [current_epoch, previous_epoch] - assert target.epoch == compute_epoch_at_slot(attestation.data.slot) - - # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found - assert target.root in store.blocks - - # Attestations must be for a known block. If block is unknown, delay consideration until the block is found - assert attestation.data.beacon_block_root in store.blocks - # Attestations must not be for blocks in the future. If not, the attestation should not be considered - assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - - # LMD vote must be consistent with FFG vote target - target_slot = compute_start_slot_at_epoch(target.epoch) - assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) - - # Attestations can only affect the fork choice of subsequent slots. - # Delay consideration in the fork choice until their slot is in the past. - assert get_current_slot(store) >= attestation.data.slot + 1 -``` -```python -def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: - # Store target checkpoint state if not yet seen - if target not in store.checkpoint_states: - base_state = copy(store.block_states[target.root]) - if base_state.slot < compute_start_slot_at_epoch(target.epoch): - process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) - store.checkpoint_states[target] = base_state -``` -```python -def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: - target = attestation.data.target - beacon_block_root = attestation.data.beacon_block_root - for i in attesting_indices: - if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: - store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) -``` -```python -def on_tick(store: Store, time: uint64) -> None: - previous_slot = get_current_slot(store) - - # update store time - store.time = time - - current_slot = get_current_slot(store) - # Not a new epoch, return - if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): - return - # Update store.justified_checkpoint if a better checkpoint is known - if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = store.best_justified_checkpoint -``` -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - pre_state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root - - # Check the block is valid and compute the post-state - state = pre_state.copy() - state_transition(state, signed_block, True) - # Add new block to the store - store.blocks[hash_tree_root(block)] = block - # Add new state for this block to the store - store.block_states[hash_tree_root(block)] = state - - # Update justified checkpoint - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: - store.best_justified_checkpoint = state.current_justified_checkpoint - if should_update_justified_checkpoint(store, state.current_justified_checkpoint): - store.justified_checkpoint = state.current_justified_checkpoint - - # Update finalized checkpoint - if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: - store.finalized_checkpoint = state.finalized_checkpoint - - # Potentially update justified if different from store - if store.justified_checkpoint != state.current_justified_checkpoint: - # Update justified if new justified is later than store justified - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = state.current_justified_checkpoint - return - - # Update justified if store justified is not in chain with finalized checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) - if ancestor_at_finalized_slot != store.finalized_checkpoint.root: - store.justified_checkpoint = state.current_justified_checkpoint -``` -```python -def on_attestation(store: Store, attestation: Attestation) -> None: - """ - Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. - - An ``attestation`` that is asserted as invalid may be valid at a later time, - consider scheduling it for later processing in such case. - """ - validate_on_attestation(store, attestation) - store_target_checkpoint_state(store, attestation.data.target) - - # Get state at the `target` to fully validate attestation - target_state = store.checkpoint_states[attestation.data.target] - indexed_attestation = get_indexed_attestation(target_state, attestation) - assert is_valid_indexed_attestation(target_state, indexed_attestation) - - # Update latest messages for attesting indices - update_latest_messages(store, indexed_attestation.attesting_indices, attestation) -``` -```python -class Eth1Block(Container): - timestamp: uint64 - deposit_root: Root - deposit_count: uint64 - # All other eth1 block fields -``` -```python -class AggregateAndProof(Container): - aggregator_index: ValidatorIndex - aggregate: Attestation - selection_proof: BLSSignature -``` -```python -class SignedAggregateAndProof(Container): - message: AggregateAndProof - signature: BLSSignature -``` -```python -def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: - validator = state.validators[validator_index] - return is_active_validator(validator, get_current_epoch(state)) -``` -```python -def get_committee_assignment(state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex - ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the index to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - Return None if no assignment. - """ - next_epoch = Epoch(get_current_epoch(state) + 1) - assert epoch <= next_epoch - - start_slot = compute_start_slot_at_epoch(epoch) - committee_count_per_slot = get_committee_count_per_slot(state, epoch) - for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): - for index in range(committee_count_per_slot): - committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) - if validator_index in committee: - return committee, CommitteeIndex(index), Slot(slot) - return None -``` -```python -def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: - return get_beacon_proposer_index(state) == validator_index -``` -```python -def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: - return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) -``` -```python -def voting_period_start_time(state: BeaconState) -> uint64: - eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) - return compute_time_at_slot(state, eth1_voting_period_start_slot) -``` -```python -def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: - return ( - block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start - and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start - ) -``` -```python -def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: - period_start = voting_period_start_time(state) - # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [ - get_eth1_data(block) for block in eth1_chain - if ( - is_candidate_block(block, period_start) - # Ensure cannot move back to earlier deposit contract states - and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count - ) - ] - - # Valid votes already cast during this period - valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] - - # Default vote on latest eth1 block data in the period range unless eth1 chain is not live - # Non-substantive casting for linter - state_eth1_data: Eth1Data = state.eth1_data - default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data - - return max( - valid_votes, - key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance - default=default_vote - ) -``` -```python -def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - temp_state: BeaconState = state.copy() - signed_block = SignedBeaconBlock(message=block) - state_transition(temp_state, signed_block, validate_result=False) - return hash_tree_root(temp_state) -``` -```python -def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(block, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = compute_signing_root(attestation_data, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: - """ - Compute the correct subnet for an attestation for Phase 0. - Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. - """ - slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) - committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - - return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) -``` -```python -def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) - signing_root = compute_signing_root(slot, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: - committee = get_beacon_committee(state, slot, index) - modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) - return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 -``` -```python -def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: - signatures = [attestation.signature for attestation in attestations] - return bls.Aggregate(signatures) -``` -```python -def get_aggregate_and_proof(state: BeaconState, - aggregator_index: ValidatorIndex, - aggregate: Attestation, - privkey: int) -> AggregateAndProof: - return AggregateAndProof( - aggregator_index=aggregator_index, - aggregate=aggregate, - selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), - ) -``` -```python -def get_aggregate_and_proof_signature(state: BeaconState, - aggregate_and_proof: AggregateAndProof, - privkey: int) -> BLSSignature: - aggregate = aggregate_and_proof.aggregate - domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) - signing_root = compute_signing_root(aggregate_and_proof, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_weak_subjectivity_period(state: BeaconState) -> uint64: - """ - Returns the weak subjectivity period for the current ``state``. - This computation takes into account the effect of: - - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and - - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). - A detailed calculation can be found at: - https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf - """ - ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - N = len(get_active_validator_indices(state, get_current_epoch(state))) - t = get_total_active_balance(state) // N // ETH_TO_GWEI - T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI - delta = get_validator_churn_limit(state) - Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH - D = SAFETY_DECAY - - if T * (200 + 3 * D) < t * (200 + 12 * D): - epochs_for_validator_set_churn = ( - N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) - ) - epochs_for_balance_top_ups = ( - N * (200 + 3 * D) // (600 * Delta) - ) - ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) - else: - ws_period += ( - 3 * N * D * t // (200 * Delta * (T - t)) - ) - - return ws_period -``` -```python -def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: - # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint - assert ws_state.latest_block_header.state_root == ws_checkpoint.root - assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch - - ws_period = compute_weak_subjectivity_period(ws_state) - ws_state_epoch = compute_epoch_at_slot(ws_state.slot) - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - return current_epoch <= ws_state_epoch + ws_period -``` From a404228509855c8132ff2878d2ddc74c93c0a872 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Thu, 8 Apr 2021 16:22:22 +0300 Subject: [PATCH 13/27] add back phase0 defs --- tools/specs-checker/data/phase0/all-defs.md | 1708 +++++++++++++++++++ 1 file changed, 1708 insertions(+) create mode 100644 tools/specs-checker/data/phase0/all-defs.md diff --git a/tools/specs-checker/data/phase0/all-defs.md b/tools/specs-checker/data/phase0/all-defs.md new file mode 100644 index 00000000000..9485b68f7e3 --- /dev/null +++ b/tools/specs-checker/data/phase0/all-defs.md @@ -0,0 +1,1708 @@ +```python +class Fork(Container): + previous_version: Version + current_version: Version + epoch: Epoch # Epoch of latest fork +``` +```python +class ForkData(Container): + current_version: Version + genesis_validators_root: Root +``` +```python +class Checkpoint(Container): + epoch: Epoch + root: Root +``` +```python +class Validator(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals + effective_balance: Gwei # Balance at stake + slashed: boolean + # Status epochs + activation_eligibility_epoch: Epoch # When criteria for activation were met + activation_epoch: Epoch + exit_epoch: Epoch + withdrawable_epoch: Epoch # When validator can withdraw funds +``` +```python +class AttestationData(Container): + slot: Slot + index: CommitteeIndex + # LMD GHOST vote + beacon_block_root: Root + # FFG vote + source: Checkpoint + target: Checkpoint +``` +```python +class IndexedAttestation(Container): + attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature +``` +```python +class PendingAttestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + inclusion_delay: Slot + proposer_index: ValidatorIndex +``` +```python +class Eth1Data(Container): + deposit_root: Root + deposit_count: uint64 + block_hash: Bytes32 +``` +```python +class HistoricalBatch(Container): + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] +``` +```python +class DepositMessage(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei +``` +```python +class DepositData(Container): + pubkey: BLSPubkey + withdrawal_credentials: Bytes32 + amount: Gwei + signature: BLSSignature # Signing over DepositMessage +``` +```python +class BeaconBlockHeader(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body_root: Root +``` +```python +class SigningData(Container): + object_root: Root + domain: Domain +``` +```python +class ProposerSlashing(Container): + signed_header_1: SignedBeaconBlockHeader + signed_header_2: SignedBeaconBlockHeader +``` +```python +class AttesterSlashing(Container): + attestation_1: IndexedAttestation + attestation_2: IndexedAttestation +``` +```python +class Attestation(Container): + aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] + data: AttestationData + signature: BLSSignature +``` +```python +class Deposit(Container): + proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root + data: DepositData +``` +```python +class VoluntaryExit(Container): + epoch: Epoch # Earliest epoch when voluntary exit can be processed + validator_index: ValidatorIndex +``` +```python +class BeaconBlockBody(Container): + randao_reveal: BLSSignature + eth1_data: Eth1Data # Eth1 data vote + graffiti: Bytes32 # Arbitrary data + # Operations + proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] + attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] + attestations: List[Attestation, MAX_ATTESTATIONS] + deposits: List[Deposit, MAX_DEPOSITS] + voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] +``` +```python +class BeaconBlock(Container): + slot: Slot + proposer_index: ValidatorIndex + parent_root: Root + state_root: Root + body: BeaconBlockBody +``` +```python +class BeaconState(Container): + # Versioning + genesis_time: uint64 + genesis_validators_root: Root + slot: Slot + fork: Fork + # History + latest_block_header: BeaconBlockHeader + block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] + historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] + # Eth1 + eth1_data: Eth1Data + eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] + eth1_deposit_index: uint64 + # Registry + validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] + balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] + # Randomness + randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] + # Slashings + slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances + # Attestations + previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] + # Finality + justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch + previous_justified_checkpoint: Checkpoint # Previous epoch snapshot + current_justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint +``` +```python +class SignedVoluntaryExit(Container): + message: VoluntaryExit + signature: BLSSignature +``` +```python +class SignedBeaconBlock(Container): + message: BeaconBlock + signature: BLSSignature +``` +```python +class SignedBeaconBlockHeader(Container): + message: BeaconBlockHeader + signature: BLSSignature +``` +```python +def integer_squareroot(n: uint64) -> uint64: + """ + Return the largest integer ``x`` such that ``x**2 <= n``. + """ + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +``` +```python +def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: + """ + Return the exclusive-or of two 32-byte strings. + """ + return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) +``` +```python +def bytes_to_uint64(data: bytes) -> uint64: + """ + Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian. + """ + return uint64(int.from_bytes(data, ENDIANNESS)) +``` +```python +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +``` +```python +def is_eligible_for_activation_queue(validator: Validator) -> bool: + """ + Check if ``validator`` is eligible to be placed into the activation queue. + """ + return ( + validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH + and validator.effective_balance == MAX_EFFECTIVE_BALANCE + ) +``` +```python +def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool: + """ + Check if ``validator`` is eligible for activation. + """ + return ( + # Placement in queue is finalized + validator.activation_eligibility_epoch <= state.finalized_checkpoint.epoch + # Has not yet been activated + and validator.activation_epoch == FAR_FUTURE_EPOCH + ) +``` +```python +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) +``` +```python +def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: + """ + Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. + """ + return ( + # Double vote + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or + # Surround vote + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) + ) +``` +```python +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. + """ + # Verify indices are sorted and unique + indices = indexed_attestation.attesting_indices + if len(indices) == 0 or not indices == sorted(set(indices)): + return False + # Verify aggregate signature + pubkeys = [state.validators[i].pubkey for i in indices] + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) + signing_root = compute_signing_root(indexed_attestation.data, domain) + return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) +``` +```python +def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint64, index: uint64, root: Root) -> bool: + """ + Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(branch[i] + value) + else: + value = hash(value + branch[i]) + return value == root +``` +```python +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: + """ + Return the shuffled index corresponding to ``seed`` (and ``index_count``). + """ + assert index < index_count + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_uint64(hash(seed + uint_to_bytes(uint8(current_round)))[0:8]) % index_count + flip = (pivot + index_count - index) % index_count + position = max(index, flip) + source = hash( + seed + + uint_to_bytes(uint8(current_round)) + + uint_to_bytes(uint32(position // 256)) + ) + byte = uint8(source[(position % 256) // 8]) + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index +``` +```python +def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32) -> ValidatorIndex: + """ + Return from ``indices`` a random index sampled by effective balance. + """ + assert len(indices) > 0 + MAX_RANDOM_BYTE = 2**8 - 1 + i = uint64(0) + total = uint64(len(indices)) + while True: + candidate_index = indices[compute_shuffled_index(i % total, total, seed)] + random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return candidate_index + i += 1 +``` +```python +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Bytes32, + index: uint64, + count: uint64) -> Sequence[ValidatorIndex]: + """ + Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. + """ + start = (len(indices) * index) // count + end = (len(indices) * uint64(index + 1)) // count + return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)] +``` +```python +def compute_epoch_at_slot(slot: Slot) -> Epoch: + """ + Return the epoch number at ``slot``. + """ + return Epoch(slot // SLOTS_PER_EPOCH) +``` +```python +def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: + """ + Return the start slot of ``epoch``. + """ + return Slot(epoch * SLOTS_PER_EPOCH) +``` +```python +def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. + """ + return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) +``` +```python +def compute_fork_data_root(current_version: Version, genesis_validators_root: Root) -> Root: + """ + Return the 32-byte fork data root for the ``current_version`` and ``genesis_validators_root``. + This is used primarily in signature domains to avoid collisions across forks/chains. + """ + return hash_tree_root(ForkData( + current_version=current_version, + genesis_validators_root=genesis_validators_root, + )) +``` +```python +def compute_fork_digest(current_version: Version, genesis_validators_root: Root) -> ForkDigest: + """ + Return the 4-byte fork digest for the ``current_version`` and ``genesis_validators_root``. + This is a digest primarily used for domain separation on the p2p layer. + 4-bytes suffices for practical separation of forks/chains. + """ + return ForkDigest(compute_fork_data_root(current_version, genesis_validators_root)[:4]) +``` +```python +def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_validators_root: Root=None) -> Domain: + """ + Return the domain for the ``domain_type`` and ``fork_version``. + """ + if fork_version is None: + fork_version = GENESIS_FORK_VERSION + if genesis_validators_root is None: + genesis_validators_root = Root() # all bytes zero by default + fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) + return Domain(domain_type + fork_data_root[:28]) +``` +```python +def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: + """ + Return the signing root for the corresponding signing data. + """ + return hash_tree_root(SigningData( + object_root=hash_tree_root(ssz_object), + domain=domain, + )) +``` +```python +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch. + """ + return compute_epoch_at_slot(state.slot) +``` +```python +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). + """ + current_epoch = get_current_epoch(state) + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) +``` +```python +def get_block_root(state: BeaconState, epoch: Epoch) -> Root: + """ + Return the block root at the start of a recent ``epoch``. + """ + return get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) +``` +```python +def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +``` +```python +def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] +``` +```python +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + """ + Return the sequence of active validator indices at ``epoch``. + """ + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] +``` +```python +def get_validator_churn_limit(state: BeaconState) -> uint64: + """ + Return the validator churn limit for the current epoch. + """ + active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) + return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT) +``` +```python +def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes32: + """ + Return the seed at ``epoch``. + """ + mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow + return hash(domain_type + uint_to_bytes(epoch) + mix) +``` +```python +def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: + """ + Return the number of committees in each slot for the given ``epoch``. + """ + return max(uint64(1), min( + MAX_COMMITTEES_PER_SLOT, + uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + )) +``` +```python +def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]: + """ + Return the beacon committee at ``slot`` for ``index``. + """ + epoch = compute_epoch_at_slot(slot) + committees_per_slot = get_committee_count_per_slot(state, epoch) + return compute_committee( + indices=get_active_validator_indices(state, epoch), + seed=get_seed(state, epoch, DOMAIN_BEACON_ATTESTER), + index=(slot % SLOTS_PER_EPOCH) * committees_per_slot + index, + count=committees_per_slot * SLOTS_PER_EPOCH, + ) +``` +```python +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: + """ + Return the beacon proposer index at the current slot. + """ + epoch = get_current_epoch(state) + seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot)) + indices = get_active_validator_indices(state, epoch) + return compute_proposer_index(state, indices, seed) +``` +```python +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of the ``indices``. + ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + Math safe up to ~10B ETH, afterwhich this overflows uint64. + """ + return Gwei(max(EFFECTIVE_BALANCE_INCREMENT, sum([state.validators[index].effective_balance for index in indices]))) +``` +```python +def get_total_active_balance(state: BeaconState) -> Gwei: + """ + Return the combined effective balance of the active validators. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) +``` +```python +def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: + """ + Return the signature domain (fork version concatenated with domain type) of a message. + """ + epoch = get_current_epoch(state) if epoch is None else epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return compute_domain(domain_type, fork_version, state.genesis_validators_root) +``` +```python +def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: + """ + Return the indexed attestation corresponding to ``attestation``. + """ + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + + return IndexedAttestation( + attesting_indices=sorted(attesting_indices), + data=attestation.data, + signature=attestation.signature, + ) +``` +```python +def get_attesting_indices(state: BeaconState, + data: AttestationData, + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: + """ + Return the set of attesting indices corresponding to ``data`` and ``bits``. + """ + committee = get_beacon_committee(state, data.slot, data.index) + return set(index for i, index in enumerate(committee) if bits[i]) +``` +```python +def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Increase the validator balance at index ``index`` by ``delta``. + """ + state.balances[index] += delta +``` +```python +def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Decrease the validator balance at index ``index`` by ``delta``, with underflow protection. + """ + state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta +``` +```python +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the exit of the validator with index ``index``. + """ + # Return if validator already initiated exit + validator = state.validators[index] + if validator.exit_epoch != FAR_FUTURE_EPOCH: + return + + # Compute exit queue epoch + exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] + exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) + if exit_queue_churn >= get_validator_churn_limit(state): + exit_queue_epoch += Epoch(1) + + # Set validator exit epoch and withdrawable epoch + validator.exit_epoch = exit_queue_epoch + validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) +``` +```python +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: + """ + Slash the validator with index ``slashed_index``. + """ + epoch = get_current_epoch(state) + initiate_validator_exit(state, slashed_index) + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + + # Apply proposer and whistleblower rewards + proposer_index = get_beacon_proposer_index(state) + if whistleblower_index is None: + whistleblower_index = proposer_index + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) +``` +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit]) -> BeaconState: + fork = Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + return state +``` +```python +def is_valid_genesis_state(state: BeaconState) -> bool: + if state.genesis_time < MIN_GENESIS_TIME: + return False + if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: + return False + return True +``` +```python +def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> None: + block = signed_block.message + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Verify signature + if validate_result: + assert verify_block_signature(state, signed_block) + # Process block + process_block(state, block) + # Verify state root + if validate_result: + assert block.state_root == hash_tree_root(state) +``` +```python +def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: + proposer = state.validators[signed_block.message.proposer_index] + signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) +``` +```python +def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot < slot + while state.slot < slot: + process_slot(state) + # Process epoch on the start slot of the next epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch(state) + state.slot = Slot(state.slot + 1) +``` +```python +def process_slot(state: BeaconState) -> None: + # Cache state root + previous_state_root = hash_tree_root(state) + state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + # Cache latest block header state root + if state.latest_block_header.state_root == Bytes32(): + state.latest_block_header.state_root = previous_state_root + # Cache block root + previous_block_root = hash_tree_root(state.latest_block_header) + state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root +``` +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) + process_participation_record_updates(state) +``` +```python +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) + return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations +``` +```python +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_source_attestations(state, epoch) + if a.data.target.root == get_block_root(state, epoch) + ] +``` +```python +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_target_attestations(state, epoch) + if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) + ] +``` +```python +def get_unslashed_attesting_indices(state: BeaconState, + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: + output = set() # type: Set[ValidatorIndex] + for a in attestations: + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) + return set(filter(lambda index: not state.validators[index].slashed, output)) +``` +```python +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: + """ + Return the combined effective balance of the set of unslashed validators participating in ``attestations``. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +``` +```python +def process_justification_and_finalization(state: BeaconState) -> None: + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + previous_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + current_attestations = get_matching_target_attestations(state, get_current_epoch(state)) + total_active_balance = get_total_active_balance(state) + previous_target_balance = get_attesting_balance(state, previous_attestations) + current_target_balance = get_attesting_balance(state, current_attestations) + weigh_justification_and_finalization(state, total_active_balance, previous_target_balance, current_target_balance) +``` +```python +def weigh_justification_and_finalization(state: BeaconState, + total_active_balance: Gwei, + previous_epoch_target_balance: Gwei, + current_epoch_target_balance: Gwei) -> None: + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 + if previous_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) + state.justification_bits[1] = 0b1 + if current_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) + state.justification_bits[0] = 0b1 + + # Process finalizations + bits = state.justification_bits + # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint +``` +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) +``` +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` +```python +def is_in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` +```python +def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: + previous_epoch = get_previous_epoch(state) + return [ + ValidatorIndex(index) for index, v in enumerate(state.validators) + if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) + ] +``` +```python +def get_attestation_component_deltas(state: BeaconState, + attestations: Sequence[PendingAttestation] + ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Helper with shared logic for use by get source, target, and head deltas functions + """ + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + total_balance = get_total_active_balance(state) + unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) + attesting_balance = get_total_balance(state, unslashed_attesting_indices) + for index in get_eligible_validator_indices(state): + if index in unslashed_attesting_indices: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) + else: + penalties[index] += get_base_reward(state, index) + return rewards, penalties +``` +```python +def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for source-vote for each validator. + """ + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_source_attestations) +``` +```python +def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for target-vote for each validator. + """ + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_target_attestations) +``` +```python +def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for head-vote for each validator. + """ + matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_head_attestations) +``` +```python +def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return proposer and inclusion delay micro-rewards/penalties for each validator. + """ + rewards = [Gwei(0) for _ in range(len(state.validators))] + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + for index in get_unslashed_attesting_indices(state, matching_source_attestations): + attestation = min([ + a for a in matching_source_attestations + if index in get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = Gwei(get_base_reward(state, index) - get_proposer_reward(state, index)) + rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) + + # No penalties associated with inclusion delay + penalties = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return inactivity reward/penalty deltas for each validator. + """ + penalties = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) + for index in get_eligible_validator_indices(state): + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) + if index not in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + # No rewards associated with inactivity penalties + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` +```python +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attestation reward/penalty deltas for each validator. + """ + source_rewards, source_penalties = get_source_deltas(state) + target_rewards, target_penalties = get_target_deltas(state) + head_rewards, head_penalties = get_head_deltas(state) + inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) + _, inactivity_penalties = get_inactivity_penalty_deltas(state) + + rewards = [ + source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] + for i in range(len(state.validators)) + ] + + penalties = [ + source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] + for i in range(len(state.validators)) + ] + + return rewards, penalties +``` +```python +def process_rewards_and_penalties(state: BeaconState) -> None: + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch + if get_current_epoch(state) == GENESIS_EPOCH: + return + + rewards, penalties = get_attestation_deltas(state) + for index in range(len(state.validators)): + increase_balance(state, ValidatorIndex(index), rewards[index]) + decrease_balance(state, ValidatorIndex(index), penalties[index]) +``` +```python +def process_registry_updates(state: BeaconState) -> None: + # Process activation eligibility and ejections + for index, validator in enumerate(state.validators): + if is_eligible_for_activation_queue(validator): + validator.activation_eligibility_epoch = get_current_epoch(state) + 1 + + if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: + initiate_validator_exit(state, ValidatorIndex(index)) + + # Queue validators eligible for activation and not yet dequeued for activation + activation_queue = sorted([ + index for index, validator in enumerate(state.validators) + if is_eligible_for_activation(state, validator) + # Order by the sequence of activation_eligibility_epoch setting and then index + ], key=lambda index: (state.validators[index].activation_eligibility_epoch, index)) + # Dequeued validators for activation up to churn limit + for index in activation_queue[:get_validator_churn_limit(state)]: + validator = state.validators[index] + validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) +``` +```python +def process_slashings(state: BeaconState) -> None: + epoch = get_current_epoch(state) + total_balance = get_total_active_balance(state) + adjusted_total_slashing_balance = min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + for index, validator in enumerate(state.validators): + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance + penalty = penalty_numerator // total_balance * increment + decrease_balance(state, ValidatorIndex(index), penalty) +``` +```python +def process_eth1_data_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset eth1 data votes + if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] +``` +```python +def process_effective_balance_updates(state: BeaconState) -> None: + # Update effective balances with hysteresis + for index, validator in enumerate(state.validators): + balance = state.balances[index] + HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) + DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER + UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER + if ( + balance + DOWNWARD_THRESHOLD < validator.effective_balance + or validator.effective_balance + UPWARD_THRESHOLD < balance + ): + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) +``` +```python +def process_slashings_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) +``` +```python +def process_randao_mixes_reset(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) + # Set randao mix + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) +``` +```python +def process_historical_roots_update(state: BeaconState) -> None: + # Set historical root accumulator + next_epoch = Epoch(get_current_epoch(state) + 1) + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) + state.historical_roots.append(hash_tree_root(historical_batch)) +``` +```python +def process_participation_record_updates(state: BeaconState) -> None: + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] +``` +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) +``` +```python +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + # Verify that proposer index is the correct index + assert block.proposer_index == get_beacon_proposer_index(state) + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), + ) + + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed +``` +```python +def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: + epoch = get_current_epoch(state) + # Verify RANDAO reveal + proposer = state.validators[get_beacon_proposer_index(state)] + signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) + assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal) + # Mix in RANDAO reveal + mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) + state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix +``` +```python +def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: + state.eth1_data_votes.append(body.eth1_data) + if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.eth1_data = body.eth1_data +``` +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) +``` +```python +def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: + header_1 = proposer_slashing.signed_header_1.message + header_2 = proposer_slashing.signed_header_2.message + + # Verify header slots match + assert header_1.slot == header_2.slot + # Verify header proposer indices match + assert header_1.proposer_index == header_2.proposer_index + # Verify the headers are different + assert header_1 != header_2 + # Verify the proposer is slashable + proposer = state.validators[header_1.proposer_index] + assert is_slashable_validator(proposer, get_current_epoch(state)) + # Verify signatures + for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) + signing_root = compute_signing_root(signed_header.message, domain) + assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) + + slash_validator(state, header_1.proposer_index) +``` +```python +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 + assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) + assert is_valid_indexed_attestation(state, attestation_1) + assert is_valid_indexed_attestation(state, attestation_2) + + slashed_any = False + indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices) + for index in sorted(indices): + if is_slashable_validator(state.validators[index], get_current_epoch(state)): + slash_validator(state, index) + slashed_any = True + assert slashed_any +``` +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + assert data.index < get_committee_count_per_slot(state, data.target.epoch) + + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + + pending_attestation = PendingAttestation( + data=data, + aggregation_bits=attestation.aggregation_bits, + inclusion_delay=state.slot - data.slot, + proposer_index=get_beacon_proposer_index(state), + ) + + if data.target.epoch == get_current_epoch(state): + assert data.source == state.current_justified_checkpoint + state.current_epoch_attestations.append(pending_attestation) + else: + assert data.source == state.previous_justified_checkpoint + state.previous_epoch_attestations.append(pending_attestation) + + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) +``` +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + ) +``` +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` +```python +def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: + voluntary_exit = signed_voluntary_exit.message + validator = state.validators[voluntary_exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify exit has not been initiated + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= voluntary_exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD + # Verify signature + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) + # Initiate exit + initiate_validator_exit(state, voluntary_exit.validator_index) +``` +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root +``` +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) +``` +```python +def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: + assert anchor_block.state_root == hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: copy(anchor_block)}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, + ) +``` +```python +def get_slots_since_genesis(store: Store) -> int: + return (store.time - store.genesis_time) // SECONDS_PER_SLOT +``` +```python +def get_current_slot(store: Store) -> Slot: + return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) +``` +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` +```python +def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: + block = store.blocks[root] + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` +```python +def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + )) +``` +```python +def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: + block = store.blocks[block_root] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == block_root + ] + + # If any children branches contain expected finalized/justified checkpoints, + # add to filtered block-tree and signal viability to parent. + if any(children): + filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + # If leaf block, check finalized/justified checkpoints as matching latest. + head_state = store.block_states[block_root] + + correct_justified = ( + store.justified_checkpoint.epoch == GENESIS_EPOCH + or head_state.current_justified_checkpoint == store.justified_checkpoint + ) + correct_finalized = ( + store.finalized_checkpoint.epoch == GENESIS_EPOCH + or head_state.finalized_checkpoint == store.finalized_checkpoint + ) + # If expected finalized/justified, add to viable block-tree and signal viability to parent. + if correct_justified and correct_finalized: + blocks[block_root] = block + return True + + # Otherwise, branch not viable + return False +``` +```python +def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: + """ + Retrieve a filtered block tree from ``store``, only returning branches + whose leaf state's justified/finalized info agrees with that in ``store``. + """ + base = store.justified_checkpoint.root + blocks: Dict[Root, BeaconBlock] = {} + filter_block_tree(store, base, blocks) + return blocks +``` +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].parent_root == head + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) +``` +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: + return False + + return True +``` +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # LMD vote must be consistent with FFG vote target + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = copy(store.block_states[target.root]) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` +```python +def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time + store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint +``` +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint +``` +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) +``` +```python +class Eth1Block(Container): + timestamp: uint64 + deposit_root: Root + deposit_count: uint64 + # All other eth1 block fields +``` +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` +```python +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) +``` +```python +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex + ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the index to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. + """ + next_epoch = Epoch(get_current_epoch(state) + 1) + assert epoch <= next_epoch + + start_slot = compute_start_slot_at_epoch(epoch) + committee_count_per_slot = get_committee_count_per_slot(state, epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): + for index in range(committee_count_per_slot): + committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) + if validator_index in committee: + return committee, CommitteeIndex(index), Slot(slot) + return None +``` +```python +def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` +```python +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) +``` +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) + return compute_time_at_slot(state, eth1_voting_period_start_slot) +``` +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start + ) +``` +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] + + # Valid votes already cast during this period + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data + + return max( + valid_votes, + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance + default=default_vote + ) +``` +```python +def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) +``` +```python +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) + committees_since_epoch_start = committees_per_slot * slots_since_epoch_start + + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) +``` +```python +def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: + committee = get_beacon_committee(state, slot, index) + modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) + return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 +``` +```python +def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: + signatures = [attestation.signature for attestation in attestations] + return bls.Aggregate(signatures) +``` +```python +def get_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + if T * (200 + 3 * D) < t * (200 + 12 * D): + epochs_for_validator_set_churn = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + epochs_for_balance_top_ups = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) + else: + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) + + return ws_period +``` +```python +def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: + # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint + assert ws_state.latest_block_header.state_root == ws_checkpoint.root + assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch + + ws_period = compute_weak_subjectivity_period(ws_state) + ws_state_epoch = compute_epoch_at_slot(ws_state.slot) + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + return current_epoch <= ws_state_epoch + ws_period +``` From d6f013d757b6419d417c99b6e673d42e8a21f6ca Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 00:35:08 +0300 Subject: [PATCH 14/27] update spec texts --- .../phase0/beacon-chain.md} | 490 ------------------ .../data/specs/phase0/deposit-contract.md | 0 .../data/specs/phase0/fork-choice.md | 278 ++++++++++ .../data/specs/phase0/p2p-interface.md | 0 .../data/specs/phase0/validator.md | 168 ++++++ .../data/specs/phase0/weak-subjectivity.md | 44 ++ 6 files changed, 490 insertions(+), 490 deletions(-) rename tools/specs-checker/data/{phase0/all-defs.md => specs/phase0/beacon-chain.md} (69%) create mode 100644 tools/specs-checker/data/specs/phase0/deposit-contract.md create mode 100644 tools/specs-checker/data/specs/phase0/fork-choice.md create mode 100644 tools/specs-checker/data/specs/phase0/p2p-interface.md create mode 100644 tools/specs-checker/data/specs/phase0/validator.md create mode 100644 tools/specs-checker/data/specs/phase0/weak-subjectivity.md diff --git a/tools/specs-checker/data/phase0/all-defs.md b/tools/specs-checker/data/specs/phase0/beacon-chain.md similarity index 69% rename from tools/specs-checker/data/phase0/all-defs.md rename to tools/specs-checker/data/specs/phase0/beacon-chain.md index 9485b68f7e3..0cb6d82bd03 100644 --- a/tools/specs-checker/data/phase0/all-defs.md +++ b/tools/specs-checker/data/specs/phase0/beacon-chain.md @@ -1216,493 +1216,3 @@ def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVolu # Initiate exit initiate_validator_exit(state, voluntary_exit.validator_index) ``` -```python -@dataclass(eq=True, frozen=True) -class LatestMessage(object): - epoch: Epoch - root: Root -``` -```python -@dataclass -class Store(object): - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) -``` -```python -def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: - assert anchor_block.state_root == hash_tree_root(anchor_state) - anchor_root = hash_tree_root(anchor_block) - anchor_epoch = get_current_epoch(anchor_state) - justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) - return Store( - time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), - genesis_time=anchor_state.genesis_time, - justified_checkpoint=justified_checkpoint, - finalized_checkpoint=finalized_checkpoint, - best_justified_checkpoint=justified_checkpoint, - blocks={anchor_root: copy(anchor_block)}, - block_states={anchor_root: copy(anchor_state)}, - checkpoint_states={justified_checkpoint: copy(anchor_state)}, - ) -``` -```python -def get_slots_since_genesis(store: Store) -> int: - return (store.time - store.genesis_time) // SECONDS_PER_SLOT -``` -```python -def get_current_slot(store: Store) -> Slot: - return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) -``` -```python -def compute_slots_since_epoch_start(slot: Slot) -> int: - return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) -``` -```python -def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: - block = store.blocks[root] - if block.slot > slot: - return get_ancestor(store, block.parent_root, slot) - elif block.slot == slot: - return root - else: - # root is older than queried slot, thus a skip slot. Return most recent root prior to slot - return root -``` -```python -def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: - state = store.checkpoint_states[store.justified_checkpoint] - active_indices = get_active_validator_indices(state, get_current_epoch(state)) - return Gwei(sum( - state.validators[i].effective_balance for i in active_indices - if (i in store.latest_messages - and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) - )) -``` -```python -def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: - block = store.blocks[block_root] - children = [ - root for root in store.blocks.keys() - if store.blocks[root].parent_root == block_root - ] - - # If any children branches contain expected finalized/justified checkpoints, - # add to filtered block-tree and signal viability to parent. - if any(children): - filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] - if any(filter_block_tree_result): - blocks[block_root] = block - return True - return False - - # If leaf block, check finalized/justified checkpoints as matching latest. - head_state = store.block_states[block_root] - - correct_justified = ( - store.justified_checkpoint.epoch == GENESIS_EPOCH - or head_state.current_justified_checkpoint == store.justified_checkpoint - ) - correct_finalized = ( - store.finalized_checkpoint.epoch == GENESIS_EPOCH - or head_state.finalized_checkpoint == store.finalized_checkpoint - ) - # If expected finalized/justified, add to viable block-tree and signal viability to parent. - if correct_justified and correct_finalized: - blocks[block_root] = block - return True - - # Otherwise, branch not viable - return False -``` -```python -def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: - """ - Retrieve a filtered block tree from ``store``, only returning branches - whose leaf state's justified/finalized info agrees with that in ``store``. - """ - base = store.justified_checkpoint.root - blocks: Dict[Root, BeaconBlock] = {} - filter_block_tree(store, base, blocks) - return blocks -``` -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - children = [ - root for root in blocks.keys() - if blocks[root].parent_root == head - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) -``` -```python -def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: - """ - To address the bouncing attack, only update conflicting justified - checkpoints in the fork choice if in the early slots of the epoch. - Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. - - See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. - """ - if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: - return True - - justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) - if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: - return False - - return True -``` -```python -def validate_on_attestation(store: Store, attestation: Attestation) -> None: - target = attestation.data.target - - # Attestations must be from the current or previous epoch - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - # Use GENESIS_EPOCH for previous when genesis to avoid underflow - previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH - # If attestation target is from a future epoch, delay consideration until the epoch arrives - assert target.epoch in [current_epoch, previous_epoch] - assert target.epoch == compute_epoch_at_slot(attestation.data.slot) - - # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found - assert target.root in store.blocks - - # Attestations must be for a known block. If block is unknown, delay consideration until the block is found - assert attestation.data.beacon_block_root in store.blocks - # Attestations must not be for blocks in the future. If not, the attestation should not be considered - assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot - - # LMD vote must be consistent with FFG vote target - target_slot = compute_start_slot_at_epoch(target.epoch) - assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) - - # Attestations can only affect the fork choice of subsequent slots. - # Delay consideration in the fork choice until their slot is in the past. - assert get_current_slot(store) >= attestation.data.slot + 1 -``` -```python -def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: - # Store target checkpoint state if not yet seen - if target not in store.checkpoint_states: - base_state = copy(store.block_states[target.root]) - if base_state.slot < compute_start_slot_at_epoch(target.epoch): - process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) - store.checkpoint_states[target] = base_state -``` -```python -def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: - target = attestation.data.target - beacon_block_root = attestation.data.beacon_block_root - for i in attesting_indices: - if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: - store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) -``` -```python -def on_tick(store: Store, time: uint64) -> None: - previous_slot = get_current_slot(store) - - # update store time - store.time = time - - current_slot = get_current_slot(store) - # Not a new epoch, return - if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): - return - # Update store.justified_checkpoint if a better checkpoint is known - if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = store.best_justified_checkpoint -``` -```python -def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: - block = signed_block.message - # Parent block must be known - assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - pre_state = copy(store.block_states[block.parent_root]) - # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. - assert get_current_slot(store) >= block.slot - - # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert block.slot > finalized_slot - # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root - - # Check the block is valid and compute the post-state - state = pre_state.copy() - state_transition(state, signed_block, True) - # Add new block to the store - store.blocks[hash_tree_root(block)] = block - # Add new state for this block to the store - store.block_states[hash_tree_root(block)] = state - - # Update justified checkpoint - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: - store.best_justified_checkpoint = state.current_justified_checkpoint - if should_update_justified_checkpoint(store, state.current_justified_checkpoint): - store.justified_checkpoint = state.current_justified_checkpoint - - # Update finalized checkpoint - if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: - store.finalized_checkpoint = state.finalized_checkpoint - - # Potentially update justified if different from store - if store.justified_checkpoint != state.current_justified_checkpoint: - # Update justified if new justified is later than store justified - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - store.justified_checkpoint = state.current_justified_checkpoint - return - - # Update justified if store justified is not in chain with finalized checkpoint - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) - if ancestor_at_finalized_slot != store.finalized_checkpoint.root: - store.justified_checkpoint = state.current_justified_checkpoint -``` -```python -def on_attestation(store: Store, attestation: Attestation) -> None: - """ - Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. - - An ``attestation`` that is asserted as invalid may be valid at a later time, - consider scheduling it for later processing in such case. - """ - validate_on_attestation(store, attestation) - store_target_checkpoint_state(store, attestation.data.target) - - # Get state at the `target` to fully validate attestation - target_state = store.checkpoint_states[attestation.data.target] - indexed_attestation = get_indexed_attestation(target_state, attestation) - assert is_valid_indexed_attestation(target_state, indexed_attestation) - - # Update latest messages for attesting indices - update_latest_messages(store, indexed_attestation.attesting_indices, attestation) -``` -```python -class Eth1Block(Container): - timestamp: uint64 - deposit_root: Root - deposit_count: uint64 - # All other eth1 block fields -``` -```python -class AggregateAndProof(Container): - aggregator_index: ValidatorIndex - aggregate: Attestation - selection_proof: BLSSignature -``` -```python -class SignedAggregateAndProof(Container): - message: AggregateAndProof - signature: BLSSignature -``` -```python -def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: - validator = state.validators[validator_index] - return is_active_validator(validator, get_current_epoch(state)) -``` -```python -def get_committee_assignment(state: BeaconState, - epoch: Epoch, - validator_index: ValidatorIndex - ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: - """ - Return the committee assignment in the ``epoch`` for ``validator_index``. - ``assignment`` returned is a tuple of the following form: - * ``assignment[0]`` is the list of validators in the committee - * ``assignment[1]`` is the index to which the committee is assigned - * ``assignment[2]`` is the slot at which the committee is assigned - Return None if no assignment. - """ - next_epoch = Epoch(get_current_epoch(state) + 1) - assert epoch <= next_epoch - - start_slot = compute_start_slot_at_epoch(epoch) - committee_count_per_slot = get_committee_count_per_slot(state, epoch) - for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): - for index in range(committee_count_per_slot): - committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) - if validator_index in committee: - return committee, CommitteeIndex(index), Slot(slot) - return None -``` -```python -def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: - return get_beacon_proposer_index(state) == validator_index -``` -```python -def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: - return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) -``` -```python -def voting_period_start_time(state: BeaconState) -> uint64: - eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) - return compute_time_at_slot(state, eth1_voting_period_start_slot) -``` -```python -def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: - return ( - block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start - and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start - ) -``` -```python -def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: - period_start = voting_period_start_time(state) - # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height - votes_to_consider = [ - get_eth1_data(block) for block in eth1_chain - if ( - is_candidate_block(block, period_start) - # Ensure cannot move back to earlier deposit contract states - and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count - ) - ] - - # Valid votes already cast during this period - valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] - - # Default vote on latest eth1 block data in the period range unless eth1 chain is not live - # Non-substantive casting for linter - state_eth1_data: Eth1Data = state.eth1_data - default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data - - return max( - valid_votes, - key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance - default=default_vote - ) -``` -```python -def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: - temp_state: BeaconState = state.copy() - signed_block = SignedBeaconBlock(message=block) - state_transition(temp_state, signed_block, validate_result=False) - return hash_tree_root(temp_state) -``` -```python -def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) - signing_root = compute_signing_root(block, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) - signing_root = compute_signing_root(attestation_data, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: - """ - Compute the correct subnet for an attestation for Phase 0. - Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. - """ - slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) - committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - - return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) -``` -```python -def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: - domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) - signing_root = compute_signing_root(slot, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: - committee = get_beacon_committee(state, slot, index) - modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) - return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 -``` -```python -def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: - signatures = [attestation.signature for attestation in attestations] - return bls.Aggregate(signatures) -``` -```python -def get_aggregate_and_proof(state: BeaconState, - aggregator_index: ValidatorIndex, - aggregate: Attestation, - privkey: int) -> AggregateAndProof: - return AggregateAndProof( - aggregator_index=aggregator_index, - aggregate=aggregate, - selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), - ) -``` -```python -def get_aggregate_and_proof_signature(state: BeaconState, - aggregate_and_proof: AggregateAndProof, - privkey: int) -> BLSSignature: - aggregate = aggregate_and_proof.aggregate - domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) - signing_root = compute_signing_root(aggregate_and_proof, domain) - return bls.Sign(privkey, signing_root) -``` -```python -def compute_weak_subjectivity_period(state: BeaconState) -> uint64: - """ - Returns the weak subjectivity period for the current ``state``. - This computation takes into account the effect of: - - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and - - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). - A detailed calculation can be found at: - https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf - """ - ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY - N = len(get_active_validator_indices(state, get_current_epoch(state))) - t = get_total_active_balance(state) // N // ETH_TO_GWEI - T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI - delta = get_validator_churn_limit(state) - Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH - D = SAFETY_DECAY - - if T * (200 + 3 * D) < t * (200 + 12 * D): - epochs_for_validator_set_churn = ( - N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) - ) - epochs_for_balance_top_ups = ( - N * (200 + 3 * D) // (600 * Delta) - ) - ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) - else: - ws_period += ( - 3 * N * D * t // (200 * Delta * (T - t)) - ) - - return ws_period -``` -```python -def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: - # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint - assert ws_state.latest_block_header.state_root == ws_checkpoint.root - assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch - - ws_period = compute_weak_subjectivity_period(ws_state) - ws_state_epoch = compute_epoch_at_slot(ws_state.slot) - current_epoch = compute_epoch_at_slot(get_current_slot(store)) - return current_epoch <= ws_state_epoch + ws_period -``` diff --git a/tools/specs-checker/data/specs/phase0/deposit-contract.md b/tools/specs-checker/data/specs/phase0/deposit-contract.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/specs-checker/data/specs/phase0/fork-choice.md b/tools/specs-checker/data/specs/phase0/fork-choice.md new file mode 100644 index 00000000000..d789171a03a --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/fork-choice.md @@ -0,0 +1,278 @@ +```python +@dataclass(eq=True, frozen=True) +class LatestMessage(object): + epoch: Epoch + root: Root +``` +```python +@dataclass +class Store(object): + time: uint64 + genesis_time: uint64 + justified_checkpoint: Checkpoint + finalized_checkpoint: Checkpoint + best_justified_checkpoint: Checkpoint + blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) + block_states: Dict[Root, BeaconState] = field(default_factory=dict) + checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) + latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) +``` +```python +def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: + assert anchor_block.state_root == hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: copy(anchor_block)}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, + ) +``` +```python +def get_slots_since_genesis(store: Store) -> int: + return (store.time - store.genesis_time) // SECONDS_PER_SLOT +``` +```python +def get_current_slot(store: Store) -> Slot: + return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) +``` +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` +```python +def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: + block = store.blocks[root] + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` +```python +def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + )) +``` +```python +def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: + block = store.blocks[block_root] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == block_root + ] + + # If any children branches contain expected finalized/justified checkpoints, + # add to filtered block-tree and signal viability to parent. + if any(children): + filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + # If leaf block, check finalized/justified checkpoints as matching latest. + head_state = store.block_states[block_root] + + correct_justified = ( + store.justified_checkpoint.epoch == GENESIS_EPOCH + or head_state.current_justified_checkpoint == store.justified_checkpoint + ) + correct_finalized = ( + store.finalized_checkpoint.epoch == GENESIS_EPOCH + or head_state.finalized_checkpoint == store.finalized_checkpoint + ) + # If expected finalized/justified, add to viable block-tree and signal viability to parent. + if correct_justified and correct_finalized: + blocks[block_root] = block + return True + + # Otherwise, branch not viable + return False +``` +```python +def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: + """ + Retrieve a filtered block tree from ``store``, only returning branches + whose leaf state's justified/finalized info agrees with that in ``store``. + """ + base = store.justified_checkpoint.root + blocks: Dict[Root, BeaconBlock] = {} + filter_block_tree(store, base, blocks) + return blocks +``` +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].parent_root == head + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) +``` +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: + return False + + return True +``` +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # LMD vote must be consistent with FFG vote target + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = copy(store.block_states[target.root]) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` +```python +def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time + store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint +``` +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint +``` +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) +``` diff --git a/tools/specs-checker/data/specs/phase0/p2p-interface.md b/tools/specs-checker/data/specs/phase0/p2p-interface.md new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tools/specs-checker/data/specs/phase0/validator.md b/tools/specs-checker/data/specs/phase0/validator.md new file mode 100644 index 00000000000..17c9f462846 --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/validator.md @@ -0,0 +1,168 @@ +```python +class Eth1Block(Container): + timestamp: uint64 + deposit_root: Root + deposit_count: uint64 + # All other eth1 block fields +``` +```python +class AggregateAndProof(Container): + aggregator_index: ValidatorIndex + aggregate: Attestation + selection_proof: BLSSignature +``` +```python +class SignedAggregateAndProof(Container): + message: AggregateAndProof + signature: BLSSignature +``` +```python +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) +``` +```python +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex + ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the index to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. + """ + next_epoch = Epoch(get_current_epoch(state) + 1) + assert epoch <= next_epoch + + start_slot = compute_start_slot_at_epoch(epoch) + committee_count_per_slot = get_committee_count_per_slot(state, epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): + for index in range(committee_count_per_slot): + committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) + if validator_index in committee: + return committee, CommitteeIndex(index), Slot(slot) + return None +``` +```python +def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` +```python +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) +``` +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) + return compute_time_at_slot(state, eth1_voting_period_start_slot) +``` +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start + ) +``` +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] + + # Valid votes already cast during this period + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data + + return max( + valid_votes, + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance + default=default_vote + ) +``` +```python +def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) +``` +```python +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) + committees_since_epoch_start = committees_per_slot * slots_since_epoch_start + + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) +``` +```python +def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: + committee = get_beacon_committee(state, slot, index) + modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) + return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 +``` +```python +def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: + signatures = [attestation.signature for attestation in attestations] + return bls.Aggregate(signatures) +``` +```python +def get_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` diff --git a/tools/specs-checker/data/specs/phase0/weak-subjectivity.md b/tools/specs-checker/data/specs/phase0/weak-subjectivity.md new file mode 100644 index 00000000000..88faf4f32d0 --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/weak-subjectivity.md @@ -0,0 +1,44 @@ +```python +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + if T * (200 + 3 * D) < t * (200 + 12 * D): + epochs_for_validator_set_churn = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + epochs_for_balance_top_ups = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) + else: + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) + + return ws_period +``` +```python +def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: + # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint + assert ws_state.latest_block_header.state_root == ws_checkpoint.root + assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch + + ws_period = compute_weak_subjectivity_period(ws_state) + ws_state_epoch = compute_epoch_at_slot(ws_state.slot) + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + return current_epoch <= ws_state_epoch + ws_period +``` From fcc732d5030e9a2403e4f7250a9a2a3d5ae13be8 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 00:35:53 +0300 Subject: [PATCH 15/27] re-arrange code --- tools/specs-checker/BUILD.bazel | 15 ++- tools/specs-checker/check.go | 153 +++++++++++++++++++++++++++++++ tools/specs-checker/download.go | 85 +++++++++++++++++ tools/specs-checker/main.go | 158 ++++---------------------------- 4 files changed, 269 insertions(+), 142 deletions(-) create mode 100644 tools/specs-checker/check.go create mode 100644 tools/specs-checker/download.go diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel index 4de9e4a005a..d72c466abe9 100644 --- a/tools/specs-checker/BUILD.bazel +++ b/tools/specs-checker/BUILD.bazel @@ -3,8 +3,19 @@ load("@prysm//tools/go:def.bzl", "go_library") go_library( name = "go_default_library", - srcs = ["main.go"], - embedsrcs = ["data/phase0/all-defs.md"], + srcs = [ + "check.go", + "download.go", + "main.go", + ], + embedsrcs = [ + "data/specs/phase0/beacon-chain.md", + "data/specs/phase0/deposit-contract.md", + "data/specs/phase0/fork-choice.md", + "data/specs/phase0/p2p-interface.md", + "data/specs/phase0/validator.md", + "data/specs/phase0/weak-subjectivity.md", + ], importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", visibility = ["//visibility:public"], deps = ["@com_github_urfave_cli_v2//:go_default_library"], diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go new file mode 100644 index 00000000000..45de48dd5c8 --- /dev/null +++ b/tools/specs-checker/check.go @@ -0,0 +1,153 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/urfave/cli/v2" +) + +// Regex to find Python's "def". +var reg1 = regexp.MustCompile(`def\s(.*)\(.*`) + +func check(cliCtx *cli.Context) error { + // Obtain reference snippets. + defs, err := parseSpecs() + if err != nil { + return err + } + + // Walk the path, and process all contained Golang files. + fileWalker := func(path string, info os.FileInfo, err error) error { + if info == nil { + return fmt.Errorf("invalid input dir %q", path) + } + if !strings.HasSuffix(info.Name(), ".go") { + return nil + } + return inspectFile(path, defs) + } + return filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker) +} + +func inspectFile(path string, defs map[string][]string) error { + // Parse source files, and check the pseudo code. + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + + ast.Inspect(file, func(node ast.Node) bool { + stmt, ok := node.(*ast.CommentGroup) + if !ok { + return true + } + // Ignore comment groups that do not have python pseudo-code. + chunk := stmt.Text() + if !reg1.MatchString(chunk) { + return true + } + + pos := fset.Position(node.Pos()) + + // Trim the chunk, so that it starts from Python's "def". + loc := reg1.FindStringIndex(chunk) + chunk = chunk[loc[0]:] + + // Find out Python function name. + defName, defBody := parseDefChunk(chunk) + if defName == "" { + fmt.Printf("%s: cannot parse comment pseudo code\n", pos) + return false + } + + // Calculate differences with reference implementation. + refDefs, ok := defs[defName] + if !ok { + fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) + return false + } + if !matchesRefImplementation(defName, refDefs, defBody) { + fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) + return false + } + + return true + }) + + return nil +} + +// parseSpecs parses input spec docs into map of function name -> array of function bodies +// (single entity may have several definitions). +func parseSpecs() (map[string][]string, error) { + var sb strings.Builder + for dirName, fileNames := range specDirs { + for _, fileName := range fileNames { + chunk, err := specFS.ReadFile(path.Join("data", dirName, fileName)) + if err != nil { + return nil, fmt.Errorf("cannot read specs file: %w", err) + } + _, err = sb.Write(chunk) + if err != nil { + return nil, fmt.Errorf("cannot copy specs file: %w", err) + } + } + } + chunks := strings.Split(strings.ReplaceAll(sb.String(), "```python", ""), "```") + defs := make(map[string][]string, len(chunks)) + for _, chunk := range chunks { + defName, defBody := parseDefChunk(chunk) + if defName == "" { + continue + } + defs[defName] = append(defs[defName], defBody) + } + return defs, nil +} + +func parseDefChunk(chunk string) (string, string) { + chunk = strings.TrimLeft(chunk, "\n") + if chunk == "" { + return "", "" + } + chunkLines := strings.Split(chunk, "\n") + // Ignore all snippets, that do not define functions. + if chunkLines[0][:4] != "def " { + return "", "" + } + defMatches := reg1.FindStringSubmatch(chunkLines[0]) + if len(defMatches) < 2 { + return "", "" + } + return strings.Trim(defMatches[1], " "), chunk +} + +// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). +func matchesRefImplementation(defName string, refDefs []string, input string) bool { + for _, refDef := range refDefs { + refDefLines := strings.Split(refDef, "\n") + inputLines := strings.Split(input, "\n") + + matchesPerfectly := true + for i := 0; i < len(refDefs); i++ { + a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") + if a != b { + matchesPerfectly = false + break + } + } + if matchesPerfectly { + return true + } + } + return false +} diff --git a/tools/specs-checker/download.go b/tools/specs-checker/download.go new file mode 100644 index 00000000000..f18a5119381 --- /dev/null +++ b/tools/specs-checker/download.go @@ -0,0 +1,85 @@ +package main + +import ( + _ "embed" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "regexp" + + "github.com/urfave/cli/v2" +) + +const baseUrl = "https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev" + +// Regex to find Python's code snippets in markdown. +var reg2 = regexp.MustCompile(`(?msU)^\x60\x60\x60python(.*)^\x60\x60\x60`) + +func download(cliCtx *cli.Context) error { + baseDir := cliCtx.String(dirFlag.Name) + for dirName, fileNames := range specDirs { + if err := prepareDir(path.Join(baseDir, dirName)); err != nil { + return err + } + for _, fileName := range fileNames { + outFilePath := path.Join(baseDir, dirName, fileName) + specDocUrl := fmt.Sprintf("%s/%s", baseUrl, fmt.Sprintf("%s/%s", dirName, fileName)) + if err := getAndSaveFile(specDocUrl, outFilePath); err != nil { + return err + } + } + } + + return nil +} + +func getAndSaveFile(specDocUrl, outFilePath string) error { + // Create output file. + f, err := os.Create(outFilePath) + if err != nil { + return fmt.Errorf("cannot create output file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + fmt.Printf("cannot close output file: %v", err) + } + }() + + // Download spec doc. + fmt.Printf("URL: %v\n", specDocUrl) + resp, err := http.Get(specDocUrl) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + fmt.Printf("cannot close spec doc file: %v", err) + } + }() + + // Transform and save spec docs. + specDoc, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + specDocString := string(specDoc) + for _, snippet := range reg2.FindAllString(specDocString, -1) { + fmt.Printf("Snippet:\n>>%v<<\n\n", snippet) + if _, err = f.WriteString(snippet + "\n"); err != nil { + return err + } + } + + fmt.Printf("f: %v, path: %v\n", f, outFilePath) + + return nil +} + +func prepareDir(dirPath string) error { + if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { + return err + } + return nil +} diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go index 1305e63afe7..f8881b76834 100644 --- a/tools/specs-checker/main.go +++ b/tools/specs-checker/main.go @@ -1,26 +1,13 @@ package main import ( - _ "embed" - "fmt" - "go/ast" - "go/parser" - "go/token" + "embed" "log" "os" - "path/filepath" - "regexp" - "strings" "github.com/urfave/cli/v2" ) -//go:embed data -var specText string - -// Regex to find Python's "def". -var m = regexp.MustCompile(`def\s(.*)\(.*`) - var ( dirFlag = &cli.StringFlag{ Name: "dir", @@ -30,6 +17,20 @@ var ( } ) +//go:embed data +var specFS embed.FS + +var specDirs = map[string][]string{ + "specs/phase0": { + "beacon-chain.md", + "deposit-contract.md", + "fork-choice.md", + "p2p-interface.md", + "validator.md", + "weak-subjectivity.md", + }, +} + func main() { app := &cli.App{ Name: "Specs checker utility", @@ -48,6 +49,9 @@ func main() { Name: "download", Usage: "Downloads the latest specs docs", Action: download, + Flags: []cli.Flag{ + dirFlag, + }, }, }, } @@ -57,129 +61,3 @@ func main() { log.Fatal(err) } } - -func check(cliCtx *cli.Context) error { - // Obtain reference snippets. - defs, err := parseSpecs(specText) - if err != nil { - return err - } - - // Walk the path, and process all contained Golang files. - fileWalker := func(path string, info os.FileInfo, err error) error { - if info == nil { - return fmt.Errorf("invalid input dir %q", path) - } - if !strings.HasSuffix(info.Name(), ".go") { - return nil - } - return inspectFile(path, defs) - } - return filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker) -} - -func inspectFile(path string, defs map[string][]string) error { - // Parse source files, and check the pseudo code. - fset := token.NewFileSet() - file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) - if err != nil { - return err - } - - ast.Inspect(file, func(node ast.Node) bool { - stmt, ok := node.(*ast.CommentGroup) - if !ok { - return true - } - // Ignore comment groups that do not have python pseudo-code. - chunk := stmt.Text() - if !m.MatchString(chunk) { - return true - } - - pos := fset.Position(node.Pos()) - - // Trim the chunk, so that it starts from Python's "def". - loc := m.FindStringIndex(chunk) - chunk = chunk[loc[0]:] - - // Find out Python function name. - defName, defBody := parseDefChunk(chunk) - if defName == "" { - fmt.Printf("%s: cannot parse comment pseudo code\n", pos) - return false - } - - // Calculate differences with reference implementation. - refDefs, ok := defs[defName] - if !ok { - fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) - return false - } - if !matchesRefImplementation(defName, refDefs, defBody) { - fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) - return false - } - - return true - }) - - return nil -} - -func download(cliCtx *cli.Context) error { - return nil -} - -// parseSpecs parses input spec docs into map of function name -> array of function bodies -// (single entity may have several definitions). -func parseSpecs(input string) (map[string][]string, error) { - chunks := strings.Split(strings.ReplaceAll(input, "```python", ""), "```") - defs := make(map[string][]string, len(chunks)) - for _, chunk := range chunks { - defName, defBody := parseDefChunk(chunk) - if defName == "" { - continue - } - defs[defName] = append(defs[defName], defBody) - } - return defs, nil -} - -func parseDefChunk(chunk string) (string, string) { - chunk = strings.TrimLeft(chunk, "\n") - if chunk == "" { - return "", "" - } - chunkLines := strings.Split(chunk, "\n") - // Ignore all snippets, that do not define functions. - if chunkLines[0][:4] != "def " { - return "", "" - } - defMatches := m.FindStringSubmatch(chunkLines[0]) - if len(defMatches) < 2 { - return "", "" - } - return strings.Trim(defMatches[1], " "), chunk -} - -// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). -func matchesRefImplementation(defName string, refDefs []string, input string) bool { - for _, refDef := range refDefs { - refDefLines := strings.Split(refDef, "\n") - inputLines := strings.Split(input, "\n") - - matchesPerfectly := true - for i := 0; i < len(refDefs); i++ { - a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") - if a != b { - matchesPerfectly = false - break - } - } - if matchesPerfectly { - return true - } - } - return false -} From 0754340daa8ebf50ea735e021625162eb40a94e3 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 00:45:02 +0300 Subject: [PATCH 16/27] updated spec list --- tools/specs-checker/BUILD.bazel | 2 -- tools/specs-checker/data/specs/phase0/deposit-contract.md | 0 tools/specs-checker/data/specs/phase0/p2p-interface.md | 0 tools/specs-checker/main.go | 2 -- 4 files changed, 4 deletions(-) delete mode 100644 tools/specs-checker/data/specs/phase0/deposit-contract.md delete mode 100644 tools/specs-checker/data/specs/phase0/p2p-interface.md diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel index d72c466abe9..cb90c3e889d 100644 --- a/tools/specs-checker/BUILD.bazel +++ b/tools/specs-checker/BUILD.bazel @@ -10,9 +10,7 @@ go_library( ], embedsrcs = [ "data/specs/phase0/beacon-chain.md", - "data/specs/phase0/deposit-contract.md", "data/specs/phase0/fork-choice.md", - "data/specs/phase0/p2p-interface.md", "data/specs/phase0/validator.md", "data/specs/phase0/weak-subjectivity.md", ], diff --git a/tools/specs-checker/data/specs/phase0/deposit-contract.md b/tools/specs-checker/data/specs/phase0/deposit-contract.md deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/specs-checker/data/specs/phase0/p2p-interface.md b/tools/specs-checker/data/specs/phase0/p2p-interface.md deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go index f8881b76834..b1475d004eb 100644 --- a/tools/specs-checker/main.go +++ b/tools/specs-checker/main.go @@ -23,9 +23,7 @@ var specFS embed.FS var specDirs = map[string][]string{ "specs/phase0": { "beacon-chain.md", - "deposit-contract.md", "fork-choice.md", - "p2p-interface.md", "validator.md", "weak-subjectivity.md", }, From d30eb53e7a051516efd6b0c8846190194a8de179 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 00:48:40 +0300 Subject: [PATCH 17/27] cleanup --- scripts/update-spec-docs.sh | 29 ------ tools/specs-checker/download.go | 4 - tools/specs-checker/testdata/BUILD.bazel | 8 -- .../testdata/weak_subjectivity.go | 98 ------------------- 4 files changed, 139 deletions(-) delete mode 100755 scripts/update-spec-docs.sh delete mode 100644 tools/specs-checker/testdata/BUILD.bazel delete mode 100644 tools/specs-checker/testdata/weak_subjectivity.go diff --git a/scripts/update-spec-docs.sh b/scripts/update-spec-docs.sh deleted file mode 100755 index 20f75702e74..00000000000 --- a/scripts/update-spec-docs.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -# This script will pull the latest specs from https://github.com/ethereum/eth2.0-specs repo, extract code blocks -# and save them for reference in "all-defs.md" file (in separate directories for phase0, altair etc). - -declare -a files=("phase0/beacon-chain.md" - "phase0/deposit-contract.md" - "phase0/fork-choice.md" - "phase0/p2p-interface.md" - "phase0/validator.md" - "phase0/weak-subjectivity.md" -) - -BASE_URL="https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev/specs" -OUTPUT_DIR="tools/analyzers/specdocs/data" - -# Trunc all-defs files (they will contain extracted python code blocks). -echo -n >$OUTPUT_DIR/phase0/all-defs.md - -for file in "${files[@]}"; do - OUTPUT_PATH=$OUTPUT_DIR/$file - echo "$file" - echo "- downloading" - wget -q -O "$OUTPUT_PATH" --no-check-certificate --content-disposition $BASE_URL/"$file" - echo "- extracting all code blocks" - sed -n '/^```python/,/^```/ p' <"$OUTPUT_PATH" >>"${OUTPUT_PATH%/*}"/all-defs.md - echo "- removing raw file" - rm "$OUTPUT_PATH" -done diff --git a/tools/specs-checker/download.go b/tools/specs-checker/download.go index f18a5119381..11f5096d8af 100644 --- a/tools/specs-checker/download.go +++ b/tools/specs-checker/download.go @@ -48,7 +48,6 @@ func getAndSaveFile(specDocUrl, outFilePath string) error { }() // Download spec doc. - fmt.Printf("URL: %v\n", specDocUrl) resp, err := http.Get(specDocUrl) if err != nil { return err @@ -66,14 +65,11 @@ func getAndSaveFile(specDocUrl, outFilePath string) error { } specDocString := string(specDoc) for _, snippet := range reg2.FindAllString(specDocString, -1) { - fmt.Printf("Snippet:\n>>%v<<\n\n", snippet) if _, err = f.WriteString(snippet + "\n"); err != nil { return err } } - fmt.Printf("f: %v, path: %v\n", f, outFilePath) - return nil } diff --git a/tools/specs-checker/testdata/BUILD.bazel b/tools/specs-checker/testdata/BUILD.bazel deleted file mode 100644 index d77791fe7ee..00000000000 --- a/tools/specs-checker/testdata/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@prysm//tools/go:def.bzl", "go_library") - -go_library( - name = "go_default_library", - srcs = ["weak_subjectivity.go"], - importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker/testdata", - visibility = ["//visibility:public"], -) diff --git a/tools/specs-checker/testdata/weak_subjectivity.go b/tools/specs-checker/testdata/weak_subjectivity.go deleted file mode 100644 index c4f2f6c580d..00000000000 --- a/tools/specs-checker/testdata/weak_subjectivity.go +++ /dev/null @@ -1,98 +0,0 @@ -package testdata - -// ComputeWeakSubjectivityPeriod returns weak subjectivity period for the active validator count and finalized epoch. -// -// Reference spec implementation: -// https://github.com/ethereum/eth2.0-specs/blob/master/specs/phase0/weak-subjectivity.md#calculating-the-weak-subjectivity-period -// -// def compute_weak_subjectivity_period(state: BeaconState) -> uint64: -// """ -// Returns the weak subjectivity period for the current ``state``. -// This computation takes into account the effect of: -// - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and -// - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). -// A detailed calculation can be found at: -// https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf -// """ -// ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY -// N = len(get_active_validator_indices(state, get_current_epoch(state))) -// t = get_total_active_balance(state) // N // ETH_TO_GWEI -// T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI -// delta = get_validator_churn_limit(state) -// Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH -// D = SAFETY_DECAY -// -// if T * (200 + 3 * D) < t * (200 + 12 * D): -// epochs_for_validator_set_churn = ( -// N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) -// ) -// epochs_for_balance_top_ups = ( -// N * (200 + 3 * D) // (600 * Delta) -// ) -// ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) -// else: -// ws_period += ( -// 3 * N * D * t // (200 * Delta * (T - t)) -// ) -// -// return ws_period -func ComputeWeakSubjectivityPeriod(st string) (uint64, error) { - return 0, nil -} - -// IsWithinWeakSubjectivityPeriod verifies if a given weak subjectivity checkpoint is not stale i.e. -// the current node is so far beyond, that a given state and checkpoint are not for the latest weak -// subjectivity point. Provided checkpoint still can be used to double-check that node's block root -// at a given epoch matches that of the checkpoint. -// -// Reference implementation: -// https://github.com/ethereum/eth2.0-specs/blob/master/specs/phase0/weak-subjectivity.md#checking-for-stale-weak-subjectivity-checkpoint - -// def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: -// # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint -// assert ws_state.latest_block_header.state_root == ws_checkpoint.root -// assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch -// -// ws_period = compute_weak_subjectivity_period(ws_state) -// ws_state_epoch = compute_epoch_at_slot(ws_state.slot) -// current_epoch = compute_epoch_at_slot(get_current_slot(store)) -// return current_epoch <= ws_state_epoch + ws_period -func IsWithinWeakSubjectivityPeriod(st string) (bool, error) { - return false, nil -} - -// SlotToEpoch returns the epoch number of the input slot. -// -// Spec pseudocode definition: -// def compute_epoch_at_slot(slot: Slot) -> Epoch: -// """ -// Return the epoch number of ``slot``. -// """ -// return Epoch(slot // SLOTS_PER_EPOCH) -func SlotToEpoch(slot uint64) uint64 { - return slot / 32 -} - -// CurrentEpoch returns the current epoch number calculated from -// the slot number stored in beacon state. -// -// Spec pseudocode definition: -// def get_current_epoch(state: BeaconState) -> Epoch: -// """ -// Return the current epoch. -// """ -// return compute_epoch_of_slot(state.slot) -// We might have further comments, they shouldn't trigger analyzer. -func CurrentEpoch(state string) uint64 { - return 42 -} - -func FuncWithoutComment() { - -} - -// FuncWithNoSpecComment is just a function that has comments, but none of those is from specs. -// So, parser should ignore it. -func FuncWithNoSpecComment() { - -} From fd738f8c138497a8433057b908eedfe21c1e904c Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 01:14:46 +0300 Subject: [PATCH 18/27] more comments and readme --- tools/specs-checker/README.md | 38 +++++++++++++++++++++++++++++++++++ tools/specs-checker/check.go | 16 ++++++++------- 2 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 tools/specs-checker/README.md diff --git a/tools/specs-checker/README.md b/tools/specs-checker/README.md new file mode 100644 index 00000000000..02df26f7e8f --- /dev/null +++ b/tools/specs-checker/README.md @@ -0,0 +1,38 @@ +# Specs checker tool + +This simple tool helps downloading and parsing [ETH2 specs](https://github.com/ethereum/eth2.0-specs/tree/dev/specs), +to be later used for making sure that our reference comments match specs definitions precisely. + +### Updating the reference specs +See `main.go` for a list of files to be downloaded, currently: +```golang +var specDirs = map[string][]string{ + "specs/phase0": { + "beacon-chain.md", + "fork-choice.md", + "validator.md", + "weak-subjectivity.md", + }, +} +``` + +To download/update specs: +```bash +bazel run //tools/specs-checker download -- --dir=$PWD/tools/specs-checker/data +``` + +This will pull the files defined in `specDirs`, parse them (extract Python code snippets, discarding any other text), +and save them to the folder from which `bazel run //tools/specs-checker check` will be able to embed. + +### Checking against the reference specs + +To check whether reference comments have the matching version of Python specs: +```bash +bazel run //tools/specs-checker check -- --dir $PWD/beacon-chain +bazel run //tools/specs-checker check -- --dir $PWD/validator +bazel run //tools/specs-checker check -- --dir $PWD/shared +``` +Or, to check whole project: +```bash +bazel run //tools/specs-checker check -- --dir $PWD +``` diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go index 45de48dd5c8..62a40903602 100644 --- a/tools/specs-checker/check.go +++ b/tools/specs-checker/check.go @@ -56,8 +56,6 @@ func inspectFile(path string, defs map[string][]string) error { return true } - pos := fset.Position(node.Pos()) - // Trim the chunk, so that it starts from Python's "def". loc := reg1.FindStringIndex(chunk) chunk = chunk[loc[0]:] @@ -65,18 +63,18 @@ func inspectFile(path string, defs map[string][]string) error { // Find out Python function name. defName, defBody := parseDefChunk(chunk) if defName == "" { - fmt.Printf("%s: cannot parse comment pseudo code\n", pos) + fmt.Printf("%s: cannot parse comment pseudo code\n", fset.Position(node.Pos())) return false } // Calculate differences with reference implementation. refDefs, ok := defs[defName] if !ok { - fmt.Printf("%s: %q is not found in spec docs\n", pos, defName) + fmt.Printf("%s: %q is not found in spec docs\n", fset.Position(node.Pos()), defName) return false } if !matchesRefImplementation(defName, refDefs, defBody) { - fmt.Printf("%s: %q code does not match reference implementation in specs\n", pos, defName) + fmt.Printf("%s: %q code does not match reference implementation in specs\n", fset.Position(node.Pos()), defName) return false } @@ -89,7 +87,8 @@ func inspectFile(path string, defs map[string][]string) error { // parseSpecs parses input spec docs into map of function name -> array of function bodies // (single entity may have several definitions). func parseSpecs() (map[string][]string, error) { - var sb strings.Builder + // Traverse all spec files, and aggregate them within as single string. + var sb strings.Builder for dirName, fileNames := range specDirs { for _, fileName := range fileNames { chunk, err := specFS.ReadFile(path.Join("data", dirName, fileName)) @@ -102,6 +101,8 @@ func parseSpecs() (map[string][]string, error) { } } } + + // Parse docs into function name -> array of function bodies map. chunks := strings.Split(strings.ReplaceAll(sb.String(), "```python", ""), "```") defs := make(map[string][]string, len(chunks)) for _, chunk := range chunks { @@ -114,6 +115,7 @@ func parseSpecs() (map[string][]string, error) { return defs, nil } +// parseDefChunk extract function name and function body from a Python's "def" chunk. func parseDefChunk(chunk string) (string, string) { chunk = strings.TrimLeft(chunk, "\n") if chunk == "" { @@ -138,7 +140,7 @@ func matchesRefImplementation(defName string, refDefs []string, input string) bo inputLines := strings.Split(input, "\n") matchesPerfectly := true - for i := 0; i < len(refDefs); i++ { + for i := 0; i < len(refDefLines); i++ { a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") if a != b { matchesPerfectly = false From 5bb06b5f7220f84f04fa31f165dc15759810805c Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 03:25:49 +0300 Subject: [PATCH 19/27] add merkle proofs specs --- tools/specs-checker/BUILD.bazel | 1 + tools/specs-checker/data/ssz/merkle-proofs.md | 240 ++++++++++++++++++ tools/specs-checker/download.go | 2 + tools/specs-checker/main.go | 3 + 4 files changed, 246 insertions(+) create mode 100644 tools/specs-checker/data/ssz/merkle-proofs.md diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel index cb90c3e889d..bbe668646ee 100644 --- a/tools/specs-checker/BUILD.bazel +++ b/tools/specs-checker/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "data/specs/phase0/fork-choice.md", "data/specs/phase0/validator.md", "data/specs/phase0/weak-subjectivity.md", + "data/ssz/merkle-proofs.md", ], importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", visibility = ["//visibility:public"], diff --git a/tools/specs-checker/data/ssz/merkle-proofs.md b/tools/specs-checker/data/ssz/merkle-proofs.md new file mode 100644 index 00000000000..fa09733f407 --- /dev/null +++ b/tools/specs-checker/data/ssz/merkle-proofs.md @@ -0,0 +1,240 @@ +```python +def get_power_of_two_ceil(x: int) -> int: + """ + Get the power of 2 for given input, or the closest higher power of 2 if the input is not a power of 2. + Commonly used for "how many nodes do I need for a bottom tree layer fitting x elements?" + Example: 0->1, 1->1, 2->2, 3->4, 4->4, 5->8, 6->8, 7->8, 8->8, 9->16. + """ + if x <= 1: + return 1 + elif x == 2: + return 2 + else: + return 2 * get_power_of_two_ceil((x + 1) // 2) +``` +```python +def get_power_of_two_floor(x: int) -> int: + """ + Get the power of 2 for given input, or the closest lower power of 2 if the input is not a power of 2. + The zero case is a placeholder and not used for math with generalized indices. + Commonly used for "what power of two makes up the root bit of the generalized index?" + Example: 0->1, 1->1, 2->2, 3->2, 4->4, 5->4, 6->4, 7->4, 8->8, 9->8 + """ + if x <= 1: + return 1 + if x == 2: + return x + else: + return 2 * get_power_of_two_floor(x // 2) +``` +```python +def merkle_tree(leaves: Sequence[Bytes32]) -> Sequence[Bytes32]: + """ + Return an array representing the tree nodes by generalized index: + [0, 1, 2, 3, 4, 5, 6, 7], where each layer is a power of 2. The 0 index is ignored. The 1 index is the root. + The result will be twice the size as the padded bottom layer for the input leaves. + """ + bottom_length = get_power_of_two_ceil(len(leaves)) + o = [Bytes32()] * bottom_length + list(leaves) + [Bytes32()] * (bottom_length - len(leaves)) + for i in range(bottom_length - 1, 0, -1): + o[i] = hash(o[i * 2] + o[i * 2 + 1]) + return o +``` +```python +def item_length(typ: SSZType) -> int: + """ + Return the number of bytes in a basic type, or 32 (a full hash) for compound types. + """ + if issubclass(typ, BasicValue): + return typ.byte_len + else: + return 32 +``` +```python +def get_elem_type(typ: Union[BaseBytes, BaseList, Container], + index_or_variable_name: Union[int, SSZVariableName]) -> SSZType: + """ + Return the type of the element of an object of the given type with the given index + or member variable name (eg. `7` for `x[7]`, `"foo"` for `x.foo`) + """ + return typ.get_fields()[index_or_variable_name] if issubclass(typ, Container) else typ.elem_type +``` +```python +def chunk_count(typ: SSZType) -> int: + """ + Return the number of hashes needed to represent the top-level elements in the given type + (eg. `x.foo` or `x[7]` but not `x[7].bar` or `x.foo.baz`). In all cases except lists/vectors + of basic types, this is simply the number of top-level elements, as each element gets one + hash. For lists/vectors of basic types, it is often fewer because multiple basic elements + can be packed into one 32-byte chunk. + """ + # typ.length describes the limit for list types, or the length for vector types. + if issubclass(typ, BasicValue): + return 1 + elif issubclass(typ, Bits): + return (typ.length + 255) // 256 + elif issubclass(typ, Elements): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + elif issubclass(typ, Container): + return len(typ.get_fields()) + else: + raise Exception(f"Type not supported: {typ}") +``` +```python +def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariableName]) -> Tuple[int, int, int]: + """ + Return three variables: + (i) the index of the chunk in which the given element of the item is represented; + (ii) the starting byte position within the chunk; + (iii) the ending byte position within the chunk. + For example: for a 6-item list of uint64 values, index=2 will return (0, 16, 24), index=5 will return (1, 8, 16) + """ + if issubclass(typ, Elements): + index = int(index_or_variable_name) + start = index * item_length(typ.elem_type) + return start // 32, start % 32, start % 32 + item_length(typ.elem_type) + elif issubclass(typ, Container): + variable_name = index_or_variable_name + return typ.get_field_names().index(variable_name), 0, item_length(get_elem_type(typ, variable_name)) + else: + raise Exception("Only lists/vectors/containers supported") +``` +```python +def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: + """ + Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for + `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. + """ + root = GeneralizedIndex(1) + for p in path: + assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further + if p == '__len__': + typ = uint64 + assert issubclass(typ, (List, ByteList)) + root = GeneralizedIndex(root * 2 + 1) + else: + pos, _, _ = get_item_position(typ, p) + base_index = (GeneralizedIndex(2) if issubclass(typ, (List, ByteList)) else GeneralizedIndex(1)) + root = GeneralizedIndex(root * base_index * get_power_of_two_ceil(chunk_count(typ)) + pos) + typ = get_elem_type(typ, p) + return root +``` +```python +def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: + """ + Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns + the generalized index for A -> Z. + """ + o = GeneralizedIndex(1) + for i in indices: + o = GeneralizedIndex(o * get_power_of_two_floor(i) + (i - get_power_of_two_floor(i))) + return o +``` +```python +def get_generalized_index_length(index: GeneralizedIndex) -> int: + """ + Return the length of a path represented by a generalized index. + """ + return int(log2(index)) +``` +```python +def get_generalized_index_bit(index: GeneralizedIndex, position: int) -> bool: + """ + Return the given bit of a generalized index. + """ + return (index & (1 << position)) > 0 +``` +```python +def generalized_index_sibling(index: GeneralizedIndex) -> GeneralizedIndex: + return GeneralizedIndex(index ^ 1) +``` +```python +def generalized_index_child(index: GeneralizedIndex, right_side: bool) -> GeneralizedIndex: + return GeneralizedIndex(index * 2 + right_side) +``` +```python +def generalized_index_parent(index: GeneralizedIndex) -> GeneralizedIndex: + return GeneralizedIndex(index // 2) +``` +```python +def get_branch_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of the sister chunks along the path from the chunk with the + given tree index to the root. + """ + o = [generalized_index_sibling(tree_index)] + while o[-1] > 1: + o.append(generalized_index_sibling(generalized_index_parent(o[-1]))) + return o[:-1] +``` +```python +def get_path_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of the chunks along the path from the chunk with the + given tree index to the root. + """ + o = [tree_index] + while o[-1] > 1: + o.append(generalized_index_parent(o[-1])) + return o[:-1] +``` +```python +def get_helper_indices(indices: Sequence[GeneralizedIndex]) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of all "extra" chunks in the tree needed to prove the chunks with the given + generalized indices. Note that the decreasing order is chosen deliberately to ensure equivalence to the + order of hashes in a regular single-item Merkle proof in the single-item case. + """ + all_helper_indices: Set[GeneralizedIndex] = set() + all_path_indices: Set[GeneralizedIndex] = set() + for index in indices: + all_helper_indices = all_helper_indices.union(set(get_branch_indices(index))) + all_path_indices = all_path_indices.union(set(get_path_indices(index))) + + return sorted(all_helper_indices.difference(all_path_indices), reverse=True) +``` +```python +def calculate_merkle_root(leaf: Bytes32, proof: Sequence[Bytes32], index: GeneralizedIndex) -> Root: + assert len(proof) == get_generalized_index_length(index) + for i, h in enumerate(proof): + if get_generalized_index_bit(index, i): + leaf = hash(h + leaf) + else: + leaf = hash(leaf + h) + return leaf +``` +```python +def verify_merkle_proof(leaf: Bytes32, proof: Sequence[Bytes32], index: GeneralizedIndex, root: Root) -> bool: + return calculate_merkle_root(leaf, proof, index) == root +``` +```python +def calculate_multi_merkle_root(leaves: Sequence[Bytes32], + proof: Sequence[Bytes32], + indices: Sequence[GeneralizedIndex]) -> Root: + assert len(leaves) == len(indices) + helper_indices = get_helper_indices(indices) + assert len(proof) == len(helper_indices) + objects = { + **{index: node for index, node in zip(indices, leaves)}, + **{index: node for index, node in zip(helper_indices, proof)} + } + keys = sorted(objects.keys(), reverse=True) + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in objects and k ^ 1 in objects and k // 2 not in objects: + objects[GeneralizedIndex(k // 2)] = hash( + objects[GeneralizedIndex((k | 1) ^ 1)] + + objects[GeneralizedIndex(k | 1)] + ) + keys.append(GeneralizedIndex(k // 2)) + pos += 1 + return objects[GeneralizedIndex(1)] +``` +```python +def verify_merkle_multiproof(leaves: Sequence[Bytes32], + proof: Sequence[Bytes32], + indices: Sequence[GeneralizedIndex], + root: Root) -> bool: + return calculate_multi_merkle_root(leaves, proof, indices) == root +``` diff --git a/tools/specs-checker/download.go b/tools/specs-checker/download.go index 11f5096d8af..05a5d228e07 100644 --- a/tools/specs-checker/download.go +++ b/tools/specs-checker/download.go @@ -18,6 +18,7 @@ const baseUrl = "https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev" var reg2 = regexp.MustCompile(`(?msU)^\x60\x60\x60python(.*)^\x60\x60\x60`) func download(cliCtx *cli.Context) error { + fmt.Print("Downloading specs:\n") baseDir := cliCtx.String(dirFlag.Name) for dirName, fileNames := range specDirs { if err := prepareDir(path.Join(baseDir, dirName)); err != nil { @@ -26,6 +27,7 @@ func download(cliCtx *cli.Context) error { for _, fileName := range fileNames { outFilePath := path.Join(baseDir, dirName, fileName) specDocUrl := fmt.Sprintf("%s/%s", baseUrl, fmt.Sprintf("%s/%s", dirName, fileName)) + fmt.Printf("- %s\n", specDocUrl) if err := getAndSaveFile(specDocUrl, outFilePath); err != nil { return err } diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go index b1475d004eb..822e1307d1d 100644 --- a/tools/specs-checker/main.go +++ b/tools/specs-checker/main.go @@ -27,6 +27,9 @@ var specDirs = map[string][]string{ "validator.md", "weak-subjectivity.md", }, + "ssz": { + "merkle-proofs.md", + }, } func main() { From ef0b3cc110d2d9e9d55b2468d18ad2567d6063a1 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 04:04:28 +0300 Subject: [PATCH 20/27] add extra.md --- tools/specs-checker/BUILD.bazel | 1 + tools/specs-checker/check.go | 30 +++++++++++++++++++++--------- tools/specs-checker/data/extra.md | 15 +++++++++++++++ 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 tools/specs-checker/data/extra.md diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel index bbe668646ee..691d1dc96e7 100644 --- a/tools/specs-checker/BUILD.bazel +++ b/tools/specs-checker/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "data/specs/phase0/validator.md", "data/specs/phase0/weak-subjectivity.md", "data/ssz/merkle-proofs.md", + "data/extra.md", ], importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", visibility = ["//visibility:public"], diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go index 62a40903602..47b3fecace3 100644 --- a/tools/specs-checker/check.go +++ b/tools/specs-checker/check.go @@ -73,7 +73,7 @@ func inspectFile(path string, defs map[string][]string) error { fmt.Printf("%s: %q is not found in spec docs\n", fset.Position(node.Pos()), defName) return false } - if !matchesRefImplementation(defName, refDefs, defBody) { + if !matchesRefImplementation(refDefs, defBody) { fmt.Printf("%s: %q code does not match reference implementation in specs\n", fset.Position(node.Pos()), defName) return false } @@ -87,21 +87,33 @@ func inspectFile(path string, defs map[string][]string) error { // parseSpecs parses input spec docs into map of function name -> array of function bodies // (single entity may have several definitions). func parseSpecs() (map[string][]string, error) { + loadSpecsFile := func(sb *strings.Builder, specFilePath string) error { + chunk, err := specFS.ReadFile(specFilePath) + if err != nil { + return fmt.Errorf("cannot read specs file: %w", err) + } + _, err = sb.Write(chunk) + if err != nil { + return fmt.Errorf("cannot copy specs file: %w", err) + } + return nil + } + // Traverse all spec files, and aggregate them within as single string. var sb strings.Builder for dirName, fileNames := range specDirs { for _, fileName := range fileNames { - chunk, err := specFS.ReadFile(path.Join("data", dirName, fileName)) - if err != nil { - return nil, fmt.Errorf("cannot read specs file: %w", err) - } - _, err = sb.Write(chunk) - if err != nil { - return nil, fmt.Errorf("cannot copy specs file: %w", err) + if err := loadSpecsFile(&sb, path.Join("data", dirName, fileName)); err != nil { + return nil, err } } } + // Load file with extra definitions (this allows us to use pseudo-code that is not from specs). + if err := loadSpecsFile(&sb, path.Join("data", "extra.md")); err != nil { + return nil, err + } + // Parse docs into function name -> array of function bodies map. chunks := strings.Split(strings.ReplaceAll(sb.String(), "```python", ""), "```") defs := make(map[string][]string, len(chunks)) @@ -134,7 +146,7 @@ func parseDefChunk(chunk string) (string, string) { } // matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). -func matchesRefImplementation(defName string, refDefs []string, input string) bool { +func matchesRefImplementation(refDefs []string, input string) bool { for _, refDef := range refDefs { refDefLines := strings.Split(refDef, "\n") inputLines := strings.Split(input, "\n") diff --git a/tools/specs-checker/data/extra.md b/tools/specs-checker/data/extra.md new file mode 100644 index 00000000000..0b83f4fc215 --- /dev/null +++ b/tools/specs-checker/data/extra.md @@ -0,0 +1,15 @@ +```python +def Sign(SK: int, message: Bytes) -> BLSSignature +``` +```python +def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool +``` +```python +def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool +``` +```python +def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool +``` +```python +def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature +``` \ No newline at end of file From f0b083adaceb6824c360fc624938bfefe047ebfc Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 18:31:50 +0300 Subject: [PATCH 21/27] mark wrong length issue --- tools/specs-checker/check.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go index 47b3fecace3..8cef527db5c 100644 --- a/tools/specs-checker/check.go +++ b/tools/specs-checker/check.go @@ -73,7 +73,7 @@ func inspectFile(path string, defs map[string][]string) error { fmt.Printf("%s: %q is not found in spec docs\n", fset.Position(node.Pos()), defName) return false } - if !matchesRefImplementation(refDefs, defBody) { + if !matchesRefImplementation(defName, refDefs, defBody, fset.Position(node.Pos())) { fmt.Printf("%s: %q code does not match reference implementation in specs\n", fset.Position(node.Pos()), defName) return false } @@ -146,7 +146,7 @@ func parseDefChunk(chunk string) (string, string) { } // matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). -func matchesRefImplementation(refDefs []string, input string) bool { +func matchesRefImplementation(defName string, refDefs []string, input string, pos token.Position) bool { for _, refDef := range refDefs { refDefLines := strings.Split(refDef, "\n") inputLines := strings.Split(input, "\n") @@ -159,6 +159,11 @@ func matchesRefImplementation(refDefs []string, input string) bool { break } } + // Mark potential issues, when there's some more comments in our code (which might be ok, as we are not required + // to put specs comments as the last one in the doc block). + if len(refDefLines) != len(inputLines) { + fmt.Printf("%s: %q potentially has issues (comment is longer than reference implementation)\n", pos, defName) + } if matchesPerfectly { return true } From 32433ddd4392fadd0e4a1e227cf89cc4d9f78c93 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 18:38:16 +0300 Subject: [PATCH 22/27] update readme --- tools/specs-checker/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/specs-checker/README.md b/tools/specs-checker/README.md index 02df26f7e8f..765297f0154 100644 --- a/tools/specs-checker/README.md +++ b/tools/specs-checker/README.md @@ -13,6 +13,9 @@ var specDirs = map[string][]string{ "validator.md", "weak-subjectivity.md", }, + "ssz": { + "merkle-proofs.md", + }, } ``` From 8dc9da56c2dddf7d4c0090858326259cea35b35d Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 18:41:51 +0300 Subject: [PATCH 23/27] update readme --- tools/specs-checker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/specs-checker/README.md b/tools/specs-checker/README.md index 765297f0154..cc4bbc77477 100644 --- a/tools/specs-checker/README.md +++ b/tools/specs-checker/README.md @@ -35,7 +35,7 @@ bazel run //tools/specs-checker check -- --dir $PWD/beacon-chain bazel run //tools/specs-checker check -- --dir $PWD/validator bazel run //tools/specs-checker check -- --dir $PWD/shared ``` -Or, to check whole project: +Or, to check the whole project: ```bash bazel run //tools/specs-checker check -- --dir $PWD ``` From 14a112cffd86b764db6a19b78b83612ea31b971c Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 19:05:21 +0300 Subject: [PATCH 24/27] remove non-def snippets --- .../data/specs/phase0/beacon-chain.md | 181 ------------------ .../data/specs/phase0/fork-choice.md | 19 -- .../data/specs/phase0/validator.md | 18 -- tools/specs-checker/download.go | 2 +- 4 files changed, 1 insertion(+), 219 deletions(-) diff --git a/tools/specs-checker/data/specs/phase0/beacon-chain.md b/tools/specs-checker/data/specs/phase0/beacon-chain.md index 0cb6d82bd03..8ec14411687 100644 --- a/tools/specs-checker/data/specs/phase0/beacon-chain.md +++ b/tools/specs-checker/data/specs/phase0/beacon-chain.md @@ -1,185 +1,4 @@ ```python -class Fork(Container): - previous_version: Version - current_version: Version - epoch: Epoch # Epoch of latest fork -``` -```python -class ForkData(Container): - current_version: Version - genesis_validators_root: Root -``` -```python -class Checkpoint(Container): - epoch: Epoch - root: Root -``` -```python -class Validator(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 # Commitment to pubkey for withdrawals - effective_balance: Gwei # Balance at stake - slashed: boolean - # Status epochs - activation_eligibility_epoch: Epoch # When criteria for activation were met - activation_epoch: Epoch - exit_epoch: Epoch - withdrawable_epoch: Epoch # When validator can withdraw funds -``` -```python -class AttestationData(Container): - slot: Slot - index: CommitteeIndex - # LMD GHOST vote - beacon_block_root: Root - # FFG vote - source: Checkpoint - target: Checkpoint -``` -```python -class IndexedAttestation(Container): - attesting_indices: List[ValidatorIndex, MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - signature: BLSSignature -``` -```python -class PendingAttestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - inclusion_delay: Slot - proposer_index: ValidatorIndex -``` -```python -class Eth1Data(Container): - deposit_root: Root - deposit_count: uint64 - block_hash: Bytes32 -``` -```python -class HistoricalBatch(Container): - block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] -``` -```python -class DepositMessage(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei -``` -```python -class DepositData(Container): - pubkey: BLSPubkey - withdrawal_credentials: Bytes32 - amount: Gwei - signature: BLSSignature # Signing over DepositMessage -``` -```python -class BeaconBlockHeader(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body_root: Root -``` -```python -class SigningData(Container): - object_root: Root - domain: Domain -``` -```python -class ProposerSlashing(Container): - signed_header_1: SignedBeaconBlockHeader - signed_header_2: SignedBeaconBlockHeader -``` -```python -class AttesterSlashing(Container): - attestation_1: IndexedAttestation - attestation_2: IndexedAttestation -``` -```python -class Attestation(Container): - aggregation_bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE] - data: AttestationData - signature: BLSSignature -``` -```python -class Deposit(Container): - proof: Vector[Bytes32, DEPOSIT_CONTRACT_TREE_DEPTH + 1] # Merkle path to deposit root - data: DepositData -``` -```python -class VoluntaryExit(Container): - epoch: Epoch # Earliest epoch when voluntary exit can be processed - validator_index: ValidatorIndex -``` -```python -class BeaconBlockBody(Container): - randao_reveal: BLSSignature - eth1_data: Eth1Data # Eth1 data vote - graffiti: Bytes32 # Arbitrary data - # Operations - proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS] - attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS] - attestations: List[Attestation, MAX_ATTESTATIONS] - deposits: List[Deposit, MAX_DEPOSITS] - voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS] -``` -```python -class BeaconBlock(Container): - slot: Slot - proposer_index: ValidatorIndex - parent_root: Root - state_root: Root - body: BeaconBlockBody -``` -```python -class BeaconState(Container): - # Versioning - genesis_time: uint64 - genesis_validators_root: Root - slot: Slot - fork: Fork - # History - latest_block_header: BeaconBlockHeader - block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT] - historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT] - # Eth1 - eth1_data: Eth1Data - eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH] - eth1_deposit_index: uint64 - # Registry - validators: List[Validator, VALIDATOR_REGISTRY_LIMIT] - balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT] - # Randomness - randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR] - # Slashings - slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR] # Per-epoch sums of slashed effective balances - # Attestations - previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH] - # Finality - justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH] # Bit set for every recent justified epoch - previous_justified_checkpoint: Checkpoint # Previous epoch snapshot - current_justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint -``` -```python -class SignedVoluntaryExit(Container): - message: VoluntaryExit - signature: BLSSignature -``` -```python -class SignedBeaconBlock(Container): - message: BeaconBlock - signature: BLSSignature -``` -```python -class SignedBeaconBlockHeader(Container): - message: BeaconBlockHeader - signature: BLSSignature -``` -```python def integer_squareroot(n: uint64) -> uint64: """ Return the largest integer ``x`` such that ``x**2 <= n``. diff --git a/tools/specs-checker/data/specs/phase0/fork-choice.md b/tools/specs-checker/data/specs/phase0/fork-choice.md index d789171a03a..f2f53070ebd 100644 --- a/tools/specs-checker/data/specs/phase0/fork-choice.md +++ b/tools/specs-checker/data/specs/phase0/fork-choice.md @@ -1,23 +1,4 @@ ```python -@dataclass(eq=True, frozen=True) -class LatestMessage(object): - epoch: Epoch - root: Root -``` -```python -@dataclass -class Store(object): - time: uint64 - genesis_time: uint64 - justified_checkpoint: Checkpoint - finalized_checkpoint: Checkpoint - best_justified_checkpoint: Checkpoint - blocks: Dict[Root, BeaconBlock] = field(default_factory=dict) - block_states: Dict[Root, BeaconState] = field(default_factory=dict) - checkpoint_states: Dict[Checkpoint, BeaconState] = field(default_factory=dict) - latest_messages: Dict[ValidatorIndex, LatestMessage] = field(default_factory=dict) -``` -```python def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: assert anchor_block.state_root == hash_tree_root(anchor_state) anchor_root = hash_tree_root(anchor_block) diff --git a/tools/specs-checker/data/specs/phase0/validator.md b/tools/specs-checker/data/specs/phase0/validator.md index 17c9f462846..a7c04c19449 100644 --- a/tools/specs-checker/data/specs/phase0/validator.md +++ b/tools/specs-checker/data/specs/phase0/validator.md @@ -1,22 +1,4 @@ ```python -class Eth1Block(Container): - timestamp: uint64 - deposit_root: Root - deposit_count: uint64 - # All other eth1 block fields -``` -```python -class AggregateAndProof(Container): - aggregator_index: ValidatorIndex - aggregate: Attestation - selection_proof: BLSSignature -``` -```python -class SignedAggregateAndProof(Container): - message: AggregateAndProof - signature: BLSSignature -``` -```python def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: validator = state.validators[validator_index] return is_active_validator(validator, get_current_epoch(state)) diff --git a/tools/specs-checker/download.go b/tools/specs-checker/download.go index 05a5d228e07..3496872225b 100644 --- a/tools/specs-checker/download.go +++ b/tools/specs-checker/download.go @@ -15,7 +15,7 @@ import ( const baseUrl = "https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev" // Regex to find Python's code snippets in markdown. -var reg2 = regexp.MustCompile(`(?msU)^\x60\x60\x60python(.*)^\x60\x60\x60`) +var reg2 = regexp.MustCompile(`(?msU)^\x60\x60\x60python\n+def\s(.*)^\x60\x60\x60`) func download(cliCtx *cli.Context) error { fmt.Print("Downloading specs:\n") From e048cadc994a1d2609385dce259f19b10f277df8 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 19:06:49 +0300 Subject: [PATCH 25/27] update comment --- tools/specs-checker/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go index 822e1307d1d..fee43e56640 100644 --- a/tools/specs-checker/main.go +++ b/tools/specs-checker/main.go @@ -12,7 +12,7 @@ var ( dirFlag = &cli.StringFlag{ Name: "dir", Value: "", - Usage: "Path to a directory containing Golang files to check", + Usage: "Target directory", Required: true, } ) From a376e019a23b7d4134d967fa71147c96b26a56ee Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Sat, 10 Apr 2021 19:17:22 +0300 Subject: [PATCH 26/27] check numrows --- tools/specs-checker/check.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go index 8cef527db5c..f72f4277242 100644 --- a/tools/specs-checker/check.go +++ b/tools/specs-checker/check.go @@ -17,6 +17,10 @@ import ( // Regex to find Python's "def". var reg1 = regexp.MustCompile(`def\s(.*)\(.*`) +// checkNumRows defines whether tool should check that the spec comment is the last comment of the block, so not only +// it matches the reference snippet, but it also has the same number of rows. +const checkNumRows = false + func check(cliCtx *cli.Context) error { // Obtain reference snippets. defs, err := parseSpecs() @@ -161,7 +165,7 @@ func matchesRefImplementation(defName string, refDefs []string, input string, po } // Mark potential issues, when there's some more comments in our code (which might be ok, as we are not required // to put specs comments as the last one in the doc block). - if len(refDefLines) != len(inputLines) { + if checkNumRows && len(refDefLines) != len(inputLines) { fmt.Printf("%s: %q potentially has issues (comment is longer than reference implementation)\n", pos, defName) } if matchesPerfectly { From e4c791c66b58703accaabc45d4a377b11166a8c4 Mon Sep 17 00:00:00 2001 From: Victor Farazdagi Date: Mon, 12 Apr 2021 11:59:30 +0300 Subject: [PATCH 27/27] ignore last empty line --- tools/specs-checker/check.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go index f72f4277242..aa005257ee6 100644 --- a/tools/specs-checker/check.go +++ b/tools/specs-checker/check.go @@ -152,8 +152,8 @@ func parseDefChunk(chunk string) (string, string) { // matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). func matchesRefImplementation(defName string, refDefs []string, input string, pos token.Position) bool { for _, refDef := range refDefs { - refDefLines := strings.Split(refDef, "\n") - inputLines := strings.Split(input, "\n") + refDefLines := strings.Split(strings.TrimRight(refDef, "\n"), "\n") + inputLines := strings.Split(strings.TrimRight(input, "\n"), "\n") matchesPerfectly := true for i := 0; i < len(refDefLines); i++ {