diff --git a/tools/specs-checker/BUILD.bazel b/tools/specs-checker/BUILD.bazel new file mode 100644 index 00000000000..691d1dc96e7 --- /dev/null +++ b/tools/specs-checker/BUILD.bazel @@ -0,0 +1,28 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = [ + "check.go", + "download.go", + "main.go", + ], + embedsrcs = [ + "data/specs/phase0/beacon-chain.md", + "data/specs/phase0/fork-choice.md", + "data/specs/phase0/validator.md", + "data/specs/phase0/weak-subjectivity.md", + "data/ssz/merkle-proofs.md", + "data/extra.md", + ], + importpath = "github.com/prysmaticlabs/prysm/tools/specs-checker", + visibility = ["//visibility:public"], + deps = ["@com_github_urfave_cli_v2//:go_default_library"], +) + +go_binary( + name = "specs-checker", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/specs-checker/README.md b/tools/specs-checker/README.md new file mode 100644 index 00000000000..cc4bbc77477 --- /dev/null +++ b/tools/specs-checker/README.md @@ -0,0 +1,41 @@ +# Specs checker tool + +This simple tool helps downloading and parsing [ETH2 specs](https://github.com/ethereum/eth2.0-specs/tree/dev/specs), +to be later used for making sure that our reference comments match specs definitions precisely. + +### Updating the reference specs +See `main.go` for a list of files to be downloaded, currently: +```golang +var specDirs = map[string][]string{ + "specs/phase0": { + "beacon-chain.md", + "fork-choice.md", + "validator.md", + "weak-subjectivity.md", + }, + "ssz": { + "merkle-proofs.md", + }, +} +``` + +To download/update specs: +```bash +bazel run //tools/specs-checker download -- --dir=$PWD/tools/specs-checker/data +``` + +This will pull the files defined in `specDirs`, parse them (extract Python code snippets, discarding any other text), +and save them to the folder from which `bazel run //tools/specs-checker check` will be able to embed. + +### Checking against the reference specs + +To check whether reference comments have the matching version of Python specs: +```bash +bazel run //tools/specs-checker check -- --dir $PWD/beacon-chain +bazel run //tools/specs-checker check -- --dir $PWD/validator +bazel run //tools/specs-checker check -- --dir $PWD/shared +``` +Or, to check the whole project: +```bash +bazel run //tools/specs-checker check -- --dir $PWD +``` diff --git a/tools/specs-checker/check.go b/tools/specs-checker/check.go new file mode 100644 index 00000000000..aa005257ee6 --- /dev/null +++ b/tools/specs-checker/check.go @@ -0,0 +1,176 @@ +package main + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/urfave/cli/v2" +) + +// Regex to find Python's "def". +var reg1 = regexp.MustCompile(`def\s(.*)\(.*`) + +// checkNumRows defines whether tool should check that the spec comment is the last comment of the block, so not only +// it matches the reference snippet, but it also has the same number of rows. +const checkNumRows = false + +func check(cliCtx *cli.Context) error { + // Obtain reference snippets. + defs, err := parseSpecs() + if err != nil { + return err + } + + // Walk the path, and process all contained Golang files. + fileWalker := func(path string, info os.FileInfo, err error) error { + if info == nil { + return fmt.Errorf("invalid input dir %q", path) + } + if !strings.HasSuffix(info.Name(), ".go") { + return nil + } + return inspectFile(path, defs) + } + return filepath.Walk(cliCtx.String(dirFlag.Name), fileWalker) +} + +func inspectFile(path string, defs map[string][]string) error { + // Parse source files, and check the pseudo code. + fset := token.NewFileSet() + file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) + if err != nil { + return err + } + + ast.Inspect(file, func(node ast.Node) bool { + stmt, ok := node.(*ast.CommentGroup) + if !ok { + return true + } + // Ignore comment groups that do not have python pseudo-code. + chunk := stmt.Text() + if !reg1.MatchString(chunk) { + return true + } + + // Trim the chunk, so that it starts from Python's "def". + loc := reg1.FindStringIndex(chunk) + chunk = chunk[loc[0]:] + + // Find out Python function name. + defName, defBody := parseDefChunk(chunk) + if defName == "" { + fmt.Printf("%s: cannot parse comment pseudo code\n", fset.Position(node.Pos())) + return false + } + + // Calculate differences with reference implementation. + refDefs, ok := defs[defName] + if !ok { + fmt.Printf("%s: %q is not found in spec docs\n", fset.Position(node.Pos()), defName) + return false + } + if !matchesRefImplementation(defName, refDefs, defBody, fset.Position(node.Pos())) { + fmt.Printf("%s: %q code does not match reference implementation in specs\n", fset.Position(node.Pos()), defName) + return false + } + + return true + }) + + return nil +} + +// parseSpecs parses input spec docs into map of function name -> array of function bodies +// (single entity may have several definitions). +func parseSpecs() (map[string][]string, error) { + loadSpecsFile := func(sb *strings.Builder, specFilePath string) error { + chunk, err := specFS.ReadFile(specFilePath) + if err != nil { + return fmt.Errorf("cannot read specs file: %w", err) + } + _, err = sb.Write(chunk) + if err != nil { + return fmt.Errorf("cannot copy specs file: %w", err) + } + return nil + } + + // Traverse all spec files, and aggregate them within as single string. + var sb strings.Builder + for dirName, fileNames := range specDirs { + for _, fileName := range fileNames { + if err := loadSpecsFile(&sb, path.Join("data", dirName, fileName)); err != nil { + return nil, err + } + } + } + + // Load file with extra definitions (this allows us to use pseudo-code that is not from specs). + if err := loadSpecsFile(&sb, path.Join("data", "extra.md")); err != nil { + return nil, err + } + + // Parse docs into function name -> array of function bodies map. + chunks := strings.Split(strings.ReplaceAll(sb.String(), "```python", ""), "```") + defs := make(map[string][]string, len(chunks)) + for _, chunk := range chunks { + defName, defBody := parseDefChunk(chunk) + if defName == "" { + continue + } + defs[defName] = append(defs[defName], defBody) + } + return defs, nil +} + +// parseDefChunk extract function name and function body from a Python's "def" chunk. +func parseDefChunk(chunk string) (string, string) { + chunk = strings.TrimLeft(chunk, "\n") + if chunk == "" { + return "", "" + } + chunkLines := strings.Split(chunk, "\n") + // Ignore all snippets, that do not define functions. + if chunkLines[0][:4] != "def " { + return "", "" + } + defMatches := reg1.FindStringSubmatch(chunkLines[0]) + if len(defMatches) < 2 { + return "", "" + } + return strings.Trim(defMatches[1], " "), chunk +} + +// matchesRefImplementation compares input string to reference code snippets (there might be multiple implementations). +func matchesRefImplementation(defName string, refDefs []string, input string, pos token.Position) bool { + for _, refDef := range refDefs { + refDefLines := strings.Split(strings.TrimRight(refDef, "\n"), "\n") + inputLines := strings.Split(strings.TrimRight(input, "\n"), "\n") + + matchesPerfectly := true + for i := 0; i < len(refDefLines); i++ { + a, b := strings.Trim(refDefLines[i], " "), strings.Trim(inputLines[i], " ") + if a != b { + matchesPerfectly = false + break + } + } + // Mark potential issues, when there's some more comments in our code (which might be ok, as we are not required + // to put specs comments as the last one in the doc block). + if checkNumRows && len(refDefLines) != len(inputLines) { + fmt.Printf("%s: %q potentially has issues (comment is longer than reference implementation)\n", pos, defName) + } + if matchesPerfectly { + return true + } + } + return false +} diff --git a/tools/specs-checker/data/extra.md b/tools/specs-checker/data/extra.md new file mode 100644 index 00000000000..0b83f4fc215 --- /dev/null +++ b/tools/specs-checker/data/extra.md @@ -0,0 +1,15 @@ +```python +def Sign(SK: int, message: Bytes) -> BLSSignature +``` +```python +def Verify(PK: BLSPubkey, message: Bytes, signature: BLSSignature) -> bool +``` +```python +def AggregateVerify(pairs: Sequence[PK: BLSPubkey, message: Bytes], signature: BLSSignature) -> bool +``` +```python +def FastAggregateVerify(PKs: Sequence[BLSPubkey], message: Bytes, signature: BLSSignature) -> bool +``` +```python +def Aggregate(signatures: Sequence[BLSSignature]) -> BLSSignature +``` \ No newline at end of file diff --git a/tools/specs-checker/data/specs/phase0/beacon-chain.md b/tools/specs-checker/data/specs/phase0/beacon-chain.md new file mode 100644 index 00000000000..8ec14411687 --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/beacon-chain.md @@ -0,0 +1,1037 @@ +```python +def integer_squareroot(n: uint64) -> uint64: + """ + Return the largest integer ``x`` such that ``x**2 <= n``. + """ + x = n + y = (x + 1) // 2 + while y < x: + x = y + y = (x + n // x) // 2 + return x +``` +```python +def xor(bytes_1: Bytes32, bytes_2: Bytes32) -> Bytes32: + """ + Return the exclusive-or of two 32-byte strings. + """ + return Bytes32(a ^ b for a, b in zip(bytes_1, bytes_2)) +``` +```python +def bytes_to_uint64(data: bytes) -> uint64: + """ + Return the integer deserialization of ``data`` interpreted as ``ENDIANNESS``-endian. + """ + return uint64(int.from_bytes(data, ENDIANNESS)) +``` +```python +def is_active_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is active. + """ + return validator.activation_epoch <= epoch < validator.exit_epoch +``` +```python +def is_eligible_for_activation_queue(validator: Validator) -> bool: + """ + Check if ``validator`` is eligible to be placed into the activation queue. + """ + return ( + validator.activation_eligibility_epoch == FAR_FUTURE_EPOCH + and validator.effective_balance == MAX_EFFECTIVE_BALANCE + ) +``` +```python +def is_eligible_for_activation(state: BeaconState, validator: Validator) -> bool: + """ + Check if ``validator`` is eligible for activation. + """ + return ( + # Placement in queue is finalized + validator.activation_eligibility_epoch <= state.finalized_checkpoint.epoch + # Has not yet been activated + and validator.activation_epoch == FAR_FUTURE_EPOCH + ) +``` +```python +def is_slashable_validator(validator: Validator, epoch: Epoch) -> bool: + """ + Check if ``validator`` is slashable. + """ + return (not validator.slashed) and (validator.activation_epoch <= epoch < validator.withdrawable_epoch) +``` +```python +def is_slashable_attestation_data(data_1: AttestationData, data_2: AttestationData) -> bool: + """ + Check if ``data_1`` and ``data_2`` are slashable according to Casper FFG rules. + """ + return ( + # Double vote + (data_1 != data_2 and data_1.target.epoch == data_2.target.epoch) or + # Surround vote + (data_1.source.epoch < data_2.source.epoch and data_2.target.epoch < data_1.target.epoch) + ) +``` +```python +def is_valid_indexed_attestation(state: BeaconState, indexed_attestation: IndexedAttestation) -> bool: + """ + Check if ``indexed_attestation`` is not empty, has sorted and unique indices and has a valid aggregate signature. + """ + # Verify indices are sorted and unique + indices = indexed_attestation.attesting_indices + if len(indices) == 0 or not indices == sorted(set(indices)): + return False + # Verify aggregate signature + pubkeys = [state.validators[i].pubkey for i in indices] + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, indexed_attestation.data.target.epoch) + signing_root = compute_signing_root(indexed_attestation.data, domain) + return bls.FastAggregateVerify(pubkeys, signing_root, indexed_attestation.signature) +``` +```python +def is_valid_merkle_branch(leaf: Bytes32, branch: Sequence[Bytes32], depth: uint64, index: uint64, root: Root) -> bool: + """ + Check if ``leaf`` at ``index`` verifies against the Merkle ``root`` and ``branch``. + """ + value = leaf + for i in range(depth): + if index // (2**i) % 2: + value = hash(branch[i] + value) + else: + value = hash(value + branch[i]) + return value == root +``` +```python +def compute_shuffled_index(index: uint64, index_count: uint64, seed: Bytes32) -> uint64: + """ + Return the shuffled index corresponding to ``seed`` (and ``index_count``). + """ + assert index < index_count + + # Swap or not (https://link.springer.com/content/pdf/10.1007%2F978-3-642-32009-5_1.pdf) + # See the 'generalized domain' algorithm on page 3 + for current_round in range(SHUFFLE_ROUND_COUNT): + pivot = bytes_to_uint64(hash(seed + uint_to_bytes(uint8(current_round)))[0:8]) % index_count + flip = (pivot + index_count - index) % index_count + position = max(index, flip) + source = hash( + seed + + uint_to_bytes(uint8(current_round)) + + uint_to_bytes(uint32(position // 256)) + ) + byte = uint8(source[(position % 256) // 8]) + bit = (byte >> (position % 8)) % 2 + index = flip if bit else index + + return index +``` +```python +def compute_proposer_index(state: BeaconState, indices: Sequence[ValidatorIndex], seed: Bytes32) -> ValidatorIndex: + """ + Return from ``indices`` a random index sampled by effective balance. + """ + assert len(indices) > 0 + MAX_RANDOM_BYTE = 2**8 - 1 + i = uint64(0) + total = uint64(len(indices)) + while True: + candidate_index = indices[compute_shuffled_index(i % total, total, seed)] + random_byte = hash(seed + uint_to_bytes(uint64(i // 32)))[i % 32] + effective_balance = state.validators[candidate_index].effective_balance + if effective_balance * MAX_RANDOM_BYTE >= MAX_EFFECTIVE_BALANCE * random_byte: + return candidate_index + i += 1 +``` +```python +def compute_committee(indices: Sequence[ValidatorIndex], + seed: Bytes32, + index: uint64, + count: uint64) -> Sequence[ValidatorIndex]: + """ + Return the committee corresponding to ``indices``, ``seed``, ``index``, and committee ``count``. + """ + start = (len(indices) * index) // count + end = (len(indices) * uint64(index + 1)) // count + return [indices[compute_shuffled_index(uint64(i), uint64(len(indices)), seed)] for i in range(start, end)] +``` +```python +def compute_epoch_at_slot(slot: Slot) -> Epoch: + """ + Return the epoch number at ``slot``. + """ + return Epoch(slot // SLOTS_PER_EPOCH) +``` +```python +def compute_start_slot_at_epoch(epoch: Epoch) -> Slot: + """ + Return the start slot of ``epoch``. + """ + return Slot(epoch * SLOTS_PER_EPOCH) +``` +```python +def compute_activation_exit_epoch(epoch: Epoch) -> Epoch: + """ + Return the epoch during which validator activations and exits initiated in ``epoch`` take effect. + """ + return Epoch(epoch + 1 + MAX_SEED_LOOKAHEAD) +``` +```python +def compute_fork_data_root(current_version: Version, genesis_validators_root: Root) -> Root: + """ + Return the 32-byte fork data root for the ``current_version`` and ``genesis_validators_root``. + This is used primarily in signature domains to avoid collisions across forks/chains. + """ + return hash_tree_root(ForkData( + current_version=current_version, + genesis_validators_root=genesis_validators_root, + )) +``` +```python +def compute_fork_digest(current_version: Version, genesis_validators_root: Root) -> ForkDigest: + """ + Return the 4-byte fork digest for the ``current_version`` and ``genesis_validators_root``. + This is a digest primarily used for domain separation on the p2p layer. + 4-bytes suffices for practical separation of forks/chains. + """ + return ForkDigest(compute_fork_data_root(current_version, genesis_validators_root)[:4]) +``` +```python +def compute_domain(domain_type: DomainType, fork_version: Version=None, genesis_validators_root: Root=None) -> Domain: + """ + Return the domain for the ``domain_type`` and ``fork_version``. + """ + if fork_version is None: + fork_version = GENESIS_FORK_VERSION + if genesis_validators_root is None: + genesis_validators_root = Root() # all bytes zero by default + fork_data_root = compute_fork_data_root(fork_version, genesis_validators_root) + return Domain(domain_type + fork_data_root[:28]) +``` +```python +def compute_signing_root(ssz_object: SSZObject, domain: Domain) -> Root: + """ + Return the signing root for the corresponding signing data. + """ + return hash_tree_root(SigningData( + object_root=hash_tree_root(ssz_object), + domain=domain, + )) +``` +```python +def get_current_epoch(state: BeaconState) -> Epoch: + """ + Return the current epoch. + """ + return compute_epoch_at_slot(state.slot) +``` +```python +def get_previous_epoch(state: BeaconState) -> Epoch: + """` + Return the previous epoch (unless the current epoch is ``GENESIS_EPOCH``). + """ + current_epoch = get_current_epoch(state) + return GENESIS_EPOCH if current_epoch == GENESIS_EPOCH else Epoch(current_epoch - 1) +``` +```python +def get_block_root(state: BeaconState, epoch: Epoch) -> Root: + """ + Return the block root at the start of a recent ``epoch``. + """ + return get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch)) +``` +```python +def get_block_root_at_slot(state: BeaconState, slot: Slot) -> Root: + """ + Return the block root at a recent ``slot``. + """ + assert slot < state.slot <= slot + SLOTS_PER_HISTORICAL_ROOT + return state.block_roots[slot % SLOTS_PER_HISTORICAL_ROOT] +``` +```python +def get_randao_mix(state: BeaconState, epoch: Epoch) -> Bytes32: + """ + Return the randao mix at a recent ``epoch``. + """ + return state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] +``` +```python +def get_active_validator_indices(state: BeaconState, epoch: Epoch) -> Sequence[ValidatorIndex]: + """ + Return the sequence of active validator indices at ``epoch``. + """ + return [ValidatorIndex(i) for i, v in enumerate(state.validators) if is_active_validator(v, epoch)] +``` +```python +def get_validator_churn_limit(state: BeaconState) -> uint64: + """ + Return the validator churn limit for the current epoch. + """ + active_validator_indices = get_active_validator_indices(state, get_current_epoch(state)) + return max(MIN_PER_EPOCH_CHURN_LIMIT, uint64(len(active_validator_indices)) // CHURN_LIMIT_QUOTIENT) +``` +```python +def get_seed(state: BeaconState, epoch: Epoch, domain_type: DomainType) -> Bytes32: + """ + Return the seed at ``epoch``. + """ + mix = get_randao_mix(state, Epoch(epoch + EPOCHS_PER_HISTORICAL_VECTOR - MIN_SEED_LOOKAHEAD - 1)) # Avoid underflow + return hash(domain_type + uint_to_bytes(epoch) + mix) +``` +```python +def get_committee_count_per_slot(state: BeaconState, epoch: Epoch) -> uint64: + """ + Return the number of committees in each slot for the given ``epoch``. + """ + return max(uint64(1), min( + MAX_COMMITTEES_PER_SLOT, + uint64(len(get_active_validator_indices(state, epoch))) // SLOTS_PER_EPOCH // TARGET_COMMITTEE_SIZE, + )) +``` +```python +def get_beacon_committee(state: BeaconState, slot: Slot, index: CommitteeIndex) -> Sequence[ValidatorIndex]: + """ + Return the beacon committee at ``slot`` for ``index``. + """ + epoch = compute_epoch_at_slot(slot) + committees_per_slot = get_committee_count_per_slot(state, epoch) + return compute_committee( + indices=get_active_validator_indices(state, epoch), + seed=get_seed(state, epoch, DOMAIN_BEACON_ATTESTER), + index=(slot % SLOTS_PER_EPOCH) * committees_per_slot + index, + count=committees_per_slot * SLOTS_PER_EPOCH, + ) +``` +```python +def get_beacon_proposer_index(state: BeaconState) -> ValidatorIndex: + """ + Return the beacon proposer index at the current slot. + """ + epoch = get_current_epoch(state) + seed = hash(get_seed(state, epoch, DOMAIN_BEACON_PROPOSER) + uint_to_bytes(state.slot)) + indices = get_active_validator_indices(state, epoch) + return compute_proposer_index(state, indices, seed) +``` +```python +def get_total_balance(state: BeaconState, indices: Set[ValidatorIndex]) -> Gwei: + """ + Return the combined effective balance of the ``indices``. + ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + Math safe up to ~10B ETH, afterwhich this overflows uint64. + """ + return Gwei(max(EFFECTIVE_BALANCE_INCREMENT, sum([state.validators[index].effective_balance for index in indices]))) +``` +```python +def get_total_active_balance(state: BeaconState) -> Gwei: + """ + Return the combined effective balance of the active validators. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, set(get_active_validator_indices(state, get_current_epoch(state)))) +``` +```python +def get_domain(state: BeaconState, domain_type: DomainType, epoch: Epoch=None) -> Domain: + """ + Return the signature domain (fork version concatenated with domain type) of a message. + """ + epoch = get_current_epoch(state) if epoch is None else epoch + fork_version = state.fork.previous_version if epoch < state.fork.epoch else state.fork.current_version + return compute_domain(domain_type, fork_version, state.genesis_validators_root) +``` +```python +def get_indexed_attestation(state: BeaconState, attestation: Attestation) -> IndexedAttestation: + """ + Return the indexed attestation corresponding to ``attestation``. + """ + attesting_indices = get_attesting_indices(state, attestation.data, attestation.aggregation_bits) + + return IndexedAttestation( + attesting_indices=sorted(attesting_indices), + data=attestation.data, + signature=attestation.signature, + ) +``` +```python +def get_attesting_indices(state: BeaconState, + data: AttestationData, + bits: Bitlist[MAX_VALIDATORS_PER_COMMITTEE]) -> Set[ValidatorIndex]: + """ + Return the set of attesting indices corresponding to ``data`` and ``bits``. + """ + committee = get_beacon_committee(state, data.slot, data.index) + return set(index for i, index in enumerate(committee) if bits[i]) +``` +```python +def increase_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Increase the validator balance at index ``index`` by ``delta``. + """ + state.balances[index] += delta +``` +```python +def decrease_balance(state: BeaconState, index: ValidatorIndex, delta: Gwei) -> None: + """ + Decrease the validator balance at index ``index`` by ``delta``, with underflow protection. + """ + state.balances[index] = 0 if delta > state.balances[index] else state.balances[index] - delta +``` +```python +def initiate_validator_exit(state: BeaconState, index: ValidatorIndex) -> None: + """ + Initiate the exit of the validator with index ``index``. + """ + # Return if validator already initiated exit + validator = state.validators[index] + if validator.exit_epoch != FAR_FUTURE_EPOCH: + return + + # Compute exit queue epoch + exit_epochs = [v.exit_epoch for v in state.validators if v.exit_epoch != FAR_FUTURE_EPOCH] + exit_queue_epoch = max(exit_epochs + [compute_activation_exit_epoch(get_current_epoch(state))]) + exit_queue_churn = len([v for v in state.validators if v.exit_epoch == exit_queue_epoch]) + if exit_queue_churn >= get_validator_churn_limit(state): + exit_queue_epoch += Epoch(1) + + # Set validator exit epoch and withdrawable epoch + validator.exit_epoch = exit_queue_epoch + validator.withdrawable_epoch = Epoch(validator.exit_epoch + MIN_VALIDATOR_WITHDRAWABILITY_DELAY) +``` +```python +def slash_validator(state: BeaconState, + slashed_index: ValidatorIndex, + whistleblower_index: ValidatorIndex=None) -> None: + """ + Slash the validator with index ``slashed_index``. + """ + epoch = get_current_epoch(state) + initiate_validator_exit(state, slashed_index) + validator = state.validators[slashed_index] + validator.slashed = True + validator.withdrawable_epoch = max(validator.withdrawable_epoch, Epoch(epoch + EPOCHS_PER_SLASHINGS_VECTOR)) + state.slashings[epoch % EPOCHS_PER_SLASHINGS_VECTOR] += validator.effective_balance + decrease_balance(state, slashed_index, validator.effective_balance // MIN_SLASHING_PENALTY_QUOTIENT) + + # Apply proposer and whistleblower rewards + proposer_index = get_beacon_proposer_index(state) + if whistleblower_index is None: + whistleblower_index = proposer_index + whistleblower_reward = Gwei(validator.effective_balance // WHISTLEBLOWER_REWARD_QUOTIENT) + proposer_reward = Gwei(whistleblower_reward // PROPOSER_REWARD_QUOTIENT) + increase_balance(state, proposer_index, proposer_reward) + increase_balance(state, whistleblower_index, Gwei(whistleblower_reward - proposer_reward)) +``` +```python +def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32, + eth1_timestamp: uint64, + deposits: Sequence[Deposit]) -> BeaconState: + fork = Fork( + previous_version=GENESIS_FORK_VERSION, + current_version=GENESIS_FORK_VERSION, + epoch=GENESIS_EPOCH, + ) + state = BeaconState( + genesis_time=eth1_timestamp + GENESIS_DELAY, + fork=fork, + eth1_data=Eth1Data(block_hash=eth1_block_hash, deposit_count=uint64(len(deposits))), + latest_block_header=BeaconBlockHeader(body_root=hash_tree_root(BeaconBlockBody())), + randao_mixes=[eth1_block_hash] * EPOCHS_PER_HISTORICAL_VECTOR, # Seed RANDAO with Eth1 entropy + ) + + # Process deposits + leaves = list(map(lambda deposit: deposit.data, deposits)) + for index, deposit in enumerate(deposits): + deposit_data_list = List[DepositData, 2**DEPOSIT_CONTRACT_TREE_DEPTH](*leaves[:index + 1]) + state.eth1_data.deposit_root = hash_tree_root(deposit_data_list) + process_deposit(state, deposit) + + # Process activations + for index, validator in enumerate(state.validators): + balance = state.balances[index] + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + if validator.effective_balance == MAX_EFFECTIVE_BALANCE: + validator.activation_eligibility_epoch = GENESIS_EPOCH + validator.activation_epoch = GENESIS_EPOCH + + # Set genesis validators root for domain separation and chain versioning + state.genesis_validators_root = hash_tree_root(state.validators) + + return state +``` +```python +def is_valid_genesis_state(state: BeaconState) -> bool: + if state.genesis_time < MIN_GENESIS_TIME: + return False + if len(get_active_validator_indices(state, GENESIS_EPOCH)) < MIN_GENESIS_ACTIVE_VALIDATOR_COUNT: + return False + return True +``` +```python +def state_transition(state: BeaconState, signed_block: SignedBeaconBlock, validate_result: bool=True) -> None: + block = signed_block.message + # Process slots (including those with no blocks) since block + process_slots(state, block.slot) + # Verify signature + if validate_result: + assert verify_block_signature(state, signed_block) + # Process block + process_block(state, block) + # Verify state root + if validate_result: + assert block.state_root == hash_tree_root(state) +``` +```python +def verify_block_signature(state: BeaconState, signed_block: SignedBeaconBlock) -> bool: + proposer = state.validators[signed_block.message.proposer_index] + signing_root = compute_signing_root(signed_block.message, get_domain(state, DOMAIN_BEACON_PROPOSER)) + return bls.Verify(proposer.pubkey, signing_root, signed_block.signature) +``` +```python +def process_slots(state: BeaconState, slot: Slot) -> None: + assert state.slot < slot + while state.slot < slot: + process_slot(state) + # Process epoch on the start slot of the next epoch + if (state.slot + 1) % SLOTS_PER_EPOCH == 0: + process_epoch(state) + state.slot = Slot(state.slot + 1) +``` +```python +def process_slot(state: BeaconState) -> None: + # Cache state root + previous_state_root = hash_tree_root(state) + state.state_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_state_root + # Cache latest block header state root + if state.latest_block_header.state_root == Bytes32(): + state.latest_block_header.state_root = previous_state_root + # Cache block root + previous_block_root = hash_tree_root(state.latest_block_header) + state.block_roots[state.slot % SLOTS_PER_HISTORICAL_ROOT] = previous_block_root +``` +```python +def process_epoch(state: BeaconState) -> None: + process_justification_and_finalization(state) + process_rewards_and_penalties(state) + process_registry_updates(state) + process_slashings(state) + process_eth1_data_reset(state) + process_effective_balance_updates(state) + process_slashings_reset(state) + process_randao_mixes_reset(state) + process_historical_roots_update(state) + process_participation_record_updates(state) +``` +```python +def get_matching_source_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + assert epoch in (get_previous_epoch(state), get_current_epoch(state)) + return state.current_epoch_attestations if epoch == get_current_epoch(state) else state.previous_epoch_attestations +``` +```python +def get_matching_target_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_source_attestations(state, epoch) + if a.data.target.root == get_block_root(state, epoch) + ] +``` +```python +def get_matching_head_attestations(state: BeaconState, epoch: Epoch) -> Sequence[PendingAttestation]: + return [ + a for a in get_matching_target_attestations(state, epoch) + if a.data.beacon_block_root == get_block_root_at_slot(state, a.data.slot) + ] +``` +```python +def get_unslashed_attesting_indices(state: BeaconState, + attestations: Sequence[PendingAttestation]) -> Set[ValidatorIndex]: + output = set() # type: Set[ValidatorIndex] + for a in attestations: + output = output.union(get_attesting_indices(state, a.data, a.aggregation_bits)) + return set(filter(lambda index: not state.validators[index].slashed, output)) +``` +```python +def get_attesting_balance(state: BeaconState, attestations: Sequence[PendingAttestation]) -> Gwei: + """ + Return the combined effective balance of the set of unslashed validators participating in ``attestations``. + Note: ``get_total_balance`` returns ``EFFECTIVE_BALANCE_INCREMENT`` Gwei minimum to avoid divisions by zero. + """ + return get_total_balance(state, get_unslashed_attesting_indices(state, attestations)) +``` +```python +def process_justification_and_finalization(state: BeaconState) -> None: + # Initial FFG checkpoint values have a `0x00` stub for `root`. + # Skip FFG updates in the first two epochs to avoid corner cases that might result in modifying this stub. + if get_current_epoch(state) <= GENESIS_EPOCH + 1: + return + previous_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + current_attestations = get_matching_target_attestations(state, get_current_epoch(state)) + total_active_balance = get_total_active_balance(state) + previous_target_balance = get_attesting_balance(state, previous_attestations) + current_target_balance = get_attesting_balance(state, current_attestations) + weigh_justification_and_finalization(state, total_active_balance, previous_target_balance, current_target_balance) +``` +```python +def weigh_justification_and_finalization(state: BeaconState, + total_active_balance: Gwei, + previous_epoch_target_balance: Gwei, + current_epoch_target_balance: Gwei) -> None: + previous_epoch = get_previous_epoch(state) + current_epoch = get_current_epoch(state) + old_previous_justified_checkpoint = state.previous_justified_checkpoint + old_current_justified_checkpoint = state.current_justified_checkpoint + + # Process justifications + state.previous_justified_checkpoint = state.current_justified_checkpoint + state.justification_bits[1:] = state.justification_bits[:JUSTIFICATION_BITS_LENGTH - 1] + state.justification_bits[0] = 0b0 + if previous_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=previous_epoch, + root=get_block_root(state, previous_epoch)) + state.justification_bits[1] = 0b1 + if current_epoch_target_balance * 3 >= total_active_balance * 2: + state.current_justified_checkpoint = Checkpoint(epoch=current_epoch, + root=get_block_root(state, current_epoch)) + state.justification_bits[0] = 0b1 + + # Process finalizations + bits = state.justification_bits + # The 2nd/3rd/4th most recent epochs are justified, the 2nd using the 4th as source + if all(bits[1:4]) and old_previous_justified_checkpoint.epoch + 3 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 2nd/3rd most recent epochs are justified, the 2nd using the 3rd as source + if all(bits[1:3]) and old_previous_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_previous_justified_checkpoint + # The 1st/2nd/3rd most recent epochs are justified, the 1st using the 3rd as source + if all(bits[0:3]) and old_current_justified_checkpoint.epoch + 2 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint + # The 1st/2nd most recent epochs are justified, the 1st using the 2nd as source + if all(bits[0:2]) and old_current_justified_checkpoint.epoch + 1 == current_epoch: + state.finalized_checkpoint = old_current_justified_checkpoint +``` +```python +def get_base_reward(state: BeaconState, index: ValidatorIndex) -> Gwei: + total_balance = get_total_active_balance(state) + effective_balance = state.validators[index].effective_balance + return Gwei(effective_balance * BASE_REWARD_FACTOR // integer_squareroot(total_balance) // BASE_REWARDS_PER_EPOCH) +``` +```python +def get_proposer_reward(state: BeaconState, attesting_index: ValidatorIndex) -> Gwei: + return Gwei(get_base_reward(state, attesting_index) // PROPOSER_REWARD_QUOTIENT) +``` +```python +def get_finality_delay(state: BeaconState) -> uint64: + return get_previous_epoch(state) - state.finalized_checkpoint.epoch +``` +```python +def is_in_inactivity_leak(state: BeaconState) -> bool: + return get_finality_delay(state) > MIN_EPOCHS_TO_INACTIVITY_PENALTY +``` +```python +def get_eligible_validator_indices(state: BeaconState) -> Sequence[ValidatorIndex]: + previous_epoch = get_previous_epoch(state) + return [ + ValidatorIndex(index) for index, v in enumerate(state.validators) + if is_active_validator(v, previous_epoch) or (v.slashed and previous_epoch + 1 < v.withdrawable_epoch) + ] +``` +```python +def get_attestation_component_deltas(state: BeaconState, + attestations: Sequence[PendingAttestation] + ) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Helper with shared logic for use by get source, target, and head deltas functions + """ + rewards = [Gwei(0)] * len(state.validators) + penalties = [Gwei(0)] * len(state.validators) + total_balance = get_total_active_balance(state) + unslashed_attesting_indices = get_unslashed_attesting_indices(state, attestations) + attesting_balance = get_total_balance(state, unslashed_attesting_indices) + for index in get_eligible_validator_indices(state): + if index in unslashed_attesting_indices: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from balance totals to avoid uint64 overflow + if is_in_inactivity_leak(state): + # Since full base reward will be canceled out by inactivity penalty deltas, + # optimal participation receives full base reward compensation here. + rewards[index] += get_base_reward(state, index) + else: + reward_numerator = get_base_reward(state, index) * (attesting_balance // increment) + rewards[index] += reward_numerator // (total_balance // increment) + else: + penalties[index] += get_base_reward(state, index) + return rewards, penalties +``` +```python +def get_source_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for source-vote for each validator. + """ + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_source_attestations) +``` +```python +def get_target_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for target-vote for each validator. + """ + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_target_attestations) +``` +```python +def get_head_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attester micro-rewards/penalties for head-vote for each validator. + """ + matching_head_attestations = get_matching_head_attestations(state, get_previous_epoch(state)) + return get_attestation_component_deltas(state, matching_head_attestations) +``` +```python +def get_inclusion_delay_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return proposer and inclusion delay micro-rewards/penalties for each validator. + """ + rewards = [Gwei(0) for _ in range(len(state.validators))] + matching_source_attestations = get_matching_source_attestations(state, get_previous_epoch(state)) + for index in get_unslashed_attesting_indices(state, matching_source_attestations): + attestation = min([ + a for a in matching_source_attestations + if index in get_attesting_indices(state, a.data, a.aggregation_bits) + ], key=lambda a: a.inclusion_delay) + rewards[attestation.proposer_index] += get_proposer_reward(state, index) + max_attester_reward = Gwei(get_base_reward(state, index) - get_proposer_reward(state, index)) + rewards[index] += Gwei(max_attester_reward // attestation.inclusion_delay) + + # No penalties associated with inclusion delay + penalties = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` +```python +def get_inactivity_penalty_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return inactivity reward/penalty deltas for each validator. + """ + penalties = [Gwei(0) for _ in range(len(state.validators))] + if is_in_inactivity_leak(state): + matching_target_attestations = get_matching_target_attestations(state, get_previous_epoch(state)) + matching_target_attesting_indices = get_unslashed_attesting_indices(state, matching_target_attestations) + for index in get_eligible_validator_indices(state): + # If validator is performing optimally this cancels all rewards for a neutral balance + base_reward = get_base_reward(state, index) + penalties[index] += Gwei(BASE_REWARDS_PER_EPOCH * base_reward - get_proposer_reward(state, index)) + if index not in matching_target_attesting_indices: + effective_balance = state.validators[index].effective_balance + penalties[index] += Gwei(effective_balance * get_finality_delay(state) // INACTIVITY_PENALTY_QUOTIENT) + + # No rewards associated with inactivity penalties + rewards = [Gwei(0) for _ in range(len(state.validators))] + return rewards, penalties +``` +```python +def get_attestation_deltas(state: BeaconState) -> Tuple[Sequence[Gwei], Sequence[Gwei]]: + """ + Return attestation reward/penalty deltas for each validator. + """ + source_rewards, source_penalties = get_source_deltas(state) + target_rewards, target_penalties = get_target_deltas(state) + head_rewards, head_penalties = get_head_deltas(state) + inclusion_delay_rewards, _ = get_inclusion_delay_deltas(state) + _, inactivity_penalties = get_inactivity_penalty_deltas(state) + + rewards = [ + source_rewards[i] + target_rewards[i] + head_rewards[i] + inclusion_delay_rewards[i] + for i in range(len(state.validators)) + ] + + penalties = [ + source_penalties[i] + target_penalties[i] + head_penalties[i] + inactivity_penalties[i] + for i in range(len(state.validators)) + ] + + return rewards, penalties +``` +```python +def process_rewards_and_penalties(state: BeaconState) -> None: + # No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch + if get_current_epoch(state) == GENESIS_EPOCH: + return + + rewards, penalties = get_attestation_deltas(state) + for index in range(len(state.validators)): + increase_balance(state, ValidatorIndex(index), rewards[index]) + decrease_balance(state, ValidatorIndex(index), penalties[index]) +``` +```python +def process_registry_updates(state: BeaconState) -> None: + # Process activation eligibility and ejections + for index, validator in enumerate(state.validators): + if is_eligible_for_activation_queue(validator): + validator.activation_eligibility_epoch = get_current_epoch(state) + 1 + + if is_active_validator(validator, get_current_epoch(state)) and validator.effective_balance <= EJECTION_BALANCE: + initiate_validator_exit(state, ValidatorIndex(index)) + + # Queue validators eligible for activation and not yet dequeued for activation + activation_queue = sorted([ + index for index, validator in enumerate(state.validators) + if is_eligible_for_activation(state, validator) + # Order by the sequence of activation_eligibility_epoch setting and then index + ], key=lambda index: (state.validators[index].activation_eligibility_epoch, index)) + # Dequeued validators for activation up to churn limit + for index in activation_queue[:get_validator_churn_limit(state)]: + validator = state.validators[index] + validator.activation_epoch = compute_activation_exit_epoch(get_current_epoch(state)) +``` +```python +def process_slashings(state: BeaconState) -> None: + epoch = get_current_epoch(state) + total_balance = get_total_active_balance(state) + adjusted_total_slashing_balance = min(sum(state.slashings) * PROPORTIONAL_SLASHING_MULTIPLIER, total_balance) + for index, validator in enumerate(state.validators): + if validator.slashed and epoch + EPOCHS_PER_SLASHINGS_VECTOR // 2 == validator.withdrawable_epoch: + increment = EFFECTIVE_BALANCE_INCREMENT # Factored out from penalty numerator to avoid uint64 overflow + penalty_numerator = validator.effective_balance // increment * adjusted_total_slashing_balance + penalty = penalty_numerator // total_balance * increment + decrease_balance(state, ValidatorIndex(index), penalty) +``` +```python +def process_eth1_data_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset eth1 data votes + if next_epoch % EPOCHS_PER_ETH1_VOTING_PERIOD == 0: + state.eth1_data_votes = [] +``` +```python +def process_effective_balance_updates(state: BeaconState) -> None: + # Update effective balances with hysteresis + for index, validator in enumerate(state.validators): + balance = state.balances[index] + HYSTERESIS_INCREMENT = uint64(EFFECTIVE_BALANCE_INCREMENT // HYSTERESIS_QUOTIENT) + DOWNWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_DOWNWARD_MULTIPLIER + UPWARD_THRESHOLD = HYSTERESIS_INCREMENT * HYSTERESIS_UPWARD_MULTIPLIER + if ( + balance + DOWNWARD_THRESHOLD < validator.effective_balance + or validator.effective_balance + UPWARD_THRESHOLD < balance + ): + validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) +``` +```python +def process_slashings_reset(state: BeaconState) -> None: + next_epoch = Epoch(get_current_epoch(state) + 1) + # Reset slashings + state.slashings[next_epoch % EPOCHS_PER_SLASHINGS_VECTOR] = Gwei(0) +``` +```python +def process_randao_mixes_reset(state: BeaconState) -> None: + current_epoch = get_current_epoch(state) + next_epoch = Epoch(current_epoch + 1) + # Set randao mix + state.randao_mixes[next_epoch % EPOCHS_PER_HISTORICAL_VECTOR] = get_randao_mix(state, current_epoch) +``` +```python +def process_historical_roots_update(state: BeaconState) -> None: + # Set historical root accumulator + next_epoch = Epoch(get_current_epoch(state) + 1) + if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0: + historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots) + state.historical_roots.append(hash_tree_root(historical_batch)) +``` +```python +def process_participation_record_updates(state: BeaconState) -> None: + # Rotate current/previous epoch attestations + state.previous_epoch_attestations = state.current_epoch_attestations + state.current_epoch_attestations = [] +``` +```python +def process_block(state: BeaconState, block: BeaconBlock) -> None: + process_block_header(state, block) + process_randao(state, block.body) + process_eth1_data(state, block.body) + process_operations(state, block.body) +``` +```python +def process_block_header(state: BeaconState, block: BeaconBlock) -> None: + # Verify that the slots match + assert block.slot == state.slot + # Verify that the block is newer than latest block header + assert block.slot > state.latest_block_header.slot + # Verify that proposer index is the correct index + assert block.proposer_index == get_beacon_proposer_index(state) + # Verify that the parent matches + assert block.parent_root == hash_tree_root(state.latest_block_header) + # Cache current block as the new latest block + state.latest_block_header = BeaconBlockHeader( + slot=block.slot, + proposer_index=block.proposer_index, + parent_root=block.parent_root, + state_root=Bytes32(), # Overwritten in the next process_slot call + body_root=hash_tree_root(block.body), + ) + + # Verify proposer is not slashed + proposer = state.validators[block.proposer_index] + assert not proposer.slashed +``` +```python +def process_randao(state: BeaconState, body: BeaconBlockBody) -> None: + epoch = get_current_epoch(state) + # Verify RANDAO reveal + proposer = state.validators[get_beacon_proposer_index(state)] + signing_root = compute_signing_root(epoch, get_domain(state, DOMAIN_RANDAO)) + assert bls.Verify(proposer.pubkey, signing_root, body.randao_reveal) + # Mix in RANDAO reveal + mix = xor(get_randao_mix(state, epoch), hash(body.randao_reveal)) + state.randao_mixes[epoch % EPOCHS_PER_HISTORICAL_VECTOR] = mix +``` +```python +def process_eth1_data(state: BeaconState, body: BeaconBlockBody) -> None: + state.eth1_data_votes.append(body.eth1_data) + if state.eth1_data_votes.count(body.eth1_data) * 2 > EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH: + state.eth1_data = body.eth1_data +``` +```python +def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: + # Verify that outstanding deposits are processed up to the maximum number of deposits + assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) + + def for_ops(operations: Sequence[Any], fn: Callable[[BeaconState, Any], None]) -> None: + for operation in operations: + fn(state, operation) + + for_ops(body.proposer_slashings, process_proposer_slashing) + for_ops(body.attester_slashings, process_attester_slashing) + for_ops(body.attestations, process_attestation) + for_ops(body.deposits, process_deposit) + for_ops(body.voluntary_exits, process_voluntary_exit) +``` +```python +def process_proposer_slashing(state: BeaconState, proposer_slashing: ProposerSlashing) -> None: + header_1 = proposer_slashing.signed_header_1.message + header_2 = proposer_slashing.signed_header_2.message + + # Verify header slots match + assert header_1.slot == header_2.slot + # Verify header proposer indices match + assert header_1.proposer_index == header_2.proposer_index + # Verify the headers are different + assert header_1 != header_2 + # Verify the proposer is slashable + proposer = state.validators[header_1.proposer_index] + assert is_slashable_validator(proposer, get_current_epoch(state)) + # Verify signatures + for signed_header in (proposer_slashing.signed_header_1, proposer_slashing.signed_header_2): + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(signed_header.message.slot)) + signing_root = compute_signing_root(signed_header.message, domain) + assert bls.Verify(proposer.pubkey, signing_root, signed_header.signature) + + slash_validator(state, header_1.proposer_index) +``` +```python +def process_attester_slashing(state: BeaconState, attester_slashing: AttesterSlashing) -> None: + attestation_1 = attester_slashing.attestation_1 + attestation_2 = attester_slashing.attestation_2 + assert is_slashable_attestation_data(attestation_1.data, attestation_2.data) + assert is_valid_indexed_attestation(state, attestation_1) + assert is_valid_indexed_attestation(state, attestation_2) + + slashed_any = False + indices = set(attestation_1.attesting_indices).intersection(attestation_2.attesting_indices) + for index in sorted(indices): + if is_slashable_validator(state.validators[index], get_current_epoch(state)): + slash_validator(state, index) + slashed_any = True + assert slashed_any +``` +```python +def process_attestation(state: BeaconState, attestation: Attestation) -> None: + data = attestation.data + assert data.target.epoch in (get_previous_epoch(state), get_current_epoch(state)) + assert data.target.epoch == compute_epoch_at_slot(data.slot) + assert data.slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= data.slot + SLOTS_PER_EPOCH + assert data.index < get_committee_count_per_slot(state, data.target.epoch) + + committee = get_beacon_committee(state, data.slot, data.index) + assert len(attestation.aggregation_bits) == len(committee) + + pending_attestation = PendingAttestation( + data=data, + aggregation_bits=attestation.aggregation_bits, + inclusion_delay=state.slot - data.slot, + proposer_index=get_beacon_proposer_index(state), + ) + + if data.target.epoch == get_current_epoch(state): + assert data.source == state.current_justified_checkpoint + state.current_epoch_attestations.append(pending_attestation) + else: + assert data.source == state.previous_justified_checkpoint + state.previous_epoch_attestations.append(pending_attestation) + + # Verify signature + assert is_valid_indexed_attestation(state, get_indexed_attestation(state, attestation)) +``` +```python +def get_validator_from_deposit(state: BeaconState, deposit: Deposit) -> Validator: + amount = deposit.data.amount + effective_balance = min(amount - amount % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE) + + return Validator( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + activation_eligibility_epoch=FAR_FUTURE_EPOCH, + activation_epoch=FAR_FUTURE_EPOCH, + exit_epoch=FAR_FUTURE_EPOCH, + withdrawable_epoch=FAR_FUTURE_EPOCH, + effective_balance=effective_balance, + ) +``` +```python +def process_deposit(state: BeaconState, deposit: Deposit) -> None: + # Verify the Merkle branch + assert is_valid_merkle_branch( + leaf=hash_tree_root(deposit.data), + branch=deposit.proof, + depth=DEPOSIT_CONTRACT_TREE_DEPTH + 1, # Add 1 for the List length mix-in + index=state.eth1_deposit_index, + root=state.eth1_data.deposit_root, + ) + + # Deposits must be processed in order + state.eth1_deposit_index += 1 + + pubkey = deposit.data.pubkey + amount = deposit.data.amount + validator_pubkeys = [v.pubkey for v in state.validators] + if pubkey not in validator_pubkeys: + # Verify the deposit signature (proof of possession) which is not checked by the deposit contract + deposit_message = DepositMessage( + pubkey=deposit.data.pubkey, + withdrawal_credentials=deposit.data.withdrawal_credentials, + amount=deposit.data.amount, + ) + domain = compute_domain(DOMAIN_DEPOSIT) # Fork-agnostic domain since deposits are valid across forks + signing_root = compute_signing_root(deposit_message, domain) + if not bls.Verify(pubkey, signing_root, deposit.data.signature): + return + + # Add validator and balance entries + state.validators.append(get_validator_from_deposit(state, deposit)) + state.balances.append(amount) + else: + # Increase balance by deposit amount + index = ValidatorIndex(validator_pubkeys.index(pubkey)) + increase_balance(state, index, amount) +``` +```python +def process_voluntary_exit(state: BeaconState, signed_voluntary_exit: SignedVoluntaryExit) -> None: + voluntary_exit = signed_voluntary_exit.message + validator = state.validators[voluntary_exit.validator_index] + # Verify the validator is active + assert is_active_validator(validator, get_current_epoch(state)) + # Verify exit has not been initiated + assert validator.exit_epoch == FAR_FUTURE_EPOCH + # Exits must specify an epoch when they become valid; they are not valid before then + assert get_current_epoch(state) >= voluntary_exit.epoch + # Verify the validator has been active long enough + assert get_current_epoch(state) >= validator.activation_epoch + SHARD_COMMITTEE_PERIOD + # Verify signature + domain = get_domain(state, DOMAIN_VOLUNTARY_EXIT, voluntary_exit.epoch) + signing_root = compute_signing_root(voluntary_exit, domain) + assert bls.Verify(validator.pubkey, signing_root, signed_voluntary_exit.signature) + # Initiate exit + initiate_validator_exit(state, voluntary_exit.validator_index) +``` diff --git a/tools/specs-checker/data/specs/phase0/fork-choice.md b/tools/specs-checker/data/specs/phase0/fork-choice.md new file mode 100644 index 00000000000..f2f53070ebd --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/fork-choice.md @@ -0,0 +1,259 @@ +```python +def get_forkchoice_store(anchor_state: BeaconState, anchor_block: BeaconBlock) -> Store: + assert anchor_block.state_root == hash_tree_root(anchor_state) + anchor_root = hash_tree_root(anchor_block) + anchor_epoch = get_current_epoch(anchor_state) + justified_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + finalized_checkpoint = Checkpoint(epoch=anchor_epoch, root=anchor_root) + return Store( + time=uint64(anchor_state.genesis_time + SECONDS_PER_SLOT * anchor_state.slot), + genesis_time=anchor_state.genesis_time, + justified_checkpoint=justified_checkpoint, + finalized_checkpoint=finalized_checkpoint, + best_justified_checkpoint=justified_checkpoint, + blocks={anchor_root: copy(anchor_block)}, + block_states={anchor_root: copy(anchor_state)}, + checkpoint_states={justified_checkpoint: copy(anchor_state)}, + ) +``` +```python +def get_slots_since_genesis(store: Store) -> int: + return (store.time - store.genesis_time) // SECONDS_PER_SLOT +``` +```python +def get_current_slot(store: Store) -> Slot: + return Slot(GENESIS_SLOT + get_slots_since_genesis(store)) +``` +```python +def compute_slots_since_epoch_start(slot: Slot) -> int: + return slot - compute_start_slot_at_epoch(compute_epoch_at_slot(slot)) +``` +```python +def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: + block = store.blocks[root] + if block.slot > slot: + return get_ancestor(store, block.parent_root, slot) + elif block.slot == slot: + return root + else: + # root is older than queried slot, thus a skip slot. Return most recent root prior to slot + return root +``` +```python +def get_latest_attesting_balance(store: Store, root: Root) -> Gwei: + state = store.checkpoint_states[store.justified_checkpoint] + active_indices = get_active_validator_indices(state, get_current_epoch(state)) + return Gwei(sum( + state.validators[i].effective_balance for i in active_indices + if (i in store.latest_messages + and get_ancestor(store, store.latest_messages[i].root, store.blocks[root].slot) == root) + )) +``` +```python +def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconBlock]) -> bool: + block = store.blocks[block_root] + children = [ + root for root in store.blocks.keys() + if store.blocks[root].parent_root == block_root + ] + + # If any children branches contain expected finalized/justified checkpoints, + # add to filtered block-tree and signal viability to parent. + if any(children): + filter_block_tree_result = [filter_block_tree(store, child, blocks) for child in children] + if any(filter_block_tree_result): + blocks[block_root] = block + return True + return False + + # If leaf block, check finalized/justified checkpoints as matching latest. + head_state = store.block_states[block_root] + + correct_justified = ( + store.justified_checkpoint.epoch == GENESIS_EPOCH + or head_state.current_justified_checkpoint == store.justified_checkpoint + ) + correct_finalized = ( + store.finalized_checkpoint.epoch == GENESIS_EPOCH + or head_state.finalized_checkpoint == store.finalized_checkpoint + ) + # If expected finalized/justified, add to viable block-tree and signal viability to parent. + if correct_justified and correct_finalized: + blocks[block_root] = block + return True + + # Otherwise, branch not viable + return False +``` +```python +def get_filtered_block_tree(store: Store) -> Dict[Root, BeaconBlock]: + """ + Retrieve a filtered block tree from ``store``, only returning branches + whose leaf state's justified/finalized info agrees with that in ``store``. + """ + base = store.justified_checkpoint.root + blocks: Dict[Root, BeaconBlock] = {} + filter_block_tree(store, base, blocks) + return blocks +``` +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + children = [ + root for root in blocks.keys() + if blocks[root].parent_root == head + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + head = max(children, key=lambda root: (get_latest_attesting_balance(store, root), root)) +``` +```python +def should_update_justified_checkpoint(store: Store, new_justified_checkpoint: Checkpoint) -> bool: + """ + To address the bouncing attack, only update conflicting justified + checkpoints in the fork choice if in the early slots of the epoch. + Otherwise, delay incorporation of new justified checkpoint until next epoch boundary. + + See https://ethresear.ch/t/prevention-of-bouncing-attack-on-ffg/6114 for more detailed analysis and discussion. + """ + if compute_slots_since_epoch_start(get_current_slot(store)) < SAFE_SLOTS_TO_UPDATE_JUSTIFIED: + return True + + justified_slot = compute_start_slot_at_epoch(store.justified_checkpoint.epoch) + if not get_ancestor(store, new_justified_checkpoint.root, justified_slot) == store.justified_checkpoint.root: + return False + + return True +``` +```python +def validate_on_attestation(store: Store, attestation: Attestation) -> None: + target = attestation.data.target + + # Attestations must be from the current or previous epoch + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + # Use GENESIS_EPOCH for previous when genesis to avoid underflow + previous_epoch = current_epoch - 1 if current_epoch > GENESIS_EPOCH else GENESIS_EPOCH + # If attestation target is from a future epoch, delay consideration until the epoch arrives + assert target.epoch in [current_epoch, previous_epoch] + assert target.epoch == compute_epoch_at_slot(attestation.data.slot) + + # Attestations target be for a known block. If target block is unknown, delay consideration until the block is found + assert target.root in store.blocks + + # Attestations must be for a known block. If block is unknown, delay consideration until the block is found + assert attestation.data.beacon_block_root in store.blocks + # Attestations must not be for blocks in the future. If not, the attestation should not be considered + assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot + + # LMD vote must be consistent with FFG vote target + target_slot = compute_start_slot_at_epoch(target.epoch) + assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + + # Attestations can only affect the fork choice of subsequent slots. + # Delay consideration in the fork choice until their slot is in the past. + assert get_current_slot(store) >= attestation.data.slot + 1 +``` +```python +def store_target_checkpoint_state(store: Store, target: Checkpoint) -> None: + # Store target checkpoint state if not yet seen + if target not in store.checkpoint_states: + base_state = copy(store.block_states[target.root]) + if base_state.slot < compute_start_slot_at_epoch(target.epoch): + process_slots(base_state, compute_start_slot_at_epoch(target.epoch)) + store.checkpoint_states[target] = base_state +``` +```python +def update_latest_messages(store: Store, attesting_indices: Sequence[ValidatorIndex], attestation: Attestation) -> None: + target = attestation.data.target + beacon_block_root = attestation.data.beacon_block_root + for i in attesting_indices: + if i not in store.latest_messages or target.epoch > store.latest_messages[i].epoch: + store.latest_messages[i] = LatestMessage(epoch=target.epoch, root=beacon_block_root) +``` +```python +def on_tick(store: Store, time: uint64) -> None: + previous_slot = get_current_slot(store) + + # update store time + store.time = time + + current_slot = get_current_slot(store) + # Not a new epoch, return + if not (current_slot > previous_slot and compute_slots_since_epoch_start(current_slot) == 0): + return + # Update store.justified_checkpoint if a better checkpoint is known + if store.best_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = store.best_justified_checkpoint +``` +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until the are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + + # Potentially update justified if different from store + if store.justified_checkpoint != state.current_justified_checkpoint: + # Update justified if new justified is later than store justified + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + store.justified_checkpoint = state.current_justified_checkpoint + return + + # Update justified if store justified is not in chain with finalized checkpoint + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = get_ancestor(store, store.justified_checkpoint.root, finalized_slot) + if ancestor_at_finalized_slot != store.finalized_checkpoint.root: + store.justified_checkpoint = state.current_justified_checkpoint +``` +```python +def on_attestation(store: Store, attestation: Attestation) -> None: + """ + Run ``on_attestation`` upon receiving a new ``attestation`` from either within a block or directly on the wire. + + An ``attestation`` that is asserted as invalid may be valid at a later time, + consider scheduling it for later processing in such case. + """ + validate_on_attestation(store, attestation) + store_target_checkpoint_state(store, attestation.data.target) + + # Get state at the `target` to fully validate attestation + target_state = store.checkpoint_states[attestation.data.target] + indexed_attestation = get_indexed_attestation(target_state, attestation) + assert is_valid_indexed_attestation(target_state, indexed_attestation) + + # Update latest messages for attesting indices + update_latest_messages(store, indexed_attestation.attesting_indices, attestation) +``` diff --git a/tools/specs-checker/data/specs/phase0/validator.md b/tools/specs-checker/data/specs/phase0/validator.md new file mode 100644 index 00000000000..a7c04c19449 --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/validator.md @@ -0,0 +1,150 @@ +```python +def check_if_validator_active(state: BeaconState, validator_index: ValidatorIndex) -> bool: + validator = state.validators[validator_index] + return is_active_validator(validator, get_current_epoch(state)) +``` +```python +def get_committee_assignment(state: BeaconState, + epoch: Epoch, + validator_index: ValidatorIndex + ) -> Optional[Tuple[Sequence[ValidatorIndex], CommitteeIndex, Slot]]: + """ + Return the committee assignment in the ``epoch`` for ``validator_index``. + ``assignment`` returned is a tuple of the following form: + * ``assignment[0]`` is the list of validators in the committee + * ``assignment[1]`` is the index to which the committee is assigned + * ``assignment[2]`` is the slot at which the committee is assigned + Return None if no assignment. + """ + next_epoch = Epoch(get_current_epoch(state) + 1) + assert epoch <= next_epoch + + start_slot = compute_start_slot_at_epoch(epoch) + committee_count_per_slot = get_committee_count_per_slot(state, epoch) + for slot in range(start_slot, start_slot + SLOTS_PER_EPOCH): + for index in range(committee_count_per_slot): + committee = get_beacon_committee(state, Slot(slot), CommitteeIndex(index)) + if validator_index in committee: + return committee, CommitteeIndex(index), Slot(slot) + return None +``` +```python +def is_proposer(state: BeaconState, validator_index: ValidatorIndex) -> bool: + return get_beacon_proposer_index(state) == validator_index +``` +```python +def get_epoch_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_RANDAO, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(compute_epoch_at_slot(block.slot), domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_time_at_slot(state: BeaconState, slot: Slot) -> uint64: + return uint64(state.genesis_time + slot * SECONDS_PER_SLOT) +``` +```python +def voting_period_start_time(state: BeaconState) -> uint64: + eth1_voting_period_start_slot = Slot(state.slot - state.slot % (EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH)) + return compute_time_at_slot(state, eth1_voting_period_start_slot) +``` +```python +def is_candidate_block(block: Eth1Block, period_start: uint64) -> bool: + return ( + block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE <= period_start + and block.timestamp + SECONDS_PER_ETH1_BLOCK * ETH1_FOLLOW_DISTANCE * 2 >= period_start + ) +``` +```python +def get_eth1_vote(state: BeaconState, eth1_chain: Sequence[Eth1Block]) -> Eth1Data: + period_start = voting_period_start_time(state) + # `eth1_chain` abstractly represents all blocks in the eth1 chain sorted by ascending block height + votes_to_consider = [ + get_eth1_data(block) for block in eth1_chain + if ( + is_candidate_block(block, period_start) + # Ensure cannot move back to earlier deposit contract states + and get_eth1_data(block).deposit_count >= state.eth1_data.deposit_count + ) + ] + + # Valid votes already cast during this period + valid_votes = [vote for vote in state.eth1_data_votes if vote in votes_to_consider] + + # Default vote on latest eth1 block data in the period range unless eth1 chain is not live + # Non-substantive casting for linter + state_eth1_data: Eth1Data = state.eth1_data + default_vote = votes_to_consider[len(votes_to_consider) - 1] if any(votes_to_consider) else state_eth1_data + + return max( + valid_votes, + key=lambda v: (valid_votes.count(v), -valid_votes.index(v)), # Tiebreak by smallest distance + default=default_vote + ) +``` +```python +def compute_new_state_root(state: BeaconState, block: BeaconBlock) -> Root: + temp_state: BeaconState = state.copy() + signed_block = SignedBeaconBlock(message=block) + state_transition(temp_state, signed_block, validate_result=False) + return hash_tree_root(temp_state) +``` +```python +def get_block_signature(state: BeaconState, block: BeaconBlock, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_PROPOSER, compute_epoch_at_slot(block.slot)) + signing_root = compute_signing_root(block, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def get_attestation_signature(state: BeaconState, attestation_data: AttestationData, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_BEACON_ATTESTER, attestation_data.target.epoch) + signing_root = compute_signing_root(attestation_data, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: + """ + Compute the correct subnet for an attestation for Phase 0. + Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. + """ + slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) + committees_since_epoch_start = committees_per_slot * slots_since_epoch_start + + return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) +``` +```python +def get_slot_signature(state: BeaconState, slot: Slot, privkey: int) -> BLSSignature: + domain = get_domain(state, DOMAIN_SELECTION_PROOF, compute_epoch_at_slot(slot)) + signing_root = compute_signing_root(slot, domain) + return bls.Sign(privkey, signing_root) +``` +```python +def is_aggregator(state: BeaconState, slot: Slot, index: CommitteeIndex, slot_signature: BLSSignature) -> bool: + committee = get_beacon_committee(state, slot, index) + modulo = max(1, len(committee) // TARGET_AGGREGATORS_PER_COMMITTEE) + return bytes_to_uint64(hash(slot_signature)[0:8]) % modulo == 0 +``` +```python +def get_aggregate_signature(attestations: Sequence[Attestation]) -> BLSSignature: + signatures = [attestation.signature for attestation in attestations] + return bls.Aggregate(signatures) +``` +```python +def get_aggregate_and_proof(state: BeaconState, + aggregator_index: ValidatorIndex, + aggregate: Attestation, + privkey: int) -> AggregateAndProof: + return AggregateAndProof( + aggregator_index=aggregator_index, + aggregate=aggregate, + selection_proof=get_slot_signature(state, aggregate.data.slot, privkey), + ) +``` +```python +def get_aggregate_and_proof_signature(state: BeaconState, + aggregate_and_proof: AggregateAndProof, + privkey: int) -> BLSSignature: + aggregate = aggregate_and_proof.aggregate + domain = get_domain(state, DOMAIN_AGGREGATE_AND_PROOF, compute_epoch_at_slot(aggregate.data.slot)) + signing_root = compute_signing_root(aggregate_and_proof, domain) + return bls.Sign(privkey, signing_root) +``` diff --git a/tools/specs-checker/data/specs/phase0/weak-subjectivity.md b/tools/specs-checker/data/specs/phase0/weak-subjectivity.md new file mode 100644 index 00000000000..88faf4f32d0 --- /dev/null +++ b/tools/specs-checker/data/specs/phase0/weak-subjectivity.md @@ -0,0 +1,44 @@ +```python +def compute_weak_subjectivity_period(state: BeaconState) -> uint64: + """ + Returns the weak subjectivity period for the current ``state``. + This computation takes into account the effect of: + - validator set churn (bounded by ``get_validator_churn_limit()`` per epoch), and + - validator balance top-ups (bounded by ``MAX_DEPOSITS * SLOTS_PER_EPOCH`` per epoch). + A detailed calculation can be found at: + https://github.com/runtimeverification/beacon-chain-verification/blob/master/weak-subjectivity/weak-subjectivity-analysis.pdf + """ + ws_period = MIN_VALIDATOR_WITHDRAWABILITY_DELAY + N = len(get_active_validator_indices(state, get_current_epoch(state))) + t = get_total_active_balance(state) // N // ETH_TO_GWEI + T = MAX_EFFECTIVE_BALANCE // ETH_TO_GWEI + delta = get_validator_churn_limit(state) + Delta = MAX_DEPOSITS * SLOTS_PER_EPOCH + D = SAFETY_DECAY + + if T * (200 + 3 * D) < t * (200 + 12 * D): + epochs_for_validator_set_churn = ( + N * (t * (200 + 12 * D) - T * (200 + 3 * D)) // (600 * delta * (2 * t + T)) + ) + epochs_for_balance_top_ups = ( + N * (200 + 3 * D) // (600 * Delta) + ) + ws_period += max(epochs_for_validator_set_churn, epochs_for_balance_top_ups) + else: + ws_period += ( + 3 * N * D * t // (200 * Delta * (T - t)) + ) + + return ws_period +``` +```python +def is_within_weak_subjectivity_period(store: Store, ws_state: BeaconState, ws_checkpoint: Checkpoint) -> bool: + # Clients may choose to validate the input state against the input Weak Subjectivity Checkpoint + assert ws_state.latest_block_header.state_root == ws_checkpoint.root + assert compute_epoch_at_slot(ws_state.slot) == ws_checkpoint.epoch + + ws_period = compute_weak_subjectivity_period(ws_state) + ws_state_epoch = compute_epoch_at_slot(ws_state.slot) + current_epoch = compute_epoch_at_slot(get_current_slot(store)) + return current_epoch <= ws_state_epoch + ws_period +``` diff --git a/tools/specs-checker/data/ssz/merkle-proofs.md b/tools/specs-checker/data/ssz/merkle-proofs.md new file mode 100644 index 00000000000..fa09733f407 --- /dev/null +++ b/tools/specs-checker/data/ssz/merkle-proofs.md @@ -0,0 +1,240 @@ +```python +def get_power_of_two_ceil(x: int) -> int: + """ + Get the power of 2 for given input, or the closest higher power of 2 if the input is not a power of 2. + Commonly used for "how many nodes do I need for a bottom tree layer fitting x elements?" + Example: 0->1, 1->1, 2->2, 3->4, 4->4, 5->8, 6->8, 7->8, 8->8, 9->16. + """ + if x <= 1: + return 1 + elif x == 2: + return 2 + else: + return 2 * get_power_of_two_ceil((x + 1) // 2) +``` +```python +def get_power_of_two_floor(x: int) -> int: + """ + Get the power of 2 for given input, or the closest lower power of 2 if the input is not a power of 2. + The zero case is a placeholder and not used for math with generalized indices. + Commonly used for "what power of two makes up the root bit of the generalized index?" + Example: 0->1, 1->1, 2->2, 3->2, 4->4, 5->4, 6->4, 7->4, 8->8, 9->8 + """ + if x <= 1: + return 1 + if x == 2: + return x + else: + return 2 * get_power_of_two_floor(x // 2) +``` +```python +def merkle_tree(leaves: Sequence[Bytes32]) -> Sequence[Bytes32]: + """ + Return an array representing the tree nodes by generalized index: + [0, 1, 2, 3, 4, 5, 6, 7], where each layer is a power of 2. The 0 index is ignored. The 1 index is the root. + The result will be twice the size as the padded bottom layer for the input leaves. + """ + bottom_length = get_power_of_two_ceil(len(leaves)) + o = [Bytes32()] * bottom_length + list(leaves) + [Bytes32()] * (bottom_length - len(leaves)) + for i in range(bottom_length - 1, 0, -1): + o[i] = hash(o[i * 2] + o[i * 2 + 1]) + return o +``` +```python +def item_length(typ: SSZType) -> int: + """ + Return the number of bytes in a basic type, or 32 (a full hash) for compound types. + """ + if issubclass(typ, BasicValue): + return typ.byte_len + else: + return 32 +``` +```python +def get_elem_type(typ: Union[BaseBytes, BaseList, Container], + index_or_variable_name: Union[int, SSZVariableName]) -> SSZType: + """ + Return the type of the element of an object of the given type with the given index + or member variable name (eg. `7` for `x[7]`, `"foo"` for `x.foo`) + """ + return typ.get_fields()[index_or_variable_name] if issubclass(typ, Container) else typ.elem_type +``` +```python +def chunk_count(typ: SSZType) -> int: + """ + Return the number of hashes needed to represent the top-level elements in the given type + (eg. `x.foo` or `x[7]` but not `x[7].bar` or `x.foo.baz`). In all cases except lists/vectors + of basic types, this is simply the number of top-level elements, as each element gets one + hash. For lists/vectors of basic types, it is often fewer because multiple basic elements + can be packed into one 32-byte chunk. + """ + # typ.length describes the limit for list types, or the length for vector types. + if issubclass(typ, BasicValue): + return 1 + elif issubclass(typ, Bits): + return (typ.length + 255) // 256 + elif issubclass(typ, Elements): + return (typ.length * item_length(typ.elem_type) + 31) // 32 + elif issubclass(typ, Container): + return len(typ.get_fields()) + else: + raise Exception(f"Type not supported: {typ}") +``` +```python +def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariableName]) -> Tuple[int, int, int]: + """ + Return three variables: + (i) the index of the chunk in which the given element of the item is represented; + (ii) the starting byte position within the chunk; + (iii) the ending byte position within the chunk. + For example: for a 6-item list of uint64 values, index=2 will return (0, 16, 24), index=5 will return (1, 8, 16) + """ + if issubclass(typ, Elements): + index = int(index_or_variable_name) + start = index * item_length(typ.elem_type) + return start // 32, start % 32, start % 32 + item_length(typ.elem_type) + elif issubclass(typ, Container): + variable_name = index_or_variable_name + return typ.get_field_names().index(variable_name), 0, item_length(get_elem_type(typ, variable_name)) + else: + raise Exception("Only lists/vectors/containers supported") +``` +```python +def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: + """ + Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for + `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. + """ + root = GeneralizedIndex(1) + for p in path: + assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further + if p == '__len__': + typ = uint64 + assert issubclass(typ, (List, ByteList)) + root = GeneralizedIndex(root * 2 + 1) + else: + pos, _, _ = get_item_position(typ, p) + base_index = (GeneralizedIndex(2) if issubclass(typ, (List, ByteList)) else GeneralizedIndex(1)) + root = GeneralizedIndex(root * base_index * get_power_of_two_ceil(chunk_count(typ)) + pos) + typ = get_elem_type(typ, p) + return root +``` +```python +def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: + """ + Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns + the generalized index for A -> Z. + """ + o = GeneralizedIndex(1) + for i in indices: + o = GeneralizedIndex(o * get_power_of_two_floor(i) + (i - get_power_of_two_floor(i))) + return o +``` +```python +def get_generalized_index_length(index: GeneralizedIndex) -> int: + """ + Return the length of a path represented by a generalized index. + """ + return int(log2(index)) +``` +```python +def get_generalized_index_bit(index: GeneralizedIndex, position: int) -> bool: + """ + Return the given bit of a generalized index. + """ + return (index & (1 << position)) > 0 +``` +```python +def generalized_index_sibling(index: GeneralizedIndex) -> GeneralizedIndex: + return GeneralizedIndex(index ^ 1) +``` +```python +def generalized_index_child(index: GeneralizedIndex, right_side: bool) -> GeneralizedIndex: + return GeneralizedIndex(index * 2 + right_side) +``` +```python +def generalized_index_parent(index: GeneralizedIndex) -> GeneralizedIndex: + return GeneralizedIndex(index // 2) +``` +```python +def get_branch_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of the sister chunks along the path from the chunk with the + given tree index to the root. + """ + o = [generalized_index_sibling(tree_index)] + while o[-1] > 1: + o.append(generalized_index_sibling(generalized_index_parent(o[-1]))) + return o[:-1] +``` +```python +def get_path_indices(tree_index: GeneralizedIndex) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of the chunks along the path from the chunk with the + given tree index to the root. + """ + o = [tree_index] + while o[-1] > 1: + o.append(generalized_index_parent(o[-1])) + return o[:-1] +``` +```python +def get_helper_indices(indices: Sequence[GeneralizedIndex]) -> Sequence[GeneralizedIndex]: + """ + Get the generalized indices of all "extra" chunks in the tree needed to prove the chunks with the given + generalized indices. Note that the decreasing order is chosen deliberately to ensure equivalence to the + order of hashes in a regular single-item Merkle proof in the single-item case. + """ + all_helper_indices: Set[GeneralizedIndex] = set() + all_path_indices: Set[GeneralizedIndex] = set() + for index in indices: + all_helper_indices = all_helper_indices.union(set(get_branch_indices(index))) + all_path_indices = all_path_indices.union(set(get_path_indices(index))) + + return sorted(all_helper_indices.difference(all_path_indices), reverse=True) +``` +```python +def calculate_merkle_root(leaf: Bytes32, proof: Sequence[Bytes32], index: GeneralizedIndex) -> Root: + assert len(proof) == get_generalized_index_length(index) + for i, h in enumerate(proof): + if get_generalized_index_bit(index, i): + leaf = hash(h + leaf) + else: + leaf = hash(leaf + h) + return leaf +``` +```python +def verify_merkle_proof(leaf: Bytes32, proof: Sequence[Bytes32], index: GeneralizedIndex, root: Root) -> bool: + return calculate_merkle_root(leaf, proof, index) == root +``` +```python +def calculate_multi_merkle_root(leaves: Sequence[Bytes32], + proof: Sequence[Bytes32], + indices: Sequence[GeneralizedIndex]) -> Root: + assert len(leaves) == len(indices) + helper_indices = get_helper_indices(indices) + assert len(proof) == len(helper_indices) + objects = { + **{index: node for index, node in zip(indices, leaves)}, + **{index: node for index, node in zip(helper_indices, proof)} + } + keys = sorted(objects.keys(), reverse=True) + pos = 0 + while pos < len(keys): + k = keys[pos] + if k in objects and k ^ 1 in objects and k // 2 not in objects: + objects[GeneralizedIndex(k // 2)] = hash( + objects[GeneralizedIndex((k | 1) ^ 1)] + + objects[GeneralizedIndex(k | 1)] + ) + keys.append(GeneralizedIndex(k // 2)) + pos += 1 + return objects[GeneralizedIndex(1)] +``` +```python +def verify_merkle_multiproof(leaves: Sequence[Bytes32], + proof: Sequence[Bytes32], + indices: Sequence[GeneralizedIndex], + root: Root) -> bool: + return calculate_multi_merkle_root(leaves, proof, indices) == root +``` diff --git a/tools/specs-checker/download.go b/tools/specs-checker/download.go new file mode 100644 index 00000000000..3496872225b --- /dev/null +++ b/tools/specs-checker/download.go @@ -0,0 +1,83 @@ +package main + +import ( + _ "embed" + "fmt" + "io/ioutil" + "net/http" + "os" + "path" + "regexp" + + "github.com/urfave/cli/v2" +) + +const baseUrl = "https://raw.githubusercontent.com/ethereum/eth2.0-specs/dev" + +// Regex to find Python's code snippets in markdown. +var reg2 = regexp.MustCompile(`(?msU)^\x60\x60\x60python\n+def\s(.*)^\x60\x60\x60`) + +func download(cliCtx *cli.Context) error { + fmt.Print("Downloading specs:\n") + baseDir := cliCtx.String(dirFlag.Name) + for dirName, fileNames := range specDirs { + if err := prepareDir(path.Join(baseDir, dirName)); err != nil { + return err + } + for _, fileName := range fileNames { + outFilePath := path.Join(baseDir, dirName, fileName) + specDocUrl := fmt.Sprintf("%s/%s", baseUrl, fmt.Sprintf("%s/%s", dirName, fileName)) + fmt.Printf("- %s\n", specDocUrl) + if err := getAndSaveFile(specDocUrl, outFilePath); err != nil { + return err + } + } + } + + return nil +} + +func getAndSaveFile(specDocUrl, outFilePath string) error { + // Create output file. + f, err := os.Create(outFilePath) + if err != nil { + return fmt.Errorf("cannot create output file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + fmt.Printf("cannot close output file: %v", err) + } + }() + + // Download spec doc. + resp, err := http.Get(specDocUrl) + if err != nil { + return err + } + defer func() { + if err := resp.Body.Close(); err != nil { + fmt.Printf("cannot close spec doc file: %v", err) + } + }() + + // Transform and save spec docs. + specDoc, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + specDocString := string(specDoc) + for _, snippet := range reg2.FindAllString(specDocString, -1) { + if _, err = f.WriteString(snippet + "\n"); err != nil { + return err + } + } + + return nil +} + +func prepareDir(dirPath string) error { + if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { + return err + } + return nil +} diff --git a/tools/specs-checker/main.go b/tools/specs-checker/main.go new file mode 100644 index 00000000000..fee43e56640 --- /dev/null +++ b/tools/specs-checker/main.go @@ -0,0 +1,64 @@ +package main + +import ( + "embed" + "log" + "os" + + "github.com/urfave/cli/v2" +) + +var ( + dirFlag = &cli.StringFlag{ + Name: "dir", + Value: "", + Usage: "Target directory", + Required: true, + } +) + +//go:embed data +var specFS embed.FS + +var specDirs = map[string][]string{ + "specs/phase0": { + "beacon-chain.md", + "fork-choice.md", + "validator.md", + "weak-subjectivity.md", + }, + "ssz": { + "merkle-proofs.md", + }, +} + +func main() { + app := &cli.App{ + Name: "Specs checker utility", + Description: "Checks that specs pseudo code used in comments is up to date", + Usage: "helps keeping specs pseudo code up to date!", + Commands: []*cli.Command{ + { + Name: "check", + Usage: "Checks that all doc strings", + Flags: []cli.Flag{ + dirFlag, + }, + Action: check, + }, + { + Name: "download", + Usage: "Downloads the latest specs docs", + Action: download, + Flags: []cli.Flag{ + dirFlag, + }, + }, + }, + } + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +}