From 3dc7430ae98d8f540dae3bbb4e9a2dcb31a2dbf2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 27 Jul 2019 08:17:06 -0400 Subject: [PATCH 01/44] Starting on phase 1 misc beacon changes --- specs/core/1_beacon_chain_misc.md | 75 +++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 specs/core/1_beacon_chain_misc.md diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md new file mode 100644 index 0000000000..09e3c2c619 --- /dev/null +++ b/specs/core/1_beacon_chain_misc.md @@ -0,0 +1,75 @@ +# Phase 1 miscellaneous beacon chain changes + +## Table of contents + + + +- [Helpers](#helpers) + - [pack_compact_validator](#pack_compact_validator) + - [unpack_compact_validator](#unpack_compact_validator) + - [committee_to_compact_committee](#committee_to_compact_committee) +- [Changes](#changes) + - [Persistent committees](#persistent-committees) + + + +## Helpers + +#### `pack_compact_validator` + +```python +def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: uint64) -> uint64: + """ + Creates a compact validator object representing index, slashed status, and compressed balance. + Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with + the unpacking function. + """ + return (index << 16) + (slashed << 15) + balance_in_increments +``` + +### `unpack_compact_validator` + +```python +def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: + """ + Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT + """ + return compact_validator >> 16, (compact_validator >> 15) % 2, compact_validator & (2**15 - 1) +``` + +#### `committee_to_compact_committee` + +```python +def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + validators = [state.validators[i] for i in committee] + compact_validators = [ + pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) + for i, v in zip(committee, validators) + ] + pubkeys = [v.pubkey for v in validators] + return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) +``` + +## Changes + +### Persistent committees + +Add to the beacon state the following fields: + +* `previous_persistent_committee_root: Hash` +* `current_persistent_committee_root: Hash` +* `next_persistent_committee_root: Hash` + +Process the following function before `process_final_updates`: + +```python +def update_persistent_committee(state: BeaconState): + if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: + state.previous_persistent_committee_root = state.current_persistent_committee_root + state.current_persistent_committee_root = state.next_persistent_committee_root + committees = Vector[CompactCommittee, SHARD_COUNT]([ + committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) + for i in range(SHARD_COUNT) + ]) + state.next_persistent_committee_root = hash_tree_root(committees) +``` From fe9fe8a1518ca890df861210f163c82be6c22852 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 10:01:34 -0400 Subject: [PATCH 02/44] [WIP] add receipt processing to phase 1 beacon chain --- specs/core/1_beacon_chain_misc.md | 60 +++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 09e3c2c619..b5b76a9a9c 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -50,6 +50,65 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` +#### `get_previous_power_of_2` + +```python +def get_previous_power_of_2(x: int) -> int: + return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) +``` + + +#### `concat_generalized_indices` + +```python +def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: + o = GeneralizedIndex(1) + for i in indices: + o = o * get_previous_power_of_2(i) + i + return o +``` + +#### `compute_historical_state_generalized_index` + +```python +def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> GeneralizedIndex: + o = GeneralizedIndex(1) + for i in range(63, -1, -1): + if (to-1) & 2**i > (frm-1) & 2**i: + to = to - ((to-1) % 2**i) - 1 + o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + return o +``` + +#### `get_generalized_index_of_crosslink_header` + +```python +def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: + MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) + return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) +``` + +#### `process_shard_receipt` + +```python +def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PLACEHOLDER], receipt: List[ShardReceiptDelta, PLACEHOLDER]): + receipt_slot = state.next_shard_receipt_period[shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + first_slot_in_last_crosslink = state.current_crosslinks[shard].start_epoch * SLOTS_PER_EPOCH + gindex = concat_generalized_indices( + get_generalized_index_of_crosslink_header(0), + get_generalized_index(ShardBlockHeader, 'state_root') + compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink) + get_generalized_index(ShardState, 'receipt_root') + ) + assert verify_merkle_proof( + leaf=hash_tree_root(receipt), + proof=proof, + index=gindex, + root=state.current_crosslinks[shard].data_root + ) +``` + ## Changes ### Persistent committees @@ -59,6 +118,7 @@ Add to the beacon state the following fields: * `previous_persistent_committee_root: Hash` * `current_persistent_committee_root: Hash` * `next_persistent_committee_root: Hash` +* `next_shard_receipt_period: Vector[uint, SHARD_COUNT]`, values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` Process the following function before `process_final_updates`: From caadc0d2349b464588b9c57a168bb58e913d2b12 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 10:15:18 -0400 Subject: [PATCH 03/44] Update 1_beacon_chain_misc.md --- specs/core/1_beacon_chain_misc.md | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index b5b76a9a9c..9e81acadfc 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -13,6 +13,17 @@ +## Classes + +#### `ShardReceiptProof` + +```python +class ShardReceiptProof(Container): + shard: Shard + proof: List[Hash, PLACEHOLDER] + receipt: List[ShardReceiptDelta, PLACEHOLDER] +``` + ## Helpers #### `pack_compact_validator` @@ -27,7 +38,7 @@ def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: return (index << 16) + (slashed << 15) + balance_in_increments ``` -### `unpack_compact_validator` +#### `unpack_compact_validator` ```python def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: @@ -92,9 +103,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: #### `process_shard_receipt` ```python -def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PLACEHOLDER], receipt: List[ShardReceiptDelta, PLACEHOLDER]): - receipt_slot = state.next_shard_receipt_period[shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[shard].start_epoch * SLOTS_PER_EPOCH +def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): + receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), get_generalized_index(ShardBlockHeader, 'state_root') @@ -102,11 +113,16 @@ def process_shard_receipt(state: BeaconState, shard: Shard, proof: List[Hash, PL get_generalized_index(ShardState, 'receipt_root') ) assert verify_merkle_proof( - leaf=hash_tree_root(receipt), - proof=proof, + leaf=hash_tree_root(receipt_proof.receipt), + proof=receipt_proof.proof, index=gindex, root=state.current_crosslinks[shard].data_root ) + for delta in receipt_proof.receipt: + increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + decrease_balance(state, delta.index, delta.block_fee) + state.next_shard_receipt_period[receipt_proof.shard] += 1 + increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) ``` ## Changes @@ -133,3 +149,11 @@ def update_persistent_committee(state: BeaconState): ]) state.next_persistent_committee_root = hash_tree_root(committees) ``` + +### Shard receipt processing + +Add to the beacon block body the following object: + +* `shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS]` + +Use `process_shard_receipt` to process each receipt. From 3f20aca65ee4759f72c865cfc6a5338a48189bf9 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:55:58 -0400 Subject: [PATCH 04/44] Added comments and ToC --- specs/core/1_beacon_chain_misc.md | 49 +++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 3 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 9e81acadfc..ceb5148786 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -4,12 +4,21 @@ +- [Classes](#classes) + - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) - [unpack_compact_validator](#unpack_compact_validator) - [committee_to_compact_committee](#committee_to_compact_committee) + - [get_previous_power_of_2](#get_previous_power_of_2) + - [verify_merkle_proof](#verify_merkle_proof) + - [concat_generalized_indices](#concat_generalized_indices) + - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) + - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) + - [process_shard_receipt](#process_shard_receipt) - [Changes](#changes) - [Persistent committees](#persistent-committees) + - [Shard receipt processing](#shard-receipt-processing) @@ -52,6 +61,9 @@ def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, u ```python def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee: + """ + Given a state and a list of validator indices, outputs the CompactCommittee representing them. + """ validators = [state.validators[i] for i in committee] compact_validators = [ pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT) @@ -68,11 +80,27 @@ def get_previous_power_of_2(x: int) -> int: return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) ``` +#### `verify_merkle_proof` + +```python +def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: + assert len(proof) == log2(index) + for i, h in enumerate(proof): + if index & 2**i: + leaf = hash(h + leaf) + else: + leaf = hash(leaf + h) + return leaf == root +``` #### `concat_generalized_indices` ```python def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: + """ + Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns + the generalized index for A -> Z. + """ o = GeneralizedIndex(1) for i in indices: o = o * get_previous_power_of_2(i) + i @@ -82,11 +110,17 @@ def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> Generali #### `compute_historical_state_generalized_index` ```python -def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> GeneralizedIndex: +def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardSlot) -> GeneralizedIndex: + """ + Computes the generalized index of the state root of slot `frm` based on the state root of slot `to`. + Relies on the `history_acc` in the `ShardState`, where `history_acc[i]` maintains the most recent 2**i'th + slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` through intermediate + blocks at the next available multiples of descending powers of two. + """ o = GeneralizedIndex(1) for i in range(63, -1, -1): - if (to-1) & 2**i > (frm-1) & 2**i: - to = to - ((to-1) % 2**i) - 1 + if (later-1) & 2**i > (earlier-1) & 2**i: + later = later - ((later-1) % 2**i) - 1 o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) return o ``` @@ -95,6 +129,9 @@ def compute_historical_state_generalized_index(frm: ShardSlot, to: ShardSlot) -> ```python def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: + """ + Gets the generalized index for the root of the index'th header in a crosslink. + """ MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) @@ -104,6 +141,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: ```python def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): + """ + Processes a ShardReceipt object. + """ receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( @@ -140,6 +180,9 @@ Process the following function before `process_final_updates`: ```python def update_persistent_committee(state: BeaconState): + """ + Updates persistent committee roots at boundary blocks. + """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: state.previous_persistent_committee_root = state.current_persistent_committee_root state.current_persistent_committee_root = state.next_persistent_committee_root From 5dad213457636a9a6d17002820a3559c6015076b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:58:33 -0400 Subject: [PATCH 05/44] Fixed up ToC --- specs/core/1_beacon_chain_misc.md | 32 ++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index ceb5148786..3798d1d40a 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -4,21 +4,23 @@ -- [Classes](#classes) - - [ShardReceiptProof](#shardreceiptproof) -- [Helpers](#helpers) - - [pack_compact_validator](#pack_compact_validator) - - [unpack_compact_validator](#unpack_compact_validator) - - [committee_to_compact_committee](#committee_to_compact_committee) - - [get_previous_power_of_2](#get_previous_power_of_2) - - [verify_merkle_proof](#verify_merkle_proof) - - [concat_generalized_indices](#concat_generalized_indices) - - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt](#process_shard_receipt) -- [Changes](#changes) - - [Persistent committees](#persistent-committees) - - [Shard receipt processing](#shard-receipt-processing) +- [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) + - [Table of contents](#table-of-contents) + - [Classes](#classes) + - [ShardReceiptProof](#shardreceiptproof) + - [Helpers](#helpers) + - [pack_compact_validator](#pack_compact_validator) + - [unpack_compact_validator](#unpack_compact_validator) + - [committee_to_compact_committee](#committee_to_compact_committee) + - [get_previous_power_of_2](#get_previous_power_of_2) + - [verify_merkle_proof](#verify_merkle_proof) + - [concat_generalized_indices](#concat_generalized_indices) + - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) + - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) + - [process_shard_receipt](#process_shard_receipt) + - [Changes](#changes) + - [Persistent committees](#persistent-committees) + - [Shard receipt processing](#shard-receipt-processing) From fab37e747a44d79f75b08d869b35fda972421bba Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 30 Jul 2019 11:59:12 -0400 Subject: [PATCH 06/44] Fixed position of Classes --- specs/core/1_beacon_chain_misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 3798d1d40a..673f99ce4c 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -6,7 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - - [Classes](#classes) + - [Classes](#classes) - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) From b9fddfe3103a77fb7502af40f2c0755aea65431d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 1 Aug 2019 10:58:15 -0400 Subject: [PATCH 07/44] Made code work with #1186 --- specs/core/1_beacon_chain_misc.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon_chain_misc.md index 673f99ce4c..4aa1796b65 100644 --- a/specs/core/1_beacon_chain_misc.md +++ b/specs/core/1_beacon_chain_misc.md @@ -86,29 +86,15 @@ def get_previous_power_of_2(x: int) -> int: ```python def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool: - assert len(proof) == log2(index) + assert len(proof) == get_generalized_index_length(index) for i, h in enumerate(proof): - if index & 2**i: + if get_generalized_index_bit(index, i): leaf = hash(h + leaf) else: leaf = hash(leaf + h) return leaf == root ``` -#### `concat_generalized_indices` - -```python -def concat_generalized_indices(*indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: - """ - Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns - the generalized index for A -> Z. - """ - o = GeneralizedIndex(1) - for i in indices: - o = o * get_previous_power_of_2(i) + i - return o -``` - #### `compute_historical_state_generalized_index` ```python From a273d9e09d27238f9617394ef3e3572915af0b39 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 09:13:09 -0600 Subject: [PATCH 08/44] minor rename of beacon chain misc to conform to other files --- specs/core/{1_beacon_chain_misc.md => 1_beacon-chain-misc.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename specs/core/{1_beacon_chain_misc.md => 1_beacon-chain-misc.md} (100%) diff --git a/specs/core/1_beacon_chain_misc.md b/specs/core/1_beacon-chain-misc.md similarity index 100% rename from specs/core/1_beacon_chain_misc.md rename to specs/core/1_beacon-chain-misc.md From 2ae7323183fca93557fee81ed8bb5ad290585d2a Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 09:42:39 -0600 Subject: [PATCH 09/44] use codeblock syntax for misc beacon updates --- specs/core/0_beacon-chain.md | 5 +++++ specs/core/1_beacon-chain-misc.md | 34 +++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index f0169f1d20..16d8048aaa 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -480,6 +480,7 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] transfers: List[Transfer, MAX_TRANSFERS] + # @shard_receipts ``` #### `BeaconBlock` @@ -533,6 +534,8 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint + + # @persistent_committee_fields ``` ## Helper functions @@ -1237,6 +1240,7 @@ def process_epoch(state: BeaconState) -> None: # @process_reveal_deadlines # @process_challenge_deadlines process_slashings(state) + # @update_persistent_committee process_final_updates(state) # @after_process_final_updates ``` @@ -1598,6 +1602,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), + # @process_shard_receipts ): for operation in operations: function(state, operation) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 4aa1796b65..f816c62595 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -159,15 +159,23 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): Add to the beacon state the following fields: -* `previous_persistent_committee_root: Hash` -* `current_persistent_committee_root: Hash` -* `next_persistent_committee_root: Hash` -* `next_shard_receipt_period: Vector[uint, SHARD_COUNT]`, values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` +```python +# begin insert @persistent_committee_fields + previous_persistent_committee_root: Hash + current_persistent_committee_root: Hash + next_persistent_committee_root: Hash + next_shard_receipt_period: Vector[uint, SHARD_COUNT] +# end insert @persistent_committee_fields +``` +`next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` -Process the following function before `process_final_updates`: +Run `update_persistent_committee` immediately before `process_final_updates`: ```python -def update_persistent_committee(state: BeaconState): +# begin insert @update_persistent_committee + update_persistent_committee(state) +# end insert @update_persistent_committee +def update_persistent_committee(state: BeaconState) -> None: """ Updates persistent committee roots at boundary blocks. """ @@ -183,8 +191,18 @@ def update_persistent_committee(state: BeaconState): ### Shard receipt processing -Add to the beacon block body the following object: +Add the `shard_receipts` operation to `BeaconBlockBody`: -* `shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS]` +```python +# begin insert @shard_receipts + shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS] +# end insert @shard_receipts +``` Use `process_shard_receipt` to process each receipt. + +```python +# begin insert @process_shard_receipts + (body.shard_receipts, process_shard_receipts), +# end insert @process_shard_receipts +``` From 6560bc42d2fc877cddf044c0a993efde74ee7b72 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 13:39:32 -0600 Subject: [PATCH 10/44] Apply suggestions from code review Co-Authored-By: Carl Beekhuizen --- specs/core/1_beacon-chain-misc.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index f816c62595..d1ea48dbaa 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -30,9 +30,9 @@ ```python class ShardReceiptProof(Container): - shard: Shard - proof: List[Hash, PLACEHOLDER] - receipt: List[ShardReceiptDelta, PLACEHOLDER] + shard: Shard + proof: List[Hash, PLACEHOLDER] + receipt: List[ShardReceiptDelta, PLACEHOLDER] ``` ## Helpers @@ -107,9 +107,9 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS """ o = GeneralizedIndex(1) for i in range(63, -1, -1): - if (later-1) & 2**i > (earlier-1) & 2**i: - later = later - ((later-1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + if (later - 1) & 2**i > (earlier - 1) & 2**i: + later = later - ((later - 1) % 2**i) - 1 + o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) return o ``` @@ -136,8 +136,8 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), - get_generalized_index(ShardBlockHeader, 'state_root') - compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink) + get_generalized_index(ShardBlockHeader, 'state_root'), + compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink), get_generalized_index(ShardState, 'receipt_root') ) assert verify_merkle_proof( From 0fa4491c412040ff3feaa9c2c76c5c07d5f63c69 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Thu, 1 Aug 2019 13:45:07 -0600 Subject: [PATCH 11/44] lint --- specs/core/0_beacon-chain.md | 1 - 1 file changed, 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 16d8048aaa..5ddceebc1e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -534,7 +534,6 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint - # @persistent_committee_fields ``` From 7132778a6993d12175c8870c933bcc7f53240848 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 6 Aug 2019 10:14:45 -0400 Subject: [PATCH 12/44] Added compact committee class --- specs/core/1_beacon-chain-misc.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index d1ea48dbaa..a8768d8b5a 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -7,6 +7,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - [Classes](#classes) + - [CompactCommittee](#compactcommittee) - [ShardReceiptProof](#shardreceiptproof) - [Helpers](#helpers) - [pack_compact_validator](#pack_compact_validator) @@ -26,6 +27,14 @@ ## Classes +#### `CompactCommittee` + +```python +class CompactCommittee(Container): + pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE] + compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE] +``` + #### `ShardReceiptProof` ```python From 36a22830526fed478a143c85c6cb2e4277b957b2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 7 Aug 2019 12:13:02 -0400 Subject: [PATCH 13/44] Shard receipts cannot penalize withdrawn validators --- specs/core/1_beacon-chain-misc.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index a8768d8b5a..c1d6283ca9 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -156,8 +156,9 @@ def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): root=state.current_crosslinks[shard].data_root ) for delta in receipt_proof.receipt: - increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) - decrease_balance(state, delta.index, delta.block_fee) + if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch: + increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) ``` From c5acddc071fd64603a0ed5ca3442b25442fb203b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 11 Aug 2019 22:47:23 +0800 Subject: [PATCH 14/44] Enable it in CI --- Makefile | 2 +- scripts/build_spec.py | 27 +++++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 318056689d..3f6cf9816c 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 96866cc8a8..5c353ede03 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -263,16 +263,18 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, def build_phase1_spec(phase0_sourcefile: str, + phase1_phase1_shard_misc_source_file: str, phase1_custody_sourcefile: str, phase1_shard_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) + phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_custody, phase1_shard_data, fork_choice_spec]: + for value in [phase1_shard_misc, phase1_custody, phase1_shard_data, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: @@ -285,17 +287,18 @@ def build_phase1_spec(phase0_sourcefile: str, description = ''' Build the specs from the md docs. If building phase 0: - 1st argument is input spec.md - 2nd argument is input fork_choice.md - 3rd argument is input validator_guide.md + 1st argument is input 0_beacon-chain.md + 2nd argument is input 0_fork-choice.md + 3rd argument is input 0_beacon-chain-validator.md 4th argument is output spec.py If building phase 1: - 1st argument is input spec_phase0.md - 2nd argument is input spec_phase1_custody.md - 3rd argument is input spec_phase1_shard_data.md - 4th argument is input fork_choice.md - 5th argument is output spec.py + 1st argument is input 0_beacon-chain.md + 2nd argument is input 1_shard-chain-misc.md + 3rd argument is input 1_custody-game.md + 4th argument is input 1_shard-data-chains.md + 5th argument is input 0_fork-choice.md + 6th argument is output spec.py ''' parser = ArgumentParser(description=description) parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #") @@ -308,10 +311,10 @@ def build_phase1_spec(phase0_sourcefile: str, else: print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.") elif args.phase == 1: - if len(args.files) == 5: + if len(args.files) == 6: build_phase1_spec(*args.files) else: - print(" Phase 1 requires 4 input files as well as an output file: " - + "(phase0.md and phase1.md, phase1.md, fork_choice.md, output.py)") + print(" Phase 1 requires 6 input files as well as an output file: " + + "(0_fork-choice.md, 0_beacon-chain.md and 1_shard-chain-misc.md, 1_custody-game.md, 1_shard-data-chains.md, output.py)") else: print("Invalid phase: {0}".format(args.phase)) From e4e6c4d8eef1839394a4a6aba183c4cf4fd77d5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 12 Aug 2019 00:45:33 +0800 Subject: [PATCH 15/44] Fix the order of build spec --- Makefile | 2 +- scripts/build_spec.py | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 3f6cf9816c..f2d953f9a6 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS) python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS) - python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/0_fork-choice.md $@ + python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $(SPEC_DIR)/core/0_fork-choice.md $@ CURRENT_DIR = ${CURDIR} diff --git a/scripts/build_spec.py b/scripts/build_spec.py index 6f0f160ac1..44e73224f1 100644 --- a/scripts/build_spec.py +++ b/scripts/build_spec.py @@ -269,12 +269,12 @@ def build_phase1_spec(phase0_sourcefile: str, fork_choice_sourcefile: str, outfile: str=None) -> Optional[str]: phase0_spec = get_spec(phase0_sourcefile) - phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) phase1_custody = get_spec(phase1_custody_sourcefile) phase1_shard_data = get_spec(phase1_shard_sourcefile) + phase1_shard_misc = get_spec(phase1_phase1_shard_misc_source_file) fork_choice_spec = get_spec(fork_choice_sourcefile) spec_objects = phase0_spec - for value in [phase1_shard_misc, phase1_custody, phase1_shard_data, fork_choice_spec]: + for value in [phase1_custody, phase1_shard_data, phase1_shard_misc, fork_choice_spec]: spec_objects = combine_spec_objects(spec_objects, value) spec = objects_to_spec(*spec_objects, PHASE1_IMPORTS) if outfile is not None: @@ -294,9 +294,9 @@ def build_phase1_spec(phase0_sourcefile: str, If building phase 1: 1st argument is input 0_beacon-chain.md - 2nd argument is input 1_shard-chain-misc.md - 3rd argument is input 1_custody-game.md - 4th argument is input 1_shard-data-chains.md + 2nd argument is input 1_custody-game.md + 3rd argument is input 1_shard-data-chains.md + 4th argument is input 1_shard-chain-misc.md 5th argument is input 0_fork-choice.md 6th argument is output spec.py ''' @@ -315,6 +315,6 @@ def build_phase1_spec(phase0_sourcefile: str, build_phase1_spec(*args.files) else: print(" Phase 1 requires 6 input files as well as an output file: " - + "(0_fork-choice.md, 0_beacon-chain.md and 1_shard-chain-misc.md, 1_custody-game.md, 1_shard-data-chains.md, output.py)") + + "(0_fork-choice.md, 0_beacon-chain.md and 1_custody-game.md, 1_shard-data-chains.md, 1_shard-chain-misc.md output.py)") else: print("Invalid phase: {0}".format(args.phase)) From cb3e0f2146387920089309bc059dc6c91306531d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 13 Aug 2019 06:12:51 -0400 Subject: [PATCH 16/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index c1d6283ca9..e82eb2a305 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -49,7 +49,7 @@ class ShardReceiptProof(Container): #### `pack_compact_validator` ```python -def pack_compact_validator(index: uint64, slashed: bool, balance_in_increments: uint64) -> uint64: +def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int: """ Creates a compact validator object representing index, slashed status, and compressed balance. Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with From f9849ca562c2e4827ab2d6af3d2eecbfc582a8fe Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 13 Aug 2019 06:14:51 -0400 Subject: [PATCH 17/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index e82eb2a305..8c1cee9007 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -61,7 +61,7 @@ def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int #### `unpack_compact_validator` ```python -def unpack_compact_validator(compact_validator: uint64) -> Tuple[uint64, bool, uint64]: +def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: """ Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT """ From 0cf454547b8b15bf88a73f466a5fbf6065ba13f4 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:27:25 +0200 Subject: [PATCH 18/44] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 8c1cee9007..56f9a223f7 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -205,7 +205,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipts: List[ShardReceipt, MAX_SHARD_RECEIPTS] + shard_receipts: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] # end insert @shard_receipts ``` From 4f92e7fe3024bc1b545c96ac48ea7fd55cf71cb2 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:31:48 +0200 Subject: [PATCH 19/44] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 56f9a223f7..8524ad51b2 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -213,6 +213,6 @@ Use `process_shard_receipt` to process each receipt. ```python # begin insert @process_shard_receipts - (body.shard_receipts, process_shard_receipts), + (body.shard_receipt_proofs, process_shard_receipt_proofs), # end insert @process_shard_receipts ``` From 49a008df605faf3ba94350b0d2a7a04fadb089b3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:31:55 +0200 Subject: [PATCH 20/44] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 8524ad51b2..3bb1fa2010 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -209,7 +209,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: # end insert @shard_receipts ``` -Use `process_shard_receipt` to process each receipt. +Use `process_shard_receipt_proof` to process each receipt. ```python # begin insert @process_shard_receipts From d424863fc0bd2166ad02707679ad80739da8a859 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:32:01 +0200 Subject: [PATCH 21/44] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 3bb1fa2010..3b32f39e60 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -205,7 +205,7 @@ Add the `shard_receipts` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipts: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] # end insert @shard_receipts ``` From fd24308d19b36a4343185047e233a23fdaacd62b Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:32:10 +0200 Subject: [PATCH 22/44] Update specs/core/1_beacon-chain-misc.md --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 3b32f39e60..be376a6ae6 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -201,7 +201,7 @@ def update_persistent_committee(state: BeaconState) -> None: ### Shard receipt processing -Add the `shard_receipts` operation to `BeaconBlockBody`: +Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts From 8255091e409968844d24e8e3ceed37c2aee3e729 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Wed, 14 Aug 2019 23:35:26 +0200 Subject: [PATCH 23/44] ShardReceipt -> ShardReceiptProof --- specs/core/1_beacon-chain-misc.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index be376a6ae6..701b38db04 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -18,7 +18,7 @@ - [concat_generalized_indices](#concat_generalized_indices) - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt](#process_shard_receipt) + - [process_shard_receipt_proof](#process_shard_receipt_proof) - [Changes](#changes) - [Persistent committees](#persistent-committees) - [Shard receipt processing](#shard-receipt-processing) @@ -134,10 +134,10 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` -#### `process_shard_receipt` +#### `process_shard_receipt_proof` ```python -def process_shard_receipt(state: BeaconState, receipt_proof: ShardReceiptProof): +def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof): """ Processes a ShardReceipt object. """ From e4a18f6fa416e854c5a5337df7b5da8ff812301d Mon Sep 17 00:00:00 2001 From: vbuterin Date: Thu, 15 Aug 2019 12:51:11 +0200 Subject: [PATCH 24/44] Made persistent committee roots a vector --- specs/core/1_beacon-chain-misc.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 701b38db04..0c23e41b76 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,6 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) + - [Configuration](#configuration) - [Classes](#classes) - [CompactCommittee](#compactcommittee) - [ShardReceiptProof](#shardreceiptproof) @@ -25,6 +26,13 @@ +## Configuration + +| Name | Value | Unit | Duration +| - | - | - | - | +| `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | +| `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | + ## Classes #### `CompactCommittee` @@ -171,9 +179,7 @@ Add to the beacon state the following fields: ```python # begin insert @persistent_committee_fields - previous_persistent_committee_root: Hash - current_persistent_committee_root: Hash - next_persistent_committee_root: Hash + persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint, SHARD_COUNT] # end insert @persistent_committee_fields ``` @@ -190,13 +196,12 @@ def update_persistent_committee(state: BeaconState) -> None: Updates persistent committee roots at boundary blocks. """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: - state.previous_persistent_committee_root = state.current_persistent_committee_root - state.current_persistent_committee_root = state.next_persistent_committee_root + period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD committees = Vector[CompactCommittee, SHARD_COUNT]([ committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) for i in range(SHARD_COUNT) ]) - state.next_persistent_committee_root = hash_tree_root(committees) + state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` ### Shard receipt processing @@ -205,7 +210,7 @@ Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: ```python # begin insert @shard_receipts - shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPTS] + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] # end insert @shard_receipts ``` From c1f2e92ad1af9e1bd88c6f85c7087beebfcbe7b7 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Fri, 23 Aug 2019 12:33:25 +0200 Subject: [PATCH 25/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Danny Ryan --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 0c23e41b76..4539be6b0b 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -180,7 +180,7 @@ Add to the beacon state the following fields: ```python # begin insert @persistent_committee_fields persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] - next_shard_receipt_period: Vector[uint, SHARD_COUNT] + next_shard_receipt_period: Vector[uint64, SHARD_COUNT] # end insert @persistent_committee_fields ``` `next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` From 1392c931d0f208ab3f156ea56e3cb667644706de Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 12:00:01 -0600 Subject: [PATCH 26/44] add GeneralizedIndex to custom types --- specs/core/1_beacon-chain-misc.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 4539be6b0b..b6f23a00e1 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,6 +6,7 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) + - [Custom types](#custom-types) - [Configuration](#configuration) - [Classes](#classes) - [CompactCommittee](#compactcommittee) @@ -26,6 +27,14 @@ +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `GeneralizedIndex` | `uint64` | a generalized index into an SSZ merkle tree | + ## Configuration | Name | Value | Unit | Duration From 56954ec5089eaaa08423a453c6a44db3d8ce1457 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 12:16:57 -0600 Subject: [PATCH 27/44] fix adding fields to phase 1 ssz objects --- specs/core/0_beacon-chain.md | 2 -- specs/core/1_beacon-chain-misc.md | 37 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 5ddceebc1e..90c179107e 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -480,7 +480,6 @@ class BeaconBlockBody(Container): deposits: List[Deposit, MAX_DEPOSITS] voluntary_exits: List[VoluntaryExit, MAX_VOLUNTARY_EXITS] transfers: List[Transfer, MAX_TRANSFERS] - # @shard_receipts ``` #### `BeaconBlock` @@ -534,7 +533,6 @@ class BeaconState(Container): previous_justified_checkpoint: Checkpoint # Previous epoch snapshot current_justified_checkpoint: Checkpoint finalized_checkpoint: Checkpoint - # @persistent_committee_fields ``` ## Helper functions diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index b6f23a00e1..65c4cdbbc4 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -182,17 +182,32 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP ## Changes -### Persistent committees +### Phase 0 container updates + +Add the following fields to the end of the specified container objects. -Add to the beacon state the following fields: +#### `BeaconState` ```python -# begin insert @persistent_committee_fields +class BeaconState(Container): + # Persistent committees persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint64, SHARD_COUNT] -# end insert @persistent_committee_fields ``` -`next_shard_receipt_period` values initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD` + +`persistent_committee_roots` values are initialized to `Bytes32()` (empty bytes value). +`next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. + +#### `BeaconBlockBody` + +```python +class BeaconBlockBody(Container): + shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] +``` + +`shard_receipt_proofs` is initialized to `[]`. + +### Persistent committees Run `update_persistent_committee` immediately before `process_final_updates`: @@ -215,18 +230,10 @@ def update_persistent_committee(state: BeaconState) -> None: ### Shard receipt processing -Add the `shard_receipt_proofs` operation to `BeaconBlockBody`: - -```python -# begin insert @shard_receipts - shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS] -# end insert @shard_receipts -``` - -Use `process_shard_receipt_proof` to process each receipt. +Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing. ```python # begin insert @process_shard_receipts - (body.shard_receipt_proofs, process_shard_receipt_proofs), + (body.shard_receipt_proofs, process_shard_receipt_proof), # end insert @process_shard_receipts ``` From bcdbf7dfc7881ede422f8e29eadaeb50caea1dd6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 02:44:22 +0800 Subject: [PATCH 28/44] Fix some flake8 errors --- specs/core/1_beacon-chain-misc.md | 53 +++++++++++++++---------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 65c4cdbbc4..b4a8d6dbe9 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -6,43 +6,37 @@ - [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes) - [Table of contents](#table-of-contents) - - [Custom types](#custom-types) - [Configuration](#configuration) - - [Classes](#classes) - - [CompactCommittee](#compactcommittee) - - [ShardReceiptProof](#shardreceiptproof) - - [Helpers](#helpers) - - [pack_compact_validator](#pack_compact_validator) - - [unpack_compact_validator](#unpack_compact_validator) - - [committee_to_compact_committee](#committee_to_compact_committee) - - [get_previous_power_of_2](#get_previous_power_of_2) - - [verify_merkle_proof](#verify_merkle_proof) - - [concat_generalized_indices](#concat_generalized_indices) - - [compute_historical_state_generalized_index](#compute_historical_state_generalized_index) - - [get_generalized_index_of_crosslink_header](#get_generalized_index_of_crosslink_header) - - [process_shard_receipt_proof](#process_shard_receipt_proof) + - [Containers](#containers) + - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptProof`](#shardreceiptproof) + - [Helper functions](#helper-functions) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`get_previous_power_of_2`](#get_previous_power_of_2) + - [`verify_merkle_proof`](#verify_merkle_proof) + - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) + - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) + - [`process_shard_receipt_proof`](#process_shard_receipt_proof) - [Changes](#changes) + - [Phase 0 container updates](#phase-0-container-updates) + - [`BeaconState`](#beaconstate) + - [`BeaconBlockBody`](#beaconblockbody) - [Persistent committees](#persistent-committees) - [Shard receipt processing](#shard-receipt-processing) -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `GeneralizedIndex` | `uint64` | a generalized index into an SSZ merkle tree | - ## Configuration | Name | Value | Unit | Duration | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | | `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | +| `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | -## Classes +## Containers #### `CompactCommittee` @@ -61,7 +55,7 @@ class ShardReceiptProof(Container): receipt: List[ShardReceiptDelta, PLACEHOLDER] ``` -## Helpers +## Helper functions #### `pack_compact_validator` @@ -146,7 +140,9 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: """ Gets the generalized index for the root of the index'th header in a crosslink. """ - MAX_CROSSLINK_SIZE = SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + MAX_CROSSLINK_SIZE = ( + SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK + ) assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` @@ -170,11 +166,14 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP leaf=hash_tree_root(receipt_proof.receipt), proof=receipt_proof.proof, index=gindex, - root=state.current_crosslinks[shard].data_root + root=state.current_crosslinks[receipt_proof.shard].data_root ) for delta in receipt_proof.receipt: if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch: - increase_balance(state, delta.index, state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE) + increase_amount = ( + state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE + ) + increase_balance(state, delta.index, increase_amount) decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) From 17043891ff39af47cca4c4867c284b8ba14735e5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 03:03:00 +0800 Subject: [PATCH 29/44] Fix some mypy errors --- specs/core/1_beacon-chain-misc.md | 13 ++++++++----- specs/light_client/merkle_proofs.md | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index b4a8d6dbe9..21b7541564 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -76,7 +76,7 @@ def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]: """ Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT """ - return compact_validator >> 16, (compact_validator >> 15) % 2, compact_validator & (2**15 - 1) + return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1) ``` #### `committee_to_compact_committee` @@ -129,7 +129,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS for i in range(63, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, 'history_acc', i)) + o = concat_generalized_indices(o, get_generalized_index(ShardState, ['history_acc', i])) return o ``` @@ -150,7 +150,7 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: #### `process_shard_receipt_proof` ```python -def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof): +def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof) -> None: """ Processes a ShardReceipt object. """ @@ -221,8 +221,11 @@ def update_persistent_committee(state: BeaconState) -> None: if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD committees = Vector[CompactCommittee, SHARD_COUNT]([ - committee_to_compact_committee(state, get_period_committee(state, get_current_epoch(state) + 1, i)) - for i in range(SHARD_COUNT) + committee_to_compact_committee( + state, + get_period_committee(state, Epoch(get_current_epoch(state) + 1), Shard(shard)), + ) + for shard in range(SHARD_COUNT) ]) state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index ce7dc647ca..344f365b02 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -181,7 +181,7 @@ _Usage note: functions outside this section should manipulate generalized indice #### `concat_generalized_indices` ```python -def concat_generalized_indices(indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex: +def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex: """ Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns the generalized index for A -> Z. From 6923bdc46a1d8785092dcee7f61a65df81e2041f Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Fri, 23 Aug 2019 14:31:26 -0600 Subject: [PATCH 30/44] remove Optional None from get_generalized_index. instead throw --- specs/core/1_beacon-chain-misc.md | 6 +++--- specs/light_client/merkle_proofs.md | 8 +++----- .../test/merkle_proofs/test_merkle_proofs.py | 14 +++++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 21b7541564..bd7130f2ca 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -129,7 +129,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS for i in range(63, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 - o = concat_generalized_indices(o, get_generalized_index(ShardState, ['history_acc', i])) + o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) return o ``` @@ -158,9 +158,9 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), - get_generalized_index(ShardBlockHeader, 'state_root'), + GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink), - get_generalized_index(ShardState, 'receipt_root') + GeneralizedIndex(get_generalized_index(ShardState, 'receipt_root')) ) assert verify_merkle_proof( leaf=hash_tree_root(receipt_proof.receipt), diff --git a/specs/light_client/merkle_proofs.md b/specs/light_client/merkle_proofs.md index 344f365b02..bbd03d3795 100644 --- a/specs/light_client/merkle_proofs.md +++ b/specs/light_client/merkle_proofs.md @@ -152,7 +152,7 @@ def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariab ``` ```python -def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> Optional[GeneralizedIndex]: +def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex: """ Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree. @@ -162,10 +162,8 @@ def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableNam assert not issubclass(typ, BasicValue) # If we descend to a basic type, the path cannot continue further if p == '__len__': typ = uint64 - if issubclass(typ, (List, Bytes)): - root = GeneralizedIndex(root * 2 + 1) - else: - return None + assert issubclass(typ, (List, Bytes)) + root = GeneralizedIndex(root * 2 + 1) else: pos, _, _ = get_item_position(typ, p) base_index = (GeneralizedIndex(2) if issubclass(typ, (List, Bytes)) else GeneralizedIndex(1)) diff --git a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py index 91c861de3b..62a2f63793 100644 --- a/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py +++ b/test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py @@ -1,10 +1,10 @@ - import re from eth_utils import ( to_tuple, ) from eth2spec.test.context import ( + expect_assertion_error, spec_state_test, with_all_phases_except, ) @@ -89,10 +89,14 @@ def test_to_path(): @spec_state_test def test_get_generalized_index(spec, state): for typ, path, generalized_index in generalized_index_cases: - assert spec.get_generalized_index( - typ=typ, - path=path, - ) == generalized_index + if generalized_index is not None: + assert spec.get_generalized_index( + typ=typ, + path=path, + ) == generalized_index + else: + expect_assertion_error(lambda: spec.get_generalized_index(typ=typ, path=path)) + yield 'typ', typ yield 'path', path yield 'generalized_index', generalized_index From b6d854de097cdfd7b63124a45b0f2b7070a56db9 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:50:57 +0200 Subject: [PATCH 31/44] Fix ToC --- specs/core/1_beacon-chain-misc.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index bd7130f2ca..720de40968 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -8,17 +8,17 @@ - [Table of contents](#table-of-contents) - [Configuration](#configuration) - [Containers](#containers) - - [`CompactCommittee`](#compactcommittee) - - [`ShardReceiptProof`](#shardreceiptproof) + - [`CompactCommittee`](#compactcommittee) + - [`ShardReceiptProof`](#shardreceiptproof) - [Helper functions](#helper-functions) - - [`pack_compact_validator`](#pack_compact_validator) - - [`unpack_compact_validator`](#unpack_compact_validator) - - [`committee_to_compact_committee`](#committee_to_compact_committee) - - [`get_previous_power_of_2`](#get_previous_power_of_2) - - [`verify_merkle_proof`](#verify_merkle_proof) - - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) - - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) - - [`process_shard_receipt_proof`](#process_shard_receipt_proof) + - [`pack_compact_validator`](#pack_compact_validator) + - [`unpack_compact_validator`](#unpack_compact_validator) + - [`committee_to_compact_committee`](#committee_to_compact_committee) + - [`get_previous_power_of_2`](#get_previous_power_of_2) + - [`verify_merkle_proof`](#verify_merkle_proof) + - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) + - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) + - [`process_shard_receipt_proof`](#process_shard_receipt_proof) - [Changes](#changes) - [Phase 0 container updates](#phase-0-container-updates) - [`BeaconState`](#beaconstate) From fb59160e6a32d437c032a33c00124a74de2327fb Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:54:48 +0200 Subject: [PATCH 32/44] Persistent -> period, process_shard_receipt: add _proofs --- specs/core/1_beacon-chain-misc.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 720de40968..a105fc8a03 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -33,7 +33,7 @@ | Name | Value | Unit | Duration | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | -| `PERSISTENT_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | +| `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | | `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | ## Containers @@ -189,12 +189,12 @@ Add the following fields to the end of the specified container objects. ```python class BeaconState(Container): - # Persistent committees - persistent_committee_roots: Vector[Hash, PERSISTENT_COMMITTEE_ROOT_LENGTH] + # Period committees + period_committee_roots: Vector[Hash, PERIOD_COMMITTEE_ROOT_LENGTH] next_shard_receipt_period: Vector[uint64, SHARD_COUNT] ``` -`persistent_committee_roots` values are initialized to `Bytes32()` (empty bytes value). +`period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). `next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` @@ -208,15 +208,15 @@ class BeaconBlockBody(Container): ### Persistent committees -Run `update_persistent_committee` immediately before `process_final_updates`: +Run `update_period_committee` immediately before `process_final_updates`: ```python -# begin insert @update_persistent_committee - update_persistent_committee(state) -# end insert @update_persistent_committee -def update_persistent_committee(state: BeaconState) -> None: +# begin insert @update_period_committee + update_period_committee(state) +# end insert @update_period_committee +def update_period_committee(state: BeaconState) -> None: """ - Updates persistent committee roots at boundary blocks. + Updates period committee roots at boundary blocks. """ if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0: period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD @@ -227,7 +227,7 @@ def update_persistent_committee(state: BeaconState) -> None: ) for shard in range(SHARD_COUNT) ]) - state.persistent_committee_roots[period % PERSISTENT_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) + state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees) ``` ### Shard receipt processing @@ -235,7 +235,7 @@ def update_persistent_committee(state: BeaconState) -> None: Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing. ```python -# begin insert @process_shard_receipts +# begin insert @process_shard_receipt_proofs (body.shard_receipt_proofs, process_shard_receipt_proof), -# end insert @process_shard_receipts +# end insert @process_shard_receipt_proofs ``` From 7175ac55935a8460b24a0105ac3b39510fe26ba1 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:55:18 +0200 Subject: [PATCH 33/44] Update specs/core/0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index 320bb8fd83..a26b4ae88a 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1550,7 +1550,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: (body.deposits, process_deposit), (body.voluntary_exits, process_voluntary_exit), (body.transfers, process_transfer), - # @process_shard_receipts + # @process_shard_receipt_proofs ): for operation in operations: function(state, operation) From a509c68c302916cb75b7063065a9d8d59cbb4bb3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Sat, 24 Aug 2019 13:55:35 +0200 Subject: [PATCH 34/44] Update specs/core/0_beacon-chain.md --- specs/core/0_beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md index a26b4ae88a..102525d253 100644 --- a/specs/core/0_beacon-chain.md +++ b/specs/core/0_beacon-chain.md @@ -1196,7 +1196,7 @@ def process_epoch(state: BeaconState) -> None: # @process_reveal_deadlines # @process_challenge_deadlines process_slashings(state) - # @update_persistent_committee + # @update_period_committee process_final_updates(state) # @after_process_final_updates ``` From 178dd23314f5db4ad15992bf761115df0f515ba3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 24 Aug 2019 20:26:35 +0800 Subject: [PATCH 35/44] `MINOR_REWARD_QUOTIENT` for rewarding the proposer for including shard receipt proof Co-Authored-By: vbuterin --- specs/core/1_beacon-chain-misc.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index a105fc8a03..d07f1c217d 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -34,7 +34,7 @@ | - | - | - | - | | `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - | | `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months | -| `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - | +| `MINOR_REWARD_QUOTIENT` | `2**8` (=256) | - | - | ## Containers @@ -176,7 +176,8 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP increase_balance(state, delta.index, increase_amount) decrease_balance(state, delta.index, delta.block_fee) state.next_shard_receipt_period[receipt_proof.shard] += 1 - increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD) + proposer_index = get_beacon_proposer_index(state) + increase_balance(state, proposer_index, Gwei(get_base_reward(state, proposer_index) // MINOR_REWARD_QUOTIENT)) ``` ## Changes From 01af8e6297770f0599ac26efb49e89d6803efcc3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sun, 25 Aug 2019 17:19:19 +0800 Subject: [PATCH 36/44] Use `get_previous_power_of_two` from merkle proofs spec --- specs/core/1_beacon-chain-misc.md | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index d07f1c217d..fcd004b95f 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -14,7 +14,6 @@ - [`pack_compact_validator`](#pack_compact_validator) - [`unpack_compact_validator`](#unpack_compact_validator) - [`committee_to_compact_committee`](#committee_to_compact_committee) - - [`get_previous_power_of_2`](#get_previous_power_of_2) - [`verify_merkle_proof`](#verify_merkle_proof) - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index) - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header) @@ -95,13 +94,6 @@ def committee_to_compact_committee(state: BeaconState, committee: Sequence[Valid return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators) ``` -#### `get_previous_power_of_2` - -```python -def get_previous_power_of_2(x: int) -> int: - return x if x <= 2 else 2 * get_previous_power_of_2(x // 2) -``` - #### `verify_merkle_proof` ```python @@ -143,7 +135,7 @@ def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex: MAX_CROSSLINK_SIZE = ( SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK ) - assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE) + assert MAX_CROSSLINK_SIZE == get_previous_power_of_two(MAX_CROSSLINK_SIZE) return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index) ``` From 9b3cb306e3ce5dca1e3d3df1584ca2c29d2ff9b6 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 26 Aug 2019 10:09:13 +0200 Subject: [PATCH 37/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index fcd004b95f..ee63eca645 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -118,7 +118,7 @@ def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardS blocks at the next available multiples of descending powers of two. """ o = GeneralizedIndex(1) - for i in range(63, -1, -1): + for i in range(HISTORY_ACCUMULATOR_VECTOR - 1, -1, -1): if (later - 1) & 2**i > (earlier - 1) & 2**i: later = later - ((later - 1) % 2**i) - 1 o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i]))) From f1caa85aaf260fbd35774efff8eb057100ac3875 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Mon, 26 Aug 2019 10:09:43 +0200 Subject: [PATCH 38/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index ee63eca645..9e9bfddb99 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,7 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), From ffdc36920e9acc2472a9ba7657b13219ab77279e Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Mon, 26 Aug 2019 08:55:50 -0600 Subject: [PATCH 39/44] lint --- specs/core/1_beacon-chain-misc.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 9e9bfddb99..c23b018f00 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -146,8 +146,9 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP """ Processes a ShardReceipt object. """ + SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD - first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH + first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')), From ab4820ced6ed85effc871de712064cbb2a531ca3 Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:13:47 +0200 Subject: [PATCH 40/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: John Adler --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index c23b018f00..41d0de2334 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,7 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH - receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0), From 0b38ff0fe26f7291f771be9b65b6bf9353ccbf7e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:17:38 +0200 Subject: [PATCH 41/44] Update specs/core/1_beacon-chain-misc.md Co-Authored-By: Hsiao-Wei Wang --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 41d0de2334..ddf06a1f05 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -189,7 +189,7 @@ class BeaconState(Container): ``` `period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). -`next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`. +`next_shard_receipt_period` values are initialized to `compute_epoch_of_shard_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` From 0f2e814c63238f14339dd1a30723ced8336cf63e Mon Sep 17 00:00:00 2001 From: vbuterin Date: Tue, 27 Aug 2019 13:19:04 +0200 Subject: [PATCH 42/44] Shard slot -> slot for PHASE_1_FORK_SLOT --- specs/core/1_beacon-chain-misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index ddf06a1f05..35a58e53bd 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -189,7 +189,7 @@ class BeaconState(Container): ``` `period_committee_roots` values are initialized to `Bytes32()` (empty bytes value). -`next_shard_receipt_period` values are initialized to `compute_epoch_of_shard_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. +`next_shard_receipt_period` values are initialized to `compute_epoch_of_slot(PHASE_1_FORK_SLOT) // EPOCHS_PER_SHARD_PERIOD`. #### `BeaconBlockBody` From 17702e6d88515435a2a7c017d3d026d47acdbd8d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 27 Aug 2019 20:10:39 +0800 Subject: [PATCH 43/44] Shard slot -> slot for PHASE_1_FORK_SLOT part2 --- configs/minimal.yaml | 4 ++-- specs/core/1_shard-data-chains.md | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 15b749b9dd..be787ca3c9 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -141,5 +141,5 @@ SHARD_SLOTS_PER_BEACON_SLOT: 2 EPOCHS_PER_SHARD_PERIOD: 4 # PHASE_1_FORK_EPOCH >= EPOCHS_PER_SHARD_PERIOD * 2 PHASE_1_FORK_EPOCH: 8 -# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH -PHASE_1_FORK_SLOT: 128 +# PHASE_1_FORK_SLOT = PHASE_1_FORK_EPOCH * SLOTS_PER_EPOCH +PHASE_1_FORK_SLOT: 64 diff --git a/specs/core/1_shard-data-chains.md b/specs/core/1_shard-data-chains.md index 8e1532f17e..3dc5498161 100644 --- a/specs/core/1_shard-data-chains.md +++ b/specs/core/1_shard-data-chains.md @@ -73,10 +73,10 @@ We define the following Python custom types for type hinting and readability: ### Initial values -| Name | Value | -| - | - | -| `PHASE_1_FORK_EPOCH` | **TBD** | -| `PHASE_1_FORK_SLOT` | **TBD** | +| Name | Value | Unit | +| - | - | - | +| `PHASE_1_FORK_EPOCH` | **TBD** | Epoch | +| `PHASE_1_FORK_SLOT` | **TBD** | Slot | ### Time parameters @@ -359,7 +359,7 @@ def get_default_shard_state(beacon_state: BeaconState, shard: Shard) -> ShardSta return ShardState( basefee=1, shard=shard, - slot=PHASE_1_FORK_SLOT, + slot=PHASE_1_FORK_SLOT * SHARD_SLOTS_PER_BEACON_SLOT, earlier_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(earlier_committee))], later_committee_rewards=[REWARD_COEFFICIENT_BASE for _ in range(len(later_committee))], earlier_committee_fees=[Gwei(0) for _ in range(len(earlier_committee))], From 979fa38ae8a7e2ebdba82a993baa0e1cde3fb639 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 27 Aug 2019 20:10:55 +0800 Subject: [PATCH 44/44] fix linter error --- specs/core/1_beacon-chain-misc.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specs/core/1_beacon-chain-misc.md b/specs/core/1_beacon-chain-misc.md index 35a58e53bd..5bb0f6da09 100644 --- a/specs/core/1_beacon-chain-misc.md +++ b/specs/core/1_beacon-chain-misc.md @@ -147,7 +147,10 @@ def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptP Processes a ShardReceipt object. """ SHARD_SLOTS_PER_EPOCH = SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH - receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + receipt_slot = ( + state.next_shard_receipt_period[receipt_proof.shard] * + SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD + ) first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SHARD_SLOTS_PER_EPOCH gindex = concat_generalized_indices( get_generalized_index_of_crosslink_header(0),