Starting on phase 1 misc beacon changes

Makefile

@@ -89,7 +89,7 @@ $(PY_SPEC_PHASE_0_TARGETS): $(PY_SPEC_PHASE_0_DEPS)
 	python3 $(SCRIPT_DIR)/build_spec.py -p0 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/validator/0_beacon-chain-validator.md $@
 
 $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
-	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/light_client/merkle_proofs.md $@
+	python3 $(SCRIPT_DIR)/build_spec.py -p1 $(SPEC_DIR)/core/0_beacon-chain.md $(SPEC_DIR)/core/0_fork-choice.md $(SPEC_DIR)/light_client/merkle_proofs.md $(SPEC_DIR)/core/1_custody-game.md $(SPEC_DIR)/core/1_shard-data-chains.md $(SPEC_DIR)/core/1_beacon-chain-misc.md $@
 
 CURRENT_DIR = ${CURDIR}
 

scripts/build_spec.py

@@ -289,18 +289,20 @@ def build_phase0_spec(phase0_sourcefile: str, fork_choice_sourcefile: str,
     return spec
 
 
-def build_phase1_spec(phase0_sourcefile: str,
-                      fork_choice_sourcefile: str,
+def build_phase1_spec(phase0_beacon_sourcefile: str,
+                      phase0_fork_choice_sourcefile: str,
+                      merkle_proofs_sourcefile: str,
                       phase1_custody_sourcefile: str,
                       phase1_shard_sourcefile: str,
-                      merkle_proofs_sourcefile: str,
+                      phase1_beacon_misc_sourcefile: str,
                       outfile: str=None) -> Optional[str]:
     all_sourcefiles = (
-        phase0_sourcefile,
-        fork_choice_sourcefile,
+        phase0_beacon_sourcefile,
+        phase0_fork_choice_sourcefile,
+        merkle_proofs_sourcefile,
         phase1_custody_sourcefile,
         phase1_shard_sourcefile,
-        merkle_proofs_sourcefile,
+        phase1_beacon_misc_sourcefile,
     )
     all_spescs = [get_spec(spec) for spec in all_sourcefiles]
     for spec in all_spescs:
@@ -327,10 +329,11 @@ def build_phase1_spec(phase0_sourcefile: str,
 If building phase 1:
     1st argument is input /core/0_beacon-chain.md
     2nd argument is input /core/0_fork-choice.md
-    3rd argument is input /core/1_custody-game.md
-    4th argument is input /core/1_shard-data-chains.md
-    5th argument is input /light_client/merkle_proofs.md
-    6th argument is output spec.py
+    3rd argument is input /light_client/merkle_proofs.md
+    4th argument is input /core/1_custody-game.md
+    5th argument is input /core/1_shard-data-chains.md
+    6th argument is input /core/1_beacon-chain-misc.md
+    7th argument is output spec.py
 '''
     parser = ArgumentParser(description=description)
     parser.add_argument("-p", "--phase", dest="phase", type=int, default=0, help="Build for phase #")
@@ -343,14 +346,14 @@ def build_phase1_spec(phase0_sourcefile: str,
         else:
             print(" Phase 0 requires spec, forkchoice, and v-guide inputs as well as an output file.")
     elif args.phase == 1:
-        if len(args.files) == 6:
+        if len(args.files) == 7:
             build_phase1_spec(*args.files)
         else:
             print(
                 " Phase 1 requires input files as well as an output file:\n"
                 "\t core/phase_0: (0_beacon-chain.md, 0_fork-choice.md)\n"
-                "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md)\n"
                 "\t light_client: (merkle_proofs.md)\n"
+                "\t core/phase_1: (1_custody-game.md, 1_shard-data-chains.md, 1_beacon-chain-misc.md)\n"
                 "\t and output.py"
             )
     else:

specs/core/0_beacon-chain.md

@@ -1196,6 +1196,7 @@ def process_epoch(state: BeaconState) -> None:
     # @process_reveal_deadlines
     # @process_challenge_deadlines
     process_slashings(state)
+    # @update_persistent_committee
     process_final_updates(state)
     # @after_process_final_updates
 ```
@@ -1549,6 +1550,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None:
         (body.deposits, process_deposit),
         (body.voluntary_exits, process_voluntary_exit),
         (body.transfers, process_transfer),
+        # @process_shard_receipts
     ):
         for operation in operations:
             function(state, operation)

specs/core/1_beacon-chain-misc.md

@@ -0,0 +1,241 @@
+# Phase 1 miscellaneous beacon chain changes
+
+## Table of contents
+
+<!-- TOC -->
+
+- [Phase 1 miscellaneous beacon chain changes](#phase-1-miscellaneous-beacon-chain-changes)
+    - [Table of contents](#table-of-contents)
+    - [Configuration](#configuration)
+    - [Containers](#containers)
+        - [`CompactCommittee`](#compactcommittee)
+        - [`ShardReceiptProof`](#shardreceiptproof)
+    - [Helper functions](#helper-functions)
+        - [`pack_compact_validator`](#pack_compact_validator)
+        - [`unpack_compact_validator`](#unpack_compact_validator)
+        - [`committee_to_compact_committee`](#committee_to_compact_committee)
+        - [`get_previous_power_of_2`](#get_previous_power_of_2)
+        - [`verify_merkle_proof`](#verify_merkle_proof)
+        - [`compute_historical_state_generalized_index`](#compute_historical_state_generalized_index)
+        - [`get_generalized_index_of_crosslink_header`](#get_generalized_index_of_crosslink_header)
+        - [`process_shard_receipt_proof`](#process_shard_receipt_proof)
+    - [Changes](#changes)
+        - [Phase 0 container updates](#phase-0-container-updates)
+            - [`BeaconState`](#beaconstate)
+            - [`BeaconBlockBody`](#beaconblockbody)
+        - [Persistent committees](#persistent-committees)
+        - [Shard receipt processing](#shard-receipt-processing)
+
+<!-- /TOC -->
+
+## Configuration
+
+| Name | Value | Unit | Duration
+| - | - | - | - |
+| `MAX_SHARD_RECEIPT_PROOFS` | `2**0` (= 1) | - | - |
+| `PERIOD_COMMITTEE_ROOT_LENGTH` | `2**8` (= 256) | periods | ~9 months |
+| `MICRO_REWARD` | `Gwei(2**0)` (=1) | Gwei | - |
+
+## Containers
+
+#### `CompactCommittee`
+
+```python
+class CompactCommittee(Container):
+    pubkeys: List[BLSPubkey, MAX_VALIDATORS_PER_COMMITTEE]
+    compact_validators: List[uint64, MAX_VALIDATORS_PER_COMMITTEE]
+```
+
+#### `ShardReceiptProof`
+
+```python
+class ShardReceiptProof(Container):
+    shard: Shard
+    proof: List[Hash, PLACEHOLDER]
+    receipt: List[ShardReceiptDelta, PLACEHOLDER]
+```
+
+## Helper functions
+
+#### `pack_compact_validator`
+
+```python
+def pack_compact_validator(index: int, slashed: bool, balance_in_increments: int) -> int:
+    """
+    Creates a compact validator object representing index, slashed status, and compressed balance.
+    Takes as input balance-in-increments (// EFFECTIVE_BALANCE_INCREMENT) to preserve symmetry with
+    the unpacking function.
+    """
+    return (index << 16) + (slashed << 15) + balance_in_increments
+```
+
+#### `unpack_compact_validator`
+
+```python
+def unpack_compact_validator(compact_validator: int) -> Tuple[int, bool, int]:
+    """
+    Returns validator index, slashed, balance // EFFECTIVE_BALANCE_INCREMENT
+    """
+    return compact_validator >> 16, bool((compact_validator >> 15) % 2), compact_validator & (2**15 - 1)
+```
+
+#### `committee_to_compact_committee`
+
+```python
+def committee_to_compact_committee(state: BeaconState, committee: Sequence[ValidatorIndex]) -> CompactCommittee:
+    """
+    Given a state and a list of validator indices, outputs the CompactCommittee representing them.
+    """
+    validators = [state.validators[i] for i in committee]
+    compact_validators = [
+        pack_compact_validator(i, v.slashed, v.effective_balance // EFFECTIVE_BALANCE_INCREMENT)
+        for i, v in zip(committee, validators)
+    ]
+    pubkeys = [v.pubkey for v in validators]
+    return CompactCommittee(pubkeys=pubkeys, compact_validators=compact_validators)
+```
+
+#### `get_previous_power_of_2`
+
+```python
+def get_previous_power_of_2(x: int) -> int:
+    return x if x <= 2 else 2 * get_previous_power_of_2(x // 2)
+```
+
+#### `verify_merkle_proof`
+
+```python
+def verify_merkle_proof(leaf: Hash, proof: Sequence[Hash], index: GeneralizedIndex, root: Hash) -> bool:
+    assert len(proof) == get_generalized_index_length(index)
+    for i, h in enumerate(proof):
+        if get_generalized_index_bit(index, i):
+            leaf = hash(h + leaf)
+        else:
+            leaf = hash(leaf + h)
+    return leaf == root
+```
+
+#### `compute_historical_state_generalized_index`
+
+```python
+def compute_historical_state_generalized_index(earlier: ShardSlot, later: ShardSlot) -> GeneralizedIndex:
+    """
+    Computes the generalized index of the state root of slot `frm` based on the state root of slot `to`.
+    Relies on the `history_acc` in the `ShardState`, where `history_acc[i]` maintains the most recent 2**i'th
+    slot state. Works by tracing a `log(later-earlier)` step path from `later` to `earlier` through intermediate
+    blocks at the next available multiples of descending powers of two.
+    """
+    o = GeneralizedIndex(1)
+    for i in range(63, -1, -1):
+        if (later - 1) & 2**i > (earlier - 1) & 2**i:
+            later = later - ((later - 1) % 2**i) - 1
+            o = concat_generalized_indices(o, GeneralizedIndex(get_generalized_index(ShardState, ['history_acc', i])))
+    return o
+```
+
+#### `get_generalized_index_of_crosslink_header`
+
+```python
+def get_generalized_index_of_crosslink_header(index: int) -> GeneralizedIndex:
+    """
+    Gets the generalized index for the root of the index'th header in a crosslink.
+    """
+    MAX_CROSSLINK_SIZE = (
+        SHARD_BLOCK_SIZE_LIMIT * SHARD_SLOTS_PER_BEACON_SLOT * SLOTS_PER_EPOCH * MAX_EPOCHS_PER_CROSSLINK
+    )
+    assert MAX_CROSSLINK_SIZE == get_previous_power_of_2(MAX_CROSSLINK_SIZE)
+    return GeneralizedIndex(MAX_CROSSLINK_SIZE // SHARD_HEADER_SIZE + index)
+```
+
+#### `process_shard_receipt_proof`
+
+```python
+def process_shard_receipt_proof(state: BeaconState, receipt_proof: ShardReceiptProof) -> None:
+    """
+    Processes a ShardReceipt object.
+    """
+    receipt_slot = state.next_shard_receipt_period[receipt_proof.shard] * SLOTS_PER_EPOCH * EPOCHS_PER_SHARD_PERIOD
+    first_slot_in_last_crosslink = state.current_crosslinks[receipt_proof.shard].start_epoch * SLOTS_PER_EPOCH
+    gindex = concat_generalized_indices(
+        get_generalized_index_of_crosslink_header(0),
+        GeneralizedIndex(get_generalized_index(ShardBlockHeader, 'state_root')),
+        compute_historical_state_generalized_index(receipt_slot, first_slot_in_last_crosslink),
+        GeneralizedIndex(get_generalized_index(ShardState, 'receipt_root'))
+    )
+    assert verify_merkle_proof(
+        leaf=hash_tree_root(receipt_proof.receipt),
+        proof=receipt_proof.proof,
+        index=gindex,
+        root=state.current_crosslinks[receipt_proof.shard].data_root
+    )
+    for delta in receipt_proof.receipt:
+        if get_current_epoch(state) < state.validators[delta.index].withdrawable_epoch:
+            increase_amount = (
+                state.validators[delta.index].effective_balance * delta.reward_coefficient // REWARD_COEFFICIENT_BASE
+            )
+            increase_balance(state, delta.index, increase_amount)
+            decrease_balance(state, delta.index, delta.block_fee)
+    state.next_shard_receipt_period[receipt_proof.shard] += 1
+    increase_balance(state, get_beacon_proposer_index(state), MICRO_REWARD)
+```
+
+## Changes
+
+### Phase 0 container updates
+
+Add the following fields to the end of the specified container objects.
+
+#### `BeaconState`
+
+```python
+class BeaconState(Container):
+    # Period committees
+    period_committee_roots: Vector[Hash, PERIOD_COMMITTEE_ROOT_LENGTH]
+    next_shard_receipt_period: Vector[uint64, SHARD_COUNT]
+```
+
+`period_committee_roots` values are initialized to `Bytes32()` (empty bytes value).
+`next_shard_receipt_period` values are initialized to `PHASE_1_FORK_SLOT // SLOTS_PER_EPOCH // EPOCHS_PER_SHARD_PERIOD`.
+
+#### `BeaconBlockBody`
+
+```python
+class BeaconBlockBody(Container):
+    shard_receipt_proofs: List[ShardReceiptProof, MAX_SHARD_RECEIPT_PROOFS]
+```
+
+`shard_receipt_proofs` is initialized to `[]`.
+
+### Persistent committees
+
+Run `update_period_committee` immediately before `process_final_updates`:
+
+```python
+# begin insert @update_period_committee
+    update_period_committee(state)
+# end insert @update_period_committee
+def update_period_committee(state: BeaconState) -> None:
+    """
+    Updates period committee roots at boundary blocks.
+    """
+    if (get_current_epoch(state) + 1) % EPOCHS_PER_SHARD_PERIOD == 0:
+        period = (get_current_epoch(state) + 1) // EPOCHS_PER_SHARD_PERIOD
+        committees = Vector[CompactCommittee, SHARD_COUNT]([
+            committee_to_compact_committee(
+                state,
+                get_period_committee(state, Epoch(get_current_epoch(state) + 1), Shard(shard)),
+            )
+            for shard in range(SHARD_COUNT)
+        ])
+        state.period_committee_roots[period % PERIOD_COMMITTEE_ROOT_LENGTH] = hash_tree_root(committees)
+```
+
+### Shard receipt processing
+
+Run `process_shard_receipt_proof` on each `ShardReceiptProof` during block processing.
+
+```python
+# begin insert @process_shard_receipt_proofs
+        (body.shard_receipt_proofs, process_shard_receipt_proof),
+# end insert @process_shard_receipt_proofs
+```

specs/light_client/merkle_proofs.md

@@ -152,7 +152,7 @@ def get_item_position(typ: SSZType, index_or_variable_name: Union[int, SSZVariab
 ```
 
 ```python
-def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> Optional[GeneralizedIndex]:
+def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex:
     """
     Converts a path (eg. `[7, "foo", 3]` for `x[7].foo[3]`, `[12, "bar", "__len__"]` for
     `len(x[12].bar)`) into the generalized index representing its position in the Merkle tree.
@@ -162,10 +162,8 @@ def get_generalized_index(typ: SSZType, path: Sequence[Union[int, SSZVariableNam
         assert not issubclass(typ, BasicValue)  # If we descend to a basic type, the path cannot continue further
         if p == '__len__':
             typ = uint64
-            if issubclass(typ, (List, Bytes)):
-                root = GeneralizedIndex(root * 2 + 1)
-            else:
-                return None
+            assert issubclass(typ, (List, Bytes))
+            root = GeneralizedIndex(root * 2 + 1)
         else:
             pos, _, _ = get_item_position(typ, p)
             base_index = (GeneralizedIndex(2) if issubclass(typ, (List, Bytes)) else GeneralizedIndex(1))
@@ -181,7 +179,7 @@ _Usage note: functions outside this section should manipulate generalized indice
 #### `concat_generalized_indices`
 
 ```python
-def concat_generalized_indices(indices: Sequence[GeneralizedIndex]) -> GeneralizedIndex:
+def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex:
     """
     Given generalized indices i1 for A -> B, i2 for B -> C .... i_n for Y -> Z, returns
     the generalized index for A -> Z.

test_libs/pyspec/eth2spec/test/merkle_proofs/test_merkle_proofs.py

@@ -1,10 +1,10 @@
-
 import re
 from eth_utils import (
     to_tuple,
 )
 
 from eth2spec.test.context import (
+    expect_assertion_error,
     spec_state_test,
     with_all_phases_except,
 )
@@ -89,10 +89,14 @@ def test_to_path():
 @spec_state_test
 def test_get_generalized_index(spec, state):
     for typ, path, generalized_index in generalized_index_cases:
-        assert spec.get_generalized_index(
-            typ=typ,
-            path=path,
-        ) == generalized_index
+        if generalized_index is not None:
+            assert spec.get_generalized_index(
+                typ=typ,
+                path=path,
+            ) == generalized_index
+        else:
+            expect_assertion_error(lambda: spec.get_generalized_index(typ=typ, path=path))
+
         yield 'typ', typ
         yield 'path', path
         yield 'generalized_index', generalized_index