Rebase The Merge onto Altair base functionality

setup.py

@@ -447,7 +447,7 @@ class AltairSpecBuilder(Phase0SpecBuilder):
     @classmethod
     def imports(cls, preset_name: str) -> str:
         return super().imports(preset_name) + '\n' + f'''
-from typing import NewType, Union
+from typing import NewType, Union as PyUnion
 
 from eth2spec.phase0 import {preset_name} as phase0
 from eth2spec.utils.ssz.ssz_typing import Path
@@ -463,7 +463,7 @@ def preparations(cls):
     @classmethod
     def sundry_functions(cls) -> str:
         return super().sundry_functions() + '\n\n' + '''
-def get_generalized_index(ssz_class: Any, *path: Sequence[Union[int, SSZVariableName]]) -> GeneralizedIndex:
+def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex:
     ssz_path = Path(ssz_class)
     for item in path:
         ssz_path = ssz_path / item
@@ -487,14 +487,14 @@ def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
 #
 # MergeSpecBuilder
 #
-class MergeSpecBuilder(Phase0SpecBuilder):
+class MergeSpecBuilder(AltairSpecBuilder):
     fork: str = MERGE
 
     @classmethod
     def imports(cls, preset_name: str):
         return super().imports(preset_name) + f'''
 from typing import Protocol
-from eth2spec.phase0 import {preset_name} as phase0
+from eth2spec.altair import {preset_name} as altair
 from eth2spec.utils.ssz.ssz_typing import Bytes20, ByteList, ByteVector, uint256, Union
 '''
 
@@ -844,38 +844,30 @@ def finalize_options(self):
         if len(self.md_doc_paths) == 0:
             print("no paths were specified, using default markdown file paths for pyspec"
                   " build (spec fork: %s)" % self.spec_fork)
-            if self.spec_fork == PHASE0:
+            if self.spec_fork in (PHASE0, ALTAIR, MERGE):
                 self.md_doc_paths = """
                     specs/phase0/beacon-chain.md
                     specs/phase0/fork-choice.md
                     specs/phase0/validator.md
                     specs/phase0/weak-subjectivity.md
                 """
-            elif self.spec_fork == ALTAIR:
-                self.md_doc_paths = """
-                    specs/phase0/beacon-chain.md
-                    specs/phase0/fork-choice.md
-                    specs/phase0/validator.md
-                    specs/phase0/weak-subjectivity.md
+            if self.spec_fork in (ALTAIR, MERGE):
+                self.md_doc_paths += """
                     specs/altair/beacon-chain.md
                     specs/altair/bls.md
                     specs/altair/fork.md
                     specs/altair/validator.md
                     specs/altair/p2p-interface.md
                     specs/altair/sync-protocol.md
                 """
-            elif self.spec_fork == MERGE:
-                self.md_doc_paths = """
-                    specs/phase0/beacon-chain.md
-                    specs/phase0/fork-choice.md
-                    specs/phase0/validator.md
-                    specs/phase0/weak-subjectivity.md
+            if self.spec_fork == MERGE:
+                self.md_doc_paths += """
                     specs/merge/beacon-chain.md
                     specs/merge/fork.md
                     specs/merge/fork-choice.md
                     specs/merge/validator.md
                 """
-            else:
+            if len(self.md_doc_paths) == 0:
                 raise Exception('no markdown files specified, and spec fork "%s" is unknown', self.spec_fork)
 
         self.parsed_md_doc_paths = self.md_doc_paths.split()

specs/merge/beacon-chain.md

@@ -1,7 +1,5 @@
 # Ethereum 2.0 The Merge
 
-**Warning**: This document is currently based on [Phase 0](../phase0/beacon-chain.md) and will be rebased on [Altair](../altair/beacon-chain.md).
-
 **Notice**: This document is a work-in-progress for researchers and implementers.
 
 ## Table of contents
@@ -69,15 +67,59 @@ This patch adds transaction execution to the beacon chain as part of the Merge f
 #### `BeaconBlockBody`
 
 ```python
-class BeaconBlockBody(phase0.BeaconBlockBody):
+class BeaconBlockBody(Container):
+    randao_reveal: BLSSignature
+    eth1_data: Eth1Data  # Eth1 data vote
+    graffiti: Bytes32  # Arbitrary data
+    # Operations
+    proposer_slashings: List[ProposerSlashing, MAX_PROPOSER_SLASHINGS]
+    attester_slashings: List[AttesterSlashing, MAX_ATTESTER_SLASHINGS]
+    attestations: List[Attestation, MAX_ATTESTATIONS]
+    deposits: List[Deposit, MAX_DEPOSITS]
+    voluntary_exits: List[SignedVoluntaryExit, MAX_VOLUNTARY_EXITS]
+    sync_aggregate: SyncAggregate
     # Execution
     execution_payload: ExecutionPayload  # [New in Merge]
 ```
 
 #### `BeaconState`
 
 ```python
-class BeaconState(phase0.BeaconState):
+class BeaconState(Container):
+    # Versioning
+    genesis_time: uint64
+    genesis_validators_root: Root
+    slot: Slot
+    fork: Fork
+    # History
+    latest_block_header: BeaconBlockHeader
+    block_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    state_roots: Vector[Root, SLOTS_PER_HISTORICAL_ROOT]
+    historical_roots: List[Root, HISTORICAL_ROOTS_LIMIT]
+    # Eth1
+    eth1_data: Eth1Data
+    eth1_data_votes: List[Eth1Data, EPOCHS_PER_ETH1_VOTING_PERIOD * SLOTS_PER_EPOCH]
+    eth1_deposit_index: uint64
+    # Registry
+    validators: List[Validator, VALIDATOR_REGISTRY_LIMIT]
+    balances: List[Gwei, VALIDATOR_REGISTRY_LIMIT]
+    # Randomness
+    randao_mixes: Vector[Bytes32, EPOCHS_PER_HISTORICAL_VECTOR]
+    # Slashings
+    slashings: Vector[Gwei, EPOCHS_PER_SLASHINGS_VECTOR]  # Per-epoch sums of slashed effective balances
+    # Participation
+    previous_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    current_epoch_participation: List[ParticipationFlags, VALIDATOR_REGISTRY_LIMIT]
+    # Finality
+    justification_bits: Bitvector[JUSTIFICATION_BITS_LENGTH]  # Bit set for every recent justified epoch
+    previous_justified_checkpoint: Checkpoint
+    current_justified_checkpoint: Checkpoint
+    finalized_checkpoint: Checkpoint
+    # Inactivity
+    inactivity_scores: List[uint64, VALIDATOR_REGISTRY_LIMIT]
+    # Sync
+    current_sync_committee: SyncCommittee
+    next_sync_committee: SyncCommittee
     # Execution
     latest_execution_payload_header: ExecutionPayloadHeader  # [New in Merge]
 ```
@@ -190,6 +232,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None:
     process_randao(state, block.body)
     process_eth1_data(state, block.body)
     process_operations(state, block.body)
+    process_sync_aggregate(state, block.body.sync_aggregate)
     if is_execution_enabled(state, block.body):
         process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE)  # [New in Merge]
 ```
@@ -232,7 +275,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe
 
 *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Merge testing only.
 
-*Note*: The function `initialize_beacon_state_from_eth1` is modified to use `MERGE_FORK_VERSION` and initialize `latest_execution_payload_header`.
+*Note*: The function `initialize_beacon_state_from_eth1` is modified: (1) using `MERGE_FORK_VERSION` as the current fork version, (2) utilizing the Merge `BeaconBlockBody` when constructing the initial `latest_block_header`, and (3) initialize `latest_execution_payload_header`.
 
 ```python
 def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32,
@@ -269,6 +312,11 @@ def initialize_beacon_state_from_eth1(eth1_block_hash: Bytes32,
     # Set genesis validators root for domain separation and chain versioning
     state.genesis_validators_root = hash_tree_root(state.validators)
 
+    # Fill in sync committees
+    # Note: A duplicate committee is assigned for the current and next committee at genesis
+    state.current_sync_committee = get_next_sync_committee(state)
+    state.next_sync_committee = get_next_sync_committee(state)
+
     # [New in Merge] Initialize the execution payload header (with block number set to 0)
     state.latest_execution_payload_header.block_hash = eth1_block_hash
     state.latest_execution_payload_header.timestamp = eth1_timestamp

specs/merge/fork.md

@@ -43,15 +43,18 @@ Note that for the pure Merge networks, we don't apply `upgrade_to_merge` since i
 
 ### Upgrading the state
 
-If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == MERGE_FORK_EPOCH`, an irregular state change is made to upgrade to Merge.
+As with the Phase0-to-Altair upgrade, the `state_transition` is modified to upgrade the `BeaconState`.
+The `BeaconState` upgrade runs as part of `process_slots`, slots with missing block proposals do not affect the upgrade time.
 
+If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == MERGE_FORK_EPOCH`, an irregular state change is made to upgrade to Merge.
 The upgrade occurs after the completion of the inner loop of `process_slots` that sets `state.slot` equal to `MERGE_FORK_EPOCH * SLOTS_PER_EPOCH`.
-Care must be taken when transitioning through the fork boundary as implementations will need a modified [state transition function](../phase0/beacon-chain.md#beacon-chain-state-transition-function) that deviates from the Phase 0 document.
-In particular, the outer `state_transition` function defined in the Phase 0 document will not expose the precise fork slot to execute the upgrade in the presence of skipped slots at the fork boundary. Instead the logic must be within `process_slots`.
+
+When multiple upgrades are scheduled for the same epoch (common for test-networks),
+all the upgrades run in sequence before resuming the regular state transition.
 
 ```python
-def upgrade_to_merge(pre: phase0.BeaconState) -> BeaconState:
-    epoch = phase0.get_current_epoch(pre)
+def upgrade_to_merge(pre: altair.BeaconState) -> BeaconState:
+    epoch = altair.get_current_epoch(pre)
     post = BeaconState(
         # Versioning
         genesis_time=pre.genesis_time,
@@ -78,14 +81,19 @@ def upgrade_to_merge(pre: phase0.BeaconState) -> BeaconState:
         randao_mixes=pre.randao_mixes,
         # Slashings
         slashings=pre.slashings,
-        # Attestations
-        previous_epoch_attestations=pre.previous_epoch_attestations,
-        current_epoch_attestations=pre.current_epoch_attestations,
+        # Participation
+        previous_epoch_participation=pre.previous_epoch_participation,
+        current_epoch_participation=pre.current_epoch_participation,
         # Finality
         justification_bits=pre.justification_bits,
         previous_justified_checkpoint=pre.previous_justified_checkpoint,
         current_justified_checkpoint=pre.current_justified_checkpoint,
         finalized_checkpoint=pre.finalized_checkpoint,
+        # Inactivity
+        inactivity_scores=pre.inactivity_scores,
+        # Sync
+        current_sync_committee=pre.current_sync_committee,
+        next_sync_committee=pre.next_sync_committee,
         # Execution-layer
         latest_execution_payload_header=ExecutionPayloadHeader(),
     )

specs/merge/validator.md

@@ -1,7 +1,5 @@
 # Ethereum 2.0 The Merge
 
-**Warning:** This document is currently based on [Phase 0](../phase0/validator.md) but will be rebased to [Altair](../altair/validator.md) once the latter is shipped.
-
 **Notice**: This document is a work-in-progress for researchers and implementers.
 
 ## Table of contents
@@ -29,9 +27,11 @@ This document represents the changes to be made in the code of an "honest valida
 
 ## Prerequisites
 
-This document is an extension of the [Phase 0 -- Validator](../phase0/validator.md). All behaviors and definitions defined in the Phase 0 doc carry over unless explicitly noted or overridden.
+This document is an extension of the [Altair -- Honest Validator](../altair/validator.md) guide.
+All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden.
 
-All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [The Merge](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout.
+All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [The Merge](./beacon-chain.md) are requisite for this document and used throughout.
+Please see related Beacon Chain doc before continuing and use them as a reference throughout.
 
 ## Protocols
 

specs/sharding/beacon-chain.md

@@ -179,7 +179,7 @@ class BeaconBlockBody(merge.BeaconBlockBody):  # [extends The Merge block body]
 ### `BeaconState`
 
 ```python
-class BeaconState(merge.BeaconState):  # [extends The Merge state]
+class BeaconState(merge.BeaconState):
     # [Updated fields] (Warning: this changes with Altair, Sharding will rebase to use participation-flags)
     previous_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH]
     current_epoch_attestations: List[PendingAttestation, MAX_ATTESTATIONS * SLOTS_PER_EPOCH]
@@ -494,9 +494,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None:
     process_randao(state, block.body)
     process_eth1_data(state, block.body)
     process_operations(state, block.body)  # [Modified in Sharding]
-    # Pre-merge, skip execution payload processing
-    if is_execution_enabled(state, block):
-        process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE)  # [New in Merge]
+    process_sync_aggregate(state, block.body.sync_aggregate)
+    # is_execution_enabled is omitted, execution is enabled by default. 
+    process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE)
 ```
 
 #### Operations
@@ -527,7 +527,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None:
 
 ```python
 def process_attestation(state: BeaconState, attestation: Attestation) -> None:
-    phase0.process_attestation(state, attestation)
+    altair.process_attestation(state, attestation)
     update_pending_shard_work(state, attestation)
 ```
 
@@ -681,25 +681,26 @@ This epoch transition overrides the Merge epoch transition:
 
 ```python
 def process_epoch(state: BeaconState) -> None:
-    # Sharding
+    # Sharding pre-processing
     process_pending_shard_confirmations(state)
     charge_confirmed_shard_fees(state)
     reset_pending_shard_work(state)
 
-    # Phase0
+    # Base functionality
     process_justification_and_finalization(state)
+    process_inactivity_updates(state)
     process_rewards_and_penalties(state)
     process_registry_updates(state)
     process_slashings(state)
-
-    # Final updates
     process_eth1_data_reset(state)
     process_effective_balance_updates(state)
     process_slashings_reset(state)
     process_randao_mixes_reset(state)
     process_historical_roots_update(state)
-    process_participation_record_updates(state)
+    process_participation_flag_updates(state)
+    process_sync_committee_updates(state)
 
+    # Sharding post-processing
     process_shard_epoch_increment(state)
 ```
 

specs/sharding/p2p-interface.md

@@ -29,8 +29,7 @@
 
 ## Introduction
 
-The specification of these changes continues in the same format as the [Phase0](../phase0/p2p-interface.md) and
-[Altair](../altair/p2p-interface.md) network specifications, and assumes them as pre-requisite.
+The specification of these changes continues in the same format as the network specifications of previous upgrades, and assumes them as pre-requisite.
 The adjustments and additions for Shards are outlined in this document.
 
 ## Constants

tests/core/pyspec/eth2spec/test/context.py

@@ -347,10 +347,6 @@ def wrapper(*args, **kw):
                 preset_name = kw.pop('preset')
             targets = spec_targets[preset_name]
 
-            # TODO: test state is dependent on phase0 but is immediately transitioned to later phases.
-            #  A new state-creation helper for later phases may be in place, and then tests can run without phase0
-            available_phases.add(PHASE0)
-
             # Populate all phases for multi-phase tests
             phase_dir = {}
             if PHASE0 in available_phases:
@@ -433,23 +429,15 @@ def wrapper(*args, spec: Spec, **kw):
 
 
 def is_post_altair(spec):
-    if spec.fork == MERGE:  # TODO: remove parallel Altair-Merge condition after rebase.
-        return False
-    if spec.fork in FORKS_BEFORE_ALTAIR:
-        return False
-    return True
+    return spec.fork not in FORKS_BEFORE_ALTAIR
 
 
 def is_post_merge(spec):
-    if spec.fork == ALTAIR:  # TODO: remove parallel Altair-Merge condition after rebase.
-        return False
-    if spec.fork in FORKS_BEFORE_MERGE:
-        return False
-    return True
+    return spec.fork not in FORKS_BEFORE_MERGE
 
 
-with_altair_and_later = with_phases([ALTAIR])  # TODO: include Merge, but not until Merge work is rebased.
-with_merge_and_later = with_phases([MERGE])
+with_altair_and_later = with_phases([ALTAIR, MERGE])
+with_merge_and_later = with_phases([MERGE])  # TODO: include sharding when spec stabilizes.
 
 
 def fork_transition_test(pre_fork_name, post_fork_name, fork_epoch=None):

tests/core/pyspec/eth2spec/test/helpers/block_processing.py

@@ -30,6 +30,7 @@ def get_process_calls(spec):
         # Merge
         'process_application_payload':
             lambda state, block: spec.process_application_payload(state, block.body),
+        # TODO: add sharding processing functions when spec stabilizes.
         # Custody Game
         'process_custody_game_operations':
             lambda state, block: spec.process_custody_game_operations(state, block.body),

tests/core/pyspec/eth2spec/test/helpers/constants.py

@@ -18,12 +18,9 @@
 ALL_PHASES = (PHASE0, ALTAIR, MERGE)
 # The forks that output to the test vectors.
 TESTGEN_FORKS = (PHASE0, ALTAIR, MERGE)
-# TODO: everything runs in parallel to Altair.
-# After features are rebased on the Altair fork, this can be reduced to just PHASE0.
-FORKS_BEFORE_ALTAIR = (PHASE0, MERGE, SHARDING, CUSTODY_GAME, DAS)
 
-# TODO: when rebasing Merge onto Altair, add ALTAIR to this tuple.
-FORKS_BEFORE_MERGE = (PHASE0,)
+FORKS_BEFORE_ALTAIR = (PHASE0,)
+FORKS_BEFORE_MERGE = (PHASE0, ALTAIR)
 
 #
 # Config

tests/core/pyspec/eth2spec/test/helpers/epoch_processing.py

@@ -28,7 +28,7 @@ def get_process_calls(spec):
             'process_participation_record_updates'
         ),
         'process_sync_committee_updates',  # altair
-        'process_shard_epoch_increment'  # sharding
+        # TODO: add sharding processing functions when spec stabilizes.
     ]