Modularize Phase1 into post-Merge features: Sharding, Custody Game, Data Availability Sampling

Makefile

@@ -20,7 +20,11 @@ GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENER
 # To check generator matching:
 #$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
 
-MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/phase1/*.md) $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SSZ_DIR)/*.md) $(wildcard $(SPEC_DIR)/networking/*.md) $(wildcard $(SPEC_DIR)/validator/*.md)
+MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/phase0/*.md) $(wildcard $(SPEC_DIR)/altair/*.md) $(wildcard $(SSZ_DIR)/*.md) \
+                 $(wildcard $(SPEC_DIR)/merge/*.md) \
+                 $(wildcard $(SPEC_DIR)/custody/*.md) \
+                 $(wildcard $(SPEC_DIR)/das/*.md) \
+                 $(wildcard $(SPEC_DIR)/sharding/*.md)
 
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
@@ -88,11 +92,11 @@ install_test:
 
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n 4 --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 find_test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -k=$(K) --disable-bls --cov=eth2spec.phase0.spec --cov=eth2spec.altair.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: pyspec
 	mkdir -p tests/core/pyspec/test-reports/eth2spec; . venv/bin/activate; cd $(PY_SPEC_DIR); \
@@ -112,10 +116,11 @@ check_toc: $(MARKDOWN_FILES:=.toc)
 codespell:
 	codespell . --skip ./.git -I .codespell-whitelist
 
+# TODO: add future merge, sharding, etc. packages to linting.
 lint: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
 	flake8  --config $(LINTER_CONFIG_FILE) ./eth2spec \
-	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.phase1 -p eth2spec.altair
+	&& mypy --config-file $(LINTER_CONFIG_FILE) -p eth2spec.phase0 -p eth2spec.altair
 
 lint_generators: pyspec
 	. venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

README.md

@@ -11,7 +11,10 @@ This repository hosts the current Eth2 specifications. Discussions about design
 
 [![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec)
 
-Core specifications for Eth2 clients be found in [specs](specs/). These are divided into phases. Each subsequent phase depends upon the prior. The current phases specified are:
+Core specifications for Eth2 clients be found in [specs](specs/). These are divided into features.
+Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready.
+
+The current features are:
 
 ### Phase 0
 
@@ -27,13 +30,35 @@ Core specifications for Eth2 clients be found in [specs](specs/). These are divi
 * [Altair fork](specs/altair/fork.md)
 * [Light client sync protocol](specs/altair/sync-protocol.md)
 
-### Sharding
+### Merge
 
-The sharding spec is still actively in R&D; see the most recent available pull request [here](https://github.com/ethereum/eth2.0-specs/pull/2146) and some technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD).
+The merge is still actively in R&D. The specifications outline a general direction for engineering work,
+while the details are in review and may change.
 
-### Merge
+* Background material:
+  * An [ethresear.ch](https://ethresear.ch) post [describing the basic mechanism](https://ethresear.ch/t/the-eth1-eth2-transition/6265)
+  * [ethereum.org](https://ethereum.org) high-level description of the merge [here](https://ethereum.org/en/eth2/docking/)
+* Specifications:
+  * [Beacon Chain changes](specs/merge/beacon-chain.md)
+  * [Fork Choice changes](specs/merge/fork-choice.md)
+  * [Validator additions](specs/merge/validator.md)
+
+### Sharding
 
-The merge is still actively in R&D; see an [ethresear.ch](https://ethresear.ch) post describing the proposed basic mechanism [here](https://ethresear.ch/t/the-eth1-eth2-transition/6265) and the section of [ethereum.org](https://ethereum.org) describing the merge at a high level [here](https://ethereum.org/en/eth2/docking/).
+Sharding follows the merge, and is divided into three parts:
+
+* Sharding base functionality - In early engineering phase
+  * [Beacon Chain changes](specs/sharding/beacon-chain.md)
+  * [P2P Network changes](specs/sharding/p2p-interface.md)
+* Custody Game - Ready, dependent on sharding 
+  * [Custody Game](specs/custody_game/custody-game.md)
+  * [Validator custody work](specs/custody_game/validator.md)
+* Data Availability Sampling - In active R&D
+  * Technical details [here](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD).
+  * [Core types and functions](specs/das/das-core.md)
+  * [P2P Networking](specs/das/p2p-interface.md)
+  * [Fork Choice](specs/das/fork-choice.md)
+  * [Sampling process](specs/das/sampling.md)
 
 ### Accompanying documents can be found in [specs](specs) and include:
 

configs/mainnet/custody_game.yaml

@@ -0,0 +1,48 @@
+# Mainnet preset - Custody Game
+
+# Time parameters
+# ---------------------------------------------------------------
+# 2**1 (= 2) epochs, 12.8 minutes
+RANDAO_PENALTY_EPOCHS: 2
+# 2**15 (= 32,768) epochs, ~146 days
+EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 32768
+# 2**14 (= 16,384) epochs ~73 days
+EPOCHS_PER_CUSTODY_PERIOD: 16384
+# 2**11 (= 2,048) epochs, ~9 days
+CUSTODY_PERIOD_TO_RANDAO_PADDING: 2048
+# 2**15 (= 32,768) epochs, ~146 days
+MAX_CHUNK_CHALLENGE_DELAY: 32768
+
+# Misc parameters
+# ---------------------------------------------------------------
+# 2**256 - 189
+CUSTODY_PRIME: 115792089237316195423570985008687907853269984665640564039457584007913129639747
+# 3
+CUSTODY_SECRETS: 3
+# 1/1024 chance of custody bit 1
+CUSTODY_PROBABILITY_EXPONENT: 10
+
+# Max operations
+# ---------------------------------------------------------------
+# 2**8 (= 256)
+MAX_CUSTODY_KEY_REVEALS: 256
+# 2**0 (= 1)
+MAX_EARLY_DERIVED_SECRET_REVEALS: 1
+# 2**2 (= 2)
+MAX_CUSTODY_CHUNK_CHALLENGES: 4
+# 2** 4 (= 16)
+MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 16
+# 2**0 (= 1)
+MAX_CUSTODY_SLASHINGS: 1
+
+# Reward and penalty quotients
+# ---------------------------------------------------------------
+EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2
+# 2**8 (= 256)
+MINOR_REWARD_QUOTIENT: 256
+
+# Signature domains
+# ---------------------------------------------------------------
+DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
+DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000
+DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000

configs/mainnet/merge.yaml

@@ -0,0 +1,7 @@
+# Mainnet preset - The Merge
+
+# Fork
+# ---------------------------------------------------------------
+MERGE_FORK_VERSION: 0x02000000
+# TBD
+MERGE_FORK_SLOT: 0

configs/mainnet/sharding.yaml

@@ -0,0 +1,43 @@
+# Mainnet preset - Sharding
+
+# Fork
+# ---------------------------------------------------------------
+SHARDING_FORK_VERSION: 0x03000000
+# TBD
+SHARDING_FORK_SLOT: 0
+
+
+# Beacon-chain
+# ---------------------------------------------------------------
+# Misc
+# 2**10 (= 1,024)
+MAX_SHARDS: 1024
+# 2**6 = 64
+INITIAL_ACTIVE_SHARDS: 64
+# 2**3 (= 8)
+GASPRICE_ADJUSTMENT_COEFFICIENT: 8
+
+# Shard block configs
+# ---------------------------------------------------------------
+MAX_SHARD_HEADERS_PER_SHARD: 4
+# 2**11 (= 2,048)
+MAX_SAMPLES_PER_BLOCK: 2048
+# 2**10 (= 1,1024)
+TARGET_SAMPLES_PER_BLOCK: 1024
+
+# Gwei values
+# ---------------------------------------------------------------
+# 2**33 (= 8,589,934,592) Gwei
+MAX_GASPRICE: 8589934592
+# 2**3 (= 8) Gwei
+MIN_GASPRICE: 8
+
+# Time parameters
+# ---------------------------------------------------------------
+# 2**8 (= 256) | epochs
+SHARD_COMMITTEE_PERIOD: 256
+
+# Signature domains
+# ---------------------------------------------------------------
+DOMAIN_SHARD_PROPOSAL: 0x80000000
+DOMAIN_SHARD_COMMITTEE: 0x81000000

configs/minimal/custody_game.yaml

@@ -0,0 +1,48 @@
+# Minimal preset - Custody Game
+
+# Time parameters
+# ---------------------------------------------------------------
+# 2**1 (= 2) epochs, 12.8 minutes
+RANDAO_PENALTY_EPOCHS: 2
+# [customized] quicker for testing
+EARLY_DERIVED_SECRET_PENALTY_MAX_FUTURE_EPOCHS: 64
+# [customized] quicker for testing
+EPOCHS_PER_CUSTODY_PERIOD: 32
+# [customized] quicker for testing
+CUSTODY_PERIOD_TO_RANDAO_PADDING: 8
+# [customize for faster testing]
+MAX_CHUNK_CHALLENGE_DELAY: 64
+
+# Misc parameters
+# ---------------------------------------------------------------
+# 2**256 - 189
+CUSTODY_PRIME: 115792089237316195423570985008687907853269984665640564039457584007913129639747
+# 3
+CUSTODY_SECRETS: 3
+# 1/4 chance of custody bit 1 [customized for faster testing]
+CUSTODY_PROBABILITY_EXPONENT: 2
+
+# Max operations
+# ---------------------------------------------------------------
+# 2**8 (= 256)
+MAX_CUSTODY_KEY_REVEALS: 256
+# 2**0 (= 1)
+MAX_EARLY_DERIVED_SECRET_REVEALS: 1
+# [customized]
+MAX_CUSTODY_CHUNK_CHALLENGES: 2
+# [customized]
+MAX_CUSTODY_CHUNK_CHALLENGE_RESP: 8
+# 2**0 (= 1)
+MAX_CUSTODY_SLASHINGS: 1
+
+# Reward and penalty quotients
+# ---------------------------------------------------------------
+EARLY_DERIVED_SECRET_REVEAL_SLOT_REWARD_MULTIPLE: 2
+# 2**8 (= 256)
+MINOR_REWARD_QUOTIENT: 256
+
+# Signature domains
+# ---------------------------------------------------------------
+DOMAIN_CUSTODY_BIT_SLASHING: 0x83000000
+DOMAIN_LIGHT_SELECTION_PROOF: 0x84000000
+DOMAIN_LIGHT_AGGREGATE_AND_PROOF: 0x85000000

configs/minimal/merge.yaml

@@ -0,0 +1,7 @@
+# Minimal preset - The Merge
+
+# Fork
+# ---------------------------------------------------------------
+MERGE_FORK_VERSION: 0x02000001
+# TBD
+MERGE_FORK_SLOT: 0