ethereum · djrtwo · Aug 12, 2019 · Jul 10, 2019 · Jul 10, 2019 · Jul 20, 2019
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,13 +35,13 @@ commands:
     description: "Restore the cache with pyspec keys"
     steps:
       - restore_cached_venv:
-          venv_name: v3-pyspec-bump2
+          venv_name: v4-pyspec
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
   save_pyspec_cached_venv:
     description: Save a venv into a cache with pyspec keys"
     steps:
       - save_cached_venv:
-          venv_name: v3-pyspec-bump2
+          venv_name: v4-pyspec
           reqs_checksum: cache-{{ checksum "test_libs/pyspec/requirements.txt" }}-{{ checksum "test_libs/pyspec/requirements-testing.txt" }}
           venv_path: ./test_libs/pyspec/venv
   restore_deposit_contract_cached_venv:

diff --git a/Makefile b/Makefile
@@ -2,17 +2,20 @@ SPEC_DIR = ./specs
 SCRIPT_DIR = ./scripts
 TEST_LIBS_DIR = ./test_libs
 PY_SPEC_DIR = $(TEST_LIBS_DIR)/pyspec
-YAML_TEST_DIR = ./eth2.0-spec-tests/tests
+TEST_VECTOR_DIR = ./eth2.0-spec-tests/tests
 GENERATOR_DIR = ./test_generators
 DEPOSIT_CONTRACT_DIR = ./deposit_contract
 CONFIGS_DIR = ./configs
 
 # Collect a list of generator names
-GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/)))
-# Map this list of generator paths to a list of test output paths
-YAML_TEST_TARGETS = $(patsubst $(GENERATOR_DIR)/%, $(YAML_TEST_DIR)/%, $(GENERATORS))
+GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
+# Map this list of generator paths to "gen_{generator name}" entries
+GENERATOR_TARGETS = $(patsubst $(GENERATOR_DIR)/%/, gen_%, $(GENERATORS))
 GENERATOR_VENVS = $(patsubst $(GENERATOR_DIR)/%, $(GENERATOR_DIR)/%venv, $(GENERATORS))
 
+# To check generator matching:
+#$(info $$GENERATOR_TARGETS is [${GENERATOR_TARGETS}])
+
 PY_SPEC_PHASE_0_TARGETS = $(PY_SPEC_DIR)/eth2spec/phase0/spec.py
 PY_SPEC_PHASE_0_DEPS = $(SPEC_DIR)/core/0_*.md
 
@@ -24,14 +27,14 @@ PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
 COV_HTML_OUT=.htmlcov
 COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
-.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \
+.PHONY: clean partial_clean all test citest lint generate_tests pyspec phase0 phase1 install_test open_cov \
         install_deposit_contract_test test_deposit_contract compile_deposit_contract
 
-all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
+all: $(PY_SPEC_ALL_TARGETS)
 
 # deletes everything except the venvs
 partial_clean:
-	rm -rf $(YAML_TEST_DIR)
+	rm -rf $(TEST_VECTOR_DIR)
 	rm -rf $(GENERATOR_VENVS)
 	rm -rf $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
@@ -44,8 +47,8 @@ clean: partial_clean
 	rm -rf $(PY_SPEC_DIR)/venv
 	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv
 
-# "make gen_yaml_tests" to run generators
-gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
+# "make generate_tests" to run all generators
+generate_tests: $(PY_SPEC_ALL_TARGETS) $(GENERATOR_TARGETS)
 
 # installs the packages to run pyspec tests
 install_test:
@@ -90,8 +93,8 @@ $(PY_SPEC_DIR)/eth2spec/phase1/spec.py: $(PY_SPEC_PHASE_1_DEPS)
 
 CURRENT_DIR = ${CURDIR}
 
-# The function that builds a set of suite files, by calling a generator for the given type (param 1)
-define build_yaml_tests
+# Runs a generator, identified by param 1
+define run_generator
 	# Started!
 	# Create output directory
 	# Navigate to the generator
@@ -100,24 +103,25 @@ define build_yaml_tests
 	# Install all the necessary requirements
 	# Run the generator. The generator is assumed to have an "main.py" file.
 	# We output to the tests dir (generator program should accept a "-o <filepath>" argument.
+	# `-l minimal general` can be added to the generator call to filter to smaller configs, when testing.
 	echo "generator $(1) started"; \
-	mkdir -p $(YAML_TEST_DIR)$(1); \
-	cd $(GENERATOR_DIR)$(1); \
+	mkdir -p $(TEST_VECTOR_DIR); \
+	cd $(GENERATOR_DIR)/$(1); \
 	if ! test -d venv; then python3 -m venv venv; fi; \
 	. venv/bin/activate; \
 	pip3 install -r requirements.txt; \
-	python3 main.py -o $(CURRENT_DIR)/$(YAML_TEST_DIR)$(1) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
+	python3 main.py -o $(CURRENT_DIR)/$(TEST_VECTOR_DIR) -c $(CURRENT_DIR)/$(CONFIGS_DIR); \
 	echo "generator $(1) finished"
 endef
 
 # The tests dir itself is simply build by creating the directory (recursively creating deeper directories if necessary)
-$(YAML_TEST_DIR):
-	$(info creating directory, to output yaml targets to: ${YAML_TEST_TARGETS})
+$(TEST_VECTOR_DIR):
+	$(info creating test output directory, for generators: ${GENERATOR_TARGETS})
 	mkdir -p $@
-$(YAML_TEST_DIR)/:
-	$(info ignoring duplicate yaml tests dir)
+$(TEST_VECTOR_DIR)/:
+	$(info ignoring duplicate tests dir)
 
-# For any target within the tests dir, build it using the build_yaml_tests function.
+# For any generator, build it using the run_generator function.
 # (creation of output dir is a dependency)
-$(YAML_TEST_DIR)%: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR)
-	$(call build_yaml_tests,$*)
+gen_%: $(PY_SPEC_ALL_TARGETS) $(TEST_VECTOR_DIR)
+	$(call run_generator,$*)
diff --git a/configs/README.md b/configs/README.md
@@ -0,0 +1,36 @@
+# Configs
+
+This directory contains a set of constants presets used for testing, testnets, and mainnet.
+
+A preset file contains all the constants known for its target.
+Later-fork constants can be ignored, e.g. ignore phase1 constants as a client that only supports phase 0 currently.
+
+
+## Forking
+
+Configs are not replaced, but extended with forks. This is to support syncing from one state to the other over a fork boundary, without hot-swapping a config.
+Instead, for forks that introduce changes in a constant, the constant name is prefixed with a short abbreviation of the fork.
+
+Over time, the need to sync an older state may be deprecated.
+In this case, the prefix on the new constant may be removed, and the old constant will keep a special name before completely being removed.
+
+A previous iteration of forking made use of "timelines", but this collides with the definitions used in the spec (constants for special forking slots etc.),
+ and was not integrated sufficiently in any of the spec tools or implementations.
+Instead, the config essentially doubles as fork definition now, changing the value for e.g. `PHASE_1_GENESIS_SLOT` changes the fork.
+
+Another reason to prefer forking through constants is the ability to program a forking moment based on context, instead of being limited to a static slot number.
+
+
+## Format
+
+Each preset is a key-value mapping.
+
+**Key**: an `UPPER_SNAKE_CASE` (a.k.a. "macro case") formatted string, name of the constant.
+
+**Value** can be either:
+ - an unsigned integer number, can be up to 64 bits (incl.)
+ - a hexadecimal string, prefixed with `0x`
+
+Presets may contain comments to describe the values.
+
+See [`mainnet.yaml`](./mainnet.yaml) for a complete example.
diff --git a/configs/constant_presets/README.md b/configs/constant_presets/README.md
diff --git a/configs/fork_timelines/README.md b/configs/fork_timelines/README.md
diff --git a/configs/fork_timelines/mainnet.yaml b/configs/fork_timelines/mainnet.yaml
diff --git a/configs/fork_timelines/testing.yaml b/configs/fork_timelines/testing.yaml
diff --git a/configs/constant_presets/mainnet.yaml → configs/mainnet.yaml b/configs/constant_presets/mainnet.yaml → configs/mainnet.yaml
diff --git a/configs/constant_presets/minimal.yaml → configs/minimal.yaml b/configs/constant_presets/minimal.yaml → configs/minimal.yaml
diff --git a/scripts/build_spec.py b/scripts/build_spec.py
@@ -48,7 +48,7 @@
 from eth2spec.utils.ssz.ssz_impl import (
     hash_tree_root,
     signing_root,
-    is_empty,
+    is_zero,
 )
 from eth2spec.utils.ssz.ssz_typing import (
     uint64, bit, boolean, Container, List, Vector, Bytes, BytesN,

diff --git a/specs/core/0_beacon-chain.md b/specs/core/0_beacon-chain.md
@@ -1504,8 +1504,6 @@ def process_final_updates(state: BeaconState) -> None:
         HALF_INCREMENT = EFFECTIVE_BALANCE_INCREMENT // 2
         if balance < validator.effective_balance or validator.effective_balance + 3 * HALF_INCREMENT < balance:
             validator.effective_balance = min(balance - balance % EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE)
-    # Update start shard
-    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Set active index root
     index_epoch = Epoch(next_epoch + ACTIVATION_EXIT_DELAY)
     index_root_position = index_epoch % EPOCHS_PER_HISTORICAL_VECTOR
@@ -1522,6 +1520,8 @@ def process_final_updates(state: BeaconState) -> None:
     if next_epoch % (SLOTS_PER_HISTORICAL_ROOT // SLOTS_PER_EPOCH) == 0:
         historical_batch = HistoricalBatch(block_roots=state.block_roots, state_roots=state.state_roots)
         state.historical_roots.append(hash_tree_root(historical_batch))
+    # Update start shard
+    state.start_shard = Shard((state.start_shard + get_shard_delta(state, current_epoch)) % SHARD_COUNT)
     # Rotate current/previous epoch attestations
     state.previous_epoch_attestations = state.current_epoch_attestations
     state.current_epoch_attestations = []
@@ -1654,6 +1654,9 @@ def process_attestation(state: BeaconState, attestation: Attestation) -> None:
     attestation_slot = get_attestation_data_slot(state, data)
     assert attestation_slot + MIN_ATTESTATION_INCLUSION_DELAY <= state.slot <= attestation_slot + SLOTS_PER_EPOCH
 
+    committee = get_crosslink_committee(state, data.target.epoch, data.crosslink.shard)
+    assert len(attestation.aggregation_bits) == len(attestation.custody_bits) == len(committee)
+
     pending_attestation = PendingAttestation(
         data=data,
         aggregation_bits=attestation.aggregation_bits,

diff --git a/specs/core/0_fork-choice.md b/specs/core/0_fork-choice.md
@@ -91,8 +91,12 @@ def get_genesis_store(genesis_state: BeaconState) -> Store:
 ```python
 def get_ancestor(store: Store, root: Hash, slot: Slot) -> Hash:
     block = store.blocks[root]
-    assert block.slot >= slot
-    return root if block.slot == slot else get_ancestor(store, block.parent_root, slot)
+    if block.slot > slot:
+        return get_ancestor(store, block.parent_root, slot)
+    elif block.slot == slot:
+        return root
+    else:
+        return Bytes32()  # root is older than queried slot: no results. 
 ```
 
 #### `get_latest_attesting_balance`

diff --git a/specs/core/1_custody-game.md b/specs/core/1_custody-game.md
@@ -391,7 +391,7 @@ def get_custody_period_for_validator(state: BeaconState, validator_index: Valida
 ```python
 def replace_empty_or_append(list: MutableSequence[Any], new_element: Any) -> int:
     for i in range(len(list)):
-        if is_empty(list[i]):
+        if is_zero(list[i]):
             list[i] = new_element
             return i
     list.append(new_element)

diff --git a/specs/simple-serialize.md b/specs/simple-serialize.md
@@ -14,7 +14,7 @@
         - [Variable-size and fixed-size](#variable-size-and-fixed-size)
         - [Aliases](#aliases)
         - [Default values](#default-values)
-            - [`is_empty`](#is_empty)
+            - [`is_zero`](#is_zero)
         - [Illegal types](#illegal-types)
     - [Serialization](#serialization)
         - [`uintN`](#uintn)
@@ -75,19 +75,21 @@ For convenience we alias:
 * `bit` to `boolean`
 * `byte` to `uint8` (this is a basic type)
 * `BytesN` to `Vector[byte, N]` (this is *not* a basic type)
-* `null`: `{}`, i.e. the empty container
+* `null`: `{}`
 
 ### Default values
 
 The default value of a type upon initialization is recursively defined using `0` for `uintN`, `False` for `boolean` and the elements of `Bitvector`, and `[]` for lists and `Bitlist`. Unions default to the first type in the union (with type index zero), which is `null` if present in the union.
 
-#### `is_empty`
+#### `is_zero`
 
-An SSZ object is called empty (and thus, `is_empty(object)` returns true) if it is equal to the default value for that type.
+An SSZ object is called zeroed (and thus, `is_zero(object)` returns true) if it is equal to the default value for that type.
 
 ### Illegal types
 
-Empty vector types (i.e. `[subtype, 0]` for some `subtype`) are not legal. The `null` type is only legal as the first type in a union subtype (i.e. with type index zero).
+- Empty vector types (`Vector[type, 0]`, `Bitvector[0]`) are illegal.
+- Containers with no fields are illegal.
+- The `null` type is only legal as the first type in a union subtype (i.e. with type index zero).
 
 ## Serialization