release v0.7.1

.gitignore

@@ -14,3 +14,10 @@ eth2.0-spec-tests/
 # Dynamically built from Markdown spec
 test_libs/pyspec/eth2spec/phase0/spec.py
 test_libs/pyspec/eth2spec/phase1/spec.py
+
+# coverage reports
+.htmlcov
+.coverage
+
+# local CI testing output
+test_libs/pyspec/test-reports

Makefile

@@ -21,8 +21,11 @@ PY_SPEC_PHASE_1_DEPS = $(SPEC_DIR)/core/1_*.md
 
 PY_SPEC_ALL_TARGETS = $(PY_SPEC_PHASE_0_TARGETS) $(PY_SPEC_PHASE_1_TARGETS)
 
+COV_HTML_OUT=.htmlcov
+COV_INDEX_FILE=$(PY_SPEC_DIR)/$(COV_HTML_OUT)/index.html
 
-.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test install_deposit_contract_test test_deposit_contract compile_deposit_contract 
+.PHONY: clean all test citest lint gen_yaml_tests pyspec phase0 phase1 install_test open_cov \
+        install_deposit_contract_test test_deposit_contract compile_deposit_contract
 
 all: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_DIR) $(YAML_TEST_TARGETS)
 
@@ -32,6 +35,9 @@ clean:
 	rm -rf $(PY_SPEC_DIR)/venv $(PY_SPEC_DIR)/.pytest_cache
 	rm -rf $(PY_SPEC_ALL_TARGETS)
 	rm -rf $(DEPOSIT_CONTRACT_DIR)/venv $(DEPOSIT_CONTRACT_DIR)/.pytest_cache
+	rm -rf $(PY_SPEC_DIR)/$(COV_HTML_OUT)
+	rm -rf $(PY_SPEC_DIR)/.coverage
+	rm -rf $(PY_SPEC_DIR)/test-reports
 
 # "make gen_yaml_tests" to run generators
 gen_yaml_tests: $(PY_SPEC_ALL_TARGETS) $(YAML_TEST_TARGETS)
@@ -41,11 +47,15 @@ install_test:
 	cd $(PY_SPEC_DIR); python3 -m venv venv; . venv/bin/activate; pip3 install -r requirements-testing.txt;
 
 test: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); . venv/bin/activate; python -m pytest eth2spec
+	cd $(PY_SPEC_DIR); . venv/bin/activate;	export PYTHONPATH="./"; \
+	python -m pytest --cov=eth2spec.phase0.spec --cov=eth2spec.phase1.spec --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 citest: $(PY_SPEC_ALL_TARGETS)
-	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate;	\
-	python -m pytest --junitxml=test-reports/eth2spec/test_results_phase0.xml eth2spec
+	cd $(PY_SPEC_DIR); mkdir -p test-reports/eth2spec; . venv/bin/activate; \
+	python -m pytest --junitxml=test-reports/eth2spec/test_results.xml eth2spec
+
+open_cov:
+	((open "$(COV_INDEX_FILE)" || xdg-open "$(COV_INDEX_FILE)") &> /dev/null) &
 
 lint: $(PY_SPEC_ALL_TARGETS)
 	cd $(PY_SPEC_DIR); . venv/bin/activate; \

test_generators/bls/main.py

@@ -12,8 +12,15 @@
 from py_ecc import bls
 
 
-def int_to_hex(n: int) -> str:
-    return '0x' + int_to_big_endian(n).hex()
+F2Q_COEFF_LEN = 48
+G2_COMPRESSED_Z_LEN = 48
+
+
+def int_to_hex(n: int, byte_length: int=None) -> str:
+    byte_value = int_to_big_endian(n)
+    if byte_length:
+        byte_value = byte_value.rjust(byte_length, b'\x00')
+    return '0x' + byte_value.hex()
 
 
 def hex_to_int(x: str) -> int:
@@ -58,8 +65,8 @@ def hash_message(msg: bytes,
     """
     return [
         [
-            int_to_hex(fq2.coeffs[0]),
-            int_to_hex(fq2.coeffs[1]),
+            int_to_hex(fq2.coeffs[0], F2Q_COEFF_LEN),
+            int_to_hex(fq2.coeffs[1], F2Q_COEFF_LEN),
         ]
         for fq2 in bls.utils.hash_to_G2(msg, domain)
     ]
@@ -75,8 +82,7 @@ def hash_message_compressed(msg: bytes, domain: int) -> Tuple[str, str]:
         - Message hash as a compressed G2 point
     """
     z1, z2 = bls.utils.compress_G2(bls.utils.hash_to_G2(msg, domain))
-    return [int_to_hex(z1), int_to_hex(z2)]
-
+    return [int_to_hex(z1, G2_COMPRESSED_Z_LEN), int_to_hex(z2, G2_COMPRESSED_Z_LEN)]
 
 
 @to_tuple

test_libs/pyspec/README.md

@@ -28,7 +28,7 @@ These tests are sanity tests, to verify if the spec itself is consistent.
 
 #### Automated
 
-Run `make test` from the root of the specs repository.
+Run `make test` from the root of the specs repository (after running `make install_test` if have not before).
 
 #### Manual
 
@@ -50,6 +50,10 @@ pytest --config=minimal eth2spec
 ```
 Note the package-name, this is to locate the tests.
 
+### How to view code coverage report
+
+Run `make open_cov` from the root of the specs repository after running `make test` to open the html code coverage report.
+
 
 ## Contributing
 

test_libs/pyspec/eth2spec/fuzzing/decoder.py

@@ -0,0 +1,84 @@
+from eth2spec.utils.ssz import ssz_typing as spec_ssz
+import ssz
+
+
+def translate_typ(typ) -> ssz.BaseSedes:
+    """
+    Translates a spec type to a Py-SSZ type description (sedes).
+    :param typ: The spec type, a class.
+    :return: The Py-SSZ equivalent.
+    """
+    if spec_ssz.is_container_type(typ):
+        return ssz.Container(
+            [translate_typ(field_typ) for (field_name, field_typ) in typ.get_fields()])
+    elif spec_ssz.is_bytesn_type(typ):
+        return ssz.ByteVector(typ.length)
+    elif spec_ssz.is_bytes_type(typ):
+        return ssz.ByteList()
+    elif spec_ssz.is_vector_type(typ):
+        return ssz.Vector(translate_typ(spec_ssz.read_vector_elem_type(typ)), typ.length)
+    elif spec_ssz.is_list_type(typ):
+        return ssz.List(translate_typ(spec_ssz.read_list_elem_type(typ)))
+    elif spec_ssz.is_bool_type(typ):
+        return ssz.boolean
+    elif spec_ssz.is_uint_type(typ):
+        size = spec_ssz.uint_byte_size(typ)
+        if size == 1:
+            return ssz.uint8
+        elif size == 2:
+            return ssz.uint16
+        elif size == 4:
+            return ssz.uint32
+        elif size == 8:
+            return ssz.uint64
+        elif size == 16:
+            return ssz.uint128
+        elif size == 32:
+            return ssz.uint256
+        else:
+            raise TypeError("invalid uint size")
+    else:
+        raise TypeError("Type not supported: {}".format(typ))
+
+
+def translate_value(value, typ):
+    """
+    Translate a value output from Py-SSZ deserialization into the given spec type.
+    :param value: The PySSZ value
+    :param typ: The type from the spec to translate into
+    :return: the translated value
+    """
+    if spec_ssz.is_uint_type(typ):
+        size = spec_ssz.uint_byte_size(typ)
+        if size == 1:
+            return spec_ssz.uint8(value)
+        elif size == 2:
+            return spec_ssz.uint16(value)
+        elif size == 4:
+            return spec_ssz.uint32(value)
+        elif size == 8:
+            # uint64 is default (TODO this is changing soon)
+            return value
+        elif size == 16:
+            return spec_ssz.uint128(value)
+        elif size == 32:
+            return spec_ssz.uint256(value)
+        else:
+            raise TypeError("invalid uint size")
+    elif spec_ssz.is_list_type(typ):
+        elem_typ = spec_ssz.read_elem_type(typ)
+        return [translate_value(elem, elem_typ) for elem in value]
+    elif spec_ssz.is_bool_type(typ):
+        return value
+    elif spec_ssz.is_vector_type(typ):
+        elem_typ = spec_ssz.read_elem_type(typ)
+        return typ(*(translate_value(elem, elem_typ) for elem in value))
+    elif spec_ssz.is_bytesn_type(typ):
+        return typ(value)
+    elif spec_ssz.is_bytes_type(typ):
+        return value
+    elif spec_ssz.is_container_type(typ):
+        return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ)
+                      in zip(typ.get_field_names(), value, typ.get_field_types())})
+    else:
+        raise TypeError("Type not supported: {}".format(typ))

test_libs/pyspec/eth2spec/fuzzing/test_decoder.py

@@ -0,0 +1,33 @@
+from eth2spec.fuzzing.decoder import translate_typ, translate_value
+from eth2spec.phase0 import spec
+from eth2spec.utils.ssz import ssz_impl as spec_ssz_impl
+from random import Random
+from eth2spec.debug import random_value
+
+
+def test_decoder():
+    rng = Random(123)
+
+    # check these types only, Block covers a lot of operation types already.
+    for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]:
+        # create a random pyspec value
+        original = random_value.get_random_ssz_object(rng, typ, 100, 10,
+                                                      mode=random_value.RandomizationMode.mode_random,
+                                                      chaos=True)
+        # serialize it, using pyspec
+        pyspec_data = spec_ssz_impl.serialize(original)
+        # get the py-ssz type for it
+        block_sedes = translate_typ(typ)
+        # try decoding using the py-ssz type
+        raw_value = block_sedes.deserialize(pyspec_data)
+
+        # serialize it using py-ssz
+        pyssz_data = block_sedes.serialize(raw_value)
+        # now check if the serialized form is equal. If so, we confirmed decoding and encoding to work.
+        assert pyspec_data == pyssz_data
+
+        # now translate the py-ssz value in a pyspec-value
+        block = translate_value(raw_value, typ)
+
+        # and see if the hash-tree-root of the original matches the hash-tree-root of the decoded & translated value.
+        assert spec_ssz_impl.hash_tree_root(original) == spec_ssz_impl.hash_tree_root(block)

test_libs/pyspec/eth2spec/test/helpers/state.py

@@ -1,3 +1,6 @@
+from eth2spec.test.helpers.block import sign_block
+
+
 def get_balance(state, index):
     return state.balances[index]
 
@@ -23,3 +26,13 @@ def get_state_root(spec, state, slot) -> bytes:
     """
     assert slot < state.slot <= slot + spec.SLOTS_PER_HISTORICAL_ROOT
     return state.latest_state_roots[slot % spec.SLOTS_PER_HISTORICAL_ROOT]
+
+
+def state_transition_and_sign_block(spec, state, block):
+    """
+    State transition via the provided ``block``
+    then package the block with the state root and signature.
+    """
+    spec.state_transition(state, block)
+    block.state_root = state.hash_tree_root()
+    sign_block(spec, state, block)

test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_crosslinks.py

@@ -3,7 +3,8 @@
 from eth2spec.test.context import spec_state_test, with_all_phases
 from eth2spec.test.helpers.state import (
     next_epoch,
-    next_slot
+    next_slot,
+    state_transition_and_sign_block,
 )
 from eth2spec.test.helpers.block import apply_empty_block, sign_block
 from eth2spec.test.helpers.attestations import (
@@ -27,11 +28,14 @@ def run_process_crosslinks(spec, state, valid=True):
     block = build_empty_block_for_next_slot(spec, state)
     block.slot = slot
     sign_block(spec, state, block)
-    spec.state_transition(state, block)
+    state_transition_and_sign_block(spec, state, block)
 
     # cache state before epoch transition
     spec.process_slot(state)
 
+    # process components of epoch transition before processing crosslinks
+    spec.process_justification_and_finalization(state)
+
     yield 'pre', state
     spec.process_crosslinks(state)
     yield 'post', state

test_libs/pyspec/eth2spec/test/phase_0/epoch_processing/test_process_registry_updates.py

@@ -1,6 +1,5 @@
-from eth2spec.phase0.spec import state_transition
 from eth2spec.test.helpers.block import build_empty_block_for_next_slot, sign_block
-from eth2spec.test.helpers.state import next_epoch
+from eth2spec.test.helpers.state import next_epoch, state_transition_and_sign_block
 from eth2spec.test.context import spec_state_test, with_all_phases
 
 
@@ -16,7 +15,7 @@ def run_process_registry_updates(spec, state, valid=True):
     block = build_empty_block_for_next_slot(spec, state)
     block.slot = slot
     sign_block(spec, state, block)
-    state_transition(state, block)
+    state_transition_and_sign_block(spec, state, block)
 
     # cache state before epoch transition
     spec.process_slot(state)