From e7bb9bf19bd387842158e8e852bd1df7d2f11e8f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 01:56:36 +0200 Subject: [PATCH 1/6] SSZ decoding through pyssz, with translation of types/values --- test_libs/fuzzing/decoder.py | 84 ++++++++++++++++++++++++++++++ test_libs/fuzzing/requirements.txt | 3 ++ 2 files changed, 87 insertions(+) create mode 100644 test_libs/fuzzing/decoder.py create mode 100644 test_libs/fuzzing/requirements.txt diff --git a/test_libs/fuzzing/decoder.py b/test_libs/fuzzing/decoder.py new file mode 100644 index 0000000000..16d798f6f3 --- /dev/null +++ b/test_libs/fuzzing/decoder.py @@ -0,0 +1,84 @@ +from eth2spec.utils.ssz import ssz_typing as spec_ssz +import ssz + + +def translate_typ(typ) -> ssz.BaseSedes: + """ + Translates a spec type to a Py-SSZ type description (sedes). + :param typ: The spec type, a class. + :return: The Py-SSZ equivalent. + """ + if spec_ssz.is_container_type(typ): + return ssz.Container( + [(field_name, translate_typ(field_typ)) for (field_name, field_typ) in typ.get_fields()]) + elif spec_ssz.is_bytesn_type(typ): + return ssz.ByteVector(typ.length) + elif spec_ssz.is_bytes_type(typ): + return ssz.ByteList() + elif spec_ssz.is_vector_type(typ): + return ssz.Vector(translate_typ(spec_ssz.read_vector_elem_type(typ)), typ.length) + elif spec_ssz.is_list_type(typ): + return ssz.List(translate_typ(spec_ssz.read_list_elem_type(typ))) + elif spec_ssz.is_bool_type(typ): + return ssz.boolean + elif spec_ssz.is_uint_type(typ): + size = spec_ssz.uint_byte_size(typ) + if size == 1: + return ssz.uint8 + elif size == 2: + return ssz.uint16 + elif size == 4: + return ssz.uint32 + elif size == 8: + return ssz.uint64 + elif size == 16: + return ssz.uint128 + elif size == 32: + return ssz.uint256 + else: + raise TypeError("invalid uint size") + else: + raise Exception("Type not supported: {}".format(typ)) + + +def translate_value(value, typ): + """ + Translate a value output from Py-SSZ deserialization into the given spec type. + :param value: The PySSZ value + :param typ: The type from the spec to translate into + :return: the translated value + """ + if spec_ssz.is_uint_type(typ): + size = spec_ssz.uint_byte_size(typ) + if size == 1: + return spec_ssz.uint8(value) + elif size == 2: + return spec_ssz.uint16(value) + elif size == 4: + return spec_ssz.uint32(value) + elif size == 8: + # uint64 is default (TODO this is changing soon) + return value + elif size == 16: + return spec_ssz.uint128(value) + elif size == 32: + return spec_ssz.uint256(value) + else: + raise TypeError("invalid uint size") + elif spec_ssz.is_list_type(typ): + elem_typ = spec_ssz.read_elem_type(typ) + return [translate_value(elem, elem_typ) for elem in value] + elif spec_ssz.is_bool_type(typ): + return False + elif spec_ssz.is_vector_type(typ): + elem_typ = spec_ssz.read_elem_type(typ) + return typ(translate_value(elem, elem_typ) for elem in value) + elif spec_ssz.is_bytesn_type(typ): + return typ(value) + elif spec_ssz.is_bytes_type(typ): + return value + elif spec_ssz.is_container_type(typ): + return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ) + in zip(typ.get_field_names(), value.values(), typ.get_field_types())}) + else: + raise Exception("Type not supported: {}".format(typ)) diff --git a/test_libs/fuzzing/requirements.txt b/test_libs/fuzzing/requirements.txt new file mode 100644 index 0000000000..98dcd9564c --- /dev/null +++ b/test_libs/fuzzing/requirements.txt @@ -0,0 +1,3 @@ +../../test_libs/pyspec +../../test_libs/config_helpers +ssz==0.1.0a8 From 7b0ffc1ace8eefb61d459cddff94969a9601df52 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 19:09:49 +0200 Subject: [PATCH 2/6] move decoder for fuzzing, minor fixes, update dependency to support SOS style offsets --- test_libs/fuzzing/requirements.txt | 3 --- test_libs/pyspec/eth2spec/fuzzing/__init__.py | 0 test_libs/{ => pyspec/eth2spec}/fuzzing/decoder.py | 8 ++++---- test_libs/pyspec/requirements.txt | 1 + test_libs/pyspec/setup.py | 3 ++- 5 files changed, 7 insertions(+), 8 deletions(-) delete mode 100644 test_libs/fuzzing/requirements.txt create mode 100644 test_libs/pyspec/eth2spec/fuzzing/__init__.py rename test_libs/{ => pyspec/eth2spec}/fuzzing/decoder.py (89%) diff --git a/test_libs/fuzzing/requirements.txt b/test_libs/fuzzing/requirements.txt deleted file mode 100644 index 98dcd9564c..0000000000 --- a/test_libs/fuzzing/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -../../test_libs/pyspec -../../test_libs/config_helpers -ssz==0.1.0a8 diff --git a/test_libs/pyspec/eth2spec/fuzzing/__init__.py b/test_libs/pyspec/eth2spec/fuzzing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test_libs/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py similarity index 89% rename from test_libs/fuzzing/decoder.py rename to test_libs/pyspec/eth2spec/fuzzing/decoder.py index 16d798f6f3..bb8aceb615 100644 --- a/test_libs/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -10,7 +10,7 @@ def translate_typ(typ) -> ssz.BaseSedes: """ if spec_ssz.is_container_type(typ): return ssz.Container( - [(field_name, translate_typ(field_typ)) for (field_name, field_typ) in typ.get_fields()]) + [translate_typ(field_typ) for (field_name, field_typ) in typ.get_fields()]) elif spec_ssz.is_bytesn_type(typ): return ssz.ByteVector(typ.length) elif spec_ssz.is_bytes_type(typ): @@ -38,7 +38,7 @@ def translate_typ(typ) -> ssz.BaseSedes: else: raise TypeError("invalid uint size") else: - raise Exception("Type not supported: {}".format(typ)) + raise TypeError("Type not supported: {}".format(typ)) def translate_value(value, typ): @@ -79,6 +79,6 @@ def translate_value(value, typ): return value elif spec_ssz.is_container_type(typ): return typ(**{f_name: translate_value(f_val, f_typ) for (f_name, f_val, f_typ) - in zip(typ.get_field_names(), value.values(), typ.get_field_types())}) + in zip(typ.get_field_names(), value, typ.get_field_types())}) else: - raise Exception("Type not supported: {}".format(typ)) + raise TypeError("Type not supported: {}".format(typ)) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index 3b38930bda..eed0d5a7da 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -3,3 +3,4 @@ eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 typing_inspect==0.4.0 +ssz==0.1.0a9 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index e99b911eeb..49f3c84a10 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -9,6 +9,7 @@ "eth-typing>=2.1.0,<3.0.0", "pycryptodome==3.7.3", "py_ecc>=1.6.0", - "typing_inspect==0.4.0" + "typing_inspect==0.4.0", + "ssz==0.1.0a9" ] ) From 01be8b7e656de4bfce86136dac79774eefeea3e5 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 19:16:15 +0200 Subject: [PATCH 3/6] minor fix --- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index bb8aceb615..5722ed2975 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -69,7 +69,7 @@ def translate_value(value, typ): elem_typ = spec_ssz.read_elem_type(typ) return [translate_value(elem, elem_typ) for elem in value] elif spec_ssz.is_bool_type(typ): - return False + return value elif spec_ssz.is_vector_type(typ): elem_typ = spec_ssz.read_elem_type(typ) return typ(translate_value(elem, elem_typ) for elem in value) From 895ab67815866c2a3f78022fc8dc6ad575f4b1fc Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 20:41:08 +0200 Subject: [PATCH 4/6] fix decoder, also fix bug in pyssz, see PR 74 --- test_libs/pyspec/eth2spec/fuzzing/decoder.py | 2 +- .../pyspec/eth2spec/fuzzing/test_decoder.py | 39 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 test_libs/pyspec/eth2spec/fuzzing/test_decoder.py diff --git a/test_libs/pyspec/eth2spec/fuzzing/decoder.py b/test_libs/pyspec/eth2spec/fuzzing/decoder.py index 5722ed2975..a5d3dfd97a 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/decoder.py @@ -72,7 +72,7 @@ def translate_value(value, typ): return value elif spec_ssz.is_vector_type(typ): elem_typ = spec_ssz.read_elem_type(typ) - return typ(translate_value(elem, elem_typ) for elem in value) + return typ(*(translate_value(elem, elem_typ) for elem in value)) elif spec_ssz.is_bytesn_type(typ): return typ(value) elif spec_ssz.is_bytes_type(typ): diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py new file mode 100644 index 0000000000..3747b3535e --- /dev/null +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -0,0 +1,39 @@ +from eth2spec.fuzzing.decoder import translate_typ, translate_value +from eth2spec.phase0 import spec +from preset_loader import loader +from eth2spec.utils.ssz import ssz_impl as spec_ssz_impl +from random import Random +from eth2spec.debug import random_value + + +def test_decoder(): + configs_path = "../../../../configs/" + config_name = "minimal" + presets = loader.load_presets(configs_path, config_name) + spec.apply_constants_preset(presets) + + rng = Random(123) + + # check these types only, Block covers a lot of operation types already. + for typ in [spec.BeaconBlock, spec.BeaconState, spec.IndexedAttestation, spec.AttestationDataAndCustodyBit]: + # create a random pyspec value + original = random_value.get_random_ssz_object(rng, typ, 100, 10, + mode=random_value.RandomizationMode.mode_random, + chaos=True) + # serialize it, using pyspec + pyspec_data = spec_ssz_impl.serialize(original) + # get the py-ssz type for it + block_sedes = translate_typ(typ) + # try decoding using the py-ssz type + raw_value = block_sedes.deserialize(pyspec_data) + + # serialize it using py-ssz + pyssz_data = block_sedes.serialize(raw_value) + # now check if the serialized form is equal. If so, we confirmed decoding and encoding to work. + assert pyspec_data == pyssz_data + + # now translate the py-ssz value in a pyspec-value + block = translate_value(raw_value, typ) + + # and see if the hash-tree-root of the original matches the hash-tree-root of the decoded & translated value. + assert spec_ssz_impl.hash_tree_root(original) == spec_ssz_impl.hash_tree_root(block) From d4bf55e5a10f505f6e767e2864d9632f805c5763 Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 21:24:24 +0200 Subject: [PATCH 5/6] update pyssz to include deserialization bugfix --- test_libs/pyspec/requirements.txt | 2 +- test_libs/pyspec/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_libs/pyspec/requirements.txt b/test_libs/pyspec/requirements.txt index eed0d5a7da..2de2aa84bb 100644 --- a/test_libs/pyspec/requirements.txt +++ b/test_libs/pyspec/requirements.txt @@ -3,4 +3,4 @@ eth-typing>=2.1.0,<3.0.0 pycryptodome==3.7.3 py_ecc>=1.6.0 typing_inspect==0.4.0 -ssz==0.1.0a9 +ssz==0.1.0a10 diff --git a/test_libs/pyspec/setup.py b/test_libs/pyspec/setup.py index 49f3c84a10..3856640abf 100644 --- a/test_libs/pyspec/setup.py +++ b/test_libs/pyspec/setup.py @@ -10,6 +10,6 @@ "pycryptodome==3.7.3", "py_ecc>=1.6.0", "typing_inspect==0.4.0", - "ssz==0.1.0a9" + "ssz==0.1.0a10" ] ) From 367586d888ae5bc901a81588227926971b40585f Mon Sep 17 00:00:00 2001 From: protolambda Date: Fri, 14 Jun 2019 21:31:33 +0200 Subject: [PATCH 6/6] remove need for presets loading, just test mainnet, not too many/large objects anyway --- test_libs/pyspec/eth2spec/fuzzing/test_decoder.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py index 3747b3535e..26ee6e913a 100644 --- a/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py +++ b/test_libs/pyspec/eth2spec/fuzzing/test_decoder.py @@ -1,17 +1,11 @@ from eth2spec.fuzzing.decoder import translate_typ, translate_value from eth2spec.phase0 import spec -from preset_loader import loader from eth2spec.utils.ssz import ssz_impl as spec_ssz_impl from random import Random from eth2spec.debug import random_value def test_decoder(): - configs_path = "../../../../configs/" - config_name = "minimal" - presets = loader.load_presets(configs_path, config_name) - spec.apply_constants_preset(presets) - rng = Random(123) # check these types only, Block covers a lot of operation types already.