Merge pull request ethereum#19 from ericsson49/fc-compliance2

Add new check description to F/C test format
ericsson49 · Jul 2, 2024 · 4a0745b · 4a0745b
2 parents 8545060 + ce897c8
commit 4a0745b
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 25 deletions.
diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md
@@ -177,6 +177,10 @@ finalized_checkpoint: {
     root: string,             -- Encoded 32-byte value from store.finalized_checkpoint.root
 }
 proposer_boost_root: string   -- Encoded 32-byte value from store.proposer_boost_root
+viable_for_head_roots_and_weights: [{
+    root: string,             -- Encoded 32-byte value of filtered_block_tree leaf blocks
+    weight: int               -- Integer value from get_weight(store, viable_block_root)
+}]
 ```
 
 Additionally, these fields if `get_proposer_head` and `should_override_forkchoice_update` features are implemented:
@@ -189,13 +193,6 @@ should_override_forkchoice_update: {  -- [New in Bellatrix]
 }
 ```
 
-Additional check utilized by the generated fork choice tests:
-```yaml
-viable_for_head_roots_and_weights: {
-    <head-root>: <weight>             -- A <bytes32-hex-str, int> map listing roots of all viable for head blocks and their fork choice weights
-}
-```
-
 For example:
 ```yaml
 - checks:
@@ -206,8 +203,10 @@ For example:
     proposer_boost_root: '0xdaa1d49d57594ced0c35688a6da133abb086d191a2ebdfd736fad95299325aeb'
     get_proposer_head: '0xdaa1d49d57594ced0c35688a6da133abb086d191a2ebdfd736fad95299325aeb'
     should_override_forkchoice_update: {validator_is_connected: false, result: false}
-    viable_for_head_roots_and_weights: {'0x533290b6f44d31c925acd08dfc8448624979d48c40b877d4e6714648866c9ddb': 192000000000,
-      '0x5cfb9d9099cdf1d8ab68ce96cdae9f0fa6eef16914a01070580dfdc1d2d59ec3': 544000000000}
+    viable_for_head_roots_and_weights: [
+      {root: '0x533290b6f44d31c925acd08dfc8448624979d48c40b877d4e6714648866c9ddb', weight: 192000000000},
+      {root: '0x5cfb9d9099cdf1d8ab68ce96cdae9f0fa6eef16914a01070580dfdc1d2d59ec3', weight: 544000000000}
+    ]
 ```
 
 *Note*: Each `checks` step may include one or multiple items. Each item has to be checked against the current store.

diff --git a/tests/generators/fork_choice_generated/README.md b/tests/generators/fork_choice_generated/README.md
@@ -7,3 +7,45 @@ Implementation of the approach described in the [Fork Choice compliance testing
 Preliminary research has been also performed in this [repo](https://github.com/txrx-research/fork_choice_test_generation/tree/main).
 
 To simplfy adoption of the tests, we follow the test format described in the [fork choice test formats documentation](../../formats/fork_choice/README.md), with a minor exception (new check added).
+
+# Pre-requisites
+
+Install requirements (preferrably, in a dedicated Python environment)
+
+```
+> pip install -r requirements.txt
+```
+
+In order to run tests, install `tqdm` additionally.
+```
+> pip install tqdm
+```
+
+# Generating tests
+
+```
+> python test_gen.py -o ${test_dir} --fc-gen-config ${config_dir}/test_gen.yaml
+```
+
+There are three configurations in the repo: [tiny](tiny/), [small](small/) and [standard](standard/).
+
+# Running tests
+
+Install `tqdm` library (to show progress)
+```
+> pip install tqdm
+```
+
+and then
+```
+> python test_run.py -i ${test_dir}
+```
+
+# Generating configurations
+
+Files in [tiny](tiny/), [small](small/) and [standard](standard/) are generated with [generate_test_instances.py](generate_test_instances.py), e.g.
+```
+> python generate_test_instances.py
+```
+
+But one normally doesn't need to generate them.
diff --git a/tests/generators/fork_choice_generated/generate_test_instances.py b/tests/generators/fork_choice_generated/generate_test_instances.py
@@ -115,26 +115,26 @@ def generate_block_cover(params):
 
 gen_params = {
     ###################
-    # small instances #
+    # tiny instances #
     ###################
 
-    'block_tree_tree_small': {
-        'out_path': 'small/block_tree_tree.yaml',
+    'block_tree_tree_tiny': {
+        'out_path': 'tiny/block_tree_tree.yaml',
         'models': ['sm_link', 'block_tree'],
         'params': [
             ({'anchor_epoch': 0, 'number_of_epochs': 4, 'number_of_links': 3}, {'number_of_blocks': 8, 'max_children': 2, 'number_of_solutions': 3}),
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 16, 'max_children': 2, 'number_of_solutions': 3}),
         ]
     },
-    'block_tree_other_small': {
-        'out_path': 'small/block_tree_other.yaml',
+    'block_tree_other_tiny': {
+        'out_path': 'tiny/block_tree_other.yaml',
         'models': ['sm_link', 'block_tree'],
         'params': [
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 12, 'max_children': 2, 'number_of_solutions': 3}),
         ]
     },
-    'block_cover_small': {
-        'out_path': 'small/block_cover.yaml',
+    'block_cover_tiny': {
+        'out_path': 'tiny/block_cover.yaml',
         'models': ['block_cover'],
         'params': [
             ({'anchor_epoch': 0, 'number_of_solutions': 1},),
@@ -143,36 +143,36 @@ def generate_block_cover(params):
     },
 
     ###################
-    # smoke instances #
+    # small instances #
     ###################
 
-    'block_tree_tree_smoke': {
-        'out_path': 'smoke/block_tree_tree.yaml',
+    'block_tree_tree_small': {
+        'out_path': 'small/block_tree_tree.yaml',
         'models': ['sm_link', 'block_tree'],
         'params': [
             ({'anchor_epoch': 0, 'number_of_epochs': 5, 'number_of_links': 3}, {'number_of_blocks': 16, 'max_children': 2, 'number_of_solutions': 2}),
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 5, 'max_children': 3, 'number_of_solutions': None}),
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 12, 'max_children': 2, 'number_of_solutions': 73}),
         ]
     },
-    'block_tree_tree_smoke_2': {
-        'out_path': 'smoke/block_tree_tree_2.yaml',
+    'block_tree_tree_small_2': {
+        'out_path': 'small/block_tree_tree_2.yaml',
         'models': ['sm_link', 'block_tree'],
         'params': [
             ({'anchor_epoch': 0, 'number_of_epochs': 6, 'number_of_links': 4}, {'number_of_blocks': 16, 'max_children': 2, 'number_of_solutions': 2}),
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 6, 'max_children': 4, 'number_of_solutions': None}),
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 12, 'max_children': 2, 'number_of_solutions': 283}),
         ]
     },
-    'block_tree_other_smoke': {
-        'out_path': 'smoke/block_tree_other.yaml',
+    'block_tree_other_small': {
+        'out_path': 'small/block_tree_other.yaml',
         'models': ['sm_link', 'block_tree'],
         'params': [
             ([{'sm_links': [[0, 1], [0, 2], [2, 3], [3, 4]]}],  {'number_of_blocks': 12, 'max_children': 2, 'number_of_solutions': 4}),
         ]
     },
-    'block_cover_smoke': {
-        'out_path': 'smoke/block_cover.yaml',
+    'block_cover_small': {
+        'out_path': 'small/block_cover.yaml',
         'models': ['block_cover'],
         'params': [
             ({'anchor_epoch': 0, 'number_of_solutions': 2},),