From a12e16b739b8abbc3cffaaed64fc528e0bce2a09 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Apr 2024 17:01:08 +0900 Subject: [PATCH 1/4] Add EIP-7549 aggregation logic to testing tools --- .../test_should_override_forkchoice_update.py | 6 +- .../eth2spec/test/helpers/attestations.py | 121 ++++++++++++------ .../fork_choice/test_get_proposer_head.py | 6 +- .../test/phase0/fork_choice/test_reorg.py | 8 +- 4 files changed, 95 insertions(+), 46 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/fork_choice/test_should_override_forkchoice_update.py b/tests/core/pyspec/eth2spec/test/bellatrix/fork_choice/test_should_override_forkchoice_update.py index b40cc5bbe2..6c8e33e954 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/fork_choice/test_should_override_forkchoice_update.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/fork_choice/test_should_override_forkchoice_update.py @@ -7,7 +7,7 @@ MINIMAL, ) from eth2spec.test.helpers.attestations import ( - get_valid_attestation_at_slot, + get_valid_attestations_at_slot, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -109,7 +109,7 @@ def test_should_override_forkchoice_update__true(spec, state): # Fill a slot with attestations to its parent block = build_empty_block_for_next_slot(spec, state) parent_block_slot = block.slot - 1 - block.body.attestations = get_valid_attestation_at_slot( + block.body.attestations = get_valid_attestations_at_slot( state, spec, parent_block_slot, @@ -135,7 +135,7 @@ def test_should_override_forkchoice_update__true(spec, state): # Add attestations to the parent block temp_state = state.copy() next_slot(spec, temp_state) - attestations = get_valid_attestation_at_slot( + attestations = get_valid_attestations_at_slot( temp_state, spec, slot_to_attest=temp_state.slot - 1, diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 528de7f67c..e57eb8318e 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -84,7 +84,6 @@ def build_attestation_data(spec, state, slot, index, beacon_block_root=None, sha target=spec.Checkpoint(epoch=spec.compute_epoch_at_slot(slot), root=epoch_boundary_root), ) - # if spec.fork == SHARDING # TODO: add extra data for shard voting return data @@ -95,8 +94,12 @@ def get_valid_attestation(spec, filter_participant_set=None, beacon_block_root=None, signed=False): - # If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. - # Thus strictly speaking invalid when no participant is added later. + """ + Return a valid attestation at `slot` and committee index `index`. + + If filter_participant_set filters everything, the attestation has 0 participants, and cannot be signed. + Thus strictly speaking invalid when no participant is added later. + """ if slot is None: slot = state.slot if index is None: @@ -104,18 +107,8 @@ def get_valid_attestation(spec, attestation_data = build_attestation_data(spec, state, slot=slot, index=index, beacon_block_root=beacon_block_root) - beacon_committee = spec.get_beacon_committee(state, slot, index) + attestation = spec.Attestation(data=attestation_data) - if is_post_eip7549(spec): - # will fill aggregation_bits later - attestation = spec.Attestation(data=attestation_data) - else: - committee_size = len(beacon_committee) - aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) - attestation = spec.Attestation( - aggregation_bits=aggregation_bits, - data=attestation_data, - ) # fill the attestation with (optionally filtered) participants, and optionally sign it fill_aggregate_attestation(spec, state, attestation, signed=signed, filter_participant_set=filter_participant_set, committee_index=index) @@ -132,7 +125,7 @@ def sign_aggregate_attestation(spec, state, attestation_data, participants: List spec, state, attestation_data, - privkey + privkey, ) ) return bls.Aggregate(signatures) @@ -180,11 +173,16 @@ def fill_aggregate_attestation(spec, state, attestation, committee_index, signed if filter_participant_set is not None: participants = filter_participant_set(participants) + # initialize `aggregation_bits` if is_post_eip7549(spec): - attestation.committee_bits = spec.Bitvector[spec.MAX_COMMITTEES_PER_SLOT]() attestation.committee_bits[committee_index] = True attestation.aggregation_bits = get_empty_eip7549_aggregation_bits( spec, state, attestation.committee_bits, attestation.data.slot) + else: + committee_size = len(beacon_committee) + attestation.aggregation_bits = Bitlist[spec.MAX_VALIDATORS_PER_COMMITTEE](*([0] * committee_size)) + + # fill in the `aggregation_bits` for i in range(len(beacon_committee)): if is_post_eip7549(spec): offset = get_eip7549_aggregation_bits_offset( @@ -205,7 +203,10 @@ def add_attestations_to_state(spec, state, attestations, slot): spec.process_attestation(state, attestation) -def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): +def get_valid_attestations_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): + """ + Return attestations at slot `slot_to_attest`. + """ committees_per_slot = spec.get_committee_count_per_slot(state, spec.compute_epoch_at_slot(slot_to_attest)) for index in range(committees_per_slot): def participants_filter(comm): @@ -213,7 +214,6 @@ def participants_filter(comm): return comm else: return participation_fn(state.slot, index, comm) - # if spec.fork == SHARDING: TODO: add shard data to attestation, include shard headers in block yield get_valid_attestation( spec, state, @@ -225,6 +225,48 @@ def participants_filter(comm): ) +def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): + """ + Return the aggregate attestation + """ + assert is_post_eip7549(spec) + attestations = list(get_valid_attestations_at_slot( + state, spec, slot_to_attest, + participation_fn=participation_fn, + beacon_block_root=beacon_block_root, + )) + if len(attestations) == 0: + return None + aggregate = spec.Attestation(data=attestations[0].data) + + # fill in committee_bits + all_committee_indices = [] + for attestation in attestations: + committee_indices_of_attestation = spec.get_committee_indices(attestation.committee_bits) + assert len(committee_indices_of_attestation) == 1 + all_committee_indices += committee_indices_of_attestation + + all_committee_indices = set(all_committee_indices) + for committee_index in all_committee_indices: + aggregate.committee_bits[committee_index] = True + + # aggregate the attestation data and sigs + aggregate.aggregation_bits = get_empty_eip7549_aggregation_bits( + spec, state, aggregate.committee_bits, slot_to_attest) + for attestation in attestations: + committee_indices_of_attestation = spec.get_committee_indices(attestation.committee_bits) + assert len(committee_indices_of_attestation) == 1 + committee_index = committee_indices_of_attestation[0] + offset = get_eip7549_aggregation_bits_offset( + spec, state, attestation.data.slot, aggregate.committee_bits, committee_index) + for i in range(len(attestation.aggregation_bits)): + aggregation_bits_index = offset + i + aggregate.aggregation_bits[aggregation_bits_index] = attestation.aggregation_bits[i] + aggregate.signature = bls.Aggregate([attestation.signature for attestation in attestations]) + + return aggregate + + def next_slots_with_attestations(spec, state, slot_count, @@ -249,6 +291,26 @@ def next_slots_with_attestations(spec, return state, signed_blocks, post_state +def _add_valid_attestations(spec, state, block, slot_to_attest, participation_fn=None): + if is_post_eip7549(spec): + attestation = get_valid_attestation_at_slot( + state, + spec, + slot_to_attest, + participation_fn=participation_fn, + ) + block.body.attestations.append(attestation) + else: + attestations = get_valid_attestations_at_slot( + state, + spec, + slot_to_attest, + participation_fn=participation_fn, + ) + for attestation in attestations: + block.body.attestations.append(attestation) + + def next_epoch_with_attestations(spec, state, fill_cur_epoch, @@ -281,24 +343,10 @@ def state_transition_with_full_block(spec, if fill_cur_epoch and state.slot >= spec.MIN_ATTESTATION_INCLUSION_DELAY: slot_to_attest = state.slot - spec.MIN_ATTESTATION_INCLUSION_DELAY + 1 if slot_to_attest >= spec.compute_start_slot_at_epoch(spec.get_current_epoch(state)): - attestations = get_valid_attestation_at_slot( - state, - spec, - slot_to_attest, - participation_fn=participation_fn - ) - for attestation in attestations: - block.body.attestations.append(attestation) + _add_valid_attestations(spec, state, block, slot_to_attest, participation_fn=participation_fn) if fill_prev_epoch and state.slot >= spec.SLOTS_PER_EPOCH: slot_to_attest = state.slot - spec.SLOTS_PER_EPOCH + 1 - attestations = get_valid_attestation_at_slot( - state, - spec, - slot_to_attest, - participation_fn=participation_fn - ) - for attestation in attestations: - block.body.attestations.append(attestation) + _add_valid_attestations(spec, state, block, slot_to_attest, participation_fn=participation_fn) if sync_aggregate is not None: block.body.sync_aggregate = sync_aggregate @@ -319,7 +367,7 @@ def state_transition_with_full_attestations_block(spec, state, fill_cur_epoch, f slots = state.slot % spec.SLOTS_PER_EPOCH for slot_offset in range(slots): target_slot = state.slot - slot_offset - attestations += get_valid_attestation_at_slot( + attestations += get_valid_attestations_at_slot( state, spec, target_slot, @@ -330,7 +378,7 @@ def state_transition_with_full_attestations_block(spec, state, fill_cur_epoch, f slots = spec.SLOTS_PER_EPOCH - state.slot % spec.SLOTS_PER_EPOCH for slot_offset in range(1, slots): target_slot = state.slot - (state.slot % spec.SLOTS_PER_EPOCH) - slot_offset - attestations += get_valid_attestation_at_slot( + attestations += get_valid_attestations_at_slot( state, spec, target_slot, @@ -423,6 +471,7 @@ def get_empty_eip7549_aggregation_bits(spec, state, committee_bits, slot): def get_eip7549_aggregation_bits_offset(spec, state, slot, committee_bits, committee_index): + # FIXME: it's not efficient to use it as an aggregator committee_indices = spec.get_committee_indices(committee_bits) assert committee_index in committee_indices offset = 0 diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_proposer_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_proposer_head.py index 9419a18dfc..81e5b4f563 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_proposer_head.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_proposer_head.py @@ -4,7 +4,7 @@ with_altair_and_later, ) from eth2spec.test.helpers.attestations import ( - get_valid_attestation_at_slot, + get_valid_attestations_at_slot, ) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -101,7 +101,7 @@ def test_basic_is_parent_root(spec, state): # Fill a slot with attestations to its parent block = build_empty_block_for_next_slot(spec, state) parent_block_slot = block.slot - 1 - block.body.attestations = get_valid_attestation_at_slot( + block.body.attestations = get_valid_attestations_at_slot( state, spec, parent_block_slot, @@ -128,7 +128,7 @@ def test_basic_is_parent_root(spec, state): slot = state.slot # Add attestations to the parent block - attestations = get_valid_attestation_at_slot( + attestations = get_valid_attestations_at_slot( state, spec, slot_to_attest=slot - 1, diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_reorg.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_reorg.py index ca9b483f88..983a8ee661 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_reorg.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_reorg.py @@ -9,7 +9,7 @@ from eth2spec.test.helpers.attestations import ( state_transition_with_full_block, get_valid_attestation, - get_valid_attestation_at_slot, + get_valid_attestations_at_slot, ) from eth2spec.test.helpers.block import ( build_empty_block, @@ -218,7 +218,7 @@ def _run_delayed_justification(spec, state, attemped_reorg, is_justifying_previo # add attestations of y temp_state = state.copy() next_slot(spec, temp_state) - attestations_for_y = list(get_valid_attestation_at_slot(temp_state, spec, signed_block_y.message.slot)) + attestations_for_y = list(get_valid_attestations_at_slot(temp_state, spec, signed_block_y.message.slot)) current_time = temp_state.slot * spec.config.SECONDS_PER_SLOT + store.genesis_time on_tick_and_append_step(spec, store, current_time, test_steps) yield from add_attestations(spec, store, attestations_for_y, test_steps) @@ -345,10 +345,10 @@ def _run_include_votes_of_another_empty_chain(spec, state, enough_ffg, is_justif # create 2/3 votes for the empty chain attestations_for_y = [] # target_is_current = not is_justifying_previous_epoch - attestations = list(get_valid_attestation_at_slot(state, spec, state_a.slot)) + attestations = list(get_valid_attestations_at_slot(state, spec, state_a.slot)) attestations_for_y.append(attestations) for state in states_of_empty_chain: - attestations = list(get_valid_attestation_at_slot(state, spec, state.slot)) + attestations = list(get_valid_attestations_at_slot(state, spec, state.slot)) attestations_for_y.append(attestations) state = state_a.copy() From 265788be58cfbcdd7280d17eedc309eb0bef0347 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 Apr 2024 18:01:08 +0900 Subject: [PATCH 2/4] refactor --- .../eth2spec/test/helpers/attestations.py | 75 +++++++++++++------ 1 file changed, 53 insertions(+), 22 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index e57eb8318e..567bbd4c49 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -225,6 +225,51 @@ def participants_filter(comm): ) +def _get_aggregate_committee_indices(spec, attestations): + """ + Aggregate all unique committee indices from the given attestations. + """ + all_committee_indices = set() + for attestation in attestations: + committee_indices = spec.get_committee_indices(attestation.committee_bits) + assert len(committee_indices) == 1 + all_committee_indices.add(committee_indices[0]) + + return all_committee_indices + + +def _aggregate_aggregation_bits_and_signatures(spec, state, slot, aggregate, attestations): + """ + Aggregate the aggregation bits and signatures from the attestations, + incorporating the calculation of aggregation bits offset directly. + """ + # initialize aggregation bits for the aggregate attestation + aggregate.aggregation_bits = get_empty_eip7549_aggregation_bits( + spec, state, aggregate.committee_bits, slot) + + signatures = [] + + offset = 0 + attestations = sorted(attestations, key=lambda att: spec.get_committee_indices(att.committee_bits)[0]) + for attestation in attestations: + # retrieve the single committee index for the attestation. + committee_index = spec.get_committee_indices(attestation.committee_bits)[0] + + # update the aggregate's aggregation bits based on each attestation. + for i, bit in enumerate(attestation.aggregation_bits): + aggregate.aggregation_bits[offset + i] = bit + + # collect signatures for aggregation. + signatures.append(attestation.signature) + + # update offset + committee = spec.get_beacon_committee(state, slot, committee_index) + offset += len(committee) + + # aggregate signatures from all attestations. + aggregate.signature = bls.Aggregate(signatures) + + def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): """ Return the aggregate attestation @@ -235,34 +280,18 @@ def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn= participation_fn=participation_fn, beacon_block_root=beacon_block_root, )) - if len(attestations) == 0: + if not attestations: return None + + # initialize the aggregate attestation. aggregate = spec.Attestation(data=attestations[0].data) # fill in committee_bits - all_committee_indices = [] - for attestation in attestations: - committee_indices_of_attestation = spec.get_committee_indices(attestation.committee_bits) - assert len(committee_indices_of_attestation) == 1 - all_committee_indices += committee_indices_of_attestation - - all_committee_indices = set(all_committee_indices) + all_committee_indices = _get_aggregate_committee_indices(spec, attestations) for committee_index in all_committee_indices: aggregate.committee_bits[committee_index] = True - # aggregate the attestation data and sigs - aggregate.aggregation_bits = get_empty_eip7549_aggregation_bits( - spec, state, aggregate.committee_bits, slot_to_attest) - for attestation in attestations: - committee_indices_of_attestation = spec.get_committee_indices(attestation.committee_bits) - assert len(committee_indices_of_attestation) == 1 - committee_index = committee_indices_of_attestation[0] - offset = get_eip7549_aggregation_bits_offset( - spec, state, attestation.data.slot, aggregate.committee_bits, committee_index) - for i in range(len(attestation.aggregation_bits)): - aggregation_bits_index = offset + i - aggregate.aggregation_bits[aggregation_bits_index] = attestation.aggregation_bits[i] - aggregate.signature = bls.Aggregate([attestation.signature for attestation in attestations]) + _aggregate_aggregation_bits_and_signatures(spec, state, slot_to_attest, aggregate, attestations) return aggregate @@ -471,7 +500,9 @@ def get_empty_eip7549_aggregation_bits(spec, state, committee_bits, slot): def get_eip7549_aggregation_bits_offset(spec, state, slot, committee_bits, committee_index): - # FIXME: it's not efficient to use it as an aggregator + """ + Calculate the offset for the aggregation bits based on the committee index. + """ committee_indices = spec.get_committee_indices(committee_bits) assert committee_index in committee_indices offset = 0 From 3d3ec8fbba55c8067e8cbae1dce03777b9c0dd70 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 5 Apr 2024 09:38:32 -0600 Subject: [PATCH 3/4] Update tests/core/pyspec/eth2spec/test/helpers/attestations.py --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 567bbd4c49..0562ec9b8b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -272,7 +272,8 @@ def _aggregate_aggregation_bits_and_signatures(spec, state, slot, aggregate, att def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): """ - Return the aggregate attestation + Return the aggregate attestation post EIP-7549. + Note: this EIP supports dense packing of on-chain aggregates so we can just return a single `Attestation`. """ assert is_post_eip7549(spec) attestations = list(get_valid_attestations_at_slot( From a56bd85674d4c35164a4877bf7878326fef8e35f Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Fri, 5 Apr 2024 09:43:48 -0600 Subject: [PATCH 4/4] Update tests/core/pyspec/eth2spec/test/helpers/attestations.py --- tests/core/pyspec/eth2spec/test/helpers/attestations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/attestations.py b/tests/core/pyspec/eth2spec/test/helpers/attestations.py index 0562ec9b8b..916a7ca00d 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/attestations.py +++ b/tests/core/pyspec/eth2spec/test/helpers/attestations.py @@ -272,7 +272,7 @@ def _aggregate_aggregation_bits_and_signatures(spec, state, slot, aggregate, att def get_valid_attestation_at_slot(state, spec, slot_to_attest, participation_fn=None, beacon_block_root=None): """ - Return the aggregate attestation post EIP-7549. + Return the aggregate attestation post EIP-7549. Note: this EIP supports dense packing of on-chain aggregates so we can just return a single `Attestation`. """ assert is_post_eip7549(spec)