Rpc blobs bug fix + blobsUtil endianness + integration tests

ethereum/spec/src/main/java/tech/pegasys/teku/spec/Spec.java

@@ -876,6 +876,10 @@ public Optional<Integer> getMaxBlobsPerBlock() {
         .map(SpecConfigDeneb::getMaxBlobsPerBlock);
   }
 
+  public Optional<Integer> getMaxBlobsPerBlock(final UInt64 slot) {
+    return atSlot(slot).getConfig().toVersionDeneb().map(SpecConfigDeneb::getMaxBlobsPerBlock);
+  }
+
   public UInt64 computeSubnetForBlobSidecar(final SignedBlobSidecar signedBlobSidecar) {
     return signedBlobSidecar.getBlobSidecar().getIndex().mod(BLOB_SIDECAR_SUBNET_COUNT);
   }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/networking/libp2p/rpc/BlobSidecarsByRangeRequestMessage.java

@@ -13,6 +13,7 @@
 
 package tech.pegasys.teku.spec.datastructures.networking.libp2p.rpc;
 
+import java.util.Optional;
 import tech.pegasys.teku.infrastructure.ssz.containers.Container2;
 import tech.pegasys.teku.infrastructure.ssz.containers.ContainerSchema2;
 import tech.pegasys.teku.infrastructure.ssz.primitive.SszUInt64;
@@ -24,6 +25,8 @@ public class BlobSidecarsByRangeRequestMessage
     extends Container2<BlobSidecarsByRangeRequestMessage, SszUInt64, SszUInt64>
     implements RpcRequest {
 
+  private final Optional<Integer> maxBlobsPerBlock;
+
   public static class BlobSidecarsByRangeRequestMessageSchema
       extends ContainerSchema2<BlobSidecarsByRangeRequestMessage, SszUInt64, SszUInt64> {
 
@@ -47,10 +50,13 @@ private BlobSidecarsByRangeRequestMessage(
       final BlobSidecarsByRangeRequestMessage.BlobSidecarsByRangeRequestMessageSchema type,
       final TreeNode backingNode) {
     super(type, backingNode);
+    this.maxBlobsPerBlock = Optional.empty();
   }
 
-  public BlobSidecarsByRangeRequestMessage(final UInt64 startSlot, final UInt64 count) {
+  public BlobSidecarsByRangeRequestMessage(
+      final UInt64 startSlot, final UInt64 count, final int maxBlobsPerBlock) {
     super(SSZ_SCHEMA, SszUInt64.of(startSlot), SszUInt64.of(count));
+    this.maxBlobsPerBlock = Optional.of(maxBlobsPerBlock);
   }
 
   public UInt64 getStartSlot() {
@@ -67,6 +73,7 @@ public UInt64 getMaxSlot() {
 
   @Override
   public int getMaximumResponseChunks() {
-    return getCount().intValue();
+    return getCount().intValue()
+        * maxBlobsPerBlock.orElseThrow(() -> new IllegalStateException("Unexpected method usage"));
   }
 }

ethereum/spec/src/main/java/tech/pegasys/teku/spec/datastructures/util/BlobsUtil.java

@@ -100,7 +100,7 @@ private Blob generateBlob(final UInt64 slot) {
     final Bytes rawBlob =
         IntStream.range(0, fieldElementsPerBlob)
             .mapToObj(__ -> randomBLSFieldElement())
-            .map(fieldElement -> Bytes.wrap(fieldElement.toArray(ByteOrder.LITTLE_ENDIAN)))
+            .map(fieldElement -> Bytes.wrap(fieldElement.toArray(ByteOrder.BIG_ENDIAN)))
             .reduce(Bytes::wrap)
             .orElse(Bytes.EMPTY);
 

ethereum/spec/src/test/java/tech/pegasys/teku/spec/datastructures/networking/libp2p/rpc/BlobSidecarsByRangeRequestMessageTest.java

@@ -29,7 +29,7 @@ public class BlobSidecarsByRangeRequestMessageTest {
   @Test
   public void shouldRoundTripViaSsz() {
     final BlobSidecarsByRangeRequestMessage request =
-        new BlobSidecarsByRangeRequestMessage(UInt64.valueOf(2), UInt64.valueOf(3));
+        new BlobSidecarsByRangeRequestMessage(UInt64.valueOf(2), UInt64.valueOf(3), 4);
     final Bytes data = request.sszSerialize();
     final BlobSidecarsByRangeRequestMessage result =
         BlobSidecarsByRangeRequestMessage.SSZ_SCHEMA.sszDeserialize(data);
@@ -41,11 +41,19 @@ public void shouldRoundTripViaSsz() {
   @MethodSource("getMaxSlotParams")
   public void getMaxSlot(final long startSlot, final long count, final long expected) {
     final BlobSidecarsByRangeRequestMessage request =
-        new BlobSidecarsByRangeRequestMessage(UInt64.valueOf(startSlot), UInt64.valueOf(count));
+        new BlobSidecarsByRangeRequestMessage(UInt64.valueOf(startSlot), UInt64.valueOf(count), 4);
 
     assertThat(request.getMaxSlot()).isEqualTo(UInt64.valueOf(expected));
   }
 
+  @Test
+  public void getMaximumResponseChunks() {
+    final BlobSidecarsByRangeRequestMessage request =
+        new BlobSidecarsByRangeRequestMessage(UInt64.valueOf(19), UInt64.valueOf(23), 4);
+
+    assertThat(request.getMaximumResponseChunks()).isEqualTo(23 * 4);
+  }
+
   public static Stream<Arguments> getMaxSlotParams() {
     return Stream.of(
         Arguments.of(0, 1, 0),

ethereum/spec/src/testFixtures/java/tech/pegasys/teku/spec/generator/ChainBuilder.java

@@ -662,7 +662,9 @@ private SignedBlockAndState generateBlockWithRandomBlobSidecars(
       final SszList<Attestation> attestations,
       final SszList<AttesterSlashing> attesterSlashings)
       throws EpochProcessingException, SlotProcessingException {
-    final List<Blob> randomBlobs = blobsUtil.generateBlobs(slot, RANDOM_BLOBS_COUNT);
+    final List<Blob> randomBlobs =
+        blobsUtil.generateBlobs(
+            slot, options.getGenerateRandomBlobsCount().orElse(RANDOM_BLOBS_COUNT));
     final MiscHelpersDeneb miscHelpers =
         spec.forMilestone(SpecMilestone.DENEB).miscHelpers().toVersionDeneb().orElseThrow();
     final List<KZGCommitment> kzgCommitments =
@@ -852,6 +854,7 @@ public static final class BlockOptions {
     private Optional<KZGProof> kzgProof = Optional.empty();
     private Optional<List<BlobSidecar>> blobSidecars = Optional.empty();
     private boolean generateRandomBlobs = false;
+    private Optional<Integer> generateRandomBlobsCount = Optional.empty();
     private boolean storeBlobSidecars = true;
     private boolean skipStateTransition = false;
     private boolean wrongProposer = false;
@@ -918,6 +921,12 @@ public BlockOptions setGenerateRandomBlobs(final boolean generateRandomBlobs) {
       return this;
     }
 
+    public BlockOptions setGenerateRandomBlobsCount(
+        final Optional<Integer> generateRandomBlobsCount) {
+      this.generateRandomBlobsCount = generateRandomBlobsCount;
+      return this;
+    }
+
     public BlockOptions setStoreBlobSidecars(final boolean storeBlobSidecars) {
       this.storeBlobSidecars = storeBlobSidecars;
       return this;
@@ -999,6 +1008,10 @@ public boolean getGenerateRandomBlobs() {
       return generateRandomBlobs;
     }
 
+    public Optional<Integer> getGenerateRandomBlobsCount() {
+      return generateRandomBlobsCount;
+    }
+
     public boolean getWrongProposer() {
       return wrongProposer;
     }

ethereum/statetransition/src/main/java/tech/pegasys/teku/statetransition/util/BlobSidecarPoolImpl.java

@@ -128,12 +128,7 @@ private static BlockBlobSidecarsTracker createTracker(
       final Spec spec, final SlotAndBlockRoot slotAndBlockRoot) {
     return new BlockBlobSidecarsTracker(
         slotAndBlockRoot,
-        UInt64.valueOf(
-            spec.atSlot(slotAndBlockRoot.getSlot())
-                .getConfig()
-                .toVersionDeneb()
-                .orElseThrow()
-                .getMaxBlobsPerBlock()));
+        UInt64.valueOf(spec.getMaxBlobsPerBlock(slotAndBlockRoot.getSlot()).orElseThrow()));
   }
 
   @Override

networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/AbstractRpcMethodIntegrationTest.java

@@ -16,16 +16,22 @@
 import static org.assertj.core.util.Preconditions.checkState;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.params.provider.Arguments;
+import tech.pegasys.teku.infrastructure.async.Waiter;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
 import tech.pegasys.teku.networking.eth2.rpc.core.encodings.RpcEncoding;
 import tech.pegasys.teku.spec.Spec;
 import tech.pegasys.teku.spec.SpecMilestone;
 import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
+import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.altair.BeaconBlockBodyAltair;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.bellatrix.BeaconBlockBodyBellatrix;
 import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.capella.BeaconBlockBodyCapella;
@@ -242,6 +248,31 @@ protected static Stream<Arguments> generateSpec() {
     return Arrays.stream(SpecMilestone.values()).map(Arguments::of);
   }
 
+  protected List<BlobSidecar> retrieveCanonicalBlobSidecarsFromPeerStorage(
+      final Stream<UInt64> slots) {
+
+    return slots
+        .map(
+            slot ->
+                peerStorage
+                    .recentChainData()
+                    .getBlockRootBySlot(slot)
+                    .map(root -> new SlotAndBlockRoot(slot, root)))
+        .filter(Optional::isPresent)
+        .map(Optional::get)
+        .map(this::safeRetrieveBlobSidecars)
+        .flatMap(Collection::stream)
+        .collect(Collectors.toUnmodifiableList());
+  }
+
+  private List<BlobSidecar> safeRetrieveBlobSidecars(final SlotAndBlockRoot slotAndBlockRoot) {
+    try {
+      return Waiter.waitFor(peerStorage.recentChainData().retrieveBlobSidecars(slotAndBlockRoot));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+  }
+
   protected static Class<?> milestoneToBeaconBlockBodyClass(final SpecMilestone milestone) {
     switch (milestone) {
       case PHASE0:

networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/BlobSidecarsByRangeIntegrationTest.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright ConsenSys Software Inc., 2023
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.networking.eth2;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static tech.pegasys.teku.infrastructure.async.Waiter.waitFor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+import org.junit.jupiter.api.Test;
+import tech.pegasys.teku.infrastructure.unsigned.UInt64;
+import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
+import tech.pegasys.teku.networking.p2p.rpc.RpcResponseListener;
+import tech.pegasys.teku.spec.TestSpecFactory;
+import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
+
+public class BlobSidecarsByRangeIntegrationTest extends AbstractRpcMethodIntegrationTest {
+
+  @Test
+  public void requestBlobSidecars_shouldFailBeforeDenebMilestone() {
+    final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalCapella());
+    assertThatThrownBy(() -> requestBlobSidecarsByRange(peer))
+        .hasRootCauseInstanceOf(UnsupportedOperationException.class)
+        .hasMessageContaining("BlobSidecarsByRange method is not supported");
+  }
+
+  @Test
+  public void requestBlobSidecars_shouldReturnEmptyBlobSidecarsOnDenebMilestone()
+      throws ExecutionException, InterruptedException, TimeoutException {
+    final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalDeneb());
+    final List<BlobSidecar> blobSidecars = requestBlobSidecarsByRange(peer);
+    assertThat(blobSidecars).isEmpty();
+  }
+
+  @Test
+  public void requestBlobSidecars_shouldReturnBlobSidecarsOnDenebMilestone()
+      throws ExecutionException, InterruptedException, TimeoutException {
+    final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalDeneb());
+
+    // generate 4 blobs per block
+    peerStorage.chainUpdater().blockOptions.setGenerateRandomBlobs(true);
+    peerStorage.chainUpdater().blockOptions.setGenerateRandomBlobsCount(Optional.of(4));
+
+    // up to slot 3
+    final UInt64 targetSlot = UInt64.valueOf(3);
+    peerStorage.chainUpdater().advanceChainUntil(targetSlot);
+
+    // grab expected blobs from storage
+    final List<BlobSidecar> expectedBlobSidecars =
+        retrieveCanonicalBlobSidecarsFromPeerStorage(UInt64.rangeClosed(UInt64.ONE, targetSlot));
+
+    // call and check
+    final List<BlobSidecar> blobSidecars = requestBlobSidecarsByRange(peer);
+    assertThat(blobSidecars).containsExactlyInAnyOrderElementsOf(expectedBlobSidecars);
+  }
+
+  private List<BlobSidecar> requestBlobSidecarsByRange(final Eth2Peer peer)
+      throws InterruptedException, ExecutionException, TimeoutException {
+    final List<BlobSidecar> blobSidecars = new ArrayList<>();
+    waitFor(
+        peer.requestBlobSidecarsByRange(
+            UInt64.ONE, UInt64.valueOf(10), RpcResponseListener.from(blobSidecars::add)));
+    assertThat(peer.getOutstandingRequests()).isEqualTo(0);
+    return blobSidecars;
+  }
+}

networking/eth2/src/integration-test/java/tech/pegasys/teku/networking/eth2/BlobSidecarsByRootIntegrationTest.java

@@ -22,11 +22,12 @@
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 import org.apache.tuweni.bytes.Bytes32;
 import org.junit.jupiter.api.Test;
 import tech.pegasys.teku.infrastructure.unsigned.UInt64;
 import tech.pegasys.teku.networking.eth2.peers.Eth2Peer;
-import tech.pegasys.teku.networking.eth2.rpc.core.RpcException;
 import tech.pegasys.teku.networking.p2p.rpc.RpcResponseListener;
 import tech.pegasys.teku.spec.TestSpecFactory;
 import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar;
@@ -37,7 +38,7 @@ public class BlobSidecarsByRootIntegrationTest extends AbstractRpcMethodIntegrat
   @Test
   public void requestBlobSidecars_shouldFailBeforeDenebMilestone() {
     final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalCapella());
-    assertThatThrownBy(() -> requestBlobSidecars(peer, List.of()))
+    assertThatThrownBy(() -> requestBlobSidecarsByRoot(peer, List.of()))
         .hasRootCauseInstanceOf(UnsupportedOperationException.class)
         .hasMessageContaining("BlobSidecarsByRoot method is not supported");
   }
@@ -46,14 +47,53 @@ public void requestBlobSidecars_shouldFailBeforeDenebMilestone() {
   public void requestBlobSidecar_shouldFailBeforeDenebMilestone() {
     final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalCapella());
     assertThatThrownBy(
-            () -> requestBlobSidecar(peer, new BlobIdentifier(Bytes32.ZERO, UInt64.ZERO)))
+            () -> requestBlobSidecarByRoot(peer, new BlobIdentifier(Bytes32.ZERO, UInt64.ZERO)))
         .hasRootCauseInstanceOf(UnsupportedOperationException.class)
         .hasMessageContaining("BlobSidecarsByRoot method is not supported");
   }
 
-  private List<BlobSidecar> requestBlobSidecars(
+  @Test
+  public void requestBlobSidecars_shouldReturnEmptyBlobSidecarsOnDenebMilestone()
+      throws ExecutionException, InterruptedException, TimeoutException {
+    final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalDeneb());
+    final Optional<BlobSidecar> blobSidecar =
+        requestBlobSidecarByRoot(peer, new BlobIdentifier(Bytes32.ZERO, UInt64.ZERO));
+    assertThat(blobSidecar).isEmpty();
+  }
+
+  @Test
+  public void requestBlobSidecars_shouldReturnBlobSidecarsOnDenebMilestone()
+      throws ExecutionException, InterruptedException, TimeoutException {
+    final Eth2Peer peer = createPeer(TestSpecFactory.createMinimalDeneb());
+
+    // generate 4 blobs per block
+    peerStorage.chainUpdater().blockOptions.setGenerateRandomBlobs(true);
+    peerStorage.chainUpdater().blockOptions.setGenerateRandomBlobsCount(Optional.of(4));
+
+    // up to slot 3
+    final UInt64 targetSlot = UInt64.valueOf(3);
+    peerStorage.chainUpdater().advanceChainUntil(targetSlot);
+
+    // grab expected blobs from storage
+    final List<BlobSidecar> expectedBlobSidecars =
+        retrieveCanonicalBlobSidecarsFromPeerStorage(Stream.of(UInt64.ONE, UInt64.valueOf(3)));
+
+    // request all expected plus a non existing
+    List<BlobIdentifier> requestedBlobIds =
+        Stream.concat(
+                Stream.of(new BlobIdentifier(Bytes32.ZERO, UInt64.ZERO)),
+                expectedBlobSidecars.stream()
+                    .map(sidecar -> new BlobIdentifier(sidecar.getBlockRoot(), sidecar.getIndex())))
+            .collect(Collectors.toUnmodifiableList());
+
+    final List<BlobSidecar> blobSidecars = requestBlobSidecarsByRoot(peer, requestedBlobIds);
+
+    assertThat(blobSidecars).containsExactlyInAnyOrderElementsOf(expectedBlobSidecars);
+  }
+
+  private List<BlobSidecar> requestBlobSidecarsByRoot(
       final Eth2Peer peer, final List<BlobIdentifier> blobIdentifiers)
-      throws InterruptedException, ExecutionException, TimeoutException, RpcException {
+      throws InterruptedException, ExecutionException, TimeoutException {
     final List<BlobSidecar> blobSidecars = new ArrayList<>();
     waitFor(
         peer.requestBlobSidecarsByRoot(
@@ -62,7 +102,7 @@ private List<BlobSidecar> requestBlobSidecars(
     return blobSidecars;
   }
 
-  private Optional<BlobSidecar> requestBlobSidecar(
+  private Optional<BlobSidecar> requestBlobSidecarByRoot(
       final Eth2Peer peer, final BlobIdentifier blobIdentifier)
       throws ExecutionException, InterruptedException, TimeoutException {
     final Optional<BlobSidecar> blobSidecar =