Merge pull request #122 from harmony-dev/feature/incremental-packed

Implement incremental packed list (basic type list/vector) hashing
harmony-dev · Apr 23, 2019 · da3bec1 · da3bec1
2 parents 618502a + eb3771a
commit da3bec1
Show file tree

Hide file tree

Showing 7 changed files with 360 additions and 77 deletions.
diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java
@@ -4,6 +4,7 @@
 import net.consensys.cava.bytes.Bytes;
 import org.ethereum.beacon.ssz.access.SSZField;
 import org.ethereum.beacon.ssz.creator.CompositeObjCreator;
+import org.ethereum.beacon.ssz.type.SSZListType;
 import org.ethereum.beacon.ssz.type.SSZType;
 import org.ethereum.beacon.ssz.type.TypeResolver;
 import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer;
@@ -46,6 +47,11 @@ public SSZSerializerResult visitAny(SSZType sszType, Object value) {
     return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer());
   }
 
+  @Override
+  public SSZSerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) {
+    return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSimpleSerializer());
+  }
+
   /**
    * Restores data instance from serialization data using {@link CompositeObjCreator}
    *

diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java
@@ -1,14 +1,17 @@
 package org.ethereum.beacon.ssz.visitor;
 
 import static java.lang.Math.min;
+import static java.util.stream.Collectors.toList;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collection;
 import java.util.List;
 import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.function.BiFunction;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import org.ethereum.beacon.ssz.incremental.ObservableComposite;
 import org.ethereum.beacon.ssz.incremental.UpdateListener;
 import org.ethereum.beacon.ssz.type.SSZCompositeType;
@@ -64,10 +67,8 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue,
         tracker.merkleTree = super.visitComposite(type, rawValue, childVisitor);
       } else if (!tracker.elementsUpdated.isEmpty()){
         if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) {
-//          tracker.merkleTree =
-//              updatePackedTrie(value, childVisitor, tracker.merkleTree, tracker.elementsUpdated);
-          // TODO fallback to full recalculation for now
-          tracker.merkleTree = super.visitComposite(type, rawValue, childVisitor);
+          tracker.merkleTree =
+              updatePackedTrie((SSZListType) type, rawValue, tracker.merkleTree, tracker.elementsUpdated);
         } else {
           tracker.merkleTree =
               updateNonPackedTrie(type, rawValue, childVisitor, tracker.merkleTree, tracker.elementsUpdated);
@@ -88,19 +89,54 @@ private MerkleTrie updateNonPackedTrie(
       MerkleTrie merkleTree,
       SortedSet<Integer> elementsUpdated) {
 
-    int newChildrenCount = type.getChildrenCount(value);
-    MerkleTrie newTrie = copyWithSize(merkleTree, newChildrenCount);
-    int newChunksCount = newTrie.nodes.length / 2;
+    return updateTrie(
+        type,
+        value,
+        idx -> childVisitor.apply(idx, type.getChild(value, idx)).getFinalRoot(),
+        type.getChildrenCount(value),
+        merkleTree,
+        elementsUpdated);
+  }
+
+  private MerkleTrie updatePackedTrie(
+      SSZListType type,
+      Object value,
+      MerkleTrie oldTrie,
+      SortedSet<Integer> elementsUpdated) {
+
+    int typeSize = type.getElementType().getSize();
+    int valsPerChunk = bytesPerChunk / typeSize;
+
+    return updateTrie(
+        type,
+        value,
+        idx -> serializePackedChunk(type, value, idx),
+        (type.getChildrenCount(value) - 1) / valsPerChunk + 1,
+        oldTrie,
+        elementsUpdated.stream().map(i -> i / valsPerChunk).distinct().collect(toList()));
+  }
+
+  private MerkleTrie updateTrie(
+      SSZCompositeType type,
+      Object value,
+      Function<Integer, BytesValue> childChunkSupplier,
+      int newChunksCount,
+      MerkleTrie oldTrie,
+      Collection<Integer> chunksUpdated) {
+
+    MerkleTrie newTrie = copyWithSize(oldTrie, newChunksCount);
+    int newTrieWidth = newTrie.nodes.length / 2;
 
-    int pos = newChunksCount;
+    int pos = newTrieWidth;
 
     List<Integer> elementsToRecalc = new ArrayList<>();
-    for (int i: elementsUpdated) {
-      if (i < newChunksCount) {
+    for (int i: chunksUpdated) {
+      if (i < newTrieWidth) {
         elementsToRecalc.add(i);
-        if (i < newChildrenCount) {
-          MerkleTrie childHash = childVisitor.apply(i, type.getChild(value, i));
-          newTrie.nodes[pos + i] = childHash.getFinalRoot();
+        if (i < newChunksCount) {
+          newTrie.nodes[pos + i] = childChunkSupplier.apply(i);
+        } else {
+          newTrie.nodes[pos + i] = getZeroHash(0);
         }
       }
     }
@@ -130,15 +166,6 @@ private MerkleTrie updateNonPackedTrie(
     return newTrie;
   }
 
-  private MerkleTrie updatePackedTrie(
-      SSZCompositeType type,
-      Object value,
-      Function<Long, MerkleTrie> childVisitor,
-      MerkleTrie merkleTree,
-      SortedSet<Integer> elementsUpdated) {
-
-    throw new UnsupportedOperationException();
-  }
   private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) {
     int newSize = (int) nextPowerOf2(newChunksCount) * 2;
     if (newSize == trie.nodes.length) {
@@ -162,4 +189,21 @@ private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) {
       return new MerkleTrie(newNodes);
     }
   }
+
+  private BytesValue serializePackedChunk(SSZListType basicListType, Object listValue, int chunkIndex) {
+    int typeSize = basicListType.getElementType().getSize();
+    int valsPerChunk = bytesPerChunk / typeSize;
+    if (valsPerChunk * typeSize != bytesPerChunk) {
+      throw new UnsupportedOperationException("Type size (" + typeSize
+          + ") which is not a factor of hasher chunk size (" + bytesPerChunk + ") is not yet supported.");
+    }
+    int idx = chunkIndex * valsPerChunk;
+    int len = Math.min(valsPerChunk, basicListType.getChildrenCount(listValue) - idx);
+    BytesValue chunk = serializer.visitList(basicListType, listValue, idx, len)
+        .getSerializedBody();
+    if (len < valsPerChunk) {
+      chunk = BytesValue.concat(chunk, BytesValue.wrap(new byte[bytesPerChunk - chunk.size()]));
+    }
+    return chunk;
+  }
 }
diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java
@@ -65,14 +65,24 @@ public SSZSerializerResult visitList(SSZListType type, Object param,
     return visitComposite(type, param, childVisitor);
   }
 
+  @Override
+  public SSZSerializerResult visitSubList(SSZListType type, Object param,
+      int startIdx, int len, ChildVisitor<Object, SSZSerializerResult> childVisitor) {
+    return visitComposite(type, param, childVisitor, startIdx, len);
+  }
+
   @Override
   public SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue,
       ChildVisitor<Object, SSZSerializerResult> childVisitor) {
+    return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue));
+  }
 
+  private SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue,
+      ChildVisitor<Object, SSZSerializerResult> childVisitor, int startIdx, int len) {
     List<BytesValue> childSerializations = new ArrayList<>();
     boolean fixedSize = type.isFixedSize();
     int length = 0;
-    for (int i = 0; i < type.getChildrenCount(rawValue); i++) {
+    for (int i = startIdx; i < startIdx + len; i++) {
       SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i));
       childSerializations.add(res.serializedLength);
       childSerializations.add(res.serializedBody);

diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java
@@ -36,13 +36,22 @@ default ResultType visitList(
     return visitComposite(type, param, childVisitor);
   }
 
-  /**
-   * Invoked on the SSZ Container in the type hierarchy
-   *
-   * NOTE: you should either implement {@link #visitComposite(SSZCompositeType, Object, ChildVisitor)}
-   * method or  both {@link #visitList(SSZListType, Object, ChildVisitor)} and
-   * {@link #visitContainer(SSZContainerType, Object, ChildVisitor)} method
-   */
+  default ResultType visitSubList(
+      SSZListType type,
+      ParamType param,
+      int startIdx,
+      int len,
+      ChildVisitor<ParamType, ResultType> childVisitor) {
+    throw new UnsupportedOperationException();
+  }
+
+    /**
+     * Invoked on the SSZ Container in the type hierarchy
+     *
+     * NOTE: you should either implement {@link #visitComposite(SSZCompositeType, Object, ChildVisitor)}
+     * method or  both {@link #visitList(SSZListType, Object, ChildVisitor)} and
+     * {@link #visitContainer(SSZContainerType, Object, ChildVisitor)} method
+     */
   default ResultType visitContainer(
       SSZContainerType type, ParamType param, ChildVisitor<ParamType, ResultType> childVisitor) {
     return visitComposite(type, param, childVisitor);

diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java
@@ -1,5 +1,6 @@
 package org.ethereum.beacon.ssz.visitor;
 
+import org.ethereum.beacon.ssz.type.SSZListType;
 import org.ethereum.beacon.ssz.type.SSZType;
 
 /**
@@ -8,4 +9,6 @@
 public interface SSZVisitorHandler<ResultType> {
 
   ResultType visitAny(SSZType descriptor, Object value);
+
+  ResultType visitList(SSZListType descriptor, Object listValue, int startIdx, int len);
 }
diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java
@@ -25,4 +25,11 @@ public  <ResultType, ParamType> ResultType handleAny(
       throw new IllegalArgumentException("Unknown type: " + type);
     }
   }
+
+  public  <ResultType, ParamType> ResultType handleSubList(
+      SSZListType type, ParamType value, int startIdx, int len, SSZVisitor<ResultType, ParamType> visitor) {
+
+    return visitor.visitSubList(type, value, startIdx, len, (idx, param) ->
+        handleAny(type.getElementType(), param, visitor));
+  }
 }