From 54f72dcfd5f91b806f06cca91518ea00de9dc8af Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Mon, 22 Apr 2019 18:25:59 +0300 Subject: [PATCH 1/5] Add serializing sublist for incremental hash --- .../ethereum/beacon/ssz/SSZSerializer.java | 6 +++++ .../ssz/visitor/SSZIncrementalHasher.java | 8 +++++++ .../ssz/visitor/SSZSimpleSerializer.java | 12 +++++++++- .../beacon/ssz/visitor/SSZVisitor.java | 23 +++++++++++++------ .../beacon/ssz/visitor/SSZVisitorHandler.java | 3 +++ .../beacon/ssz/visitor/SSZVisitorHost.java | 7 ++++++ 6 files changed, 51 insertions(+), 8 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java index 3061e87c8..f612c12d1 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/SSZSerializer.java @@ -4,6 +4,7 @@ import net.consensys.cava.bytes.Bytes; import org.ethereum.beacon.ssz.access.SSZField; import org.ethereum.beacon.ssz.creator.CompositeObjCreator; +import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; import org.ethereum.beacon.ssz.visitor.SSZSimpleDeserializer; @@ -46,6 +47,11 @@ public SSZSerializerResult visitAny(SSZType sszType, Object value) { return sszVisitorHost.handleAny(sszType, value, new SSZSimpleSerializer()); } + @Override + public SSZSerializerResult visitList(SSZListType descriptor, Object listValue, int startIdx, int len) { + return sszVisitorHost.handleSubList(descriptor, listValue, startIdx, len, new SSZSimpleSerializer()); + } + /** * Restores data instance from serialization data using {@link CompositeObjCreator} * diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index 6a472d1cc..f6afb6bf2 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -162,4 +162,12 @@ private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) { return new MerkleTrie(newNodes); } } + + private BytesValue serializePackedChunk(SSZListType basicListType, Object listValue, int chunkIndex) { + return serializer.visitList( + basicListType, + listValue, + chunkIndex * basicListType.getElementType().getSize(), + basicListType.getElementType().getSize()).getSerializedBody(); + } } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java index 632110f58..42f8929e0 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZSimpleSerializer.java @@ -65,14 +65,24 @@ public SSZSerializerResult visitList(SSZListType type, Object param, return visitComposite(type, param, childVisitor); } + @Override + public SSZSerializerResult visitSubList(SSZListType type, Object param, + int startIdx, int len, ChildVisitor childVisitor) { + return visitComposite(type, param, childVisitor, startIdx, len); + } + @Override public SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, ChildVisitor childVisitor) { + return visitComposite(type, rawValue, childVisitor, 0, type.getChildrenCount(rawValue)); + } + private SSZSerializerResult visitComposite(SSZCompositeType type, Object rawValue, + ChildVisitor childVisitor, int startIdx, int len) { List childSerializations = new ArrayList<>(); boolean fixedSize = type.isFixedSize(); int length = 0; - for (int i = 0; i < type.getChildrenCount(rawValue); i++) { + for (int i = startIdx; i < startIdx + len; i++) { SSZSerializerResult res = childVisitor.apply(i, type.getChild(rawValue, i)); childSerializations.add(res.serializedLength); childSerializations.add(res.serializedBody); diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java index 2648b983a..4ba50fa4a 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitor.java @@ -36,13 +36,22 @@ default ResultType visitList( return visitComposite(type, param, childVisitor); } - /** - * Invoked on the SSZ Container in the type hierarchy - * - * NOTE: you should either implement {@link #visitComposite(SSZCompositeType, Object, ChildVisitor)} - * method or both {@link #visitList(SSZListType, Object, ChildVisitor)} and - * {@link #visitContainer(SSZContainerType, Object, ChildVisitor)} method - */ + default ResultType visitSubList( + SSZListType type, + ParamType param, + int startIdx, + int len, + ChildVisitor childVisitor) { + throw new UnsupportedOperationException(); + } + + /** + * Invoked on the SSZ Container in the type hierarchy + * + * NOTE: you should either implement {@link #visitComposite(SSZCompositeType, Object, ChildVisitor)} + * method or both {@link #visitList(SSZListType, Object, ChildVisitor)} and + * {@link #visitContainer(SSZContainerType, Object, ChildVisitor)} method + */ default ResultType visitContainer( SSZContainerType type, ParamType param, ChildVisitor childVisitor) { return visitComposite(type, param, childVisitor); diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java index f32847bbe..dda1a5d05 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHandler.java @@ -1,5 +1,6 @@ package org.ethereum.beacon.ssz.visitor; +import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; /** @@ -8,4 +9,6 @@ public interface SSZVisitorHandler { ResultType visitAny(SSZType descriptor, Object value); + + ResultType visitList(SSZListType descriptor, Object listValue, int startIdx, int len); } diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java index 9defc6bfc..10a61570b 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZVisitorHost.java @@ -25,4 +25,11 @@ public ResultType handleAny( throw new IllegalArgumentException("Unknown type: " + type); } } + + public ResultType handleSubList( + SSZListType type, ParamType value, int startIdx, int len, SSZVisitor visitor) { + + return visitor.visitSubList(type, value, startIdx, len, (idx, param) -> + handleAny(type.getElementType(), param, visitor)); + } } From a7ffb9eb48b4856532ca465ca87c4ae3647467c4 Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Mon, 22 Apr 2019 20:33:20 +0300 Subject: [PATCH 2/5] Implement incremental hashing for packed lists --- .../ssz/visitor/SSZIncrementalHasher.java | 96 +++++-- .../beacon/ssz/SSZIncrementalTest.java | 266 ++++++++++++++---- 2 files changed, 284 insertions(+), 78 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index f6afb6bf2..f6406fcfa 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -1,14 +1,17 @@ package org.ethereum.beacon.ssz.visitor; import static java.lang.Math.min; +import static java.util.stream.Collectors.toList; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.SortedSet; import java.util.TreeSet; import java.util.function.BiFunction; import java.util.function.Function; +import java.util.stream.Collectors; import org.ethereum.beacon.ssz.incremental.ObservableComposite; import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZCompositeType; @@ -64,10 +67,8 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, tracker.merkleTree = super.visitComposite(type, rawValue, childVisitor); } else if (!tracker.elementsUpdated.isEmpty()){ if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { -// tracker.merkleTree = -// updatePackedTrie(value, childVisitor, tracker.merkleTree, tracker.elementsUpdated); - // TODO fallback to full recalculation for now - tracker.merkleTree = super.visitComposite(type, rawValue, childVisitor); + tracker.merkleTree = + updatePackedTrie((SSZListType) type, rawValue, childVisitor, tracker.merkleTree, tracker.elementsUpdated); } else { tracker.merkleTree = updateNonPackedTrie(type, rawValue, childVisitor, tracker.merkleTree, tracker.elementsUpdated); @@ -88,19 +89,53 @@ private MerkleTrie updateNonPackedTrie( MerkleTrie merkleTree, SortedSet elementsUpdated) { - int newChildrenCount = type.getChildrenCount(value); - MerkleTrie newTrie = copyWithSize(merkleTree, newChildrenCount); - int newChunksCount = newTrie.nodes.length / 2; + return updateTrie( + type, + value, + idx -> childVisitor.apply(idx, type.getChild(value, idx)).getFinalRoot(), + type.getChildrenCount(value), + merkleTree, + elementsUpdated); + } + + private MerkleTrie updatePackedTrie( + SSZListType type, + Object value, + BiFunction childVisitor, + MerkleTrie oldTrie, + SortedSet elementsUpdated) { + + int typeSize = type.getElementType().getSize(); + int valsPerChunk = bytesPerChunk / typeSize; + + return updateTrie( + type, + value, + idx -> serializePackedChunk(type, value, idx), + (type.getChildrenCount(value) - 1) / valsPerChunk + 1, + oldTrie, + elementsUpdated.stream().map(i -> i / valsPerChunk).distinct().collect(toList())); + } + + private MerkleTrie updateTrie( + SSZCompositeType type, + Object value, + Function childChunkSupplier, + int newChunksCount, + MerkleTrie oldTrie, + Collection chunksUpdated) { - int pos = newChunksCount; + MerkleTrie newTrie = copyWithSize(oldTrie, newChunksCount); + int newTrieWidth = newTrie.nodes.length / 2; + + int pos = newTrieWidth; List elementsToRecalc = new ArrayList<>(); - for (int i: elementsUpdated) { - if (i < newChunksCount) { + for (int i: chunksUpdated) { + if (i < newTrieWidth) { elementsToRecalc.add(i); - if (i < newChildrenCount) { - MerkleTrie childHash = childVisitor.apply(i, type.getChild(value, i)); - newTrie.nodes[pos + i] = childHash.getFinalRoot(); + if (i < newChunksCount) { + newTrie.nodes[pos + i] = childChunkSupplier.apply(i); } } } @@ -130,20 +165,11 @@ private MerkleTrie updateNonPackedTrie( return newTrie; } - private MerkleTrie updatePackedTrie( - SSZCompositeType type, - Object value, - Function childVisitor, - MerkleTrie merkleTree, - SortedSet elementsUpdated) { - - throw new UnsupportedOperationException(); - } private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) { int newSize = (int) nextPowerOf2(newChunksCount) * 2; - if (newSize == trie.nodes.length) { - return new MerkleTrie(Arrays.copyOf(trie.nodes, newSize)); - } else { +// if (newSize == trie.nodes.length) { +// return new MerkleTrie(Arrays.copyOf(trie.nodes, newSize)); +// } else { BytesValue[] oldNodes = trie.nodes; BytesValue[] newNodes = new BytesValue[newSize]; int oldPos = oldNodes.length / 2; @@ -160,14 +186,22 @@ private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) { } return new MerkleTrie(newNodes); - } +// } } private BytesValue serializePackedChunk(SSZListType basicListType, Object listValue, int chunkIndex) { - return serializer.visitList( - basicListType, - listValue, - chunkIndex * basicListType.getElementType().getSize(), - basicListType.getElementType().getSize()).getSerializedBody(); + int typeSize = basicListType.getElementType().getSize(); + int valsPerChunk = bytesPerChunk / typeSize; + if (valsPerChunk * typeSize != bytesPerChunk) { + throw new UnsupportedOperationException(""); + } + int idx = chunkIndex * valsPerChunk; + int len = Math.min(valsPerChunk, basicListType.getChildrenCount(listValue) - idx); + BytesValue chunk = serializer.visitList(basicListType, listValue, idx, len) + .getSerializedBody(); + if (len < valsPerChunk) { + chunk = BytesValue.concat(chunk, BytesValue.wrap(new byte[bytesPerChunk - chunk.size()])); + } + return chunk; } } diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java index eb539cbc3..a011f5f5e 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java @@ -1,6 +1,7 @@ package org.ethereum.beacon.ssz; import java.util.Map; +import java.util.Random; import java.util.function.Function; import java.util.function.Supplier; import org.ethereum.beacon.crypto.Hashes; @@ -15,9 +16,9 @@ import org.ethereum.beacon.ssz.incremental.UpdateListener; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; +import org.ethereum.beacon.ssz.visitor.MerkleTrie; import org.ethereum.beacon.ssz.visitor.SSZIncrementalHasher; import org.ethereum.beacon.ssz.visitor.SSZSimpleHasher; -import org.ethereum.beacon.ssz.visitor.MerkleTrie; import org.ethereum.beacon.ssz.visitor.SSZVisitorHost; import org.junit.Assert; import org.junit.Test; @@ -25,9 +26,21 @@ import tech.pegasys.artemis.util.bytes.BytesValue; import tech.pegasys.artemis.util.collections.ReadList; import tech.pegasys.artemis.util.collections.WriteList; +import tech.pegasys.artemis.util.uint.UInt64; public class SSZIncrementalTest { + private static class CountingHash implements Function { + int counter = 0; + + @Override + public Hash32 apply(BytesValue bytesValue) { + counter++; + return Hashes.keccak256(bytesValue); + } + } + + @SSZSerializable public static class I1 implements ObservableComposite { UpdateListener updateListener; @@ -85,15 +98,6 @@ public void setA3(int a3) { @Test public void testHashIncremental1() throws Exception { - class CountingHash implements Function { - int counter = 0; - - @Override - public Hash32 apply(BytesValue bytesValue) { - counter++; - return Hashes.keccak256(bytesValue); - } - } SSZBuilder sszBuilder = new SSZBuilder(); TypeResolver typeResolver = sszBuilder.getTypeResolver(); @@ -201,15 +205,6 @@ public A1(int a1) { @Test public void testReadList() { - class CountingHash implements Function { - int counter = 0; - - @Override - public Hash32 apply(BytesValue bytesValue) { - counter++; - return Hashes.keccak256(bytesValue); - } - } SSZBuilder sszBuilder = new SSZBuilder() .addDefaultListAccessors() .addListAccessors(new ReadListAccessor()); @@ -265,15 +260,6 @@ public Hash32 apply(BytesValue bytesValue) { @Test public void testReadListBranching() { - class CountingHash implements Function { - int counter = 0; - - @Override - public Hash32 apply(BytesValue bytesValue) { - counter++; - return Hashes.keccak256(bytesValue); - } - } CountingHash countingHashSimp = new CountingHash(); CountingHash countingHashInc = new CountingHash(); @@ -358,19 +344,214 @@ public Hash32 apply(BytesValue bytesValue) { } @Test - public void testListRemove() { - class CountingHash implements Function { - int counter = 0; + public void testPackedList1() { + SSZBuilder sszBuilder = new SSZBuilder(); + TypeResolver typeResolver = sszBuilder.getTypeResolver(); + + SSZVisitorHost visitorHost = new SSZVisitorHost(); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + CountingHash countingHashSimp = new CountingHash(); + CountingHash countingHashInc = new CountingHash(); + SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, + 32); + SSZSimpleHasher simpleHasher = new SSZSimpleHasher(serializer, countingHashSimp, 32); + + WriteList list1 = new ObservableListImpl<>(WriteList.create(Integer::valueOf)); + for (int i = 0; i < 17; i++) { + list1.add(UInt64.valueOf(0xF00000000L + i)); + SSZType sszListType = typeResolver.resolveSSZType(SSZField.resolveFromValue(list1)); + + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list1, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list1, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + } + + for (int i = 0; i < 17; i++) { + list1.set(i, UInt64.valueOf(0xF000A0000L + i)); + SSZType sszListType = typeResolver.resolveSSZType(SSZField.resolveFromValue(list1)); + + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list1, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list1, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + } + } + + @Test + public void testPackedListRandom() { + packedRandomTest( + new ObservableListImpl<>(WriteList.create(Integer::valueOf)), + new Supplier() { + UInt64 val = UInt64.valueOf(0xF00000000L); + @Override + public UInt64 get() { + val = val.increment(); + return val; + } + }); + + packedRandomTest( + new ObservableListImpl<>(WriteList.create(Integer::valueOf)), + new Supplier() { + int val = 0xF000000; + @Override + public Integer get() { + val++; + return val; + } + }); + + packedRandomTest( + new ObservableListImpl<>(WriteList.create(Integer::valueOf)), + new Supplier() { + Random rnd = new Random(); + @Override + public Hash32 get() { + return Hash32.random(rnd); + } + }); + + packedRandomTest( + new ObservableListImpl<>(WriteList.create(Integer::valueOf)), + new Supplier() { + byte val = 0x00; + @Override + public Byte get() { + val++; + return val; + } + }); + } + + private void packedRandomTest(WriteList list, Supplier numSupplier) { + SSZBuilder sszBuilder = new SSZBuilder(); + TypeResolver typeResolver = sszBuilder.getTypeResolver(); + + SSZVisitorHost visitorHost = new SSZVisitorHost(); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + CountingHash countingHashSimp = new CountingHash(); + CountingHash countingHashInc = new CountingHash(); + SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, + 32); + SSZSimpleHasher simpleHasher = new SSZSimpleHasher(serializer, countingHashSimp, 32); + + list.add(numSupplier.get()); + + SSZType sszListType = typeResolver.resolveSSZType(SSZField.resolveFromValue(list)); + + Random rnd = new Random(0); + + for (int i = 0; i < 1000; i++) { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; - @Override - public Hash32 apply(BytesValue bytesValue) { - counter++; - return Hashes.keccak256(bytesValue); + if (i == 500) { + for (int j = 0; j < 1000; j++) { + list.add(numSupplier.get()); + } + } + + for (int j = 0; j < rnd.nextInt(8); j++) { + if (list.isEmpty()) break; + list.remove(rnd.nextInt(list.size())); + } + + for (int j = 0; j < rnd.nextInt(8); j++) { + list.add(rnd.nextInt(list.size() + 1), numSupplier.get()); + } + + for (int j = 0; j < rnd.nextInt(8); j++) { + if (list.isEmpty()) break; + list.set(rnd.nextInt(list.size()), numSupplier.get()); + } + + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + if (i > 500) { + Assert.assertTrue(countingHashInc.counter * 5 < countingHashSimp.counter); } } - SSZBuilder sszBuilder = new SSZBuilder() - .addDefaultListAccessors() - .addListAccessors(new ReadListAccessor()); + } + + + @Test + public void testPackedListRemove1() { + SSZBuilder sszBuilder = new SSZBuilder(); + TypeResolver typeResolver = sszBuilder.getTypeResolver(); + + SSZVisitorHost visitorHost = new SSZVisitorHost(); + SSZSerializer serializer = new SSZSerializer(visitorHost, typeResolver); + CountingHash countingHashSimp = new CountingHash(); + CountingHash countingHashInc = new CountingHash(); + SSZIncrementalHasher incrementalHasher = new SSZIncrementalHasher(serializer, countingHashInc, 32); + SSZSimpleHasher simpleHasher = new SSZSimpleHasher(serializer, countingHashSimp, 32); + + WriteList list1 = new ObservableListImpl<>(WriteList.create(Integer::valueOf)); + list1.add(UInt64.valueOf(0x1111)); + list1.add(UInt64.valueOf(0x2222)); + list1.add(UInt64.valueOf(0x3333)); + list1.add(UInt64.valueOf(0x4444)); + list1.add(UInt64.valueOf(0x5555)); + + ReadList list2 = list1.createImmutableCopy(); + + SSZType sszListType = typeResolver.resolveSSZType(SSZField.resolveFromValue(list2)); + + { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list2, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list2, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + Assert.assertTrue(countingHashInc.counter == countingHashSimp.counter); + } + + WriteList list3 = list2.createMutableCopy(); + list3.remove(4); + ReadList list4 = list3.createImmutableCopy(); + + { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list4, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list4, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + } + + WriteList list5 = list2.createMutableCopy(); + list5.remove(3); + list5.remove(3); + ReadList list6 = list5.createImmutableCopy(); + + { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list6, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list6, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + } + + WriteList list7 = list2.createMutableCopy(); + list7.remove(2); + list7.remove(2); + list7.remove(2); + ReadList list8 = list7.createImmutableCopy(); + + { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list8, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list8, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + } + } + + @Test + public void testListRemove() { + SSZBuilder sszBuilder = new SSZBuilder(); TypeResolver typeResolver = sszBuilder.getTypeResolver(); SSZVisitorHost visitorHost = new SSZVisitorHost(); @@ -530,15 +711,6 @@ public Map getAllUpdateListeners() { @Test public void testComplexStruct() { - class CountingHash implements Function { - int counter = 0; - - @Override - public Hash32 apply(BytesValue bytesValue) { - counter++; - return Hashes.keccak256(bytesValue); - } - } SSZBuilder sszBuilder = new SSZBuilder() .addDefaultListAccessors() .addListAccessors(new ReadListAccessor()); From cb37db1710d16e4fca00023f30b112487f619d6a Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Tue, 23 Apr 2019 10:01:54 +0300 Subject: [PATCH 3/5] Fix IncrementalHasher bug --- .../beacon/ssz/visitor/SSZIncrementalHasher.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index f6406fcfa..3d2ebb971 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -136,6 +136,8 @@ private MerkleTrie updateTrie( elementsToRecalc.add(i); if (i < newChunksCount) { newTrie.nodes[pos + i] = childChunkSupplier.apply(i); + } else { + newTrie.nodes[pos + i] = getZeroHash(0); } } } @@ -167,9 +169,9 @@ private MerkleTrie updateTrie( private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) { int newSize = (int) nextPowerOf2(newChunksCount) * 2; -// if (newSize == trie.nodes.length) { -// return new MerkleTrie(Arrays.copyOf(trie.nodes, newSize)); -// } else { + if (newSize == trie.nodes.length) { + return new MerkleTrie(Arrays.copyOf(trie.nodes, newSize)); + } else { BytesValue[] oldNodes = trie.nodes; BytesValue[] newNodes = new BytesValue[newSize]; int oldPos = oldNodes.length / 2; @@ -186,7 +188,7 @@ private MerkleTrie copyWithSize(MerkleTrie trie, int newChunksCount) { } return new MerkleTrie(newNodes); -// } + } } private BytesValue serializePackedChunk(SSZListType basicListType, Object listValue, int chunkIndex) { From 9d5dd7f4c137061f90cd7977dda6d54bc395c0ff Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Tue, 23 Apr 2019 10:11:11 +0300 Subject: [PATCH 4/5] Adjust random list test --- .../beacon/ssz/SSZIncrementalTest.java | 70 ++++++++++++++----- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java index a011f5f5e..2c51c81c9 100644 --- a/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java +++ b/ssz/src/test/java/org/ethereum/beacon/ssz/SSZIncrementalTest.java @@ -14,6 +14,7 @@ import org.ethereum.beacon.ssz.incremental.ObservableCompositeHelper.ObsValue; import org.ethereum.beacon.ssz.incremental.ObservableListImpl; import org.ethereum.beacon.ssz.incremental.UpdateListener; +import org.ethereum.beacon.ssz.type.SSZListType; import org.ethereum.beacon.ssz.type.SSZType; import org.ethereum.beacon.ssz.type.TypeResolver; import org.ethereum.beacon.ssz.visitor.MerkleTrie; @@ -376,9 +377,22 @@ public void testPackedList1() { } } + @Test + public void testNonPackedListRandom() { + listRandomTest( + new ObservableListImpl<>(WriteList.create(Integer::valueOf)), + new Supplier() { + int i = 0x0F000000; + @Override + public A1 get() { + return new A1(i++); + } + }); + } + @Test public void testPackedListRandom() { - packedRandomTest( + listRandomTest( new ObservableListImpl<>(WriteList.create(Integer::valueOf)), new Supplier() { UInt64 val = UInt64.valueOf(0xF00000000L); @@ -389,7 +403,7 @@ public UInt64 get() { } }); - packedRandomTest( + listRandomTest( new ObservableListImpl<>(WriteList.create(Integer::valueOf)), new Supplier() { int val = 0xF000000; @@ -400,7 +414,7 @@ public Integer get() { } }); - packedRandomTest( + listRandomTest( new ObservableListImpl<>(WriteList.create(Integer::valueOf)), new Supplier() { Random rnd = new Random(); @@ -410,7 +424,7 @@ public Hash32 get() { } }); - packedRandomTest( + listRandomTest( new ObservableListImpl<>(WriteList.create(Integer::valueOf)), new Supplier() { byte val = 0x00; @@ -422,7 +436,7 @@ public Byte get() { }); } - private void packedRandomTest(WriteList list, Supplier numSupplier) { + private void listRandomTest(WriteList list, Supplier numSupplier) { SSZBuilder sszBuilder = new SSZBuilder(); TypeResolver typeResolver = sszBuilder.getTypeResolver(); @@ -436,27 +450,44 @@ private void packedRandomTest(WriteList list, Supplier numSup list.add(numSupplier.get()); - SSZType sszListType = typeResolver.resolveSSZType(SSZField.resolveFromValue(list)); + SSZListType sszListType = (SSZListType) typeResolver.resolveSSZType(SSZField.resolveFromValue(list)); Random rnd = new Random(0); - for (int i = 0; i < 1000; i++) { - countingHashInc.counter = 0; - countingHashSimp.counter = 0; + for (int i = 0; i < 500; i++) { + for (int j = 0; j < rnd.nextInt(8); j++) { + if (list.isEmpty()) break; + list.remove(rnd.nextInt(list.size())); + } - if (i == 500) { - for (int j = 0; j < 1000; j++) { - list.add(numSupplier.get()); - } + for (int j = 0; j < rnd.nextInt(8); j++) { + list.add(rnd.nextInt(list.size() + 1), numSupplier.get()); } for (int j = 0; j < rnd.nextInt(8); j++) { if (list.isEmpty()) break; - list.remove(rnd.nextInt(list.size())); + list.set(rnd.nextInt(list.size()), numSupplier.get()); } + MerkleTrie mt2 = visitorHost.handleAny(sszListType, list, simpleHasher); + MerkleTrie mt3 = visitorHost.handleAny(sszListType, list, incrementalHasher); + Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); + } + + // adding more elements + int elementsPerChunk = sszListType.getElementType().isBasicType() ? + 32 / sszListType.getElementType().getSize() : 1; + for (int j = 0; j < 500 * elementsPerChunk; j++) { + list.add(numSupplier.get()); + } + + // checking add/set takes significantly less hashing for incremental + for (int i = 0; i < 500; i++) { + countingHashInc.counter = 0; + countingHashSimp.counter = 0; + for (int j = 0; j < rnd.nextInt(8); j++) { - list.add(rnd.nextInt(list.size() + 1), numSupplier.get()); + list.add(numSupplier.get()); } for (int j = 0; j < rnd.nextInt(8); j++) { @@ -467,10 +498,11 @@ private void packedRandomTest(WriteList list, Supplier numSup MerkleTrie mt2 = visitorHost.handleAny(sszListType, list, simpleHasher); MerkleTrie mt3 = visitorHost.handleAny(sszListType, list, incrementalHasher); Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); - if (i > 500) { + if (i > 0) { Assert.assertTrue(countingHashInc.counter * 5 < countingHashSimp.counter); } } + } @@ -516,7 +548,7 @@ public void testPackedListRemove1() { MerkleTrie mt2 = visitorHost.handleAny(sszListType, list4, simpleHasher); MerkleTrie mt3 = visitorHost.handleAny(sszListType, list4, incrementalHasher); Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); - Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + Assert.assertTrue(countingHashInc.counter <= countingHashSimp.counter); } WriteList list5 = list2.createMutableCopy(); @@ -530,7 +562,7 @@ public void testPackedListRemove1() { MerkleTrie mt2 = visitorHost.handleAny(sszListType, list6, simpleHasher); MerkleTrie mt3 = visitorHost.handleAny(sszListType, list6, incrementalHasher); Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); - Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + Assert.assertTrue(countingHashInc.counter <= countingHashSimp.counter); } WriteList list7 = list2.createMutableCopy(); @@ -545,7 +577,7 @@ public void testPackedListRemove1() { MerkleTrie mt2 = visitorHost.handleAny(sszListType, list8, simpleHasher); MerkleTrie mt3 = visitorHost.handleAny(sszListType, list8, incrementalHasher); Assert.assertEquals(mt2.getFinalRoot(), mt3.getFinalRoot()); - Assert.assertTrue(countingHashInc.counter < countingHashSimp.counter); + Assert.assertTrue(countingHashInc.counter <= countingHashSimp.counter); } } From eb3771a25ea2efc3575bde0a87e5eba2d04f9579 Mon Sep 17 00:00:00 2001 From: Anton Nashatyrev Date: Tue, 23 Apr 2019 11:30:09 +0300 Subject: [PATCH 5/5] A couple of minor incremental hasher fixes --- .../ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java index 3d2ebb971..4dea95c89 100644 --- a/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java +++ b/ssz/src/main/java/org/ethereum/beacon/ssz/visitor/SSZIncrementalHasher.java @@ -68,7 +68,7 @@ public MerkleTrie visitComposite(SSZCompositeType type, Object rawValue, } else if (!tracker.elementsUpdated.isEmpty()){ if (type.isList() && ((SSZListType) type).getElementType().isBasicType()) { tracker.merkleTree = - updatePackedTrie((SSZListType) type, rawValue, childVisitor, tracker.merkleTree, tracker.elementsUpdated); + updatePackedTrie((SSZListType) type, rawValue, tracker.merkleTree, tracker.elementsUpdated); } else { tracker.merkleTree = updateNonPackedTrie(type, rawValue, childVisitor, tracker.merkleTree, tracker.elementsUpdated); @@ -101,7 +101,6 @@ private MerkleTrie updateNonPackedTrie( private MerkleTrie updatePackedTrie( SSZListType type, Object value, - BiFunction childVisitor, MerkleTrie oldTrie, SortedSet elementsUpdated) { @@ -195,7 +194,8 @@ private BytesValue serializePackedChunk(SSZListType basicListType, Object listVa int typeSize = basicListType.getElementType().getSize(); int valsPerChunk = bytesPerChunk / typeSize; if (valsPerChunk * typeSize != bytesPerChunk) { - throw new UnsupportedOperationException(""); + throw new UnsupportedOperationException("Type size (" + typeSize + + ") which is not a factor of hasher chunk size (" + bytesPerChunk + ") is not yet supported."); } int idx = chunkIndex * valsPerChunk; int len = Math.min(valsPerChunk, basicListType.getChildrenCount(listValue) - idx);