From 4e8e433da89e303c2796ccd347a8f69d51161a31 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Fri, 22 Nov 2024 00:14:04 +0530 Subject: [PATCH 01/36] add a profiling test --- tests/rlp/all_tests.nim | 3 ++- tests/rlp/test_rlp_profiler.nim | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 tests/rlp/test_rlp_profiler.nim diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index 625768c1..cd50c8b6 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -3,4 +3,5 @@ import ./test_json_suite, ./test_empty_string, ./test_object_serialization, - ./test_optional_fields + ./test_optional_fields, + ./test_rlp_profiler diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim new file mode 100644 index 00000000..9f95bef3 --- /dev/null +++ b/tests/rlp/test_rlp_profiler.nim @@ -0,0 +1,37 @@ +{.used.} + +import + ../../eth/[rlp, common], + unittest2, + times, + os, + strutils + +template benchmark(benchmarkName: string, code: untyped) = + block: + let t0 = epochTime() + code + let elapsed = epochTime() - t0 + let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 9) + echo "CPU Time [", benchmarkName, "] ", elapsedStr, "s" + +const + accesses = @[AccessPair( + address: address"0x0000000000000000000000000000000000000001", + storageKeys: @[default(Bytes32)] + )] + +let my_tx = Transaction( + txType: TxEip1559, + chainId: 1.ChainId, + nonce: 0.AccountNonce, + gasLimit: 123457.GasInt, + maxPriorityFeePerGas: 42.GasInt, + maxFeePerGas: 10.GasInt, + accessList: accesses +) + +suite "test running time of rlp serialization": + test "transaction serialization": + benchmark "Transaction": + let my_bytes = rlp.encode(my_tx) From 59a5f66e795f89cd0e18d695a241b3cf9b117c8b Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 23 Nov 2024 09:54:29 +0530 Subject: [PATCH 02/36] basic rlp profiler --- eth.nimble | 3 ++- tests/rlp/all_tests.nim | 10 +++++----- tests/rlp/test_rlp_profiler.nim | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/eth.nimble b/eth.nimble index a0f15f8d..590f3fc9 100644 --- a/eth.nimble +++ b/eth.nimble @@ -34,7 +34,8 @@ let cfg = " --skipUserCfg --nimcache:build/nimcache -f" & " --warning[ObservableStores]:off -d:nimOldCaseObjects" & " -d:chronicles_log_level=TRACE" & - " --threads:on -d:release" + " --threads:on -d:debug" & + " --excessiveStackTrace:on" proc build(args, path, outdir: string) = diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index cd50c8b6..aabc8287 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import - ./test_api_usage, - ./test_json_suite, - ./test_empty_string, - ./test_object_serialization, - ./test_optional_fields, +# ./test_api_usage, +# ./test_json_suite, +# ./test_empty_string, +# ./test_object_serialization, +# ./test_optional_fields, ./test_rlp_profiler diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index 9f95bef3..fb4c4beb 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -11,7 +11,7 @@ template benchmark(benchmarkName: string, code: untyped) = block: let t0 = epochTime() code - let elapsed = epochTime() - t0 + let elapsed = (epochTime() - t0) let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 9) echo "CPU Time [", benchmarkName, "] ", elapsedStr, "s" @@ -34,4 +34,4 @@ let my_tx = Transaction( suite "test running time of rlp serialization": test "transaction serialization": benchmark "Transaction": - let my_bytes = rlp.encode(my_tx) + let myBytes = rlp.encode(my_tx) From 419f98fcb778f17a7c199c78e7a3bc1b9009e2e1 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 23 Nov 2024 15:59:54 +0530 Subject: [PATCH 03/36] force expansion of memory by reducing allocation size --- eth/rlp/writer.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index dff1a7f6..ae977991 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -65,7 +65,7 @@ func writeInt(outStream: var auto, i: SomeUnsignedInt) = proc initRlpWriter*: RlpWriter = # Avoid allocations during initial write of small items - since the writer is # expected to be short-lived, it doesn't hurt to allocate this buffer - result.output = newSeqOfCap[byte](2000) + result.output = newSeqOfCap[byte](2) proc maybeClosePendingLists(self: var RlpWriter) = while self.pendingLists.len > 0: From 7981555d3db0cf14252d7fe91476f61590add32e Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 23 Nov 2024 16:01:14 +0530 Subject: [PATCH 04/36] optimize using two pass --- eth/rlp/writer.nim | 146 ++++++++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 47 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index ae977991..2f9dd42e 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -17,6 +17,10 @@ type RlpWriter* = object pendingLists: seq[tuple[remainingItems, startPos: int]] output: seq[byte] + outStreamLen: int + listPrefixBytes: seq[int] + fillLevel: int + dryRun: bool RlpIntBuf* = ArrayBuf[9, byte] ## Small buffer for holding a single RLP-encoded integer @@ -28,44 +32,60 @@ func bytesNeeded(num: SomeUnsignedInt): int = # Number of non-zero bytes in the big endian encoding sizeof(num) - (num.leadingZeros() shr 3) -func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, - lastByteIdx: int, numberOfBytes: int) = - var n = number - for i in countdown(lastByteIdx, lastByteIdx - numberOfBytes + 1): - outStream[i] = byte(n and 0xff) - n = n shr 8 - -func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, +func writeBigEndian(writer: var RlpWriter, number: SomeUnsignedInt, numberOfBytes: int) {.inline.} = - outStream.setLen(outStream.len + numberOfBytes) - outStream.writeBigEndian(number, outStream.len - 1, numberOfBytes) + if writer.dryRun: + writer.outStreamLen += numberOfBytes + else: + var n = number + for i in countdown(writer.fillLevel + numberOfBytes - 1, writer.fillLevel - 1): + writer.output[i] = byte(n and 0xff) + n = n shr 8 + + writer.fillLevel += numberOfBytes -func writeCount(bytes: var auto, count: int, baseMarker: byte) = +func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: - bytes.add(baseMarker + byte(count)) + if writer.dryRun: + writer.outStreamLen += 1 + else: + writer.output[writer.fillLevel] = (baseMarker + byte(count)) + writer.fillLevel += 1 else: - let - origLen = bytes.len - lenPrefixBytes = uint64(count).bytesNeeded + let lenPrefixBytes = uint64(count).bytesNeeded - bytes.setLen(origLen + lenPrefixBytes + 1) - bytes[origLen] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - bytes.writeBigEndian(uint64(count), bytes.len - 1, lenPrefixBytes) + if writer.dryRun: + writer.outStreamLen += lenPrefixBytes + 1 + else: + writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + writer.writeBigEndian(uint64(count), lenPrefixBytes) -func writeInt(outStream: var auto, i: SomeUnsignedInt) = +func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): - outStream.add BLOB_START_MARKER + if writer.dryRun: + writer.outStreamLen += 1 + else: + writer.output[writer.fillLevel] = BLOB_START_MARKER + writer.fillLevel += 1 elif i < typeof(i)(BLOB_START_MARKER): - outStream.add byte(i) + if writer.dryRun: + writer.outStreamLen += 1 + else: + writer.output[writer.fillLevel] = byte(i) + writer.fillLevel += 1 else: let bytesNeeded = i.bytesNeeded - outStream.writeCount(bytesNeeded, BLOB_START_MARKER) - outStream.writeBigEndian(i, bytesNeeded) + writer.writeCount(bytesNeeded, BLOB_START_MARKER) + writer.writeBigEndian(i, bytesNeeded) proc initRlpWriter*: RlpWriter = +<<<<<<< Updated upstream # Avoid allocations during initial write of small items - since the writer is # expected to be short-lived, it doesn't hurt to allocate this buffer result.output = newSeqOfCap[byte](2) +======= + result.output = newSeqOfCap[byte](2000) +>>>>>>> Stashed changes proc maybeClosePendingLists(self: var RlpWriter) = while self.pendingLists.len > 0: @@ -80,55 +100,68 @@ proc maybeClosePendingLists(self: var RlpWriter) = self.pendingLists.setLen lastListIdx # How many bytes were written since the start? - let listLen = self.output.len - listStartPos + let listLen = self.fillLevel - listStartPos # Compute the number of bytes required to write down the list length let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 - # Shift the written data to make room for the prefix length - self.output.setLen(self.output.len + totalPrefixBytes) - - moveMem(addr self.output[listStartPos + totalPrefixBytes], - unsafeAddr self.output[listStartPos], - listLen) - - # Write out the prefix length - if listLen < THRESHOLD_LIST_LEN: - self.output[listStartPos] = LIST_START_MARKER + byte(listLen) + if self.dryRun: + self.outStreamLen += totalPrefixBytes + self.listPrefixBytes.add(totalPrefixBytes) else: - let listLenBytes = totalPrefixBytes - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + # Write out the prefix length + if listLen < THRESHOLD_LIST_LEN: + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = totalPrefixBytes - 1 + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + + self.writeBigEndian(uint64(listLen), listLenBytes) + self.fillLevel -= listLenBytes else: # The currently open list is not finished yet. Nothing to do. return proc appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = - self.output.setLen(self.output.len + bytes.len) - assign(self.output.toOpenArray( - self.output.len - bytes.len, self.output.len - 1), bytes) + if self.dryRun: + self.outStreamLen += bytes.len + else: + assign(self.output.toOpenArray( + self.fillLevel, self.fillLevel + bytes.len - 1), bytes) + + # increment the fill level + self.fillLevel += bytes.len self.maybeClosePendingLists() proc startList*(self: var RlpWriter, listSize: int) = if listSize == 0: - self.output.writeCount(0, LIST_START_MARKER) + self.writeCount(0, LIST_START_MARKER) self.appendRawBytes([]) else: - self.pendingLists.add((listSize, self.output.len)) + # add a list to the stack with the starting position as the current fill level + self.pendingLists.add((listSize, self.fillLevel)) + + # if not in dry run mode shift the fill level by prefixBytes (calculated during dry run) + if not self.dryRun: + self.fillLevel += self.listPrefixBytes[self.pendingLists.len - 1] proc appendBlob(self: var RlpWriter, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - self.output.add byte(data[0]) + if self.dryRun: + self.outStreamLen += 1 + else: + self.output[self.fillLevel] = byte(data[0]) + self.fillLevel += 1 self.maybeClosePendingLists() else: - self.output.writeCount(data.len, BLOB_START_MARKER) + self.writeCount(data.len, BLOB_START_MARKER) self.appendRawBytes(data) proc appendInt(self: var RlpWriter, i: SomeUnsignedInt) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. - self.output.writeInt(i) + self.writeInt(i) self.maybeClosePendingLists() @@ -287,13 +320,32 @@ func clear*(w: var RlpWriter) = proc encode*[T](v: T): seq[byte] = mixin append - var writer = initRlpWriter() + var writer: RlpWriter + writer.dryRun = true + writer.append(v) + doAssert writer.pendingLists.len == 0, $writer.pendingLists.len & "Insufficient number of elements written to a started list" + writer.dryRun = false + writer.output = newSeqOfCap[byte](writer.outStreamLen) + writer.output.setLen(writer.outStreamLen) writer.append(v) move(writer.finish) func encodeInt*(i: SomeUnsignedInt): RlpIntBuf = var buf: RlpIntBuf - buf.writeInt(i) + + if i == typeof(i)(0): + buf.add BLOB_START_MARKER + elif i < typeof(i)(BLOB_START_MARKER): + buf.add byte(i) + else: + let bytesNeeded = i.bytesNeeded + + var n = i + buf.add(BLOB_START_MARKER + byte(bytesNeeded)) + for j in countdown(1, bytesNeeded + 1): + buf.add(byte(n and 0xff)) + n = n shr 8 + buf macro encodeList*(args: varargs[untyped]): seq[byte] = From b9636aa9dc9428b4961161e031091b6e2872ff76 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 25 Nov 2024 02:40:30 +0530 Subject: [PATCH 05/36] fix two pass --- eth/rlp/writer.nim | 114 +++++++++++++++++--------------- tests/rlp/all_tests.nim | 8 +-- tests/rlp/util/json_testing.nim | 4 +- 3 files changed, 65 insertions(+), 61 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 2f9dd42e..589dff68 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -9,15 +9,14 @@ import std/options, pkg/results, stew/[arraybuf, assign2, bitops2, shims/macros], - ./priv/defs + ./priv/defs export arraybuf type RlpWriter* = object - pendingLists: seq[tuple[remainingItems, startPos: int]] + pendingLists: seq[tuple[remainingItems, startPos, prefixBytes: int]] output: seq[byte] - outStreamLen: int listPrefixBytes: seq[int] fillLevel: int dryRun: bool @@ -34,59 +33,51 @@ func bytesNeeded(num: SomeUnsignedInt): int = func writeBigEndian(writer: var RlpWriter, number: SomeUnsignedInt, numberOfBytes: int) {.inline.} = - if writer.dryRun: - writer.outStreamLen += numberOfBytes - else: + + if not writer.dryRun: var n = number - for i in countdown(writer.fillLevel + numberOfBytes - 1, writer.fillLevel - 1): + for i in countdown(writer.fillLevel + numberOfBytes - 1, writer.fillLevel): writer.output[i] = byte(n and 0xff) n = n shr 8 - writer.fillLevel += numberOfBytes + writer.fillLevel += numberOfBytes func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: - if writer.dryRun: - writer.outStreamLen += 1 - else: + if not writer.dryRun: writer.output[writer.fillLevel] = (baseMarker + byte(count)) - writer.fillLevel += 1 + + writer.fillLevel += 1 else: let lenPrefixBytes = uint64(count).bytesNeeded - if writer.dryRun: - writer.outStreamLen += lenPrefixBytes + 1 - else: + if not writer.dryRun: writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - writer.writeBigEndian(uint64(count), lenPrefixBytes) + + writer.fillLevel += 1 + writer.writeBigEndian(uint64(count), lenPrefixBytes) func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): - if writer.dryRun: - writer.outStreamLen += 1 - else: + if not writer.dryRun: writer.output[writer.fillLevel] = BLOB_START_MARKER - writer.fillLevel += 1 + + writer.fillLevel += 1 elif i < typeof(i)(BLOB_START_MARKER): - if writer.dryRun: - writer.outStreamLen += 1 - else: + if not writer.dryRun: writer.output[writer.fillLevel] = byte(i) - writer.fillLevel += 1 + + writer.fillLevel += 1 else: let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) writer.writeBigEndian(i, bytesNeeded) proc initRlpWriter*: RlpWriter = -<<<<<<< Updated upstream # Avoid allocations during initial write of small items - since the writer is # expected to be short-lived, it doesn't hurt to allocate this buffer - result.output = newSeqOfCap[byte](2) -======= - result.output = newSeqOfCap[byte](2000) ->>>>>>> Stashed changes - + result + proc maybeClosePendingLists(self: var RlpWriter) = while self.pendingLists.len > 0: let lastListIdx = self.pendingLists.len - 1 @@ -97,41 +88,48 @@ proc maybeClosePendingLists(self: var RlpWriter) = if self.pendingLists[lastListIdx].remainingItems == 0: # A list have been just finished. It was started in `startList`. let listStartPos = self.pendingLists[lastListIdx].startPos + let prefixBytes = self.pendingLists[lastListIdx].prefixBytes + + # delete the last item from pending lists self.pendingLists.setLen lastListIdx - # How many bytes were written since the start? - let listLen = self.fillLevel - listStartPos + if self.dryRun: + # How many bytes were written since the start? + let listLen = self.fillLevel - listStartPos - # Compute the number of bytes required to write down the list length - let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 + let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 - if self.dryRun: - self.outStreamLen += totalPrefixBytes self.listPrefixBytes.add(totalPrefixBytes) + self.fillLevel += totalPrefixBytes else: + # How many bytes were written since the start? + let listLen = self.fillLevel - listStartPos - prefixBytes + # Write out the prefix length if listLen < THRESHOLD_LIST_LEN: self.output[listStartPos] = LIST_START_MARKER + byte(listLen) else: - let listLenBytes = totalPrefixBytes - 1 + let listLenBytes = prefixBytes - 1 self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + # TODO: because of the fillLevel manipulation in this step we lose + # functionality when NOT using two passes for rlp encoding. We need a + # clever way to circumvent this manipulation. + let temp = self.fillLevel + self.fillLevel = listStartPos + 1 self.writeBigEndian(uint64(listLen), listLenBytes) - self.fillLevel -= listLenBytes + self.fillLevel = temp else: # The currently open list is not finished yet. Nothing to do. return proc appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = - if self.dryRun: - self.outStreamLen += bytes.len - else: + if not self.dryRun: assign(self.output.toOpenArray( self.fillLevel, self.fillLevel + bytes.len - 1), bytes) - # increment the fill level - self.fillLevel += bytes.len + self.fillLevel += bytes.len self.maybeClosePendingLists() proc startList*(self: var RlpWriter, listSize: int) = @@ -139,20 +137,20 @@ proc startList*(self: var RlpWriter, listSize: int) = self.writeCount(0, LIST_START_MARKER) self.appendRawBytes([]) else: - # add a list to the stack with the starting position as the current fill level - self.pendingLists.add((listSize, self.fillLevel)) - # if not in dry run mode shift the fill level by prefixBytes (calculated during dry run) if not self.dryRun: - self.fillLevel += self.listPrefixBytes[self.pendingLists.len - 1] + let prefixBytes = self.listPrefixBytes.pop() + self.pendingLists.add((listSize, self.fillLevel, prefixBytes)) + self.fillLevel += prefixBytes + else: + self.pendingLists.add((listSize, self.fillLevel, 0)) proc appendBlob(self: var RlpWriter, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - if self.dryRun: - self.outStreamLen += 1 - else: + if not self.dryRun: self.output[self.fillLevel] = byte(data[0]) - self.fillLevel += 1 + + self.fillLevel += 1 self.maybeClosePendingLists() else: self.writeCount(data.len, BLOB_START_MARKER) @@ -310,24 +308,32 @@ proc initRlpList*(listSize: int): RlpWriter = # TODO: This should return a lent value template finish*(self: RlpWriter): seq[byte] = doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + doAssert self.listPrefixBytes.len == 0, "Insufficient number of list prefixes accounted for" self.output func clear*(w: var RlpWriter) = # Prepare writer for reuse w.pendingLists.setLen(0) + w.listPrefixBytes.setLen(0) w.output.setLen(0) proc encode*[T](v: T): seq[byte] = mixin append var writer: RlpWriter + # first pass writer.dryRun = true + writer.append(v) - doAssert writer.pendingLists.len == 0, $writer.pendingLists.len & "Insufficient number of elements written to a started list" + + # second pass writer.dryRun = false - writer.output = newSeqOfCap[byte](writer.outStreamLen) - writer.output.setLen(writer.outStreamLen) + writer.output = newSeqOfCap[byte](writer.fillLevel) + writer.output.setLen(writer.fillLevel) + writer.fillLevel = 0 + writer.append(v) + move(writer.finish) func encodeInt*(i: SomeUnsignedInt): RlpIntBuf = diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index aabc8287..bcd1f7af 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import # ./test_api_usage, -# ./test_json_suite, -# ./test_empty_string, -# ./test_object_serialization, -# ./test_optional_fields, + ./test_json_suite, + ./test_empty_string, + ./test_object_serialization, + ./test_optional_fields, ./test_rlp_profiler diff --git a/tests/rlp/util/json_testing.nim b/tests/rlp/util/json_testing.nim index ad4592da..d9307bed 100644 --- a/tests/rlp/util/json_testing.nim +++ b/tests/rlp/util/json_testing.nim @@ -55,10 +55,8 @@ proc runTests*(filename: string) = skip() return - var outRlp = initRlpWriter() - outRlp.append input let - actual = outRlp.finish.toHex + actual = rlp.encode(input).toHex expected = output.str check actual == expected From 058dcca8a57f05e4bd501da1a57f9632b3b5717c Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sun, 1 Dec 2024 15:11:50 +0530 Subject: [PATCH 06/36] add a chunked buffer implementation --- eth/rlp/chunked_buffer.nim | 86 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 eth/rlp/chunked_buffer.nim diff --git a/eth/rlp/chunked_buffer.nim b/eth/rlp/chunked_buffer.nim new file mode 100644 index 00000000..019d9529 --- /dev/null +++ b/eth/rlp/chunked_buffer.nim @@ -0,0 +1,86 @@ +import + stew/[arraybuf, assign2, bitops2, shims/macros] + +type + # a 200-byte chunk matches the block length of keccak(1600bits)v + # TODO: use some compile time technique to abstract out the size(200) + BufferChunk = array[200, byte] + RefBufferChunk = ref BufferChunk + + ChunkedBuffer* = object + fillLevel: int + chunks: seq[RefBufferChunk] + +func `$`(chunk: RefBufferChunk): string = + $(chunk[]) + +func initChunkedBuffer(): ChunkedBuffer = + let newChunk = new(RefBufferChunk) + result.chunks.add(newChunk) + result.fillLevel = 0 + +func curChunkIdx(buffer: ChunkedBuffer): int = + return (buffer.fillLevel mod len(BufferChunk)) + +func isCurChunkFull(buffer: ChunkedBuffer): bool = + (len(buffer.chunks) * len(BufferChunk)) == buffer.fillLevel + +func curChunkRemaining(buffer: ChunkedBuffer): int = + return len(BufferChunk) - (buffer.fillLevel mod len(BufferChunk)) + +func append*(buffer: var ChunkedBuffer, data: openArray[byte]) = + var remainingBytes = len(data) + + # debugEcho buffer.fillLevel, (len(buffer.chunks) * len(BufferChunk)) + + if buffer.isCurChunkFull: + let newChunk = new(RefBufferChunk) + buffer.chunks.add(newChunk) + + while remainingBytes > 0: + let startIdx = len(data) - remainingBytes + let numBytes = if buffer.curChunkRemaining < remainingBytes: buffer.curChunkRemaining else: remainingBytes + let endIdx = startIdx + numBytes - 1 + let chunkIdx = buffer.fillLevel div len(BufferChunk) + + # debugEcho startIdx, " ", endIdx, " ", chunkIdx, " ", buffer.curChunkRemaining + + assign( + buffer.chunks[chunkIdx][].toOpenArray(buffer.curChunkIdx, buffer.curChunkIdx + numBytes - 1), + data[startIdx..endIdx] + ) + + buffer.fillLevel += numBytes + remainingBytes -= numBytes + + if remainingBytes > 0: + let newChunk = new(RefBufferChunk) + buffer.chunks.add(newChunk) + + +# TODO: idx shouldn't only be type int. Technically it should be able to accomodate SomeOrdinal +func `[]`*(buffer: ChunkedBuffer, idx: int): byte = + buffer.chunks[idx div len(BufferChunk)][idx mod len(BufferChunk)] + +# TODO: idx shouldn't only be type int. Technically it should be able to accomodate SomeOrdinal +func `[]=`*(buffer: var ChunkedBuffer, idx: int, value: byte) = + buffer.chunks[idx div len(BufferChunk)][idx mod len(BufferChunk)] = value + +func append*(buffer: var ChunkedBuffer, data: byte) = + if buffer.isCurChunkFull: + let newChunk = new(RefBufferChunk) + buffer.chunks.add(newChunk) + + buffer[buffer.fillLevel] = data + buffer.fillLevel += 1 + +func consume*(buffer: var ChunkedBuffer): BufferChunk = + let chunk = buffer.chunks[0][] # this will copy the chunk + buffer.chunks.delete(0) + buffer.fillLevel -= len(BufferChunk) + chunk + +func addGapChunk*(buffer: var ChunkedBuffer) = + let newChunk = new(RefBufferChunk) + buffer.chunks.add(newChunk) + buffer.fillLevel = len(buffer.chunks) * len(BufferChunk) From d22d9d5afbc84b9914f034883954504d8f05efbd Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sun, 1 Dec 2024 15:11:50 +0530 Subject: [PATCH 07/36] add a chunked buffer implementation --- eth/rlp/chunked_buffer.nim | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eth/rlp/chunked_buffer.nim b/eth/rlp/chunked_buffer.nim index 019d9529..9d1055db 100644 --- a/eth/rlp/chunked_buffer.nim +++ b/eth/rlp/chunked_buffer.nim @@ -84,3 +84,10 @@ func addGapChunk*(buffer: var ChunkedBuffer) = let newChunk = new(RefBufferChunk) buffer.chunks.add(newChunk) buffer.fillLevel = len(buffer.chunks) * len(BufferChunk) + +func removeChunk*(buffer: var ChunkedBuffer, chunkIdx: int) = + buffer.fillLevel -= len(BufferChunk) + buffer.chunks.delete(chunkIdx) + +func completeCurChunk*(buffer: var ChunkedBuffer) = + buffer.fillLevel = len(buffer.chunks) * len(BufferChunk) From 51a5bb9d4dd5ec866b96eb5b926eb62466f7b528 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 5 Dec 2024 19:00:56 +0530 Subject: [PATCH 08/36] multiple writers --- eth/rlp/writer.nim | 171 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 130 insertions(+), 41 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 589dff68..c1087dd3 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -14,13 +14,19 @@ import export arraybuf type - RlpWriter* = object + RlpDefaultWriter* = object + pendingLists: seq[tuple[remainingItems, startPos: int]] + output: seq[byte] + + RlpTwoPassWriter* = object pendingLists: seq[tuple[remainingItems, startPos, prefixBytes: int]] output: seq[byte] listPrefixBytes: seq[int] fillLevel: int dryRun: bool + RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter + RlpIntBuf* = ArrayBuf[9, byte] ## Small buffer for holding a single RLP-encoded integer @@ -31,18 +37,14 @@ func bytesNeeded(num: SomeUnsignedInt): int = # Number of non-zero bytes in the big endian encoding sizeof(num) - (num.leadingZeros() shr 3) -func writeBigEndian(writer: var RlpWriter, number: SomeUnsignedInt, - numberOfBytes: int) {.inline.} = - - if not writer.dryRun: - var n = number - for i in countdown(writer.fillLevel + numberOfBytes - 1, writer.fillLevel): - writer.output[i] = byte(n and 0xff) - n = n shr 8 +func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, + lastByteIdx: int, numberOfBytes: int) = + var n = number + for i in countdown(lastByteIdx, lastByteIdx - numberOfBytes + 1): + outStream[i] = byte(n and 0xff) + n = n shr 8 - writer.fillLevel += numberOfBytes - -func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = +proc writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: if not writer.dryRun: writer.output[writer.fillLevel] = (baseMarker + byte(count)) @@ -55,9 +57,13 @@ func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) writer.fillLevel += 1 - writer.writeBigEndian(uint64(count), lenPrefixBytes) -func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = + if not writer.dryRun: + writer.output.writeBigEndian(uint64(count), writer.fillLevel + lenPrefixBytes - 1, lenPrefixBytes) + + writer.fillLevel += lenPrefixBytes + +proc writeInt(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): if not writer.dryRun: writer.output[writer.fillLevel] = BLOB_START_MARKER @@ -71,14 +77,41 @@ func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = else: let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) - writer.writeBigEndian(i, bytesNeeded) + + if not writer.dryRun: + writer.output.writeBigEndian(i, writer.fillLevel + bytesNeeded - 1, bytesNeeded) + + writer.fillLevel += bytesNeeded + +func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = + if count < THRESHOLD_LIST_LEN: + writer.output.add(baseMarker + byte(count)) + else: + let lenPrefixBytes = uint64(count).bytesNeeded + + writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + + writer.output.setLen(writer.output.len + lenPrefixBytes) + writer.output.writeBigEndian(uint64(count), writer.output.len - 1,lenPrefixBytes) + +func writeInt(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = + if i == typeof(i)(0): + writer.output.add BLOB_START_MARKER + elif i < typeof(i)(BLOB_START_MARKER): + writer.output.add byte(i) + else: + let bytesNeeded = i.bytesNeeded + writer.writeCount(bytesNeeded, BLOB_START_MARKER) + + writer.output.setLen(writer.output.len + bytesNeeded) + writer.output.writeBigEndian(i, writer.output.len - 1, bytesNeeded) -proc initRlpWriter*: RlpWriter = +proc initRlpWriter*: RlpDefaultWriter = # Avoid allocations during initial write of small items - since the writer is # expected to be short-lived, it doesn't hurt to allocate this buffer result - -proc maybeClosePendingLists(self: var RlpWriter) = + +proc maybeClosePendingLists(self: var RlpTwoPassWriter) = while self.pendingLists.len > 0: let lastListIdx = self.pendingLists.len - 1 doAssert self.pendingLists[lastListIdx].remainingItems > 0 @@ -90,7 +123,6 @@ proc maybeClosePendingLists(self: var RlpWriter) = let listStartPos = self.pendingLists[lastListIdx].startPos let prefixBytes = self.pendingLists[lastListIdx].prefixBytes - # delete the last item from pending lists self.pendingLists.setLen lastListIdx if self.dryRun: @@ -98,7 +130,7 @@ proc maybeClosePendingLists(self: var RlpWriter) = let listLen = self.fillLevel - listStartPos let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 + else: int(uint64(listLen).bytesNeeded) + 1 self.listPrefixBytes.add(totalPrefixBytes) self.fillLevel += totalPrefixBytes @@ -112,27 +144,58 @@ proc maybeClosePendingLists(self: var RlpWriter) = else: let listLenBytes = prefixBytes - 1 self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + else: + # The currently open list is not finished yet. Nothing to do. + return + + +proc maybeClosePendingLists(self: var RlpDefaultWriter) = + while self.pendingLists.len > 0: + let lastListIdx = self.pendingLists.len - 1 + doAssert self.pendingLists[lastListIdx].remainingItems > 0 - # TODO: because of the fillLevel manipulation in this step we lose - # functionality when NOT using two passes for rlp encoding. We need a - # clever way to circumvent this manipulation. - let temp = self.fillLevel - self.fillLevel = listStartPos + 1 - self.writeBigEndian(uint64(listLen), listLenBytes) - self.fillLevel = temp + self.pendingLists[lastListIdx].remainingItems -= 1 + # if one last item is remaining in the list + if self.pendingLists[lastListIdx].remainingItems == 0: + # A list have been just finished. It was started in `startList`. + let listStartPos = self.pendingLists[lastListIdx].startPos + + self.pendingLists.setLen lastListIdx + + let listLen = self.output.len - listStartPos + + let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + #Shift the written data to make room for the prefix length + self.output.setLen(self.output.len + totalPrefixBytes) + + moveMem(addr self.output[listStartPos + totalPrefixBytes], + unsafeAddr self.output[listStartPos], + listLen) + + # Write out the prefix length + if listLen < THRESHOLD_LIST_LEN: + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = totalPrefixBytes - 1 + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) else: # The currently open list is not finished yet. Nothing to do. return -proc appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = +proc appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = if not self.dryRun: assign(self.output.toOpenArray( self.fillLevel, self.fillLevel + bytes.len - 1), bytes) self.fillLevel += bytes.len + self.maybeClosePendingLists() -proc startList*(self: var RlpWriter, listSize: int) = +proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) self.appendRawBytes([]) @@ -145,7 +208,7 @@ proc startList*(self: var RlpWriter, listSize: int) = else: self.pendingLists.add((listSize, self.fillLevel, 0)) -proc appendBlob(self: var RlpWriter, data: openArray[byte]) = +proc appendBlob(self: var RlpTwoPassWriter, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: if not self.dryRun: self.output[self.fillLevel] = byte(data[0]) @@ -156,6 +219,28 @@ proc appendBlob(self: var RlpWriter, data: openArray[byte]) = self.writeCount(data.len, BLOB_START_MARKER) self.appendRawBytes(data) +proc appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = + self.output.setLen(self.output.len + bytes.len) + assign(self.output.toOpenArray( + self.output.len - bytes.len, self.output.len - 1), bytes) + + self.maybeClosePendingLists() + +proc startList*(self: var RlpDefaultWriter, listSize: int) = + if listSize == 0: + self.writeCount(0, LIST_START_MARKER) + self.appendRawBytes([]) + else: + self.pendingLists.add((listSize, self.output.len)) + +proc appendBlob(self: var RlpDefaultWriter, data: openArray[byte]) = + if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: + self.output.add byte(data[0]) + self.maybeClosePendingLists() + else: + self.writeCount(data.len, BLOB_START_MARKER) + self.appendRawBytes(data) + proc appendInt(self: var RlpWriter, i: SomeUnsignedInt) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. @@ -301,12 +386,15 @@ template append*[T](w: var RlpWriter; data: T) = template append*(w: var RlpWriter; data: SomeSignedInt) = {.error: "Signed integer encoding is not defined for rlp".} -proc initRlpList*(listSize: int): RlpWriter = +proc initRlpList*(listSize: int): RlpDefaultWriter = result = initRlpWriter() startList(result, listSize) -# TODO: This should return a lent value -template finish*(self: RlpWriter): seq[byte] = +template finish*(self: RlpDefaultWriter): seq[byte] = + doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + self.output + +template finish*(self: RlpTwoPassWriter): seq[byte] = doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" doAssert self.listPrefixBytes.len == 0, "Insufficient number of list prefixes accounted for" self.output @@ -314,13 +402,16 @@ template finish*(self: RlpWriter): seq[byte] = func clear*(w: var RlpWriter) = # Prepare writer for reuse w.pendingLists.setLen(0) - w.listPrefixBytes.setLen(0) - w.output.setLen(0) + when typeof(w) is RlpDefaultWriter: + w.output.setLen(0) + elif typeof(w) is RlpTwoPassWriter: + w.output.setLen(0) + w.listPrefixBytes.setLen(0) proc encode*[T](v: T): seq[byte] = mixin append - var writer: RlpWriter + var writer: RlpTwoPassWriter # first pass writer.dryRun = true @@ -344,13 +435,11 @@ func encodeInt*(i: SomeUnsignedInt): RlpIntBuf = elif i < typeof(i)(BLOB_START_MARKER): buf.add byte(i) else: - let bytesNeeded = i.bytesNeeded + let bytesNeeded = uint64(i).bytesNeeded - var n = i buf.add(BLOB_START_MARKER + byte(bytesNeeded)) - for j in countdown(1, bytesNeeded + 1): - buf.add(byte(n and 0xff)) - n = n shr 8 + buf.setLen(buf.len + bytesNeeded) + buf.writeBigEndian(i, buf.len - 1, bytesNeeded) buf From 028fdb1ad30f7d85cb504fdfe037d1d4366e7bd9 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 7 Dec 2024 01:09:59 +0530 Subject: [PATCH 09/36] add api tests --- tests/rlp/all_tests.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index bcd1f7af..cd50c8b6 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,5 +1,5 @@ import -# ./test_api_usage, + ./test_api_usage, ./test_json_suite, ./test_empty_string, ./test_object_serialization, From 27e3d945053e1e08edfe9c5611cc409a5667dc22 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 9 Dec 2024 01:57:44 +0530 Subject: [PATCH 10/36] fixes --- eth/rlp/writer.nim | 2 + tests/rlp/all_tests.nim | 12 +- tests/rlp/test_api_usage.nim | 230 +---------------------------------- 3 files changed, 13 insertions(+), 231 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index c1087dd3..98b617d0 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -138,6 +138,8 @@ proc maybeClosePendingLists(self: var RlpTwoPassWriter) = # How many bytes were written since the start? let listLen = self.fillLevel - listStartPos - prefixBytes + debugEcho self.output + # Write out the prefix length if listLen < THRESHOLD_LIST_LEN: self.output[listStartPos] = LIST_START_MARKER + byte(listLen) diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index cd50c8b6..2a0b52bc 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import - ./test_api_usage, - ./test_json_suite, - ./test_empty_string, - ./test_object_serialization, - ./test_optional_fields, - ./test_rlp_profiler + ./test_api_usage +# ./test_json_suite, +# ./test_empty_string, +# ./test_object_serialization, +# ./test_optional_fields, +# ./test_rlp_profiler diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index cf9ad2f8..c0784acc 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -28,236 +28,16 @@ proc test_blockBodyTranscode() = Transaction(nonce: 3), Transaction(nonce: 4)])] - let trBlkSeq = blkSeq.encode.decode(typeof blkSeq) + let temp = blkSeq.encode + + debugEcho temp + + let trBlkSeq = temp.decode(typeof blkSeq) check trBlkSeq.len == blkSeq.len for n in 0 ..< min(trBlkSeq.len, trBlkSeq.len): check (n, trBlkSeq[n]) == (n, blkSeq[n]) suite "test api usage": - test "empty bytes are not a proper RLP": - var rlp = rlpFromBytes seq[byte](@[]) - - check: - not rlp.hasData - not rlp.isBlob - not rlp.isList - not rlp.isEmpty - - expect AssertionDefect: - rlp.skipElem - - test "you cannot finish a list without appending enough elements": - var writer = initRlpList(3) - writer.append "foo" - writer.append "bar" - - expect AssertionDefect: - discard writer.finish - - test "encode/decode object": - type - MyEnum = enum - foo, - bar - - MyObj = object - a: array[3, char] - b: uint64 - c: MyEnum - - IntObj = object - v: int - - var input: MyObj - input.a = ['e', 't', 'h'] - input.b = 63 - input.c = bar - - - var writer = initRlpWriter() - writer.append(input) - - check: - not compiles(writer.append(default(IntObj))) - - let bytes = writer.finish() - var rlp = rlpFromBytes(bytes) - - var output = rlp.read(MyObj) - check: - input == output - - test "encode and decode lists": - var writer = initRlpList(3) - writer.append "foo" - writer.append ["bar", "baz"] - writer.append [uint64 30, 40, 50] - - var - bytes = writer.finish - rlp = rlpFromBytes bytes - - check: - bytes.toHex == "d183666f6fc8836261728362617ac31e2832" - rlp.inspectMatch """ - { - "foo" - { - "bar" - "baz" - } - { - byte 30 - byte 40 - byte 50 - } - } - """ - - bytes = encodeList(uint64 6000, - "Lorem ipsum dolor sit amet", - "Donec ligula tortor, egestas eu est vitae") - - rlp = rlpFromBytes bytes - check: - rlp.listLen == 3 - rlp.listElem(0).toInt(uint64) == 6000 - rlp.listElem(1).toString == "Lorem ipsum dolor sit amet" - rlp.listElem(2).toString == "Donec ligula tortor, egestas eu est vitae" - - # test creating RLPs from other RLPs - var list = rlpFromBytes encodeList(rlp.listElem(1), rlp.listElem(0)) - - # test that iteration with enterList/skipElem works as expected - doAssert list.enterList # We already know that we are working with a list - check list.toString == "Lorem ipsum dolor sit amet" - list.skipElem - - check list.toInt(uint32) == 6000.uint32 - var intVar: uint32 - list >> intVar - check intVar == 6000 - - check(not list.hasData) - expect AssertionDefect: list.skipElem - test "encode and decode block body": test_blockBodyTranscode() - - test "toBytes": - let rlp = rlpFromHex("f2cb847f000001827666827666a040ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4845ab90b50") - let tok = rlp.listElem(1).toBytes() - check: - tok.len == 32 - tok.toHex == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4" - - test "nested lists": - let listBytes = encode([[uint64 1, 2, 3], [uint64 5, 6, 7]]) - let listRlp = rlpFromBytes listBytes - let sublistRlp0 = listRlp.listElem(0) - let sublistRlp1 = listRlp.listElem(1) - check sublistRlp0.listElem(0).toInt(uint64) == 1 - check sublistRlp0.listElem(1).toInt(uint64) == 2 - check sublistRlp0.listElem(2).toInt(uint64) == 3 - check sublistRlp1.listElem(0).toInt(uint64) == 5 - check sublistRlp1.listElem(1).toInt(uint64) == 6 - check sublistRlp1.listElem(2).toInt(uint64) == 7 - - test "encoding length": - let listBytes = encode([uint64 1,2,3,4,5]) - let listRlp = rlpFromBytes listBytes - check listRlp.listLen == 5 - - let emptyListBytes = encode "" - check emptyListBytes.len == 1 - let emptyListRlp = rlpFromBytes emptyListBytes - check emptyListRlp.blobLen == 0 - - test "basic decoding": - var rlp1 = rlpFromHex("856d6f6f7365") - var rlp2 = rlpFromHex("0x856d6f6f7365") - - check: - rlp1.inspect == q"moose" - rlp2.inspect == q"moose" - - test "malformed/truncated RLP": - var rlp = rlpFromHex("b8056d6f6f7365") - expect MalformedRlpError: - discard rlp.inspect - - test "encode byte arrays": - var b1 = [byte(1), 2, 5, 7, 8] - var b2 = [byte(6), 8, 12, 123] - var b3 = @[byte(122), 56, 65, 12] - - let rlp = rlpFromBytes(encode((b1, b2, b3))) - check: - rlp.listLen == 3 - rlp.listElem(0).toBytes() == b1 - rlp.listElem(1).toBytes() == b2 - rlp.listElem(2).toBytes() == b3 - - # The first byte here is the length of the datum (132 - 128 => 4) - $(rlp.listElem(1).rawData) == "[132, 6, 8, 12, 123]" - - test "empty byte arrays": - var - rlp = rlpFromBytes rlp.encode("") - b = rlp.toBytes - check $b == "@[]" - - test "invalid enum": - type - MyEnum = enum - foo = 0x00, - bar = 0x01 - - var writer = initRlpWriter() - writer.append(byte 1) # valid - writer.append(byte 2) # invalid - writer.append(cast[uint64](-1)) # invalid - let bytes = writer.finish() - var rlp = rlpFromBytes(bytes) - - check rlp.read(MyEnum) == bar - - expect RlpTypeMismatch: - discard rlp.read(MyEnum) - rlp.skipElem() - - expect RlpTypeMismatch: - discard rlp.read(MyEnum) - - test "invalid enum - enum with hole": - type - MyEnum = enum - foo = 0x00, - bar = 0x01, - baz = 0x0100 - - var writer = initRlpWriter() - writer.append(1'u64) # valid - writer.append(2'u64) # invalid - enum hole value - writer.append(256'u64) # valid - writer.append(257'u64) # invalid - too large - let bytes = writer.finish() - var rlp = rlpFromBytes(bytes) - - check rlp.read(MyEnum) == bar - - expect RlpTypeMismatch: - discard rlp.read(MyEnum) - rlp.skipElem() - - check rlp.read(MyEnum) == baz - - expect RlpTypeMismatch: - discard rlp.read(MyEnum) - rlp.skipElem() - - test "encodeInt basics": - for i in [uint64 0, 1, 10, 100, 1000, uint64.high]: - check: - encode(i) == encodeInt(i).data() From 98be45e8e6027a8bc897bb7cb80412ccc29a2581 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 10 Dec 2024 10:33:08 +0530 Subject: [PATCH 11/36] fix double stack update problem --- eth/rlp/writer.nim | 26 ++++++++++++--------- tests/rlp/all_tests.nim | 12 +++++----- tests/rlp/test_api_usage.nim | 6 +---- tests/rlp/test_rlp_profiler.nim | 40 ++++++++++++++++++++++++++++----- 4 files changed, 56 insertions(+), 28 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 98b617d0..0fa151b7 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -19,8 +19,9 @@ type output: seq[byte] RlpTwoPassWriter* = object - pendingLists: seq[tuple[remainingItems, startPos, prefixBytes: int]] + pendingLists: seq[tuple[idx, remainingItems, startPos, prefixBytes: int]] output: seq[byte] + listCount: int listPrefixBytes: seq[int] fillLevel: int dryRun: bool @@ -96,9 +97,9 @@ func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = func writeInt(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): - writer.output.add BLOB_START_MARKER + writer.output.add BLOB_START_MARKER elif i < typeof(i)(BLOB_START_MARKER): - writer.output.add byte(i) + writer.output.add byte(i) else: let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) @@ -121,25 +122,25 @@ proc maybeClosePendingLists(self: var RlpTwoPassWriter) = if self.pendingLists[lastListIdx].remainingItems == 0: # A list have been just finished. It was started in `startList`. let listStartPos = self.pendingLists[lastListIdx].startPos + let listIdx = self.pendingLists[lastListIdx].idx let prefixBytes = self.pendingLists[lastListIdx].prefixBytes + # remove the pending list tuple self.pendingLists.setLen lastListIdx if self.dryRun: - # How many bytes were written since the start? + # How many bytes were written since the start? let listLen = self.fillLevel - listStartPos let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 - self.listPrefixBytes.add(totalPrefixBytes) + self.listPrefixBytes[listIdx] = totalPrefixBytes self.fillLevel += totalPrefixBytes else: # How many bytes were written since the start? let listLen = self.fillLevel - listStartPos - prefixBytes - debugEcho self.output - # Write out the prefix length if listLen < THRESHOLD_LIST_LEN: self.output[listStartPos] = LIST_START_MARKER + byte(listLen) @@ -204,11 +205,14 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = else: # if not in dry run mode shift the fill level by prefixBytes (calculated during dry run) if not self.dryRun: - let prefixBytes = self.listPrefixBytes.pop() - self.pendingLists.add((listSize, self.fillLevel, prefixBytes)) + let prefixBytes = self.listPrefixBytes[0] + self.listPrefixBytes.delete(0) + self.pendingLists.add((self.listCount, listSize, self.fillLevel, prefixBytes)) self.fillLevel += prefixBytes else: - self.pendingLists.add((listSize, self.fillLevel, 0)) + self.pendingLists.add((self.listCount, listSize, self.fillLevel, 0)) + self.listPrefixBytes.add(0) + self.listCount += 1 proc appendBlob(self: var RlpTwoPassWriter, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: @@ -416,11 +420,11 @@ proc encode*[T](v: T): seq[byte] = var writer: RlpTwoPassWriter # first pass writer.dryRun = true - writer.append(v) # second pass writer.dryRun = false + writer.listCount = 0 writer.output = newSeqOfCap[byte](writer.fillLevel) writer.output.setLen(writer.fillLevel) writer.fillLevel = 0 diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index 2a0b52bc..cd50c8b6 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import - ./test_api_usage -# ./test_json_suite, -# ./test_empty_string, -# ./test_object_serialization, -# ./test_optional_fields, -# ./test_rlp_profiler + ./test_api_usage, + ./test_json_suite, + ./test_empty_string, + ./test_object_serialization, + ./test_optional_fields, + ./test_rlp_profiler diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index c0784acc..7d6956ca 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -28,11 +28,7 @@ proc test_blockBodyTranscode() = Transaction(nonce: 3), Transaction(nonce: 4)])] - let temp = blkSeq.encode - - debugEcho temp - - let trBlkSeq = temp.decode(typeof blkSeq) + let trBlkSeq = blkSeq.encode.decode(typeof blkSeq) check trBlkSeq.len == blkSeq.len for n in 0 ..< min(trBlkSeq.len, trBlkSeq.len): diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index fb4c4beb..c8dca027 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -9,9 +9,12 @@ import template benchmark(benchmarkName: string, code: untyped) = block: - let t0 = epochTime() - code - let elapsed = (epochTime() - t0) + var sum = 0.0 + for i in countup(1,100): + let t0 = epochTime() + code + sum += (epochTime() - t0) + let elapsed = sum/100 let elapsedStr = elapsed.formatFloat(format = ffDecimal, precision = 9) echo "CPU Time [", benchmarkName, "] ", elapsedStr, "s" @@ -21,7 +24,7 @@ const storageKeys: @[default(Bytes32)] )] -let my_tx = Transaction( +let myTx = Transaction( txType: TxEip1559, chainId: 1.ChainId, nonce: 0.AccountNonce, @@ -31,7 +34,32 @@ let my_tx = Transaction( accessList: accesses ) +let blkSeq = @[ + BlockBody( + transactions: @[ + Transaction(nonce: 1)]), + BlockBody( + uncles: @[Header(nonce: Bytes8([0x20u8,0,0,0,0,0,0,0]))]), + BlockBody(), + BlockBody( + transactions: @[ + Transaction(nonce: 3), + Transaction(nonce: 4)])] + +proc encodeOnePass[T](v: T): seq[byte] = + var writer = initRlpWriter() + + writer.append(v) + move(writer.finish) + + suite "test running time of rlp serialization": test "transaction serialization": - benchmark "Transaction": - let myBytes = rlp.encode(my_tx) + benchmark "Transaction serialization (two pass)": + let myTxBytes = rlp.encode(myTx) + benchmark "Block Sequence serialization (two pass)": + let myBlockBytes = rlp.encode(blkSeq) + benchmark "Transaction serialization (one pass)": + let myTxBytesOnePass = encodeOnePass(myTx) + benchmark "Block Sequence serailization (one pass)": + let myBlockBytesOnePass = encodeOnePass(blkSeq) From 1b3f494b57285055d166dc31aaa1fd5d4c465b5d Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 11:19:51 +0530 Subject: [PATCH 12/36] using a length tracker --- eth/rlp/writer.nim | 233 ++++++++++++++++++++++++++------------------- 1 file changed, 134 insertions(+), 99 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 0fa151b7..d4c9ec50 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -9,7 +9,10 @@ import std/options, pkg/results, stew/[arraybuf, assign2, bitops2, shims/macros], - ./priv/defs + ./priv/defs, + times, + strutils + export arraybuf @@ -19,14 +22,18 @@ type output: seq[byte] RlpTwoPassWriter* = object - pendingLists: seq[tuple[idx, remainingItems, startPos, prefixBytes: int]] + pendingLists: seq[tuple[remainingItems, startPos, prefixLen: int]] output: seq[byte] - listCount: int listPrefixBytes: seq[int] fillLevel: int - dryRun: bool - RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter + RlpLengthTracker* = object + pendingLists: seq[tuple[idx, remainingItems, length: int]] + listCount: int + listPrefixBytes: seq[int] + totalLength: int + + RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter | RlpLengthTracker RlpIntBuf* = ArrayBuf[9, byte] ## Small buffer for holding a single RLP-encoded integer @@ -45,43 +52,30 @@ func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, outStream[i] = byte(n and 0xff) n = n shr 8 -proc writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = +func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: - if not writer.dryRun: - writer.output[writer.fillLevel] = (baseMarker + byte(count)) - + writer.output[writer.fillLevel] = (baseMarker + byte(count)) writer.fillLevel += 1 else: let lenPrefixBytes = uint64(count).bytesNeeded - if not writer.dryRun: - writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - + writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) writer.fillLevel += 1 - - if not writer.dryRun: - writer.output.writeBigEndian(uint64(count), writer.fillLevel + lenPrefixBytes - 1, lenPrefixBytes) - + writer.output.writeBigEndian(uint64(count), writer.fillLevel + lenPrefixBytes - 1, lenPrefixBytes) writer.fillLevel += lenPrefixBytes -proc writeInt(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = +func writeInt(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): - if not writer.dryRun: - writer.output[writer.fillLevel] = BLOB_START_MARKER - + writer.output[writer.fillLevel] = BLOB_START_MARKER writer.fillLevel += 1 elif i < typeof(i)(BLOB_START_MARKER): - if not writer.dryRun: - writer.output[writer.fillLevel] = byte(i) - + writer.output[writer.fillLevel] = byte(i) writer.fillLevel += 1 else: let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) - - if not writer.dryRun: - writer.output.writeBigEndian(i, writer.fillLevel + bytesNeeded - 1, bytesNeeded) + writer.output.writeBigEndian(i, writer.fillLevel + bytesNeeded - 1, bytesNeeded) writer.fillLevel += bytesNeeded func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = @@ -93,7 +87,7 @@ func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) writer.output.setLen(writer.output.len + lenPrefixBytes) - writer.output.writeBigEndian(uint64(count), writer.output.len - 1,lenPrefixBytes) + writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) func writeInt(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): @@ -112,6 +106,7 @@ proc initRlpWriter*: RlpDefaultWriter = # expected to be short-lived, it doesn't hurt to allocate this buffer result +# nothing to do when serializing using tracker proc maybeClosePendingLists(self: var RlpTwoPassWriter) = while self.pendingLists.len > 0: let lastListIdx = self.pendingLists.len - 1 @@ -122,32 +117,19 @@ proc maybeClosePendingLists(self: var RlpTwoPassWriter) = if self.pendingLists[lastListIdx].remainingItems == 0: # A list have been just finished. It was started in `startList`. let listStartPos = self.pendingLists[lastListIdx].startPos - let listIdx = self.pendingLists[lastListIdx].idx - let prefixBytes = self.pendingLists[lastListIdx].prefixBytes + let prefixLen = self.pendingLists[lastListIdx].prefixLen - # remove the pending list tuple self.pendingLists.setLen lastListIdx - if self.dryRun: - # How many bytes were written since the start? - let listLen = self.fillLevel - listStartPos + let listLen = self.fillLevel - listStartPos - prefixLen - let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 - - self.listPrefixBytes[listIdx] = totalPrefixBytes - self.fillLevel += totalPrefixBytes + # Write out the prefix length + if listLen < THRESHOLD_LIST_LEN: + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) else: - # How many bytes were written since the start? - let listLen = self.fillLevel - listStartPos - prefixBytes - - # Write out the prefix length - if listLen < THRESHOLD_LIST_LEN: - self.output[listStartPos] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = prefixBytes - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + let listLenBytes = prefixLen - 1 + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) else: # The currently open list is not finished yet. Nothing to do. return @@ -189,69 +171,127 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = # The currently open list is not finished yet. Nothing to do. return -proc appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = - if not self.dryRun: - assign(self.output.toOpenArray( - self.fillLevel, self.fillLevel + bytes.len - 1), bytes) +template appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = + assign(self.output.toOpenArray(self.fillLevel, self.fillLevel + bytes.len - 1), bytes) + self.fillLevel += bytes.len - self.fillLevel += bytes.len - - self.maybeClosePendingLists() +template appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = + self.output.setLen(self.output.len + bytes.len) + assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) - self.appendRawBytes([]) + self.maybeClosePendingLists() else: - # if not in dry run mode shift the fill level by prefixBytes (calculated during dry run) - if not self.dryRun: - let prefixBytes = self.listPrefixBytes[0] - self.listPrefixBytes.delete(0) - self.pendingLists.add((self.listCount, listSize, self.fillLevel, prefixBytes)) - self.fillLevel += prefixBytes - else: - self.pendingLists.add((self.listCount, listSize, self.fillLevel, 0)) - self.listPrefixBytes.add(0) - self.listCount += 1 - -proc appendBlob(self: var RlpTwoPassWriter, data: openArray[byte]) = - if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - if not self.dryRun: - self.output[self.fillLevel] = byte(data[0]) - + let prefixLen = self.listPrefixBytes[0] + self.listPrefixBytes.delete(0) + self.pendingLists.add((listSize, self.fillLevel, prefixLen)) + self.fillLevel += prefixLen + +proc writeRawBytes(self: var RlpTwoPassWriter, bytes: openArray[byte]) = + if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: + self.output[self.fillLevel] = byte(bytes[0]) self.fillLevel += 1 - self.maybeClosePendingLists() else: - self.writeCount(data.len, BLOB_START_MARKER) - self.appendRawBytes(data) + self.writeCount(bytes.len, BLOB_START_MARKER) + self.appendRawBytes(bytes) -proc appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = - self.output.setLen(self.output.len + bytes.len) - assign(self.output.toOpenArray( - self.output.len - bytes.len, self.output.len - 1), bytes) - - self.maybeClosePendingLists() +proc writeRawBytes(self: var RlpDefaultWriter, bytes: openArray[byte]) = + if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: + self.output.add byte(bytes[0]) + else: + self.writeCount(bytes.len, BLOB_START_MARKER) + self.appendRawBytes(bytes) proc startList*(self: var RlpDefaultWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) - self.appendRawBytes([]) + self.maybeClosePendingLists() else: self.pendingLists.add((listSize, self.output.len)) -proc appendBlob(self: var RlpDefaultWriter, data: openArray[byte]) = +proc calculateListPrefix(listLen, prefixLen: int): seq[byte] = + var prefix = newSeqOfCap[byte](1) # prefix min length is 1 + prefix.setLen(prefixLen) + + if listLen < THRESHOLD_LIST_LEN: + prefix[0] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = prefixLen - 1 + prefix[0] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + prefix.writeBigEndian(uint64(listLen), listLenBytes, listLenBytes) + + move(prefix) + +proc recordLenInCurList(self: var RlpLengthTracker, length: int) = + if self.pendingLists.len <= 0: + self.totalLength += length + return + + let lastIdx = self.pendingLists.len - 1 + self.pendingLists[lastIdx].remainingItems -= 1 + self.pendingLists[lastIdx].length += length + + + if self.pendingLists[lastIdx].remainingItems == 0: + let listIdx = self.pendingLists[lastIdx].idx + let listLen = self.pendingLists[lastIdx].length + let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + # save the prefix + #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) + # take note of the prefix len + self.listPrefixBytes[listIdx] = prefixLen + # close the list by deleting + self.pendingLists.setLen(lastIdx) + + let finalListLen = listLen + prefixLen + # add the current lists length to its parent list + self.recordLenInCurList(finalListLen) + +proc startList*(self: var RlpLengthTracker, listSize: int) = + if listSize == 0: + self.recordLenInCurList(1) + else: + # open a list + self.pendingLists.add((self.listCount, listSize, 0)) + self.listCount += 1 + self.listPrefixBytes.add(0) + +func lengthCount(count: int): int = + if count < THRESHOLD_LIST_LEN: + return 1 + else: + return uint64(count).bytesNeeded + 1 + +func lengthBlob(self: var RlpLengthTracker, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - self.output.add byte(data[0]) - self.maybeClosePendingLists() + self.recordLenInCurList(1) else: - self.writeCount(data.len, BLOB_START_MARKER) - self.appendRawBytes(data) + self.recordLenInCurList(lengthCount(data.len) + data.len) + +func lengthInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = + if i < typeof(i)(BLOB_START_MARKER): + self.recordLenInCurList(1) + else: + self.recordLenInCurList(lengthCount(i.bytesNeeded) + i.bytesNeeded) + +template appendBlob(self: var RlpLengthTracker, data: openArray[byte]) = + self.lengthBlob(data) + +template appendInt(self: var RlpLengthTracker, data: SomeUnsignedInt) = + self.lengthInt(data) + +template appendBlob(self: var RlpWriter, data: openArray[byte]) = + self.writeRawBytes(data) + self.maybeClosePendingLists() proc appendInt(self: var RlpWriter, i: SomeUnsignedInt) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. self.writeInt(i) - self.maybeClosePendingLists() @@ -417,18 +457,13 @@ func clear*(w: var RlpWriter) = proc encode*[T](v: T): seq[byte] = mixin append - var writer: RlpTwoPassWriter - # first pass - writer.dryRun = true - writer.append(v) - - # second pass - writer.dryRun = false - writer.listCount = 0 - writer.output = newSeqOfCap[byte](writer.fillLevel) - writer.output.setLen(writer.fillLevel) - writer.fillLevel = 0 + var tracker: RlpLengthTracker + tracker.append(v) + var writer: RlpTwoPassWriter + writer.output = newSeqOfCap[byte](tracker.totalLength) + writer.output.setLen(tracker.totalLength) + writer.listPrefixBytes = tracker.listPrefixBytes writer.append(v) move(writer.finish) From 0bbbab5dda833ebbd3c3006977b99c9e4a0b6096 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 14:17:39 +0530 Subject: [PATCH 13/36] remove code redundancy --- eth/rlp/writer.nim | 81 ++++++++------------------------- tests/rlp/all_tests.nim | 8 ++-- tests/rlp/test_rlp_profiler.nim | 8 ++-- 3 files changed, 26 insertions(+), 71 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index d4c9ec50..006fe276 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -20,18 +20,17 @@ type RlpDefaultWriter* = object pendingLists: seq[tuple[remainingItems, startPos: int]] output: seq[byte] - - RlpTwoPassWriter* = object - pendingLists: seq[tuple[remainingItems, startPos, prefixLen: int]] - output: seq[byte] - listPrefixBytes: seq[int] - fillLevel: int RlpLengthTracker* = object pendingLists: seq[tuple[idx, remainingItems, length: int]] listCount: int listPrefixBytes: seq[int] totalLength: int + + RlpTwoPassWriter* = object + pendingLists: seq[tuple[remainingItems, startPos, prefixLen: int]] + output: seq[byte] + listPrefixBytes: seq[int] RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter | RlpLengthTracker @@ -52,33 +51,7 @@ func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, outStream[i] = byte(n and 0xff) n = n shr 8 -func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = - if count < THRESHOLD_LIST_LEN: - writer.output[writer.fillLevel] = (baseMarker + byte(count)) - writer.fillLevel += 1 - else: - let lenPrefixBytes = uint64(count).bytesNeeded - - writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - writer.fillLevel += 1 - writer.output.writeBigEndian(uint64(count), writer.fillLevel + lenPrefixBytes - 1, lenPrefixBytes) - writer.fillLevel += lenPrefixBytes - -func writeInt(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = - if i == typeof(i)(0): - writer.output[writer.fillLevel] = BLOB_START_MARKER - writer.fillLevel += 1 - elif i < typeof(i)(BLOB_START_MARKER): - writer.output[writer.fillLevel] = byte(i) - writer.fillLevel += 1 - else: - let bytesNeeded = i.bytesNeeded - writer.writeCount(bytesNeeded, BLOB_START_MARKER) - - writer.output.writeBigEndian(i, writer.fillLevel + bytesNeeded - 1, bytesNeeded) - writer.fillLevel += bytesNeeded - -func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = +func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: writer.output.add(baseMarker + byte(count)) else: @@ -89,7 +62,7 @@ func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = writer.output.setLen(writer.output.len + lenPrefixBytes) writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) -func writeInt(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = +func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): writer.output.add BLOB_START_MARKER elif i < typeof(i)(BLOB_START_MARKER): @@ -121,7 +94,7 @@ proc maybeClosePendingLists(self: var RlpTwoPassWriter) = self.pendingLists.setLen lastListIdx - let listLen = self.fillLevel - listStartPos - prefixLen + let listLen = self.output.len - listStartPos - prefixLen # Write out the prefix length if listLen < THRESHOLD_LIST_LEN: @@ -171,11 +144,7 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = # The currently open list is not finished yet. Nothing to do. return -template appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = - assign(self.output.toOpenArray(self.fillLevel, self.fillLevel + bytes.len - 1), bytes) - self.fillLevel += bytes.len - -template appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = +template appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = self.output.setLen(self.output.len + bytes.len) assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) @@ -186,18 +155,10 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = else: let prefixLen = self.listPrefixBytes[0] self.listPrefixBytes.delete(0) - self.pendingLists.add((listSize, self.fillLevel, prefixLen)) - self.fillLevel += prefixLen - -proc writeRawBytes(self: var RlpTwoPassWriter, bytes: openArray[byte]) = - if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: - self.output[self.fillLevel] = byte(bytes[0]) - self.fillLevel += 1 - else: - self.writeCount(bytes.len, BLOB_START_MARKER) - self.appendRawBytes(bytes) + self.pendingLists.add((listSize, self.output.len, prefixLen)) + self.output.setLen(self.output.len + prefixLen) -proc writeRawBytes(self: var RlpDefaultWriter, bytes: openArray[byte]) = +proc writeRawBytes(self: var RlpWriter, bytes: openArray[byte]) = if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: self.output.add byte(bytes[0]) else: @@ -260,31 +221,25 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = self.listCount += 1 self.listPrefixBytes.add(0) -func lengthCount(count: int): int = +func lengthCount(count: int): int {.inline.} = if count < THRESHOLD_LIST_LEN: return 1 else: return uint64(count).bytesNeeded + 1 -func lengthBlob(self: var RlpLengthTracker, data: openArray[byte]) = +func appendBlob(self: var RlpLengthTracker, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: self.recordLenInCurList(1) else: self.recordLenInCurList(lengthCount(data.len) + data.len) -func lengthInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = +func appendInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = if i < typeof(i)(BLOB_START_MARKER): self.recordLenInCurList(1) else: self.recordLenInCurList(lengthCount(i.bytesNeeded) + i.bytesNeeded) -template appendBlob(self: var RlpLengthTracker, data: openArray[byte]) = - self.lengthBlob(data) - -template appendInt(self: var RlpLengthTracker, data: SomeUnsignedInt) = - self.lengthInt(data) - -template appendBlob(self: var RlpWriter, data: openArray[byte]) = +func appendBlob(self: var RlpWriter, data: openArray[byte]) = self.writeRawBytes(data) self.maybeClosePendingLists() @@ -458,11 +413,11 @@ proc encode*[T](v: T): seq[byte] = mixin append var tracker: RlpLengthTracker + var writer: RlpTwoPassWriter + tracker.append(v) - var writer: RlpTwoPassWriter writer.output = newSeqOfCap[byte](tracker.totalLength) - writer.output.setLen(tracker.totalLength) writer.listPrefixBytes = tracker.listPrefixBytes writer.append(v) diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index cd50c8b6..af003598 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import ./test_api_usage, - ./test_json_suite, - ./test_empty_string, - ./test_object_serialization, - ./test_optional_fields, +# ./test_json_suite, +# ./test_empty_string, +# ./test_object_serialization, +# ./test_optional_fields, ./test_rlp_profiler diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index c8dca027..f3897918 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -55,11 +55,11 @@ proc encodeOnePass[T](v: T): seq[byte] = suite "test running time of rlp serialization": test "transaction serialization": - benchmark "Transaction serialization (two pass)": - let myTxBytes = rlp.encode(myTx) +# benchmark "Transaction serialization (two pass)": +# let myTxBytes = rlp.encode(myTx) benchmark "Block Sequence serialization (two pass)": let myBlockBytes = rlp.encode(blkSeq) - benchmark "Transaction serialization (one pass)": - let myTxBytesOnePass = encodeOnePass(myTx) +# benchmark "Transaction serialization (one pass)": +# let myTxBytesOnePass = encodeOnePass(myTx) benchmark "Block Sequence serailization (one pass)": let myBlockBytesOnePass = encodeOnePass(blkSeq) From 625f377ea1dd8fc036ee0258540733d3f4c82a2c Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 14:44:35 +0530 Subject: [PATCH 14/36] collect length in a different manner --- eth/rlp/writer.nim | 82 ++++++++++++++------------------- tests/rlp/test_rlp_profiler.nim | 8 ++-- 2 files changed, 38 insertions(+), 52 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 006fe276..d3ba2a58 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -172,72 +172,58 @@ proc startList*(self: var RlpDefaultWriter, listSize: int) = else: self.pendingLists.add((listSize, self.output.len)) -proc calculateListPrefix(listLen, prefixLen: int): seq[byte] = - var prefix = newSeqOfCap[byte](1) # prefix min length is 1 - prefix.setLen(prefixLen) +proc maybeClosePendingLists(self: var RlpLengthTracker) = + while self.pendingLists.len > 0: + let lastIdx = self.pendingLists.len - 1 + self.pendingLists[lastIdx].remainingItems -= 1 - if listLen < THRESHOLD_LIST_LEN: - prefix[0] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = prefixLen - 1 - prefix[0] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - prefix.writeBigEndian(uint64(listLen), listLenBytes, listLenBytes) - - move(prefix) - -proc recordLenInCurList(self: var RlpLengthTracker, length: int) = - if self.pendingLists.len <= 0: - self.totalLength += length - return - - let lastIdx = self.pendingLists.len - 1 - self.pendingLists[lastIdx].remainingItems -= 1 - self.pendingLists[lastIdx].length += length - - - if self.pendingLists[lastIdx].remainingItems == 0: - let listIdx = self.pendingLists[lastIdx].idx - let listLen = self.pendingLists[lastIdx].length - let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 - - # save the prefix - #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) - # take note of the prefix len - self.listPrefixBytes[listIdx] = prefixLen - # close the list by deleting - self.pendingLists.setLen(lastIdx) - - let finalListLen = listLen + prefixLen - # add the current lists length to its parent list - self.recordLenInCurList(finalListLen) + if self.pendingLists[lastIdx].remainingItems == 0: + let listIdx = self.pendingLists[lastIdx].idx + let startLen = self.pendingLists[lastIdx].length + + let listLen = self.totalLength - startLen + + let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + # save the prefix + #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) + # take note of the prefix len + self.listPrefixBytes[listIdx] = prefixLen + # close the list by deleting + self.pendingLists.setLen(lastIdx) + + self.totalLength += prefixLen + else: + return proc startList*(self: var RlpLengthTracker, listSize: int) = if listSize == 0: - self.recordLenInCurList(1) + self.totalLength += 1 + self.maybeClosePendingLists() else: # open a list - self.pendingLists.add((self.listCount, listSize, 0)) + self.pendingLists.add((self.listCount, listSize, self.totalLength)) self.listCount += 1 self.listPrefixBytes.add(0) func lengthCount(count: int): int {.inline.} = - if count < THRESHOLD_LIST_LEN: - return 1 - else: - return uint64(count).bytesNeeded + 1 + return if count < THRESHOLD_LIST_LEN: 1 + else: uint64(count).bytesNeeded + 1 func appendBlob(self: var RlpLengthTracker, data: openArray[byte]) = if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - self.recordLenInCurList(1) + self.totalLength += 1 else: - self.recordLenInCurList(lengthCount(data.len) + data.len) + self.totalLength += lengthCount(data.len) + data.len + self.maybeClosePendingLists() func appendInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = if i < typeof(i)(BLOB_START_MARKER): - self.recordLenInCurList(1) + self.totalLength += 1 else: - self.recordLenInCurList(lengthCount(i.bytesNeeded) + i.bytesNeeded) + self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded + self.maybeClosePendingLists() func appendBlob(self: var RlpWriter, data: openArray[byte]) = self.writeRawBytes(data) diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index f3897918..c8dca027 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -55,11 +55,11 @@ proc encodeOnePass[T](v: T): seq[byte] = suite "test running time of rlp serialization": test "transaction serialization": -# benchmark "Transaction serialization (two pass)": -# let myTxBytes = rlp.encode(myTx) + benchmark "Transaction serialization (two pass)": + let myTxBytes = rlp.encode(myTx) benchmark "Block Sequence serialization (two pass)": let myBlockBytes = rlp.encode(blkSeq) -# benchmark "Transaction serialization (one pass)": -# let myTxBytesOnePass = encodeOnePass(myTx) + benchmark "Transaction serialization (one pass)": + let myTxBytesOnePass = encodeOnePass(myTx) benchmark "Block Sequence serailization (one pass)": let myBlockBytesOnePass = encodeOnePass(blkSeq) From 3645271a22a4ce3eee59a4fb38d1dc2e94c60977 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 15:02:28 +0530 Subject: [PATCH 15/36] add all tests --- eth/rlp/writer.nim | 10 +- tests/rlp/all_tests.nim | 8 +- tests/rlp/test_api_usage.nim | 224 +++++++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+), 10 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index d3ba2a58..f59b8dbc 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -144,9 +144,10 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = # The currently open list is not finished yet. Nothing to do. return -template appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = +func appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = self.output.setLen(self.output.len + bytes.len) assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) + self.maybeClosePendingLists() proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: @@ -158,9 +159,10 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = self.pendingLists.add((listSize, self.output.len, prefixLen)) self.output.setLen(self.output.len + prefixLen) -proc writeRawBytes(self: var RlpWriter, bytes: openArray[byte]) = +proc appendBlob(self: var RlpWriter, bytes: openArray[byte]) = if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: self.output.add byte(bytes[0]) + self.maybeClosePendingLists() else: self.writeCount(bytes.len, BLOB_START_MARKER) self.appendRawBytes(bytes) @@ -225,10 +227,6 @@ func appendInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded self.maybeClosePendingLists() -func appendBlob(self: var RlpWriter, data: openArray[byte]) = - self.writeRawBytes(data) - self.maybeClosePendingLists() - proc appendInt(self: var RlpWriter, i: SomeUnsignedInt) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index af003598..cd50c8b6 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -1,7 +1,7 @@ import ./test_api_usage, -# ./test_json_suite, -# ./test_empty_string, -# ./test_object_serialization, -# ./test_optional_fields, + ./test_json_suite, + ./test_empty_string, + ./test_object_serialization, + ./test_optional_fields, ./test_rlp_profiler diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index 7d6956ca..cf9ad2f8 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -35,5 +35,229 @@ proc test_blockBodyTranscode() = check (n, trBlkSeq[n]) == (n, blkSeq[n]) suite "test api usage": + test "empty bytes are not a proper RLP": + var rlp = rlpFromBytes seq[byte](@[]) + + check: + not rlp.hasData + not rlp.isBlob + not rlp.isList + not rlp.isEmpty + + expect AssertionDefect: + rlp.skipElem + + test "you cannot finish a list without appending enough elements": + var writer = initRlpList(3) + writer.append "foo" + writer.append "bar" + + expect AssertionDefect: + discard writer.finish + + test "encode/decode object": + type + MyEnum = enum + foo, + bar + + MyObj = object + a: array[3, char] + b: uint64 + c: MyEnum + + IntObj = object + v: int + + var input: MyObj + input.a = ['e', 't', 'h'] + input.b = 63 + input.c = bar + + + var writer = initRlpWriter() + writer.append(input) + + check: + not compiles(writer.append(default(IntObj))) + + let bytes = writer.finish() + var rlp = rlpFromBytes(bytes) + + var output = rlp.read(MyObj) + check: + input == output + + test "encode and decode lists": + var writer = initRlpList(3) + writer.append "foo" + writer.append ["bar", "baz"] + writer.append [uint64 30, 40, 50] + + var + bytes = writer.finish + rlp = rlpFromBytes bytes + + check: + bytes.toHex == "d183666f6fc8836261728362617ac31e2832" + rlp.inspectMatch """ + { + "foo" + { + "bar" + "baz" + } + { + byte 30 + byte 40 + byte 50 + } + } + """ + + bytes = encodeList(uint64 6000, + "Lorem ipsum dolor sit amet", + "Donec ligula tortor, egestas eu est vitae") + + rlp = rlpFromBytes bytes + check: + rlp.listLen == 3 + rlp.listElem(0).toInt(uint64) == 6000 + rlp.listElem(1).toString == "Lorem ipsum dolor sit amet" + rlp.listElem(2).toString == "Donec ligula tortor, egestas eu est vitae" + + # test creating RLPs from other RLPs + var list = rlpFromBytes encodeList(rlp.listElem(1), rlp.listElem(0)) + + # test that iteration with enterList/skipElem works as expected + doAssert list.enterList # We already know that we are working with a list + check list.toString == "Lorem ipsum dolor sit amet" + list.skipElem + + check list.toInt(uint32) == 6000.uint32 + var intVar: uint32 + list >> intVar + check intVar == 6000 + + check(not list.hasData) + expect AssertionDefect: list.skipElem + test "encode and decode block body": test_blockBodyTranscode() + + test "toBytes": + let rlp = rlpFromHex("f2cb847f000001827666827666a040ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4845ab90b50") + let tok = rlp.listElem(1).toBytes() + check: + tok.len == 32 + tok.toHex == "40ef02798f211da2e8173d37f255be908871ae65060dbb2f77fb29c0421447f4" + + test "nested lists": + let listBytes = encode([[uint64 1, 2, 3], [uint64 5, 6, 7]]) + let listRlp = rlpFromBytes listBytes + let sublistRlp0 = listRlp.listElem(0) + let sublistRlp1 = listRlp.listElem(1) + check sublistRlp0.listElem(0).toInt(uint64) == 1 + check sublistRlp0.listElem(1).toInt(uint64) == 2 + check sublistRlp0.listElem(2).toInt(uint64) == 3 + check sublistRlp1.listElem(0).toInt(uint64) == 5 + check sublistRlp1.listElem(1).toInt(uint64) == 6 + check sublistRlp1.listElem(2).toInt(uint64) == 7 + + test "encoding length": + let listBytes = encode([uint64 1,2,3,4,5]) + let listRlp = rlpFromBytes listBytes + check listRlp.listLen == 5 + + let emptyListBytes = encode "" + check emptyListBytes.len == 1 + let emptyListRlp = rlpFromBytes emptyListBytes + check emptyListRlp.blobLen == 0 + + test "basic decoding": + var rlp1 = rlpFromHex("856d6f6f7365") + var rlp2 = rlpFromHex("0x856d6f6f7365") + + check: + rlp1.inspect == q"moose" + rlp2.inspect == q"moose" + + test "malformed/truncated RLP": + var rlp = rlpFromHex("b8056d6f6f7365") + expect MalformedRlpError: + discard rlp.inspect + + test "encode byte arrays": + var b1 = [byte(1), 2, 5, 7, 8] + var b2 = [byte(6), 8, 12, 123] + var b3 = @[byte(122), 56, 65, 12] + + let rlp = rlpFromBytes(encode((b1, b2, b3))) + check: + rlp.listLen == 3 + rlp.listElem(0).toBytes() == b1 + rlp.listElem(1).toBytes() == b2 + rlp.listElem(2).toBytes() == b3 + + # The first byte here is the length of the datum (132 - 128 => 4) + $(rlp.listElem(1).rawData) == "[132, 6, 8, 12, 123]" + + test "empty byte arrays": + var + rlp = rlpFromBytes rlp.encode("") + b = rlp.toBytes + check $b == "@[]" + + test "invalid enum": + type + MyEnum = enum + foo = 0x00, + bar = 0x01 + + var writer = initRlpWriter() + writer.append(byte 1) # valid + writer.append(byte 2) # invalid + writer.append(cast[uint64](-1)) # invalid + let bytes = writer.finish() + var rlp = rlpFromBytes(bytes) + + check rlp.read(MyEnum) == bar + + expect RlpTypeMismatch: + discard rlp.read(MyEnum) + rlp.skipElem() + + expect RlpTypeMismatch: + discard rlp.read(MyEnum) + + test "invalid enum - enum with hole": + type + MyEnum = enum + foo = 0x00, + bar = 0x01, + baz = 0x0100 + + var writer = initRlpWriter() + writer.append(1'u64) # valid + writer.append(2'u64) # invalid - enum hole value + writer.append(256'u64) # valid + writer.append(257'u64) # invalid - too large + let bytes = writer.finish() + var rlp = rlpFromBytes(bytes) + + check rlp.read(MyEnum) == bar + + expect RlpTypeMismatch: + discard rlp.read(MyEnum) + rlp.skipElem() + + check rlp.read(MyEnum) == baz + + expect RlpTypeMismatch: + discard rlp.read(MyEnum) + rlp.skipElem() + + test "encodeInt basics": + for i in [uint64 0, 1, 10, 100, 1000, uint64.high]: + check: + encode(i) == encodeInt(i).data() From 35c443851353169e1e5c8a3dca9d9c0e1d45714e Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 15:40:53 +0530 Subject: [PATCH 16/36] revert changes --- eth.nimble | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth.nimble b/eth.nimble index 590f3fc9..7c21f3fd 100644 --- a/eth.nimble +++ b/eth.nimble @@ -34,9 +34,7 @@ let cfg = " --skipUserCfg --nimcache:build/nimcache -f" & " --warning[ObservableStores]:off -d:nimOldCaseObjects" & " -d:chronicles_log_level=TRACE" & - " --threads:on -d:debug" & - " --excessiveStackTrace:on" - + " --threads:on -d:release" proc build(args, path, outdir: string) = exec nimc & " " & lang & " " & cfg & " " & flags & " " & args & From 9a51c27018144507adb826ca2fea8bc3bb35ca9f Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 11 Dec 2024 16:14:16 +0530 Subject: [PATCH 17/36] fix --- eth/rlp/writer.nim | 54 +++++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index f59b8dbc..df6034d1 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -144,11 +144,40 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = # The currently open list is not finished yet. Nothing to do. return +proc maybeClosePendingLists(self: var RlpLengthTracker) = + while self.pendingLists.len > 0: + let lastIdx = self.pendingLists.len - 1 + self.pendingLists[lastIdx].remainingItems -= 1 + + if self.pendingLists[lastIdx].remainingItems == 0: + let listIdx = self.pendingLists[lastIdx].idx + let startLen = self.pendingLists[lastIdx].length + + let listLen = self.totalLength - startLen + + let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + # save the prefix + #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) + # take note of the prefix len + self.listPrefixBytes[listIdx] = prefixLen + # close the list by deleting + self.pendingLists.setLen(lastIdx) + + self.totalLength += prefixLen + else: + return + func appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = self.output.setLen(self.output.len + bytes.len) assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) self.maybeClosePendingLists() +func appendRawBytes*(self: var RlpLengthTracker, bytes: openArray[byte]) = + self.totalLength += bytes.len + self.maybeClosePendingLists() + proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) @@ -174,31 +203,6 @@ proc startList*(self: var RlpDefaultWriter, listSize: int) = else: self.pendingLists.add((listSize, self.output.len)) -proc maybeClosePendingLists(self: var RlpLengthTracker) = - while self.pendingLists.len > 0: - let lastIdx = self.pendingLists.len - 1 - self.pendingLists[lastIdx].remainingItems -= 1 - - if self.pendingLists[lastIdx].remainingItems == 0: - let listIdx = self.pendingLists[lastIdx].idx - let startLen = self.pendingLists[lastIdx].length - - let listLen = self.totalLength - startLen - - let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 - - # save the prefix - #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) - # take note of the prefix len - self.listPrefixBytes[listIdx] = prefixLen - # close the list by deleting - self.pendingLists.setLen(lastIdx) - - self.totalLength += prefixLen - else: - return - proc startList*(self: var RlpLengthTracker, listSize: int) = if listSize == 0: self.totalLength += 1 From 6cdd48e42a9ba00195bb7303be6cd4370ff61187 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 12 Dec 2024 09:51:36 +0530 Subject: [PATCH 18/36] hash writer --- eth/common/eth_types_rlp.nim | 4 +- eth/rlp/default_writer.nim | 102 +++++++++++++ eth/rlp/hash_writer.nim | 70 +++++++++ eth/rlp/length_writer.nim | 95 ++++++++++++ eth/rlp/two_pass_writer.nim | 100 +++++++++++++ eth/rlp/utils.nim | 14 ++ eth/rlp/writer.nim | 252 ++++---------------------------- tests/rlp/all_tests.nim | 1 + tests/rlp/test_hashes.nim | 51 +++++++ tests/rlp/test_rlp_profiler.nim | 33 +++-- 10 files changed, 483 insertions(+), 239 deletions(-) create mode 100644 eth/rlp/default_writer.nim create mode 100644 eth/rlp/hash_writer.nim create mode 100644 eth/rlp/length_writer.nim create mode 100644 eth/rlp/two_pass_writer.nim create mode 100644 eth/rlp/utils.nim create mode 100644 tests/rlp/test_hashes.nim diff --git a/eth/common/eth_types_rlp.nim b/eth/common/eth_types_rlp.nim index 7ce4e37f..1ad65ac7 100644 --- a/eth/common/eth_types_rlp.nim +++ b/eth/common/eth_types_rlp.nim @@ -29,10 +29,10 @@ proc read*(rlp: var Rlp, T: type BlockHashOrNumber): T = BlockHashOrNumber(isHash: false, number: rlp.read(BlockNumber)) proc rlpHash*[T](v: T): Hash32 = - keccak256(rlp.encode(v)) + Hash32(rlp.encodeHash(v)) proc rlpHash*(tx: PooledTransaction): Hash32 = - keccak256(rlp.encode(tx.tx)) + Hash32(rlp.encodeHash(tx.tx)) func blockHash*(h: Header): Hash32 {.inline.} = rlpHash(h) diff --git a/eth/rlp/default_writer.nim b/eth/rlp/default_writer.nim new file mode 100644 index 00000000..7d3cf41e --- /dev/null +++ b/eth/rlp/default_writer.nim @@ -0,0 +1,102 @@ +import + std/options, + pkg/results, + stew/[arraybuf, assign2, shims/macros], + ./priv/defs, + utils + +type + RlpDefaultWriter* = object + pendingLists: seq[tuple[remainingItems, startPos: int]] + output: seq[byte] + +func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = + if count < THRESHOLD_LIST_LEN: + writer.output.add(baseMarker + byte(count)) + else: + let lenPrefixBytes = uint64(count).bytesNeeded + + writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + + writer.output.setLen(writer.output.len + lenPrefixBytes) + writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) + +proc maybeClosePendingLists(self: var RlpDefaultWriter) = + while self.pendingLists.len > 0: + let lastListIdx = self.pendingLists.len - 1 + doAssert self.pendingLists[lastListIdx].remainingItems > 0 + + self.pendingLists[lastListIdx].remainingItems -= 1 + # if one last item is remaining in the list + if self.pendingLists[lastListIdx].remainingItems == 0: + # A list have been just finished. It was started in `startList`. + let listStartPos = self.pendingLists[lastListIdx].startPos + + self.pendingLists.setLen lastListIdx + + let listLen = self.output.len - listStartPos + + let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + #Shift the written data to make room for the prefix length + self.output.setLen(self.output.len + totalPrefixBytes) + + moveMem(addr self.output[listStartPos + totalPrefixBytes], + unsafeAddr self.output[listStartPos], + listLen) + + # Write out the prefix length + if listLen < THRESHOLD_LIST_LEN: + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = totalPrefixBytes - 1 + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + else: + # The currently open list is not finished yet. Nothing to do. + return + +func writeInt*(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = + if i == typeof(i)(0): + writer.output.add BLOB_START_MARKER + elif i < typeof(i)(BLOB_START_MARKER): + writer.output.add byte(i) + else: + let bytesNeeded = i.bytesNeeded + writer.writeCount(bytesNeeded, BLOB_START_MARKER) + + writer.output.setLen(writer.output.len + bytesNeeded) + writer.output.writeBigEndian(i, writer.output.len - 1, bytesNeeded) + writer.maybeClosePendingLists() + +func appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = + self.output.setLen(self.output.len + bytes.len) + assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) + self.maybeClosePendingLists() + +proc writeBlob*(self: var RlpDefaultWriter, bytes: openArray[byte]) = + if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: + self.output.add byte(bytes[0]) + self.maybeClosePendingLists() + else: + self.writeCount(bytes.len, BLOB_START_MARKER) + self.appendRawBytes(bytes) + +proc startList*(self: var RlpDefaultWriter, listSize: int) = + if listSize == 0: + self.writeCount(0, LIST_START_MARKER) + self.maybeClosePendingLists() + else: + self.pendingLists.add((listSize, self.output.len)) + +template finish*(self: RlpDefaultWriter): seq[byte] = + doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + self.output + +func clear*(w: var RlpDefaultWriter) = + # Prepare writer for reuse + w.pendingLists.setLen(0) + w.output.setLen(0) + + diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim new file mode 100644 index 00000000..007ad289 --- /dev/null +++ b/eth/rlp/hash_writer.nim @@ -0,0 +1,70 @@ +import + std/options, + pkg/results, + nimcrypto/keccak, + stew/[arraybuf, bitops2, shims/macros], + ./priv/defs, + utils + +type + RlpHashWriter* = object + keccak: keccak.keccak256 + listPrefixBytes*: seq[seq[byte]] + +func update(writer: var RlpHashWriter, data: byte) = + writer.keccak.update([data]) + +template update(writer: var RlpHashWriter, data: openArray[byte]) = + writer.keccak.update(data) + +func writeCount(writer: var RlpHashWriter, count: int, baseMarker: byte) = + if count < THRESHOLD_LIST_LEN: + writer.update(baseMarker + byte(count)) + else: + let lenPrefixBytes = uint64(count).bytesNeeded + + writer.update baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + + var buf = newSeqOfCap[byte](lenPrefixBytes) + buf.writeBigEndian(uint64(count), buf.len - 1, lenPrefixBytes) + writer.update(buf) + +func writeInt*(writer: var RlpHashWriter, i: SomeUnsignedInt) = + if i == typeof(i)(0): + writer.update BLOB_START_MARKER + elif i < typeof(i)(BLOB_START_MARKER): + writer.update byte(i) + else: + let bytesNeeded = i.bytesNeeded + writer.writeCount(bytesNeeded, BLOB_START_MARKER) + + var buf = newSeqOfCap[byte](bytesNeeded) + buf.writeBigEndian(uint64(i), buf.len - 1, bytesNeeded) + writer.update(buf) + +template appendRawBytes*(self: var RlpHashWriter, bytes: openArray[byte]) = + self.update(bytes) + +proc writeBlob*(self: var RlpHashWriter, bytes: openArray[byte]) = + if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: + self.update byte(bytes[0]) + else: + self.writeCount(bytes.len, BLOB_START_MARKER) + self.appendRawBytes(bytes) + +proc startList*(self: var RlpHashWriter, listSize: int) = + if listSize == 0: + self.writeCount(0, LIST_START_MARKER) + else: + let prefixBytes = self.listPrefixBytes.pop() + self.update(prefixBytes) + +template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = + doAssert self.listPrefixBytes.len == 0, "Insufficient number of elements written to a started list" + self.keccak.finish() + +func clear*(w: var RlpHashWriter) = + # Prepare writer for reuse + w.listPrefixBytes.setLen(0) + + diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim new file mode 100644 index 00000000..429ab612 --- /dev/null +++ b/eth/rlp/length_writer.nim @@ -0,0 +1,95 @@ +import + std/options, + pkg/results, + stew/[arraybuf, assign2, bitops2, shims/macros], + ./priv/defs, + utils + +type + RlpLengthTracker* = object + pendingLists: seq[tuple[idx, remainingItems, length: int]] + listCount: int + listPrefixLen*: seq[int] + listPrefixBytes*: seq[seq[byte]] + totalLength*: int + +proc calculateListPrefix(listLen, prefixLen: int): seq[byte] = + var prefix = newSeqOfCap[byte](1) # prefix min length is 1 + prefix.setLen(prefixLen) + + if listLen < THRESHOLD_LIST_LEN: + prefix[0] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = prefixLen - 1 + prefix[0] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + prefix.writeBigEndian(uint64(listLen), listLenBytes, listLenBytes) + + move(prefix) + +proc maybeClosePendingLists(self: var RlpLengthTracker) = + while self.pendingLists.len > 0: + let lastIdx = self.pendingLists.len - 1 + self.pendingLists[lastIdx].remainingItems -= 1 + + if self.pendingLists[lastIdx].remainingItems == 0: + let listIdx = self.pendingLists[lastIdx].idx + let startLen = self.pendingLists[lastIdx].length + + let listLen = self.totalLength - startLen + + let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 + + # save the prefix + self.listPrefixBytes[listIdx] = calculateListPrefix(listLen, prefixLen) + # take note of the prefix len + self.listPrefixLen[listIdx] = prefixLen + # close the list by deleting + self.pendingLists.setLen(lastIdx) + + self.totalLength += prefixLen + else: + return + +func appendRawBytes*(self: var RlpLengthTracker, bytes: openArray[byte]) = + self.totalLength += bytes.len + self.maybeClosePendingLists() + +proc startList*(self: var RlpLengthTracker, listSize: int) = + if listSize == 0: + self.totalLength += 1 + self.maybeClosePendingLists() + else: + # open a list + self.pendingLists.add((self.listCount, listSize, self.totalLength)) + self.listCount += 1 + self.listPrefixLen.add(0) + self.listPrefixBytes.add(@[]) + +func lengthCount(count: int): int {.inline.} = + return if count < THRESHOLD_LIST_LEN: 1 + else: uint64(count).bytesNeeded + 1 + +func writeBlob*(self: var RlpLengthTracker, data: openArray[byte]) = + if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: + self.totalLength += 1 + else: + self.totalLength += lengthCount(data.len) + data.len + self.maybeClosePendingLists() + +func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = + if i < typeof(i)(BLOB_START_MARKER): + self.totalLength += 1 + else: + self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded + self.maybeClosePendingLists() + +template finish*(self: RlpLengthTracker): seq[seq[byte]] = + doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + self.listPrefixBytes + +func clear*(w: var RlpLengthTracker) = + # Prepare writer for reuse + w.pendingLists.setLen(0) + w.listPrefixLen.setLen(0) + w.listPrefixBytes.setLen(0) diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim new file mode 100644 index 00000000..fa0d17e4 --- /dev/null +++ b/eth/rlp/two_pass_writer.nim @@ -0,0 +1,100 @@ +import + std/options, + pkg/results, + stew/[arraybuf, assign2, shims/macros], + ./priv/defs, + utils + +type + RlpTwoPassWriter* = object + pendingLists*: seq[tuple[remainingItems, startPos, prefixLen: int]] + output*: seq[byte] + listPrefixLen*: seq[int] + +func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = + if count < THRESHOLD_LIST_LEN: + writer.output.add(baseMarker + byte(count)) + else: + let lenPrefixBytes = uint64(count).bytesNeeded + + writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + + writer.output.setLen(writer.output.len + lenPrefixBytes) + writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) + +# nothing to do when serializing using tracker +proc maybeClosePendingLists(self: var RlpTwoPassWriter) = + while self.pendingLists.len > 0: + let lastListIdx = self.pendingLists.len - 1 + doAssert self.pendingLists[lastListIdx].remainingItems > 0 + + self.pendingLists[lastListIdx].remainingItems -= 1 + # if one last item is remaining in the list + if self.pendingLists[lastListIdx].remainingItems == 0: + # A list have been just finished. It was started in `startList`. + let listStartPos = self.pendingLists[lastListIdx].startPos + let prefixLen = self.pendingLists[lastListIdx].prefixLen + + self.pendingLists.setLen lastListIdx + + let listLen = self.output.len - listStartPos - prefixLen + + # Write out the prefix length + if listLen < THRESHOLD_LIST_LEN: + self.output[listStartPos] = LIST_START_MARKER + byte(listLen) + else: + let listLenBytes = prefixLen - 1 + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + else: + # The currently open list is not finished yet. Nothing to do. + return + +func writeInt*(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = + if i == typeof(i)(0): + writer.output.add BLOB_START_MARKER + elif i < typeof(i)(BLOB_START_MARKER): + writer.output.add byte(i) + else: + let bytesNeeded = i.bytesNeeded + writer.writeCount(bytesNeeded, BLOB_START_MARKER) + + writer.output.setLen(writer.output.len + bytesNeeded) + writer.output.writeBigEndian(i, writer.output.len - 1, bytesNeeded) + writer.maybeClosePendingLists() + +func appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = + self.output.setLen(self.output.len + bytes.len) + assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) + self.maybeClosePendingLists() + +proc writeBlob*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = + if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: + self.output.add byte(bytes[0]) + self.maybeClosePendingLists() + else: + self.writeCount(bytes.len, BLOB_START_MARKER) + self.appendRawBytes(bytes) + +proc startList*(self: var RlpTwoPassWriter, listSize: int) = + mixin writeCount + + if listSize == 0: + self.writeCount(0, LIST_START_MARKER) + self.maybeClosePendingLists() + else: + let prefixLen = self.listPrefixLen[0] + self.listPrefixLen.delete(0) + self.pendingLists.add((listSize, self.output.len, prefixLen)) + self.output.setLen(self.output.len + prefixLen) + +template finish*(self: RlpTwoPassWriter): seq[byte] = + doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + doAssert self.listPrefixLen.len == 0, "Insufficient number of list prefixes accounted for" + self.output + +func clear*(w: var RlpTwoPassWriter) = + # Prepare writer for reuse + w.pendingLists.setLen(0) + w.output.setLen(0) + w.listPrefixLen.setLen(0) diff --git a/eth/rlp/utils.nim b/eth/rlp/utils.nim new file mode 100644 index 00000000..8054be9f --- /dev/null +++ b/eth/rlp/utils.nim @@ -0,0 +1,14 @@ +import stew/[bitops2] + +func bytesNeeded*(num: SomeUnsignedInt): int = + # Number of non-zero bytes in the big endian encoding + sizeof(num) - (num.leadingZeros() shr 3) + +func writeBigEndian*(outStream: var auto, number: SomeUnsignedInt, + lastByteIdx: int, numberOfBytes: int) = + var n = number + for i in countdown(lastByteIdx, lastByteIdx - numberOfBytes + 1): + outStream[i] = byte(n and 0xff) + n = n shr 8 + + diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index df6034d1..341223cb 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -8,31 +8,19 @@ import std/options, pkg/results, - stew/[arraybuf, assign2, bitops2, shims/macros], + stew/[arraybuf, shims/macros], ./priv/defs, - times, - strutils + hash_writer, + length_writer, + two_pass_writer, + default_writer, + utils, + ../common/hashes - -export arraybuf +export arraybuf, default_writer, length_writer, two_pass_writer, hash_writer type - RlpDefaultWriter* = object - pendingLists: seq[tuple[remainingItems, startPos: int]] - output: seq[byte] - - RlpLengthTracker* = object - pendingLists: seq[tuple[idx, remainingItems, length: int]] - listCount: int - listPrefixBytes: seq[int] - totalLength: int - - RlpTwoPassWriter* = object - pendingLists: seq[tuple[remainingItems, startPos, prefixLen: int]] - output: seq[byte] - listPrefixBytes: seq[int] - - RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter | RlpLengthTracker + RlpWriter* = RlpDefaultWriter | RlpTwoPassWriter | RlpLengthTracker | RlpHashWriter RlpIntBuf* = ArrayBuf[9, byte] ## Small buffer for holding a single RLP-encoded integer @@ -40,203 +28,18 @@ type const wrapObjsInList* = true -func bytesNeeded(num: SomeUnsignedInt): int = - # Number of non-zero bytes in the big endian encoding - sizeof(num) - (num.leadingZeros() shr 3) - -func writeBigEndian(outStream: var auto, number: SomeUnsignedInt, - lastByteIdx: int, numberOfBytes: int) = - var n = number - for i in countdown(lastByteIdx, lastByteIdx - numberOfBytes + 1): - outStream[i] = byte(n and 0xff) - n = n shr 8 - -func writeCount(writer: var RlpWriter, count: int, baseMarker: byte) = - if count < THRESHOLD_LIST_LEN: - writer.output.add(baseMarker + byte(count)) - else: - let lenPrefixBytes = uint64(count).bytesNeeded - - writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - - writer.output.setLen(writer.output.len + lenPrefixBytes) - writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) - -func writeInt(writer: var RlpWriter, i: SomeUnsignedInt) = - if i == typeof(i)(0): - writer.output.add BLOB_START_MARKER - elif i < typeof(i)(BLOB_START_MARKER): - writer.output.add byte(i) - else: - let bytesNeeded = i.bytesNeeded - writer.writeCount(bytesNeeded, BLOB_START_MARKER) - - writer.output.setLen(writer.output.len + bytesNeeded) - writer.output.writeBigEndian(i, writer.output.len - 1, bytesNeeded) - proc initRlpWriter*: RlpDefaultWriter = # Avoid allocations during initial write of small items - since the writer is # expected to be short-lived, it doesn't hurt to allocate this buffer result -# nothing to do when serializing using tracker -proc maybeClosePendingLists(self: var RlpTwoPassWriter) = - while self.pendingLists.len > 0: - let lastListIdx = self.pendingLists.len - 1 - doAssert self.pendingLists[lastListIdx].remainingItems > 0 - - self.pendingLists[lastListIdx].remainingItems -= 1 - # if one last item is remaining in the list - if self.pendingLists[lastListIdx].remainingItems == 0: - # A list have been just finished. It was started in `startList`. - let listStartPos = self.pendingLists[lastListIdx].startPos - let prefixLen = self.pendingLists[lastListIdx].prefixLen - - self.pendingLists.setLen lastListIdx - - let listLen = self.output.len - listStartPos - prefixLen - - # Write out the prefix length - if listLen < THRESHOLD_LIST_LEN: - self.output[listStartPos] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = prefixLen - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) - else: - # The currently open list is not finished yet. Nothing to do. - return - - -proc maybeClosePendingLists(self: var RlpDefaultWriter) = - while self.pendingLists.len > 0: - let lastListIdx = self.pendingLists.len - 1 - doAssert self.pendingLists[lastListIdx].remainingItems > 0 - - self.pendingLists[lastListIdx].remainingItems -= 1 - # if one last item is remaining in the list - if self.pendingLists[lastListIdx].remainingItems == 0: - # A list have been just finished. It was started in `startList`. - let listStartPos = self.pendingLists[lastListIdx].startPos - - self.pendingLists.setLen lastListIdx - - let listLen = self.output.len - listStartPos - - let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 - - #Shift the written data to make room for the prefix length - self.output.setLen(self.output.len + totalPrefixBytes) - - moveMem(addr self.output[listStartPos + totalPrefixBytes], - unsafeAddr self.output[listStartPos], - listLen) - - # Write out the prefix length - if listLen < THRESHOLD_LIST_LEN: - self.output[listStartPos] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = totalPrefixBytes - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) - else: - # The currently open list is not finished yet. Nothing to do. - return - -proc maybeClosePendingLists(self: var RlpLengthTracker) = - while self.pendingLists.len > 0: - let lastIdx = self.pendingLists.len - 1 - self.pendingLists[lastIdx].remainingItems -= 1 - - if self.pendingLists[lastIdx].remainingItems == 0: - let listIdx = self.pendingLists[lastIdx].idx - let startLen = self.pendingLists[lastIdx].length - - let listLen = self.totalLength - startLen - - let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 - - # save the prefix - #self.listPrefixBytes[listIdx].prefix = calculateListPrefix(listLen, prefixLen) - # take note of the prefix len - self.listPrefixBytes[listIdx] = prefixLen - # close the list by deleting - self.pendingLists.setLen(lastIdx) - - self.totalLength += prefixLen - else: - return - -func appendRawBytes*(self: var RlpWriter, bytes: openArray[byte]) = - self.output.setLen(self.output.len + bytes.len) - assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) - self.maybeClosePendingLists() - -func appendRawBytes*(self: var RlpLengthTracker, bytes: openArray[byte]) = - self.totalLength += bytes.len - self.maybeClosePendingLists() - -proc startList*(self: var RlpTwoPassWriter, listSize: int) = - if listSize == 0: - self.writeCount(0, LIST_START_MARKER) - self.maybeClosePendingLists() - else: - let prefixLen = self.listPrefixBytes[0] - self.listPrefixBytes.delete(0) - self.pendingLists.add((listSize, self.output.len, prefixLen)) - self.output.setLen(self.output.len + prefixLen) - -proc appendBlob(self: var RlpWriter, bytes: openArray[byte]) = - if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: - self.output.add byte(bytes[0]) - self.maybeClosePendingLists() - else: - self.writeCount(bytes.len, BLOB_START_MARKER) - self.appendRawBytes(bytes) - -proc startList*(self: var RlpDefaultWriter, listSize: int) = - if listSize == 0: - self.writeCount(0, LIST_START_MARKER) - self.maybeClosePendingLists() - else: - self.pendingLists.add((listSize, self.output.len)) - -proc startList*(self: var RlpLengthTracker, listSize: int) = - if listSize == 0: - self.totalLength += 1 - self.maybeClosePendingLists() - else: - # open a list - self.pendingLists.add((self.listCount, listSize, self.totalLength)) - self.listCount += 1 - self.listPrefixBytes.add(0) - -func lengthCount(count: int): int {.inline.} = - return if count < THRESHOLD_LIST_LEN: 1 - else: uint64(count).bytesNeeded + 1 - -func appendBlob(self: var RlpLengthTracker, data: openArray[byte]) = - if data.len == 1 and byte(data[0]) < BLOB_START_MARKER: - self.totalLength += 1 - else: - self.totalLength += lengthCount(data.len) + data.len - self.maybeClosePendingLists() - -func appendInt(self: var RlpLengthTracker, i: SomeUnsignedInt) = - if i < typeof(i)(BLOB_START_MARKER): - self.totalLength += 1 - else: - self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded - self.maybeClosePendingLists() +template appendBlob(self: var RlpWriter, data: openArray[byte]) = + self.writeBlob(data) proc appendInt(self: var RlpWriter, i: SomeUnsignedInt) = # this is created as a separate proc as an extra precaution against # any overloading resolution problems when matching the IntLike concept. self.writeInt(i) - self.maybeClosePendingLists() - template appendImpl(self: var RlpWriter, data: openArray[byte]) = self.appendBlob(data) @@ -379,24 +182,6 @@ proc initRlpList*(listSize: int): RlpDefaultWriter = result = initRlpWriter() startList(result, listSize) -template finish*(self: RlpDefaultWriter): seq[byte] = - doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - self.output - -template finish*(self: RlpTwoPassWriter): seq[byte] = - doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - doAssert self.listPrefixBytes.len == 0, "Insufficient number of list prefixes accounted for" - self.output - -func clear*(w: var RlpWriter) = - # Prepare writer for reuse - w.pendingLists.setLen(0) - when typeof(w) is RlpDefaultWriter: - w.output.setLen(0) - elif typeof(w) is RlpTwoPassWriter: - w.output.setLen(0) - w.listPrefixBytes.setLen(0) - proc encode*[T](v: T): seq[byte] = mixin append @@ -406,11 +191,24 @@ proc encode*[T](v: T): seq[byte] = tracker.append(v) writer.output = newSeqOfCap[byte](tracker.totalLength) - writer.listPrefixBytes = tracker.listPrefixBytes + writer.listPrefixLen = tracker.listPrefixLen writer.append(v) move(writer.finish) +proc encodeHash*[T](v: T): Hash32 = + mixin append + + var tracker: RlpLengthTracker + var writer: RlpHashWriter + + tracker.append(v) + + writer.listPrefixBytes = tracker.listPrefixBytes + writer.append(v) + + writer.finish.to(Hash32) + func encodeInt*(i: SomeUnsignedInt): RlpIntBuf = var buf: RlpIntBuf diff --git a/tests/rlp/all_tests.nim b/tests/rlp/all_tests.nim index cd50c8b6..f2debf83 100644 --- a/tests/rlp/all_tests.nim +++ b/tests/rlp/all_tests.nim @@ -4,4 +4,5 @@ import ./test_empty_string, ./test_object_serialization, ./test_optional_fields, + ./test_hashes, ./test_rlp_profiler diff --git a/tests/rlp/test_hashes.nim b/tests/rlp/test_hashes.nim new file mode 100644 index 00000000..a609cc54 --- /dev/null +++ b/tests/rlp/test_hashes.nim @@ -0,0 +1,51 @@ +{.used.} + +import + ../../eth/[rlp, common], + unittest2, + times, + os, + strutils + +const + accesses = @[AccessPair( + address: address"0x0000000000000000000000000000000000000001", + storageKeys: @[default(Bytes32)] + )] + +let myTx = Transaction( + txType: TxEip1559, + chainId: 1.ChainId, + nonce: 0.AccountNonce, + gasLimit: 123457.GasInt, + maxPriorityFeePerGas: 42.GasInt, + maxFeePerGas: 10.GasInt, + accessList: accesses +) + +let blkSeq = @[ + BlockBody( + transactions: @[ + Transaction(nonce: 1)]), + BlockBody( + uncles: @[Header(nonce: Bytes8([0x20u8,0,0,0,0,0,0,0]))]), + BlockBody(), + BlockBody( + transactions: @[ + Transaction(nonce: 3), + Transaction(nonce: 4)])] + +proc encodeAndHash[T](v: T): Hash32 = + var writer = initRlpWriter() + + writer.append(v) + + keccak256(writer.finish()) + +suite "test simulatneous encoding and hashing using hash writer": + test "sanity check - Transaction": + let hashedTx = encodeAndHash(myTx) + check rlp.encodeHash(myTx) == hashedTx + test "sanity check - Transaction": + let hashedBlockSeq = encodeAndHash(blkSeq) + check rlp.encodeHash(blkSeq) == hashedBlockSeq diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index c8dca027..fcbe5851 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -52,14 +52,27 @@ proc encodeOnePass[T](v: T): seq[byte] = writer.append(v) move(writer.finish) +proc encodeAndHash[T](v: T): Hash32 = + keccak256(encodeOnePass(v)) -suite "test running time of rlp serialization": - test "transaction serialization": - benchmark "Transaction serialization (two pass)": - let myTxBytes = rlp.encode(myTx) - benchmark "Block Sequence serialization (two pass)": - let myBlockBytes = rlp.encode(blkSeq) - benchmark "Transaction serialization (one pass)": - let myTxBytesOnePass = encodeOnePass(myTx) - benchmark "Block Sequence serailization (one pass)": - let myBlockBytesOnePass = encodeOnePass(blkSeq) +suite "test running times of rlp encode and encodeHash": + test "encoding using two pass writer": + benchmark "Transaction serialization": + let bytes1 = rlp.encode(myTx) + benchmark "Block Sequence serialization": + let bytes2 = rlp.encode(blkSeq) + test "encoding using default writer": + benchmark "Transaction serialization": + let bytes3 = encodeOnePass(myTx) + benchmark "Block Sequence serailization": + let bytes4 = encodeOnePass(blkSeq) + test "encoding and hashing using hash writer": + benchmark "Transaction serialization": + let bytes5 = rlp.encodeHash(myTx) + benchmark "Block Sequence serailization": + let bytes6 = rlp.encodeHash(blkSeq) + test "encoding and hashin using default writer": + benchmark "Transaction serialization": + let bytes7 = rlp.encodeHash(myTx) + benchmark "Block Sequence serailization": + let bytes8 = rlp.encodeHash(blkSeq) From 2fd4ba6f459b304fd497c550ddcf9d02206c68ab Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 12 Dec 2024 12:06:35 +0530 Subject: [PATCH 19/36] fix --- eth/rlp/hash_writer.nim | 7 +++++-- tests/rlp/test_rlp_profiler.nim | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index 007ad289..36c43dd6 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -11,7 +11,7 @@ type keccak: keccak.keccak256 listPrefixBytes*: seq[seq[byte]] -func update(writer: var RlpHashWriter, data: byte) = +template update(writer: var RlpHashWriter, data: byte) = writer.keccak.update([data]) template update(writer: var RlpHashWriter, data: openArray[byte]) = @@ -26,6 +26,7 @@ func writeCount(writer: var RlpHashWriter, count: int, baseMarker: byte) = writer.update baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) var buf = newSeqOfCap[byte](lenPrefixBytes) + buf.setLen(lenPrefixBytes) buf.writeBigEndian(uint64(count), buf.len - 1, lenPrefixBytes) writer.update(buf) @@ -39,6 +40,7 @@ func writeInt*(writer: var RlpHashWriter, i: SomeUnsignedInt) = writer.writeCount(bytesNeeded, BLOB_START_MARKER) var buf = newSeqOfCap[byte](bytesNeeded) + buf.setLen(bytesNeeded) buf.writeBigEndian(uint64(i), buf.len - 1, bytesNeeded) writer.update(buf) @@ -56,7 +58,8 @@ proc startList*(self: var RlpHashWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixBytes = self.listPrefixBytes.pop() + let prefixBytes = self.listPrefixBytes[0] + self.listPrefixBytes.delete(0) self.update(prefixBytes) template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index fcbe5851..64a8ebea 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -73,6 +73,6 @@ suite "test running times of rlp encode and encodeHash": let bytes6 = rlp.encodeHash(blkSeq) test "encoding and hashin using default writer": benchmark "Transaction serialization": - let bytes7 = rlp.encodeHash(myTx) + let bytes7 = encodeAndHash(myTx) benchmark "Block Sequence serailization": - let bytes8 = rlp.encodeHash(blkSeq) + let bytes8 = encodeAndHash(blkSeq) From 618ebab8c044a2454523a51be5913f474150cd27 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 12 Dec 2024 13:51:43 +0530 Subject: [PATCH 20/36] remove unwanted files --- eth/rlp/chunked_buffer.nim | 93 -------------------------------------- 1 file changed, 93 deletions(-) delete mode 100644 eth/rlp/chunked_buffer.nim diff --git a/eth/rlp/chunked_buffer.nim b/eth/rlp/chunked_buffer.nim deleted file mode 100644 index 9d1055db..00000000 --- a/eth/rlp/chunked_buffer.nim +++ /dev/null @@ -1,93 +0,0 @@ -import - stew/[arraybuf, assign2, bitops2, shims/macros] - -type - # a 200-byte chunk matches the block length of keccak(1600bits)v - # TODO: use some compile time technique to abstract out the size(200) - BufferChunk = array[200, byte] - RefBufferChunk = ref BufferChunk - - ChunkedBuffer* = object - fillLevel: int - chunks: seq[RefBufferChunk] - -func `$`(chunk: RefBufferChunk): string = - $(chunk[]) - -func initChunkedBuffer(): ChunkedBuffer = - let newChunk = new(RefBufferChunk) - result.chunks.add(newChunk) - result.fillLevel = 0 - -func curChunkIdx(buffer: ChunkedBuffer): int = - return (buffer.fillLevel mod len(BufferChunk)) - -func isCurChunkFull(buffer: ChunkedBuffer): bool = - (len(buffer.chunks) * len(BufferChunk)) == buffer.fillLevel - -func curChunkRemaining(buffer: ChunkedBuffer): int = - return len(BufferChunk) - (buffer.fillLevel mod len(BufferChunk)) - -func append*(buffer: var ChunkedBuffer, data: openArray[byte]) = - var remainingBytes = len(data) - - # debugEcho buffer.fillLevel, (len(buffer.chunks) * len(BufferChunk)) - - if buffer.isCurChunkFull: - let newChunk = new(RefBufferChunk) - buffer.chunks.add(newChunk) - - while remainingBytes > 0: - let startIdx = len(data) - remainingBytes - let numBytes = if buffer.curChunkRemaining < remainingBytes: buffer.curChunkRemaining else: remainingBytes - let endIdx = startIdx + numBytes - 1 - let chunkIdx = buffer.fillLevel div len(BufferChunk) - - # debugEcho startIdx, " ", endIdx, " ", chunkIdx, " ", buffer.curChunkRemaining - - assign( - buffer.chunks[chunkIdx][].toOpenArray(buffer.curChunkIdx, buffer.curChunkIdx + numBytes - 1), - data[startIdx..endIdx] - ) - - buffer.fillLevel += numBytes - remainingBytes -= numBytes - - if remainingBytes > 0: - let newChunk = new(RefBufferChunk) - buffer.chunks.add(newChunk) - - -# TODO: idx shouldn't only be type int. Technically it should be able to accomodate SomeOrdinal -func `[]`*(buffer: ChunkedBuffer, idx: int): byte = - buffer.chunks[idx div len(BufferChunk)][idx mod len(BufferChunk)] - -# TODO: idx shouldn't only be type int. Technically it should be able to accomodate SomeOrdinal -func `[]=`*(buffer: var ChunkedBuffer, idx: int, value: byte) = - buffer.chunks[idx div len(BufferChunk)][idx mod len(BufferChunk)] = value - -func append*(buffer: var ChunkedBuffer, data: byte) = - if buffer.isCurChunkFull: - let newChunk = new(RefBufferChunk) - buffer.chunks.add(newChunk) - - buffer[buffer.fillLevel] = data - buffer.fillLevel += 1 - -func consume*(buffer: var ChunkedBuffer): BufferChunk = - let chunk = buffer.chunks[0][] # this will copy the chunk - buffer.chunks.delete(0) - buffer.fillLevel -= len(BufferChunk) - chunk - -func addGapChunk*(buffer: var ChunkedBuffer) = - let newChunk = new(RefBufferChunk) - buffer.chunks.add(newChunk) - buffer.fillLevel = len(buffer.chunks) * len(BufferChunk) - -func removeChunk*(buffer: var ChunkedBuffer, chunkIdx: int) = - buffer.fillLevel -= len(BufferChunk) - buffer.chunks.delete(chunkIdx) - -func completeCurChunk*(buffer: var ChunkedBuffer) = - buffer.fillLevel = len(buffer.chunks) * len(BufferChunk) From e476cfca62c1e61eb0894f64b5b12d0936e6bf9c Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 12 Dec 2024 22:29:51 +0530 Subject: [PATCH 21/36] conditional length writer execution --- eth/rlp/length_writer.nim | 31 +++++++++++++++++++++---------- eth/rlp/writer.nim | 4 ++-- tests/rlp/test_rlp_profiler.nim | 16 ++++++++-------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index 429ab612..1b6aafb0 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -6,11 +6,16 @@ import utils type + TrackerKind* = enum + RecordLen, + RecordPrefix + RlpLengthTracker* = object + case kind*: TrackerKind + of RecordLen: listPrefixLen*: seq[int] + of RecordPrefix: listPrefixBytes*: seq[seq[byte]] pendingLists: seq[tuple[idx, remainingItems, length: int]] listCount: int - listPrefixLen*: seq[int] - listPrefixBytes*: seq[seq[byte]] totalLength*: int proc calculateListPrefix(listLen, prefixLen: int): seq[byte] = @@ -41,12 +46,14 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = else: int(uint64(listLen).bytesNeeded) + 1 # save the prefix - self.listPrefixBytes[listIdx] = calculateListPrefix(listLen, prefixLen) - # take note of the prefix len - self.listPrefixLen[listIdx] = prefixLen + if self.kind == TrackerKind.RecordPrefix: + self.listPrefixBytes[listIdx] = calculateListPrefix(listLen, prefixLen) + else: # take note of the prefix len + self.listPrefixLen[listIdx] = prefixLen + # close the list by deleting self.pendingLists.setLen(lastIdx) - + self.totalLength += prefixLen else: return @@ -63,8 +70,10 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = # open a list self.pendingLists.add((self.listCount, listSize, self.totalLength)) self.listCount += 1 - self.listPrefixLen.add(0) - self.listPrefixBytes.add(@[]) + if self.kind == TrackerKind.RecordLen: + self.listPrefixLen.add(0) + else: + self.listPrefixBytes.add(@[]) func lengthCount(count: int): int {.inline.} = return if count < THRESHOLD_LIST_LEN: 1 @@ -91,5 +100,7 @@ template finish*(self: RlpLengthTracker): seq[seq[byte]] = func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse w.pendingLists.setLen(0) - w.listPrefixLen.setLen(0) - w.listPrefixBytes.setLen(0) + if w.kind == TrackerKind.RecordLen: + w.listPrefixLen.setLen(0) + else: + w.listPrefixBytes.setLen(0) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 341223cb..0b34e8fb 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -185,7 +185,7 @@ proc initRlpList*(listSize: int): RlpDefaultWriter = proc encode*[T](v: T): seq[byte] = mixin append - var tracker: RlpLengthTracker + var tracker = RlpLengthTracker(kind: TrackerKind.RecordLen) var writer: RlpTwoPassWriter tracker.append(v) @@ -199,7 +199,7 @@ proc encode*[T](v: T): seq[byte] = proc encodeHash*[T](v: T): Hash32 = mixin append - var tracker: RlpLengthTracker + var tracker = RlpLengthTracker(kind: TrackerKind.RecordPrefix) var writer: RlpHashWriter tracker.append(v) diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index 64a8ebea..45a862c3 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -57,22 +57,22 @@ proc encodeAndHash[T](v: T): Hash32 = suite "test running times of rlp encode and encodeHash": test "encoding using two pass writer": - benchmark "Transaction serialization": + benchmark "Transaction serialization using two pass writer": let bytes1 = rlp.encode(myTx) - benchmark "Block Sequence serialization": + benchmark "Block Sequence serialization using two pass writer": let bytes2 = rlp.encode(blkSeq) test "encoding using default writer": - benchmark "Transaction serialization": + benchmark "Transaction serialization using default writer": let bytes3 = encodeOnePass(myTx) - benchmark "Block Sequence serailization": + benchmark "Block Sequence serailization using default writer": let bytes4 = encodeOnePass(blkSeq) test "encoding and hashing using hash writer": - benchmark "Transaction serialization": + benchmark "Transaction hashing using hash writer": let bytes5 = rlp.encodeHash(myTx) - benchmark "Block Sequence serailization": + benchmark "Block Sequence hashing using hash writer": let bytes6 = rlp.encodeHash(blkSeq) test "encoding and hashin using default writer": - benchmark "Transaction serialization": + benchmark "Transaction hashing using default writer and then hash": let bytes7 = encodeAndHash(myTx) - benchmark "Block Sequence serailization": + benchmark "Block Sequence hashing using default writer and then hash": let bytes8 = encodeAndHash(blkSeq) From e6dabc8f8ba160539cb98d8d3e105f79fb2be555 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 14 Dec 2024 13:15:19 +0530 Subject: [PATCH 22/36] perf improvements - remove adds --- eth/rlp/hash_writer.nim | 27 ++++++++--- eth/rlp/length_writer.nim | 49 ++++++++++---------- eth/rlp/two_pass_writer.nim | 90 +++++++++++++++++-------------------- eth/rlp/writer.nim | 13 +++--- 4 files changed, 94 insertions(+), 85 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index 36c43dd6..15fe6b6e 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -9,7 +9,8 @@ import type RlpHashWriter* = object keccak: keccak.keccak256 - listPrefixBytes*: seq[seq[byte]] + listLengths*: seq[int] + prefixLengths*: seq[int] template update(writer: var RlpHashWriter, data: byte) = writer.keccak.update([data]) @@ -58,16 +59,30 @@ proc startList*(self: var RlpHashWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixBytes = self.listPrefixBytes[0] - self.listPrefixBytes.delete(0) - self.update(prefixBytes) + let prefixLen = self.prefixLengths[0] + let listLen = self.listLengths[0] + self.prefixLengths.delete(0) + self.listLengths.delete(0) + + if listLen < THRESHOLD_LIST_LEN: + self.update(LIST_START_MARKER + byte(listLen)) + else: + let listLenBytes = prefixLen - 1 + self.update(LEN_PREFIXED_LIST_MARKER + byte(listLenBytes)) + + var buf = newSeqOfCap[byte](listLenBytes) + buf.setLen(listLenBytes) + buf.writeBigEndian(uint64(listLen), buf.len - 1, listLenBytes) + self.update(buf) template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = - doAssert self.listPrefixBytes.len == 0, "Insufficient number of elements written to a started list" + self.listLengths.setLen(0) + self.prefixLengths.setLen(0) self.keccak.finish() func clear*(w: var RlpHashWriter) = # Prepare writer for reuse - w.listPrefixBytes.setLen(0) + w.listLengths.setLen(0) + w.prefixLengths.setLen(0) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index 1b6aafb0..efa17dbb 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -6,15 +6,11 @@ import utils type - TrackerKind* = enum - RecordLen, - RecordPrefix - + ListAndPrefixLengths = tuple[listLengths, prefixLengths: seq[int]] RlpLengthTracker* = object - case kind*: TrackerKind - of RecordLen: listPrefixLen*: seq[int] - of RecordPrefix: listPrefixBytes*: seq[seq[byte]] - pendingLists: seq[tuple[idx, remainingItems, length: int]] + prefixLengths*: seq[int] + listLengths*: seq[int] + pendingLists: seq[tuple[idx, remainingItems, startLen: int]] listCount: int totalLength*: int @@ -38,18 +34,18 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = if self.pendingLists[lastIdx].remainingItems == 0: let listIdx = self.pendingLists[lastIdx].idx - let startLen = self.pendingLists[lastIdx].length + let startLen = self.pendingLists[lastIdx].startLen let listLen = self.totalLength - startLen let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 - # save the prefix - if self.kind == TrackerKind.RecordPrefix: - self.listPrefixBytes[listIdx] = calculateListPrefix(listLen, prefixLen) - else: # take note of the prefix len - self.listPrefixLen[listIdx] = prefixLen + # save the list lengths and prefix lengths + self.listLengths[listIdx] = listLen + self.prefixLengths[listIdx] = prefixLen + + #TODO: extend the lists if they cross length of 50 by 50. # close the list by deleting self.pendingLists.setLen(lastIdx) @@ -70,10 +66,6 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = # open a list self.pendingLists.add((self.listCount, listSize, self.totalLength)) self.listCount += 1 - if self.kind == TrackerKind.RecordLen: - self.listPrefixLen.add(0) - else: - self.listPrefixBytes.add(@[]) func lengthCount(count: int): int {.inline.} = return if count < THRESHOLD_LIST_LEN: 1 @@ -93,14 +85,23 @@ func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded self.maybeClosePendingLists() -template finish*(self: RlpLengthTracker): seq[seq[byte]] = +func initLengthTracker*(): RlpLengthTracker = + + # TODO: Don't hardcode 50 + + # we preset the lengths since we want to skip using add method for + # these lists + result.prefixLengths = newSeqOfCap[int](50) + result.prefixLengths.setLen(50) + result.listLengths = newSeqOfCap[int](50) + result.listLengths.setLen(50) + +template finish*(self: RlpLengthTracker): ListAndPrefixLengths = doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - self.listPrefixBytes + (self.listLengths, self.prefixLengths) func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse w.pendingLists.setLen(0) - if w.kind == TrackerKind.RecordLen: - w.listPrefixLen.setLen(0) - else: - w.listPrefixBytes.setLen(0) + w.prefixLengths.setLen(0) + w.listLengths.setLen(0) diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index fa0d17e4..454d31e9 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -9,69 +9,46 @@ type RlpTwoPassWriter* = object pendingLists*: seq[tuple[remainingItems, startPos, prefixLen: int]] output*: seq[byte] - listPrefixLen*: seq[int] + prefixLengths*: seq[int] + listLengths*: seq[int] + fillLevel: int func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: - writer.output.add(baseMarker + byte(count)) + writer.output[writer.fillLevel] = (baseMarker + byte(count)) + writer.fillLevel += 1 else: let lenPrefixBytes = uint64(count).bytesNeeded - writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - - writer.output.setLen(writer.output.len + lenPrefixBytes) - writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) + writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + writer.fillLevel += lenPrefixBytes + 1 + writer.output.writeBigEndian(uint64(count), writer.fillLevel - 1, lenPrefixBytes) # nothing to do when serializing using tracker -proc maybeClosePendingLists(self: var RlpTwoPassWriter) = - while self.pendingLists.len > 0: - let lastListIdx = self.pendingLists.len - 1 - doAssert self.pendingLists[lastListIdx].remainingItems > 0 - - self.pendingLists[lastListIdx].remainingItems -= 1 - # if one last item is remaining in the list - if self.pendingLists[lastListIdx].remainingItems == 0: - # A list have been just finished. It was started in `startList`. - let listStartPos = self.pendingLists[lastListIdx].startPos - let prefixLen = self.pendingLists[lastListIdx].prefixLen - - self.pendingLists.setLen lastListIdx - - let listLen = self.output.len - listStartPos - prefixLen - - # Write out the prefix length - if listLen < THRESHOLD_LIST_LEN: - self.output[listStartPos] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = prefixLen - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) - else: - # The currently open list is not finished yet. Nothing to do. - return +template maybeClosePendingLists(self: var RlpTwoPassWriter) = discard func writeInt*(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): - writer.output.add BLOB_START_MARKER + writer.output[writer.fillLevel] = BLOB_START_MARKER + writer.fillLevel += 1 elif i < typeof(i)(BLOB_START_MARKER): - writer.output.add byte(i) + writer.output[writer.fillLevel] = byte(i) + writer.fillLevel += 1 else: let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) - writer.output.setLen(writer.output.len + bytesNeeded) - writer.output.writeBigEndian(i, writer.output.len - 1, bytesNeeded) - writer.maybeClosePendingLists() + writer.fillLevel += bytesNeeded + writer.output.writeBigEndian(i, writer.fillLevel - 1, bytesNeeded) func appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = - self.output.setLen(self.output.len + bytes.len) - assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) - self.maybeClosePendingLists() + self.fillLevel += bytes.len + assign(self.output.toOpenArray(self.fillLevel - bytes.len, self.fillLevel - 1), bytes) proc writeBlob*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: - self.output.add byte(bytes[0]) - self.maybeClosePendingLists() + self.output[self.fillLevel] = byte(bytes[0]) + self.fillLevel += 1 else: self.writeCount(bytes.len, BLOB_START_MARKER) self.appendRawBytes(bytes) @@ -81,20 +58,35 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) - self.maybeClosePendingLists() else: - let prefixLen = self.listPrefixLen[0] - self.listPrefixLen.delete(0) - self.pendingLists.add((listSize, self.output.len, prefixLen)) - self.output.setLen(self.output.len + prefixLen) + let prefixLen = self.prefixLengths[0] + let listLen = self.listLengths[0] + self.prefixLengths.delete(0) + self.listLengths.delete(0) + + if listLen < THRESHOLD_LIST_LEN: + self.output[self.fillLevel] = LIST_START_MARKER + byte(listLen) + self.fillLevel += 1 + else: + let listLenBytes = prefixLen - 1 + self.output[self.fillLevel] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) + self.fillLevel += prefixLen + self.output.writeBigEndian(uint64(listLen), self.fillLevel - 1, listLenBytes) + +func initTwoPassWriter*(length: int): RlpTwoPassWriter = + result.fillLevel = 0 + result.output = newSeqOfCap[byte](length) + result.output.setLen(length) template finish*(self: RlpTwoPassWriter): seq[byte] = doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - doAssert self.listPrefixLen.len == 0, "Insufficient number of list prefixes accounted for" + self.prefixLengths.setLen(0) + self.listLengths.setLen(0) self.output func clear*(w: var RlpTwoPassWriter) = # Prepare writer for reuse w.pendingLists.setLen(0) w.output.setLen(0) - w.listPrefixLen.setLen(0) + w.prefixLengths.setLen(0) + w.listLengths.setLen(0) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 0b34e8fb..1a7b6142 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -185,13 +185,13 @@ proc initRlpList*(listSize: int): RlpDefaultWriter = proc encode*[T](v: T): seq[byte] = mixin append - var tracker = RlpLengthTracker(kind: TrackerKind.RecordLen) - var writer: RlpTwoPassWriter + var tracker = initLengthTracker() tracker.append(v) - writer.output = newSeqOfCap[byte](tracker.totalLength) - writer.listPrefixLen = tracker.listPrefixLen + var writer = initTwoPassWriter(tracker.totalLength) + writer.prefixLengths = tracker.prefixLengths + writer.listLengths = tracker.listLengths writer.append(v) move(writer.finish) @@ -199,12 +199,13 @@ proc encode*[T](v: T): seq[byte] = proc encodeHash*[T](v: T): Hash32 = mixin append - var tracker = RlpLengthTracker(kind: TrackerKind.RecordPrefix) + var tracker = initLengthTracker() var writer: RlpHashWriter tracker.append(v) - writer.listPrefixBytes = tracker.listPrefixBytes + writer.prefixLengths = tracker.prefixLengths + writer.listLengths = tracker.listLengths writer.append(v) writer.finish.to(Hash32) From 2e5f6caa9b7cffe29e0a6e03147ebc1e6fa77b84 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sat, 14 Dec 2024 14:52:20 +0530 Subject: [PATCH 23/36] remove delete operations --- eth/rlp/two_pass_writer.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index 454d31e9..8b17e7eb 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -12,6 +12,7 @@ type prefixLengths*: seq[int] listLengths*: seq[int] fillLevel: int + listCount: int func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: @@ -59,10 +60,9 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixLen = self.prefixLengths[0] - let listLen = self.listLengths[0] - self.prefixLengths.delete(0) - self.listLengths.delete(0) + let prefixLen = self.prefixLengths[self.listCount] + let listLen = self.listLengths[self.listCount] + self.listCount += 1 if listLen < THRESHOLD_LIST_LEN: self.output[self.fillLevel] = LIST_START_MARKER + byte(listLen) @@ -75,6 +75,7 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = func initTwoPassWriter*(length: int): RlpTwoPassWriter = result.fillLevel = 0 + result.listCount = 0 result.output = newSeqOfCap[byte](length) result.output.setLen(length) From 220cf2e85c3dbab97e7b456819f5a5ef00e05dda Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sun, 15 Dec 2024 13:49:39 +0530 Subject: [PATCH 24/36] do not use delete --- eth/rlp/hash_writer.nim | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index 15fe6b6e..cd015cfb 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -11,6 +11,7 @@ type keccak: keccak.keccak256 listLengths*: seq[int] prefixLengths*: seq[int] + listCount: int template update(writer: var RlpHashWriter, data: byte) = writer.keccak.update([data]) @@ -59,10 +60,10 @@ proc startList*(self: var RlpHashWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixLen = self.prefixLengths[0] - let listLen = self.listLengths[0] - self.prefixLengths.delete(0) - self.listLengths.delete(0) + let prefixLen = self.prefixLengths[self.listCount] + let listLen = self.listLengths[self.listCount] + self.listCount += 1 + if listLen < THRESHOLD_LIST_LEN: self.update(LIST_START_MARKER + byte(listLen)) @@ -85,4 +86,3 @@ func clear*(w: var RlpHashWriter) = w.listLengths.setLen(0) w.prefixLengths.setLen(0) - From 0667c925585db07e20f582ebd9170d10e401a10c Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sun, 15 Dec 2024 15:03:04 +0530 Subject: [PATCH 25/36] lint code --- eth/rlp/default_writer.nim | 30 ++++++++++++--------- eth/rlp/hash_writer.nim | 9 ++++--- eth/rlp/length_writer.nim | 54 ++++++++++++++----------------------- eth/rlp/two_pass_writer.nim | 24 ++++++++++------- 4 files changed, 56 insertions(+), 61 deletions(-) diff --git a/eth/rlp/default_writer.nim b/eth/rlp/default_writer.nim index 7d3cf41e..87f51288 100644 --- a/eth/rlp/default_writer.nim +++ b/eth/rlp/default_writer.nim @@ -16,10 +16,12 @@ func writeCount(writer: var RlpDefaultWriter, count: int, baseMarker: byte) = else: let lenPrefixBytes = uint64(count).bytesNeeded - writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - + writer.output.add baseMarker + (THRESHOLD_LIST_LEN - 1) + + byte(lenPrefixBytes) + writer.output.setLen(writer.output.len + lenPrefixBytes) - writer.output.writeBigEndian(uint64(count), writer.output.len - 1, lenPrefixBytes) + writer.output.writeBigEndian(uint64(count), writer.output.len - 1, + lenPrefixBytes) proc maybeClosePendingLists(self: var RlpDefaultWriter) = while self.pendingLists.len > 0: @@ -31,12 +33,11 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = if self.pendingLists[lastListIdx].remainingItems == 0: # A list have been just finished. It was started in `startList`. let listStartPos = self.pendingLists[lastListIdx].startPos - self.pendingLists.setLen lastListIdx - let listLen = self.output.len - listStartPos - - let totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 + let + listLen = self.output.len - listStartPos + totalPrefixBytes = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 #Shift the written data to make room for the prefix length @@ -51,8 +52,11 @@ proc maybeClosePendingLists(self: var RlpDefaultWriter) = self.output[listStartPos] = LIST_START_MARKER + byte(listLen) else: let listLenBytes = totalPrefixBytes - 1 - self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - self.output.writeBigEndian(uint64(listLen), listStartPos + listLenBytes, listLenBytes) + self.output[listStartPos] = LEN_PREFIXED_LIST_MARKER + + byte(listLenBytes) + + self.output.writeBigEndian(uint64(listLen), + listStartPos + listLenBytes, listLenBytes) else: # The currently open list is not finished yet. Nothing to do. return @@ -72,7 +76,8 @@ func writeInt*(writer: var RlpDefaultWriter, i: SomeUnsignedInt) = func appendRawBytes*(self: var RlpDefaultWriter, bytes: openArray[byte]) = self.output.setLen(self.output.len + bytes.len) - assign(self.output.toOpenArray(self.output.len - bytes.len, self.output.len - 1), bytes) + assign(self.output.toOpenArray( + self.output.len - bytes.len, self.output.len - 1), bytes) self.maybeClosePendingLists() proc writeBlob*(self: var RlpDefaultWriter, bytes: openArray[byte]) = @@ -91,12 +96,11 @@ proc startList*(self: var RlpDefaultWriter, listSize: int) = self.pendingLists.add((listSize, self.output.len)) template finish*(self: RlpDefaultWriter): seq[byte] = - doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + doAssert self.pendingLists.len == 0, + "Insufficient number of elements written to a started list" self.output func clear*(w: var RlpDefaultWriter) = # Prepare writer for reuse w.pendingLists.setLen(0) w.output.setLen(0) - - diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index cd015cfb..be5255f5 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -2,7 +2,7 @@ import std/options, pkg/results, nimcrypto/keccak, - stew/[arraybuf, bitops2, shims/macros], + stew/[arraybuf, shims/macros], ./priv/defs, utils @@ -60,10 +60,11 @@ proc startList*(self: var RlpHashWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixLen = self.prefixLengths[self.listCount] - let listLen = self.listLengths[self.listCount] - self.listCount += 1 + let + prefixLen = self.prefixLengths[self.listCount] + listLen = self.listLengths[self.listCount] + self.listCount += 1 if listLen < THRESHOLD_LIST_LEN: self.update(LIST_START_MARKER + byte(listLen)) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index efa17dbb..a10c512c 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -1,12 +1,11 @@ import std/options, pkg/results, - stew/[arraybuf, assign2, bitops2, shims/macros], + stew/[arraybuf, shims/macros], ./priv/defs, utils type - ListAndPrefixLengths = tuple[listLengths, prefixLengths: seq[int]] RlpLengthTracker* = object prefixLengths*: seq[int] listLengths*: seq[int] @@ -14,18 +13,7 @@ type listCount: int totalLength*: int -proc calculateListPrefix(listLen, prefixLen: int): seq[byte] = - var prefix = newSeqOfCap[byte](1) # prefix min length is 1 - prefix.setLen(prefixLen) - - if listLen < THRESHOLD_LIST_LEN: - prefix[0] = LIST_START_MARKER + byte(listLen) - else: - let listLenBytes = prefixLen - 1 - prefix[0] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) - prefix.writeBigEndian(uint64(listLen), listLenBytes, listLenBytes) - - move(prefix) +const LIST_LENGTH = 50 proc maybeClosePendingLists(self: var RlpLengthTracker) = while self.pendingLists.len > 0: @@ -33,20 +21,17 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = self.pendingLists[lastIdx].remainingItems -= 1 if self.pendingLists[lastIdx].remainingItems == 0: - let listIdx = self.pendingLists[lastIdx].idx - let startLen = self.pendingLists[lastIdx].startLen - - let listLen = self.totalLength - startLen - - let prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 - else: int(uint64(listLen).bytesNeeded) + 1 + let + listIdx = self.pendingLists[lastIdx].idx + startLen = self.pendingLists[lastIdx].startLen + listLen = self.totalLength - startLen + prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 + else: int(uint64(listLen).bytesNeeded) + 1 # save the list lengths and prefix lengths self.listLengths[listIdx] = listLen self.prefixLengths[listIdx] = prefixLen - #TODO: extend the lists if they cross length of 50 by 50. - # close the list by deleting self.pendingLists.setLen(lastIdx) @@ -66,6 +51,9 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = # open a list self.pendingLists.add((self.listCount, listSize, self.totalLength)) self.listCount += 1 + if self.listCount == self.listLengths.len: + self.listLengths.setLen(self.listLengths.len + LIST_LENGTH) + self.prefixLengths.setLen(self.prefixLengths.len + LIST_LENGTH) func lengthCount(count: int): int {.inline.} = return if count < THRESHOLD_LIST_LEN: 1 @@ -86,19 +74,17 @@ func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = self.maybeClosePendingLists() func initLengthTracker*(): RlpLengthTracker = - - # TODO: Don't hardcode 50 - # we preset the lengths since we want to skip using add method for # these lists - result.prefixLengths = newSeqOfCap[int](50) - result.prefixLengths.setLen(50) - result.listLengths = newSeqOfCap[int](50) - result.listLengths.setLen(50) - -template finish*(self: RlpLengthTracker): ListAndPrefixLengths = - doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - (self.listLengths, self.prefixLengths) + result.prefixLengths = newSeqOfCap[int](LIST_LENGTH) + result.prefixLengths.setLen(LIST_LENGTH) + result.listLengths = newSeqOfCap[int](LIST_LENGTH) + result.listLengths.setLen(LIST_LENGTH) + +template finish*(self: RlpLengthTracker): int = + doAssert self.pendingLists.len == 0, + "Insufficient number of elements written to a started list" + self.totalLength func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index 8b17e7eb..523c050a 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -21,12 +21,11 @@ func writeCount(writer: var RlpTwoPassWriter, count: int, baseMarker: byte) = else: let lenPrefixBytes = uint64(count).bytesNeeded - writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) + writer.output[writer.fillLevel] = baseMarker + (THRESHOLD_LIST_LEN - 1) + + byte(lenPrefixBytes) writer.fillLevel += lenPrefixBytes + 1 - writer.output.writeBigEndian(uint64(count), writer.fillLevel - 1, lenPrefixBytes) - -# nothing to do when serializing using tracker -template maybeClosePendingLists(self: var RlpTwoPassWriter) = discard + writer.output.writeBigEndian(uint64(count), + writer.fillLevel - 1, lenPrefixBytes) func writeInt*(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): @@ -44,7 +43,8 @@ func writeInt*(writer: var RlpTwoPassWriter, i: SomeUnsignedInt) = func appendRawBytes*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = self.fillLevel += bytes.len - assign(self.output.toOpenArray(self.fillLevel - bytes.len, self.fillLevel - 1), bytes) + assign(self.output.toOpenArray( + self.fillLevel - bytes.len, self.fillLevel - 1), bytes) proc writeBlob*(self: var RlpTwoPassWriter, bytes: openArray[byte]) = if bytes.len == 1 and byte(bytes[0]) < BLOB_START_MARKER: @@ -60,8 +60,10 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = if listSize == 0: self.writeCount(0, LIST_START_MARKER) else: - let prefixLen = self.prefixLengths[self.listCount] - let listLen = self.listLengths[self.listCount] + let + prefixLen = self.prefixLengths[self.listCount] + listLen = self.listLengths[self.listCount] + self.listCount += 1 if listLen < THRESHOLD_LIST_LEN: @@ -71,7 +73,8 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = let listLenBytes = prefixLen - 1 self.output[self.fillLevel] = LEN_PREFIXED_LIST_MARKER + byte(listLenBytes) self.fillLevel += prefixLen - self.output.writeBigEndian(uint64(listLen), self.fillLevel - 1, listLenBytes) + self.output.writeBigEndian(uint64(listLen), self.fillLevel - 1, + listLenBytes) func initTwoPassWriter*(length: int): RlpTwoPassWriter = result.fillLevel = 0 @@ -80,7 +83,8 @@ func initTwoPassWriter*(length: int): RlpTwoPassWriter = result.output.setLen(length) template finish*(self: RlpTwoPassWriter): seq[byte] = - doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" + doAssert self.pendingLists.len == 0, + "Insufficient number of elements written to a started list" self.prefixLengths.setLen(0) self.listLengths.setLen(0) self.output From 8af1539239c2f7e88daf9bb2b28941d245f5727b Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Sun, 15 Dec 2024 15:06:49 +0530 Subject: [PATCH 26/36] fix build warnings --- tests/rlp/test_api_usage.nim | 2 +- tests/rlp/test_hashes.nim | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index cf9ad2f8..93003853 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -1,7 +1,7 @@ {.used.} import - std/[math, strutils], + std/strutils, unittest2, stew/byteutils, ../../eth/[common, rlp] diff --git a/tests/rlp/test_hashes.nim b/tests/rlp/test_hashes.nim index a609cc54..40298df5 100644 --- a/tests/rlp/test_hashes.nim +++ b/tests/rlp/test_hashes.nim @@ -2,10 +2,7 @@ import ../../eth/[rlp, common], - unittest2, - times, - os, - strutils + unittest2 const accesses = @[AccessPair( From e59befb8b6e49fc7edb9a08cf72f2e3412a64a9f Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 19 Dec 2024 10:17:09 +0530 Subject: [PATCH 27/36] new benchmark contradicting results --- tests/rlp/test_rlp_profiler.nim | 44 ++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index 45a862c3..9e725266 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -4,8 +4,26 @@ import ../../eth/[rlp, common], unittest2, times, - os, - strutils + std/[os, strutils], + stew/io2, + results + +proc readBlock(): EthBlock = + + let + filename = "tests/common/rlps/blocks_1024_td_135112316.rlp" + res = io2.readAllBytes(filename) + + if res.isErr: + echo "failed to import", filename + return + + var + # the encoded rlp can contains one or more blocks + rlpBytes = rlpFromBytes(res.get) + + while rlpBytes.hasData: + result = rlpBytes.read(EthBlock) template benchmark(benchmarkName: string, code: untyped) = block: @@ -46,6 +64,8 @@ let blkSeq = @[ Transaction(nonce: 3), Transaction(nonce: 4)])] +let nonEmptyBlock = readBlock() + proc encodeOnePass[T](v: T): seq[byte] = var writer = initRlpWriter() @@ -61,18 +81,30 @@ suite "test running times of rlp encode and encodeHash": let bytes1 = rlp.encode(myTx) benchmark "Block Sequence serialization using two pass writer": let bytes2 = rlp.encode(blkSeq) + benchmark "Block serialization using two pass writer": + let bytes3 = rlp.encode(nonEmptyBlock) + test "encoding using default writer": benchmark "Transaction serialization using default writer": let bytes3 = encodeOnePass(myTx) benchmark "Block Sequence serailization using default writer": let bytes4 = encodeOnePass(blkSeq) + benchmark "Block serialization using default writer": + let bytes5 = encodeOnePass(nonEmptyBlock) + test "encoding and hashing using hash writer": benchmark "Transaction hashing using hash writer": - let bytes5 = rlp.encodeHash(myTx) + let bytes6 = rlp.encodeHash(myTx) benchmark "Block Sequence hashing using hash writer": - let bytes6 = rlp.encodeHash(blkSeq) + let bytes7 = rlp.encodeHash(blkSeq) + benchmark "Block hashing using hash writer": + let bytes8 = rlp.encodeHash(nonEmptyBlock) + test "encoding and hashin using default writer": benchmark "Transaction hashing using default writer and then hash": - let bytes7 = encodeAndHash(myTx) + let bytes9 = encodeAndHash(myTx) benchmark "Block Sequence hashing using default writer and then hash": - let bytes8 = encodeAndHash(blkSeq) + let bytes10 = encodeAndHash(blkSeq) + benchmark "Block hashing using defualt writer and the hash": + let bytes11 = encodeAndHash(blkSeq) + From 79e2c9bf3ee9cbb631a29bfbcc3509904f82408d Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 19 Dec 2024 10:42:38 +0530 Subject: [PATCH 28/36] use a static buffer for big endian --- eth/rlp/hash_writer.nim | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index be5255f5..adaa2d02 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -12,6 +12,7 @@ type listLengths*: seq[int] prefixLengths*: seq[int] listCount: int + bigEndianBuf: array[8, byte] template update(writer: var RlpHashWriter, data: byte) = writer.keccak.update([data]) @@ -19,6 +20,11 @@ template update(writer: var RlpHashWriter, data: byte) = template update(writer: var RlpHashWriter, data: openArray[byte]) = writer.keccak.update(data) +template updateBigEndian(writer: var RlpHashWriter, i: SomeUnsignedInt, + length: int) = + writer.bigEndianBuf.writeBigEndian(i, length - 1, length) + writer.update(writer.bigEndianBuf.toOpenArray(0, length - 1)) + func writeCount(writer: var RlpHashWriter, count: int, baseMarker: byte) = if count < THRESHOLD_LIST_LEN: writer.update(baseMarker + byte(count)) @@ -27,10 +33,7 @@ func writeCount(writer: var RlpHashWriter, count: int, baseMarker: byte) = writer.update baseMarker + (THRESHOLD_LIST_LEN - 1) + byte(lenPrefixBytes) - var buf = newSeqOfCap[byte](lenPrefixBytes) - buf.setLen(lenPrefixBytes) - buf.writeBigEndian(uint64(count), buf.len - 1, lenPrefixBytes) - writer.update(buf) + writer.updateBigEndian(uint64(count), lenPrefixBytes) func writeInt*(writer: var RlpHashWriter, i: SomeUnsignedInt) = if i == typeof(i)(0): @@ -41,11 +44,8 @@ func writeInt*(writer: var RlpHashWriter, i: SomeUnsignedInt) = let bytesNeeded = i.bytesNeeded writer.writeCount(bytesNeeded, BLOB_START_MARKER) - var buf = newSeqOfCap[byte](bytesNeeded) - buf.setLen(bytesNeeded) - buf.writeBigEndian(uint64(i), buf.len - 1, bytesNeeded) - writer.update(buf) - + writer.updateBigEndian(uint64(i), bytesNeeded) + template appendRawBytes*(self: var RlpHashWriter, bytes: openArray[byte]) = self.update(bytes) @@ -71,11 +71,8 @@ proc startList*(self: var RlpHashWriter, listSize: int) = else: let listLenBytes = prefixLen - 1 self.update(LEN_PREFIXED_LIST_MARKER + byte(listLenBytes)) - - var buf = newSeqOfCap[byte](listLenBytes) - buf.setLen(listLenBytes) - buf.writeBigEndian(uint64(listLen), buf.len - 1, listLenBytes) - self.update(buf) + + self.updateBigEndian(uint64(listLen), listLenBytes) template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = self.listLengths.setLen(0) From b10b1e1b983bba0de9d94316b04a55169201b615 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 19 Dec 2024 11:11:20 +0530 Subject: [PATCH 29/36] make lengths a tuple --- eth/rlp/hash_writer.nim | 19 ++++++++++--------- eth/rlp/length_writer.nim | 19 ++++++------------- eth/rlp/two_pass_writer.nim | 21 +++++++++------------ eth/rlp/writer.nim | 8 ++------ 4 files changed, 27 insertions(+), 40 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index adaa2d02..0d49665c 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -4,13 +4,13 @@ import nimcrypto/keccak, stew/[arraybuf, shims/macros], ./priv/defs, - utils + utils, + length_writer type RlpHashWriter* = object keccak: keccak.keccak256 - listLengths*: seq[int] - prefixLengths*: seq[int] + lengths*: seq[tuple[listLen, prefixLen: int]] listCount: int bigEndianBuf: array[8, byte] @@ -61,8 +61,8 @@ proc startList*(self: var RlpHashWriter, listSize: int) = self.writeCount(0, LIST_START_MARKER) else: let - prefixLen = self.prefixLengths[self.listCount] - listLen = self.listLengths[self.listCount] + prefixLen = self.lengths[self.listCount].prefixLen + listLen = self.lengths[self.listCount].listLen self.listCount += 1 @@ -74,13 +74,14 @@ proc startList*(self: var RlpHashWriter, listSize: int) = self.updateBigEndian(uint64(listLen), listLenBytes) +func initHashWriter*(tracker: var RlpLengthTracker): RlpHashWriter = + result.lengths = move(tracker.lengths) + template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = - self.listLengths.setLen(0) - self.prefixLengths.setLen(0) + self.lengths.setLen(0) self.keccak.finish() func clear*(w: var RlpHashWriter) = # Prepare writer for reuse - w.listLengths.setLen(0) - w.prefixLengths.setLen(0) + w.lengths.setLen(0) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index a10c512c..70a387d5 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -7,8 +7,7 @@ import type RlpLengthTracker* = object - prefixLengths*: seq[int] - listLengths*: seq[int] + lengths*: seq[tuple[listLen, prefixLen: int]] pendingLists: seq[tuple[idx, remainingItems, startLen: int]] listCount: int totalLength*: int @@ -29,8 +28,7 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = else: int(uint64(listLen).bytesNeeded) + 1 # save the list lengths and prefix lengths - self.listLengths[listIdx] = listLen - self.prefixLengths[listIdx] = prefixLen + self.lengths[listIdx] = (listLen, prefixLen) # close the list by deleting self.pendingLists.setLen(lastIdx) @@ -51,9 +49,8 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = # open a list self.pendingLists.add((self.listCount, listSize, self.totalLength)) self.listCount += 1 - if self.listCount == self.listLengths.len: - self.listLengths.setLen(self.listLengths.len + LIST_LENGTH) - self.prefixLengths.setLen(self.prefixLengths.len + LIST_LENGTH) + if self.listCount == self.lengths.len: + self.lengths.setLen(self.lengths.len + LIST_LENGTH) func lengthCount(count: int): int {.inline.} = return if count < THRESHOLD_LIST_LEN: 1 @@ -76,10 +73,7 @@ func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = func initLengthTracker*(): RlpLengthTracker = # we preset the lengths since we want to skip using add method for # these lists - result.prefixLengths = newSeqOfCap[int](LIST_LENGTH) - result.prefixLengths.setLen(LIST_LENGTH) - result.listLengths = newSeqOfCap[int](LIST_LENGTH) - result.listLengths.setLen(LIST_LENGTH) + result.lengths = newSeq[(int, int)](LIST_LENGTH) template finish*(self: RlpLengthTracker): int = doAssert self.pendingLists.len == 0, @@ -89,5 +83,4 @@ template finish*(self: RlpLengthTracker): int = func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse w.pendingLists.setLen(0) - w.prefixLengths.setLen(0) - w.listLengths.setLen(0) + w.lengths.setLen(0) diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index 523c050a..a589fe9a 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -3,14 +3,14 @@ import pkg/results, stew/[arraybuf, assign2, shims/macros], ./priv/defs, - utils + utils, + length_writer type RlpTwoPassWriter* = object pendingLists*: seq[tuple[remainingItems, startPos, prefixLen: int]] output*: seq[byte] - prefixLengths*: seq[int] - listLengths*: seq[int] + lengths*: seq[tuple[listLen, prefixLen: int]] fillLevel: int listCount: int @@ -61,8 +61,8 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = self.writeCount(0, LIST_START_MARKER) else: let - prefixLen = self.prefixLengths[self.listCount] - listLen = self.listLengths[self.listCount] + prefixLen = self.lengths[self.listCount].prefixLen + listLen = self.lengths[self.listCount].listLen self.listCount += 1 @@ -76,22 +76,19 @@ proc startList*(self: var RlpTwoPassWriter, listSize: int) = self.output.writeBigEndian(uint64(listLen), self.fillLevel - 1, listLenBytes) -func initTwoPassWriter*(length: int): RlpTwoPassWriter = +func initTwoPassWriter*(tracker: var RlpLengthTracker): RlpTwoPassWriter = result.fillLevel = 0 result.listCount = 0 - result.output = newSeqOfCap[byte](length) - result.output.setLen(length) + result.output = newSeq[byte](tracker.totalLength) + result.lengths = move(tracker.lengths) template finish*(self: RlpTwoPassWriter): seq[byte] = doAssert self.pendingLists.len == 0, "Insufficient number of elements written to a started list" - self.prefixLengths.setLen(0) - self.listLengths.setLen(0) + self.lengths.setLen(0) self.output func clear*(w: var RlpTwoPassWriter) = # Prepare writer for reuse w.pendingLists.setLen(0) w.output.setLen(0) - w.prefixLengths.setLen(0) - w.listLengths.setLen(0) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 1a7b6142..986cd5fa 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -189,9 +189,7 @@ proc encode*[T](v: T): seq[byte] = tracker.append(v) - var writer = initTwoPassWriter(tracker.totalLength) - writer.prefixLengths = tracker.prefixLengths - writer.listLengths = tracker.listLengths + var writer = initTwoPassWriter(tracker) writer.append(v) move(writer.finish) @@ -200,12 +198,10 @@ proc encodeHash*[T](v: T): Hash32 = mixin append var tracker = initLengthTracker() - var writer: RlpHashWriter tracker.append(v) - writer.prefixLengths = tracker.prefixLengths - writer.listLengths = tracker.listLengths + var writer = initHashWriter(tracker) writer.append(v) writer.finish.to(Hash32) From c3716caf960e9426dbd53dfd31ec25c15451b2c0 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 26 Dec 2024 16:47:27 +0530 Subject: [PATCH 30/36] compile time evaluation of types --- eth/rlp/length_writer.nim | 28 +++++++++++++--------------- eth/rlp/two_pass_writer.nim | 5 +---- eth/rlp/writer.nim | 34 ++++++++++++++++++++++++++++++---- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index 70a387d5..883acd1d 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -6,23 +6,23 @@ import utils type - RlpLengthTracker* = object + RlpLengthTracker*[N: static int] = object lengths*: seq[tuple[listLen, prefixLen: int]] - pendingLists: seq[tuple[idx, remainingItems, startLen: int]] + pendingLists: array[N, tuple[idx, remainingItems, startLen: int]] + listTop: int listCount: int totalLength*: int const LIST_LENGTH = 50 proc maybeClosePendingLists(self: var RlpLengthTracker) = - while self.pendingLists.len > 0: - let lastIdx = self.pendingLists.len - 1 - self.pendingLists[lastIdx].remainingItems -= 1 + while self.listTop > 0: + self.pendingLists[self.listTop - 1].remainingItems -= 1 - if self.pendingLists[lastIdx].remainingItems == 0: + if self.pendingLists[self.listTop - 1].remainingItems == 0: let - listIdx = self.pendingLists[lastIdx].idx - startLen = self.pendingLists[lastIdx].startLen + listIdx = self.pendingLists[self.listTop - 1].idx + startLen = self.pendingLists[self.listTop - 1].startLen listLen = self.totalLength - startLen prefixLen = if listLen < int(THRESHOLD_LIST_LEN): 1 else: int(uint64(listLen).bytesNeeded) + 1 @@ -31,7 +31,7 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = self.lengths[listIdx] = (listLen, prefixLen) # close the list by deleting - self.pendingLists.setLen(lastIdx) + self.listTop -= 1 self.totalLength += prefixLen else: @@ -47,7 +47,8 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = self.maybeClosePendingLists() else: # open a list - self.pendingLists.add((self.listCount, listSize, self.totalLength)) + self.pendingLists[self.listTop] = (self.listCount, listSize, self.totalLength) + self.listTop += 1 self.listCount += 1 if self.listCount == self.lengths.len: self.lengths.setLen(self.lengths.len + LIST_LENGTH) @@ -70,17 +71,14 @@ func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = self.totalLength += lengthCount(i.bytesNeeded) + i.bytesNeeded self.maybeClosePendingLists() -func initLengthTracker*(): RlpLengthTracker = +func initLengthTracker*(self: var RlpLengthTracker) = # we preset the lengths since we want to skip using add method for # these lists - result.lengths = newSeq[(int, int)](LIST_LENGTH) + self.lengths = newSeq[(int, int)](LIST_LENGTH) template finish*(self: RlpLengthTracker): int = - doAssert self.pendingLists.len == 0, - "Insufficient number of elements written to a started list" self.totalLength func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse - w.pendingLists.setLen(0) w.lengths.setLen(0) diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index a589fe9a..0f9ab704 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -8,7 +8,6 @@ import type RlpTwoPassWriter* = object - pendingLists*: seq[tuple[remainingItems, startPos, prefixLen: int]] output*: seq[byte] lengths*: seq[tuple[listLen, prefixLen: int]] fillLevel: int @@ -83,12 +82,10 @@ func initTwoPassWriter*(tracker: var RlpLengthTracker): RlpTwoPassWriter = result.lengths = move(tracker.lengths) template finish*(self: RlpTwoPassWriter): seq[byte] = - doAssert self.pendingLists.len == 0, - "Insufficient number of elements written to a started list" self.lengths.setLen(0) self.output func clear*(w: var RlpTwoPassWriter) = # Prepare writer for reuse - w.pendingLists.setLen(0) + w.lengths.setLen(0) w.output.setLen(0) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 986cd5fa..3fa8f5a7 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -6,7 +6,7 @@ # at your option. This file may not be copied, modified, or distributed except according to those terms. import - std/options, + std/[options, typetraits], pkg/results, stew/[arraybuf, shims/macros], ./priv/defs, @@ -15,6 +15,7 @@ import two_pass_writer, default_writer, utils, + stint, ../common/hashes export arraybuf, default_writer, length_writer, two_pass_writer, hash_writer @@ -67,6 +68,27 @@ proc appendImpl[T](self: var RlpWriter, list: openArray[T]) = for i in 0 ..< list.len: self.append list[i] +template innerType[T](x: Option[T] | Opt[T]): typedesc = T + +proc countNestedListsDepth(T: type): int {.compileTime.} = + mixin enumerateRlpFields + + var dummy: T + + template op(RT, fN, f) = + result += countNestedListsDepth(type f) + + when T is Option or T is Opt: + result += countNestedListsDepth(innerType(dummy)) + elif T is UInt256: + discard + elif T is object or T is tuple: + inc result + enumerateRlpFields(dummy, op) + elif T is seq or T is array: + inc result + result += countNestedListsDepth(elementType(dummy)) + proc countOptionalFields(T: type): int {.compileTime.} = mixin enumerateRlpFields @@ -184,9 +206,11 @@ proc initRlpList*(listSize: int): RlpDefaultWriter = proc encode*[T](v: T): seq[byte] = mixin append - - var tracker = initLengthTracker() + const nestedListsDepth = countNestedListsDepth(type v) + var tracker: RlpLengthTracker[nestedListsDepth] + tracker.initLengthTracker() + tracker.append(v) var writer = initTwoPassWriter(tracker) @@ -197,7 +221,9 @@ proc encode*[T](v: T): seq[byte] = proc encodeHash*[T](v: T): Hash32 = mixin append - var tracker = initLengthTracker() + const nestedListsDepth = countNestedListsDepth(type v) + var tracker: RlpLengthTracker[nestedListsDepth] + tracker.initLengthTracker() tracker.append(v) From 7c3780985314f757bb3ae457d6fc67b0e1192131 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 6 Jan 2025 21:16:02 +0530 Subject: [PATCH 31/36] static and dynamic pending lists --- eth/rlp/length_writer.nim | 23 +++++++++++++++++++++-- eth/rlp/writer.nim | 14 ++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index 883acd1d..d30d7936 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -6,13 +6,24 @@ import utils type - RlpLengthTracker*[N: static int] = object + PendingListItem = tuple[idx, remainingItems, startLen: int] + + StaticRlpLengthTracker*[N: static int] = object + pendingLists: array[N, PendingListItem] + lengths*: seq[tuple[listLen, prefixLen: int]] + listTop: int + listCount: int + totalLength*: int + + DynamicRlpLengthTracker* = object + pendingLists: seq[PendingListItem] lengths*: seq[tuple[listLen, prefixLen: int]] - pendingLists: array[N, tuple[idx, remainingItems, startLen: int]] listTop: int listCount: int totalLength*: int + RlpLengthTracker* = StaticRlpLengthTracker | DynamicRlpLengthTracker + const LIST_LENGTH = 50 proc maybeClosePendingLists(self: var RlpLengthTracker) = @@ -32,6 +43,8 @@ proc maybeClosePendingLists(self: var RlpLengthTracker) = # close the list by deleting self.listTop -= 1 + when self is DynamicRlpLengthTracker: + self.pendingLists.setLen(self.listTop) self.totalLength += prefixLen else: @@ -47,6 +60,8 @@ proc startList*(self: var RlpLengthTracker, listSize: int) = self.maybeClosePendingLists() else: # open a list + when self is DynamicRlpLengthTracker: + self.pendingLists.setLen(self.listTop + 1) self.pendingLists[self.listTop] = (self.listCount, listSize, self.totalLength) self.listTop += 1 self.listCount += 1 @@ -74,6 +89,8 @@ func writeInt*(self: var RlpLengthTracker, i: SomeUnsignedInt) = func initLengthTracker*(self: var RlpLengthTracker) = # we preset the lengths since we want to skip using add method for # these lists + when self is DynamicRlpLengthTracker: + self.pendingLists = newSeqOfCap[(int, int, int)](5) self.lengths = newSeq[(int, int)](LIST_LENGTH) template finish*(self: RlpLengthTracker): int = @@ -82,3 +99,5 @@ template finish*(self: RlpLengthTracker): int = func clear*(w: var RlpLengthTracker) = # Prepare writer for reuse w.lengths.setLen(0) + when w is DynamicRlpLengthTracker: + w.pendingLists.setLen(0) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 3fa8f5a7..226dc9d0 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -208,7 +208,12 @@ proc encode*[T](v: T): seq[byte] = mixin append const nestedListsDepth = countNestedListsDepth(type v) - var tracker: RlpLengthTracker[nestedListsDepth] + + when nestedListsDepth > 0: + var tracker = StaticRlpLengthTracker[nestedListsDepth]() + elif nestedListsDepth == 0: + var tracker = DynamicRlpLengthTracker() + tracker.initLengthTracker() tracker.append(v) @@ -222,7 +227,12 @@ proc encodeHash*[T](v: T): Hash32 = mixin append const nestedListsDepth = countNestedListsDepth(type v) - var tracker: RlpLengthTracker[nestedListsDepth] + + when nestedListsDepth > 0: + var tracker = StaticRlpLengthTracker[nestedListsDepth]() + elif nestedListsDepth == 0: + var tracker = DynamicRlpLengthTracker() + tracker.initLengthTracker() tracker.append(v) From 88570b215c591078ad2e2e093640fed75ff02092 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Mon, 6 Jan 2025 23:45:35 +0530 Subject: [PATCH 32/36] return hash32 --- eth/rlp/hash_writer.nim | 5 +++-- eth/rlp/writer.nim | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index 0d49665c..5f9eae9b 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -5,6 +5,7 @@ import stew/[arraybuf, shims/macros], ./priv/defs, utils, + ../common/hashes, length_writer type @@ -77,9 +78,9 @@ proc startList*(self: var RlpHashWriter, listSize: int) = func initHashWriter*(tracker: var RlpLengthTracker): RlpHashWriter = result.lengths = move(tracker.lengths) -template finish*(self: var RlpHashWriter): MDigest[self.keccak.bits] = +template finish*(self: var RlpHashWriter): Hash32 = self.lengths.setLen(0) - self.keccak.finish() + self.keccak.finish.to(Hash32) func clear*(w: var RlpHashWriter) = # Prepare writer for reuse diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 226dc9d0..693ccad0 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -240,7 +240,7 @@ proc encodeHash*[T](v: T): Hash32 = var writer = initHashWriter(tracker) writer.append(v) - writer.finish.to(Hash32) + writer.finish() func encodeInt*(i: SomeUnsignedInt): RlpIntBuf = var buf: RlpIntBuf From 599f89e9c32171c72dab8b3efe32c1b33b4e669e Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Tue, 7 Jan 2025 16:05:24 +0530 Subject: [PATCH 33/36] add block header hashing to profiler --- tests/rlp/test_rlp_profiler.nim | 35 +++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index 9e725266..8018fdd4 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -64,6 +64,14 @@ let blkSeq = @[ Transaction(nonce: 3), Transaction(nonce: 4)])] +let h = BlockHeader( + nonce: Bytes8([0x20u8,0,0,0,0,0,0,0]), + baseFeePerGas: Opt.some(2.u256), + withdrawalsRoot: Opt.some(default(Hash32)), + blobGasUsed: Opt.some(1'u64), + excessBlobGas: Opt.some(1'u64) +) + let nonEmptyBlock = readBlock() proc encodeOnePass[T](v: T): seq[byte] = @@ -83,28 +91,35 @@ suite "test running times of rlp encode and encodeHash": let bytes2 = rlp.encode(blkSeq) benchmark "Block serialization using two pass writer": let bytes3 = rlp.encode(nonEmptyBlock) + benchmark "Block header serialization using two pass writer": + let bytes4 = rlp.encode(h) test "encoding using default writer": benchmark "Transaction serialization using default writer": - let bytes3 = encodeOnePass(myTx) + let bytes5 = encodeOnePass(myTx) benchmark "Block Sequence serailization using default writer": - let bytes4 = encodeOnePass(blkSeq) + let bytes6 = encodeOnePass(blkSeq) benchmark "Block serialization using default writer": - let bytes5 = encodeOnePass(nonEmptyBlock) + let bytes7 = encodeOnePass(nonEmptyBlock) + benchmark "Block header serialization using default writer": + let bytes8 = encodeOnePass(h) test "encoding and hashing using hash writer": benchmark "Transaction hashing using hash writer": - let bytes6 = rlp.encodeHash(myTx) + let bytes9 = rlp.encodeHash(myTx) benchmark "Block Sequence hashing using hash writer": - let bytes7 = rlp.encodeHash(blkSeq) + let bytes10 = rlp.encodeHash(blkSeq) benchmark "Block hashing using hash writer": - let bytes8 = rlp.encodeHash(nonEmptyBlock) + let bytes11 = rlp.encodeHash(nonEmptyBlock) + benchmark "Block header hashing using hash writer": + let bytes12 = rlp.encodeHash(h) test "encoding and hashin using default writer": benchmark "Transaction hashing using default writer and then hash": - let bytes9 = encodeAndHash(myTx) + let bytes13 = encodeAndHash(myTx) benchmark "Block Sequence hashing using default writer and then hash": - let bytes10 = encodeAndHash(blkSeq) + let bytes14 = encodeAndHash(blkSeq) benchmark "Block hashing using defualt writer and the hash": - let bytes11 = encodeAndHash(blkSeq) - + let bytes15 = encodeAndHash(blkSeq) + benchmark "Block header hashing using hash writer": + let bytes16 = encodeAndHash(h) From 4fc06326ad1923b850bcc37dc07c24fcd4260ead Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Wed, 15 Jan 2025 16:12:14 +0530 Subject: [PATCH 34/36] add a more reliable profiler --- .gitignore | 5 +- tests/rlp/profiler/.gitignore | 2 + tests/rlp/profiler/build_profiler.sh | 25 ++ tests/rlp/profiler/profiler.nim | 526 +++++++++++++++++++++++++++ tests/rlp/test_rlp_profiler.nim | 6 +- 5 files changed, 560 insertions(+), 4 deletions(-) create mode 100644 tests/rlp/profiler/.gitignore create mode 100644 tests/rlp/profiler/build_profiler.sh create mode 100644 tests/rlp/profiler/profiler.nim diff --git a/.gitignore b/.gitignore index 001eb065..79857072 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ build/ *.exe *.dll *.generated.nim -nimble.paths \ No newline at end of file +nimble.paths + +#OS specific files +**/.DS_Store diff --git a/tests/rlp/profiler/.gitignore b/tests/rlp/profiler/.gitignore new file mode 100644 index 00000000..82781fe1 --- /dev/null +++ b/tests/rlp/profiler/.gitignore @@ -0,0 +1,2 @@ +!*.nim +!*.sh diff --git a/tests/rlp/profiler/build_profiler.sh b/tests/rlp/profiler/build_profiler.sh new file mode 100644 index 00000000..30a57af2 --- /dev/null +++ b/tests/rlp/profiler/build_profiler.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +objs=("tx" "header" "blk" "blk80" "blk320" "blk640" "blk1280") + +for obj in "${objs[@]}"; do + nim compile --mm:orc -d:$obj -d:release --out:bench_"$obj"_default profiler.nim + nim compile --mm:orc -d:$obj -d:opt -d:release --out:bench_"$obj"_two_pass profiler.nim + nim compile --mm:orc -d:$obj -d:hash -d:release --out:bench_"$obj"_default_hash profiler.nim + nim compile --mm:orc -d:$obj -d:hash -d:opt -d:release --out:bench_"$obj"_hash_writer profiler.nim + nim compile --mm:refc -d:$obj -d:release --out:bench_"$obj"_default_refc profiler.nim + nim compile --mm:refc -d:$obj -d:opt -d:release --out:bench_"$obj"_two_pass_refc profiler.nim + nim compile --mm:refc -d:$obj -d:hash -d:release --out:bench_"$obj"_default_hash_refc profiler.nim + nim compile --mm:refc -d:$obj -d:hash -d:opt -d:release --out:bench_"$obj"_hash_writer_refc profiler.nim +done + +for obj in "${objs[@]}"; do + source ./bench_"$obj"_default + source ./bench_"$obj"_two_pass + source ./bench_"$obj"_default_hash + source ./bench_"$obj"_hash_writer + source ./bench_"$obj"_default_refc + source ./bench_"$obj"_two_pass_refc + source ./bench_"$obj"_default_hash_refc + source ./bench_"$obj"_hash_writer_refc +done diff --git a/tests/rlp/profiler/profiler.nim b/tests/rlp/profiler/profiler.nim new file mode 100644 index 00000000..c02a1d3f --- /dev/null +++ b/tests/rlp/profiler/profiler.nim @@ -0,0 +1,526 @@ +import + ../../../eth/[rlp, common], + times, + std/[os, strutils], + stew/io2, + results + +type + Timeval {.importc: "timeval", header:"", bycopy.} = object + + Rusage* {.importc: "struct rusage", header:"", bycopy.} = object + ru_utime {.importc.}: Timeval + ru_stime {.importc.}: Timeval + ru_maxrss* {.importc.}: int32 # Maximum resident set size + # ... + ru_minflt* {.importc.}: int32 # page reclaims (soft page faults) + + RusageWho* {.size: sizeof(cint).} = enum + RusageChildren = -1 + RusageSelf = 0 + RusageThread = 1 + +when defined(debug): + var H_RUSAGE_SELF{.importc, header:" Date: Thu, 16 Jan 2025 13:37:16 +0530 Subject: [PATCH 35/36] add/update copyright --- eth/common/eth_types_rlp.nim | 2 +- eth/rlp/default_writer.nim | 7 +++++++ eth/rlp/hash_writer.nim | 7 +++++++ eth/rlp/length_writer.nim | 7 +++++++ eth/rlp/two_pass_writer.nim | 7 +++++++ eth/rlp/utils.nim | 7 +++++++ eth/rlp/writer.nim | 2 +- tests/rlp/profiler/build_profiler.sh | 7 +++++++ tests/rlp/profiler/profiler.nim | 7 +++++++ tests/rlp/test_api_usage.nim | 7 +++++++ tests/rlp/test_hashes.nim | 7 +++++++ tests/rlp/test_rlp_profiler.nim | 7 +++++++ tests/rlp/util/json_testing.nim | 7 +++++++ 13 files changed, 79 insertions(+), 2 deletions(-) diff --git a/eth/common/eth_types_rlp.nim b/eth/common/eth_types_rlp.nim index 1ad65ac7..72750035 100644 --- a/eth/common/eth_types_rlp.nim +++ b/eth/common/eth_types_rlp.nim @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024 Status Research & Development GmbH +# Copyright (c) 2022-2025 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/eth/rlp/default_writer.nim b/eth/rlp/default_writer.nim index 87f51288..c0ce47ff 100644 --- a/eth/rlp/default_writer.nim +++ b/eth/rlp/default_writer.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import std/options, pkg/results, diff --git a/eth/rlp/hash_writer.nim b/eth/rlp/hash_writer.nim index 5f9eae9b..6017e5c4 100644 --- a/eth/rlp/hash_writer.nim +++ b/eth/rlp/hash_writer.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import std/options, pkg/results, diff --git a/eth/rlp/length_writer.nim b/eth/rlp/length_writer.nim index d30d7936..981e8f4f 100644 --- a/eth/rlp/length_writer.nim +++ b/eth/rlp/length_writer.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import std/options, pkg/results, diff --git a/eth/rlp/two_pass_writer.nim b/eth/rlp/two_pass_writer.nim index 0f9ab704..752eb796 100644 --- a/eth/rlp/two_pass_writer.nim +++ b/eth/rlp/two_pass_writer.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import std/options, pkg/results, diff --git a/eth/rlp/utils.nim b/eth/rlp/utils.nim index 8054be9f..31da0a90 100644 --- a/eth/rlp/utils.nim +++ b/eth/rlp/utils.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import stew/[bitops2] func bytesNeeded*(num: SomeUnsignedInt): int = diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index 693ccad0..c49ab26b 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -1,5 +1,5 @@ # eth -# Copyright (c) 2019-2024 Status Research & Development GmbH +# Copyright (c) 2019-2025 Status Research & Development GmbH # Licensed and distributed under either of # * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). # * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). diff --git a/tests/rlp/profiler/build_profiler.sh b/tests/rlp/profiler/build_profiler.sh index 30a57af2..deab223f 100644 --- a/tests/rlp/profiler/build_profiler.sh +++ b/tests/rlp/profiler/build_profiler.sh @@ -1,5 +1,12 @@ #!/bin/bash +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + objs=("tx" "header" "blk" "blk80" "blk320" "blk640" "blk1280") for obj in "${objs[@]}"; do diff --git a/tests/rlp/profiler/profiler.nim b/tests/rlp/profiler/profiler.nim index c02a1d3f..efadcfba 100644 --- a/tests/rlp/profiler/profiler.nim +++ b/tests/rlp/profiler/profiler.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import ../../../eth/[rlp, common], times, diff --git a/tests/rlp/test_api_usage.nim b/tests/rlp/test_api_usage.nim index 93003853..b30913c2 100644 --- a/tests/rlp/test_api_usage.nim +++ b/tests/rlp/test_api_usage.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + {.used.} import diff --git a/tests/rlp/test_hashes.nim b/tests/rlp/test_hashes.nim index 40298df5..99de6c63 100644 --- a/tests/rlp/test_hashes.nim +++ b/tests/rlp/test_hashes.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + {.used.} import diff --git a/tests/rlp/test_rlp_profiler.nim b/tests/rlp/test_rlp_profiler.nim index d986e076..e6c594b8 100644 --- a/tests/rlp/test_rlp_profiler.nim +++ b/tests/rlp/test_rlp_profiler.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + {.used.} import diff --git a/tests/rlp/util/json_testing.nim b/tests/rlp/util/json_testing.nim index d9307bed..3878feb3 100644 --- a/tests/rlp/util/json_testing.nim +++ b/tests/rlp/util/json_testing.nim @@ -1,3 +1,10 @@ +# eth +# Copyright (c) 2019-2025 Status Research & Development GmbH +# Licensed and distributed under either of +# * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). +# * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). +# at your option. This file may not be copied, modified, or distributed except according to those terms. + import std/json, unittest2, From 32af6b4db978de74cae925cbc8299ab97c864486 Mon Sep 17 00:00:00 2001 From: chirag-parmar Date: Thu, 16 Jan 2025 22:45:20 +0530 Subject: [PATCH 36/36] took TOO long for TWO lines of code --- eth/rlp/writer.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/eth/rlp/writer.nim b/eth/rlp/writer.nim index c49ab26b..a8c0be3c 100644 --- a/eth/rlp/writer.nim +++ b/eth/rlp/writer.nim @@ -89,6 +89,9 @@ proc countNestedListsDepth(T: type): int {.compileTime.} = inc result result += countNestedListsDepth(elementType(dummy)) +proc countNestedListsDepth[E](T: type openArray[E]): int = + countNestedListsDepth(seq[E]) + proc countOptionalFields(T: type): int {.compileTime.} = mixin enumerateRlpFields @@ -207,7 +210,7 @@ proc initRlpList*(listSize: int): RlpDefaultWriter = proc encode*[T](v: T): seq[byte] = mixin append - const nestedListsDepth = countNestedListsDepth(type v) + const nestedListsDepth = countNestedListsDepth(T) when nestedListsDepth > 0: var tracker = StaticRlpLengthTracker[nestedListsDepth]() @@ -226,7 +229,7 @@ proc encode*[T](v: T): seq[byte] = proc encodeHash*[T](v: T): Hash32 = mixin append - const nestedListsDepth = countNestedListsDepth(type v) + const nestedListsDepth = countNestedListsDepth(T) when nestedListsDepth > 0: var tracker = StaticRlpLengthTracker[nestedListsDepth]()