add EIP-7495 implementation: StableContainer

ssz_serialization.nim

@@ -1,5 +1,5 @@
 # ssz_serialization
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -89,8 +89,7 @@ func init*(T: type SszWriter, stream: OutputStream): T =
 
 proc writeVarSizeType(w: var SszWriter, value: auto) {.gcsafe, raises: [IOError].}
 
-func beginRecord*(w: var SszWriter, TT: type): auto  =
-  type T = TT
+func beginRecord*(w: var SszWriter, T: typedesc): auto =
   when isFixedSize(T):
     FixedSizedWriterCtx()
   else:
@@ -107,7 +106,8 @@ template writeField*(w: var SszWriter,
   when ctx is FixedSizedWriterCtx:
     writeFixedSized(w.stream, toSszType(field))
   else:
-    type FieldType = type toSszType(field)
+    # `FieldType` should be `type`: https://github.com/nim-lang/Nim/issues/23564
+    template FieldType: untyped = typeof toSszType(field)
 
     when isFixedSize(FieldType):
       writeFixedSized(ctx.fixedParts, toSszType(field))
@@ -155,6 +155,7 @@ proc writeElements[T](w: var SszWriter, value: openArray[T])
 proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [IOError].} =
   trs "STARTING VAR SIZE TYPE"
 
+  mixin toSszType
   when value is HashArray|HashList:
     writeVarSizeType(w, value.data)
   elif value is array:
@@ -170,29 +171,92 @@ proc writeVarSizeType(w: var SszWriter, value: auto) {.raises: [IOError].} =
       w.writeValue 1'u8
       w.writeValue value.get
   elif value is object|tuple:
-    when isCaseObject(type(value)):
+    when typeof(value).isStableContainer:
+      static: typeof(value).ensureIsValidStableContainer()
+      const N = typeof(value).getCustomPragmaVal(sszStableContainer)
+      var
+        activeFields: BitArray[N]
+        fieldIndex = 0
+        fixedSize = 0
+      value.enumInstanceSerializedFields(_ {.used.}, field):
+        if field.isSome:
+          template F: untyped = typeof(field).T
+          when F.isFixedSize:
+            fixedSize += static(F.fixedPortionSize)
+          else:
+            fixedSize += offsetSize
+          activeFields.setBit(fieldIndex)
+        inc fieldIndex
+
+      w.writeValue activeFields
+      block:
+        var ctx = VarSizedWriterCtx(
+          offset: fixedSize,
+          fixedParts: w.stream.delayFixedSizeWrite(fixedSize))
+        value.enumInstanceSerializedFields(_ {.used.}, field):
+          if field.isSome:
+            w.writeField ctx, astToStr(field), field.unsafeGet
+        w.endRecord ctx
+    elif typeof(value).isProfile:
+      static: typeof(value).ensureIsValidProfile()
+      const O = typeof(value).numOptionalFields
+      when O > 0:
+        var
+          optionalFields: BitArray[O]
+          optIndex = 0
+      var fixedSize = 0
+      value.enumInstanceSerializedFields(_ {.used.}, field):
+        when typeof(field) is Opt:
+          let hasField = field.isSome
+          if hasField:
+            optionalFields.setBit(optIndex)
+          inc optIndex
+          template F: untyped = typeof(field).T
+        else:
+          const hasField = true
+          template F: untyped = typeof(field)
+        if hasField:
+          when F.isFixedSize:
+            fixedSize += static(F.fixedPortionSize)
+          else:
+            fixedSize += offsetSize
+
+      when O > 0:
+        w.writeValue optionalFields
+      block:
+        var ctx = VarSizedWriterCtx(
+          offset: fixedSize,
+          fixedParts: w.stream.delayFixedSizeWrite(fixedSize))
+        value.enumInstanceSerializedFields(_ {.used.}, field):
+          when typeof(field) is Opt:
+            if field.isSome:
+              w.writeField ctx, astToStr(field), field.unsafeGet
+          else:
+            w.writeField ctx, astToStr(field), field
+        w.endRecord ctx
+    elif isCaseObject(type(value)):
       isUnion(type(value))
 
       trs "WRITING SSZ Union"
 
-      # toSszType for enum is kept local here as we don't want enums to
-      # serialize in general, only for object variants.
-      # TODO: This is not sufficiant as it will still allow to parse enum
-      # fields in the object variant itself. Will probably need some macro
-      # which enumerates the fields instead to take specific action on the
-      # discriminator.
-      template toSszType[E: enum](x: E): uint8 =
-        uint8(x)
-
       # TODO: At this point, the assumption is a correct `isUnion` as described
       # above.
       enumerateSubFields(value, field):
-        type T = type toSszType(field)
-
-        when isFixedSize(T):
-          w.stream.writeFixedSized toSszType(field)
+        when typeof(field) is enum:
+          # toSszType for enum is kept local here as we don't want enums to
+          # serialize in general, only for object variants.
+          # TODO: This is not sufficiant as it will still allow to parse enum
+          # fields in the object variant itself. Will probably need some macro
+          # which enumerates the fields instead to take specific action on the
+          # discriminator.
+          w.stream.writeFixedSized uint8(field)
         else:
-          w.writeVarSizeType toSszType(field)
+          type T = type toSszType(field)
+
+          when isFixedSize(T):
+            w.stream.writeFixedSized toSszType(field)
+          else:
+            w.writeVarSizeType toSszType(field)
     else:
       trs "WRITING OBJECT"
       var ctx = beginRecord(w, type value)
@@ -222,7 +286,9 @@ func sszSizeForVarSizeList[T](value: openArray[T]): int {.gcsafe, raises:[].} =
 
 func sszSize*(value: auto): int {.gcsafe, raises:[].} =
   mixin toSszType
-  type T = type toSszType(value)
+
+  # `T` should be `type`: https://github.com/nim-lang/Nim/issues/23564
+  template T: untyped = typeof toSszType(value)
 
   when isFixedSize(T):
     anonConst fixedPortionSize(T)
@@ -248,7 +314,34 @@ func sszSize*(value: auto): int {.gcsafe, raises:[].} =
       0
 
   elif T is object|tuple:
-    when T.isCaseObject():
+    when T.isStableContainer:
+      static: T.ensureIsValidStableContainer()
+      var total = static(T.fixedPortionSize)
+      value.enumInstanceSerializedFields(_ {.used.}, field):
+        if field.isSome:
+          template F: untyped = typeof(field).T
+          when F.isFixedSize:
+            total += static(F.fixedPortionSize)
+          else:
+            total += offsetSize + field.unsafeGet.sszSize
+      total
+    elif T.isProfile:
+      static: typeof(value).ensureIsValidProfile()
+      var total = static(T.fixedPortionSize)
+      when not T.isFixedSize:
+        value.enumInstanceSerializedFields(_ {.used.}, field):
+          when typeof(field) is Opt:
+            if field.isSome:
+              template F: untyped = typeof(field).T
+              when F.isFixedSize:
+                total += static(F.fixedPortionSize)
+              else:
+                total += offsetSize + field.unsafeGet.sszSize
+          else:
+            when not typeof(field).isFixedSize:
+              total += field.sszSize
+      total
+    elif T.isCaseObject():
       isUnion(T)
       unionSize(value)
     else:

ssz_serialization/codec.nim

@@ -1,5 +1,5 @@
 # ssz_serialization
-# Copyright (c) 2018-2023 Status Research & Development GmbH
+# Copyright (c) 2018-2024 Status Research & Development GmbH
 # Licensed and distributed under either of
 #   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
 #   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
@@ -237,11 +237,19 @@ macro initSszUnionImpl(RecordType: type, input: openArray[byte]): untyped =
 func initSszUnion(T: type, input: openArray[byte]): T {.raises: [SszError].} =
   initSszUnionImpl(T, input)
 
+func read(F: typedesc, input: openArray[byte]): F {.raises: [SszError].} =
+  when compiles(F.fromSszBytes(input)):
+    F.fromSszBytes(input)
+  else:
+    var f: F
+    readSszValue(input, f)
+    f
+
 proc readSszValue*[T](
     input: openArray[byte], val: var T) {.raises: [SszError].} =
   mixin fromSszBytes, toSszType, readSszValue
 
-  template readOffsetUnchecked(n: int): uint32 {.used.}=
+  template readOffsetUnchecked(n: int): uint32 {.used.} =
     fromSszBytes(uint32, input.toOpenArray(n, n + offsetSize - 1))
 
   template readOffset(n: int): int {.used.} =
@@ -449,7 +457,153 @@ proc readSszValue*[T](
     copyMem(addr val.bytes[0], unsafeAddr input[0], input.len)
 
   elif val is object|tuple:
-    when isCaseObject(T):
+    when T.isStableContainer:
+      static: T.ensureIsValidStableContainer()
+      const N = T.getCustomPragmaVal(sszStableContainer)
+
+      let inputLen = input.len
+      const numPrefixBytes = BitArray[N].fixedPortionSize
+      if inputLen < numPrefixBytes:
+        raiseMalformedSszError(T, "Scope too small for " &
+          "`{.sszStableContainer: " & $N & ".}` active fields")
+      let activeFields = BitArray[N].read(
+        input.toOpenArray(0, numPrefixBytes - 1))
+
+      for fieldIndex in T.totalSerializedFields ..< N:
+        if activeFields[fieldIndex]:
+          raiseMalformedSszError(T, "Unknown field index " & $fieldIndex)
+
+      val.reset()
+      var
+        fieldIndex = 0
+        offset = numPrefixBytes
+        dynFieldOffsets: seq[int]
+      val.enumInstanceSerializedFields(fieldName, field):
+        if activeFields[fieldIndex]:
+          template F: untyped = typeof(field).T
+          when F.isFixedSize:
+            const fieldSize = F.fixedPortionSize
+            if inputLen - offset < fieldSize:
+              raiseMalformedSszError(T, "Scope too small for " &
+                "`" & fieldName & "` of type `" & $F & "`")
+            field.ok F.read(
+              input.toOpenArray(offset, offset + fieldSize - 1))
+            offset += fieldSize
+          else:
+            if inputLen - offset < offsetSize:
+              raiseMalformedSszError(T, "Scope too small for " &
+                "`" & fieldName & "` offset")
+            dynFieldOffsets.add readOffset(offset)
+            if dynFieldOffsets[^1] > inputLen - numPrefixBytes:
+              raiseMalformedSszError(T, "Field offset past end")
+            if dynFieldOffsets.len > 1 and
+                dynFieldOffsets[^1] < dynFieldOffsets[^2]:
+              raiseMalformedSszError(T, "Field offset not larger than previous")
+            offset += offsetSize
+        inc fieldIndex
+      if dynFieldOffsets.len > 0:
+        dynFieldOffsets.add inputLen - numPrefixBytes
+        fieldIndex = 0
+        var i = 0
+        val.enumInstanceSerializedFields(_ {.used.}, field):
+          template F: untyped = typeof(field).T
+          when not F.isFixedSize:
+            if activeFields[fieldIndex]:
+              if dynFieldOffsets[i] != offset - numPrefixBytes:
+                raiseMalformedSszError(T, "Field offset invalid")
+              let fieldSize = dynFieldOffsets[i + 1] - dynFieldOffsets[i]
+              field.ok F.read(
+                input.toOpenArray(offset, offset + fieldSize - 1))
+              offset += fieldSize
+              inc i
+          inc fieldIndex
+        doAssert i == (dynFieldOffsets.len - 1)
+      if offset != inputLen:
+        raiseMalformedSszError(T, "Unexpected extra data after object")
+    elif T.isProfile:
+      static: T.ensureIsValidProfile()
+      const O = T.numOptionalFields
+
+      let inputLen = input.len
+      when O > 0:
+        # `B` should be `type`: https://github.com/nim-lang/Nim/issues/23564
+        template B: untyped = T.getCustomPragmaVal(sszProfile)
+        const numPrefixBytes = BitArray[O].fixedPortionSize
+        if inputLen < numPrefixBytes:
+          raiseMalformedSszError(T, "Scope too small for " &
+            "`{.sszProfile: " & $B & ".}` optional fields")
+        let optionalFields = BitArray[O].read(
+          input.toOpenArray(0, numPrefixBytes - 1))
+      else:
+        const numPrefixBytes = 0
+
+      val.reset()
+      var
+        optIndex = 0
+        offset = numPrefixBytes
+        dynFieldOffsets: seq[int]
+      val.enumInstanceSerializedFields(fieldName, field):
+        when typeof(field) is Opt:
+          let hasField = optionalFields[optIndex]
+          inc optIndex
+          template F: untyped = typeof(field).T
+        else:
+          const hasField = true
+          template F: untyped = typeof(field)
+        if hasField:
+          when F.isFixedSize:
+            const fieldSize = F.fixedPortionSize
+            if inputLen - offset < fieldSize:
+              raiseMalformedSszError(T, "Scope too small for " &
+                "`" & fieldName & "` of type `" & $F & "`")
+            when typeof(field) is Opt:
+              field.ok F.read(
+                input.toOpenArray(offset, offset + fieldSize - 1))
+            else:
+              readSszValue(
+                input.toOpenArray(offset, offset + fieldSize - 1), field)
+            offset += fieldSize
+          else:
+            if inputLen - offset < offsetSize:
+              raiseMalformedSszError(T, "Scope too small for " &
+                "`" & fieldName & "` offset")
+            dynFieldOffsets.add readOffset(offset)
+            if dynFieldOffsets[^1] > inputLen - numPrefixBytes:
+              raiseMalformedSszError(T, "Field offset past end")
+            if dynFieldOffsets.len > 1 and
+                dynFieldOffsets[^1] < dynFieldOffsets[^2]:
+              raiseMalformedSszError(T, "Field offset not larger than previous")
+            offset += offsetSize
+      doAssert optIndex == O
+      if dynFieldOffsets.len > 0:
+        dynFieldOffsets.add inputLen - numPrefixBytes
+        optIndex = 0
+        var i = 0
+        val.enumInstanceSerializedFields(_ {.used.}, field):
+          when typeof(field) is Opt:
+            let hasField = optionalFields[optIndex]
+            inc optIndex
+            template F: untyped = typeof(field).T
+          else:
+            const hasField = true
+            template F: untyped = typeof(field)
+          if hasField:
+            when not F.isFixedSize:
+              if dynFieldOffsets[i] != offset - numPrefixBytes:
+                raiseMalformedSszError(T, "Field offset invalid")
+              let fieldSize = dynFieldOffsets[i + 1] - dynFieldOffsets[i]
+              when typeof(field) is Opt:
+                field.ok F.read(
+                  input.toOpenArray(offset, offset + fieldSize - 1))
+              else:
+                readSszValue(
+                  input.toOpenArray(offset, offset + fieldSize - 1), field)
+              offset += fieldSize
+              inc i
+        doAssert i == (dynFieldOffsets.len - 1)
+      if offset != inputLen:
+        raiseMalformedSszError(T, "Unexpected extra data after object")
+    elif isCaseObject(T):
       isUnion(type(val))
       val = initSszUnion(type(val), input)
     else: