add test runner for LC data collection tests (#6021)

Introduce a test runner for upcoming EF test suites related to canonical
light client data collection.

- ethereum/consensus-specs#3553

beacon_chain/spec/forks.nim

@@ -471,6 +471,11 @@ template SignedBlindedBeaconBlock*(kind: static ConsensusFork): auto =
   else:
     static: raiseAssert "Unreachable"
 
+template Forky*(
+    x: typedesc[ForkedSignedBeaconBlock],
+    kind: static ConsensusFork): auto =
+  kind.SignedBeaconBlock
+
 template withAll*(
     x: typedesc[ConsensusFork], body: untyped): untyped =
   static: doAssert ConsensusFork.high == ConsensusFork.Deneb

tests/consensus_spec/test_fixture_light_client_data_collection.nim

@@ -0,0 +1,201 @@
+# beacon_chain
+# Copyright (c) 2024 Status Research & Development GmbH
+# Licensed and distributed under either of
+#   * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT).
+#   * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0).
+# at your option. This file may not be copied, modified, or distributed except according to those terms.
+
+{.push raises: [].}
+{.used.}
+
+import
+  # Standard library
+  std/[json, sequtils],
+  # Status libraries
+  stew/byteutils,
+  chronicles,
+  taskpools,
+  # Third-party
+  yaml,
+  # Beacon chain internals
+  ../../beacon_chain/beacon_chain_db,
+  ../../beacon_chain/consensus_object_pools/[block_clearance, block_quarantine],
+  ../../beacon_chain/spec/forks,
+  # Test utilities
+  ../testutil, ../testbcutil,
+  ./fixtures_utils, ./os_ops
+
+type
+  TestStepKind {.pure.} = enum
+    NewBlock
+    NewHead
+
+  NewHeadChecks = object
+    latestFinalizedCheckpoint: Checkpoint
+    bootstraps: Table[Eth2Digest, ForkedLightClientBootstrap]
+    bestUpdates: Table[SyncCommitteePeriod, ForkedLightClientUpdate]
+    latestFinalityUpdate: ForkedLightClientFinalityUpdate
+    latestOptimisticUpdate: ForkedLightClientOptimisticUpdate
+
+  TestStep = object
+    case kind: TestStepKind
+    of TestStepKind.NewBlock:
+      blck: ForkedSignedBeaconBlock
+    of TestStepKind.NewHead:
+      headBlockRoot: Eth2Digest
+      checks: NewHeadChecks
+
+func `==`(a, b: SomeForkedLightClientObject): bool =
+  a.kind == b.kind and withForkyObject(a, (
+    when lcDataFork > LightClientDataFork.None:
+      forkyObject == b.forky(lcDataFork)
+    else:
+      true))
+
+proc loadForked[T: not Opt](
+    t: typedesc[T],
+    s: JsonNode,
+    path: string,
+    fork_digests: ForkDigests): T {.raises: [ValueError].} =
+  if s == nil:
+    when T is SomeForkedLightClientObject:
+      return default(T)
+    else:
+      raiseAssert "Unexpected nil JSON node"
+  let
+    fork_digest = ForkDigest distinctBase(ForkDigest)
+      .fromHex(s["fork_digest"].getStr())
+    consensusFork = fork_digests.consensusForkForDigest(fork_digest)
+      .expect("Unknown fork " & $fork_digest)
+    filename = s["data"].getStr()
+  when T is SomeForkedLightClientObject:
+    withLcDataFork(lcDataForkAtConsensusFork(consensusFork)):
+      when lcDataFork > LightClientDataFork.None:
+        T.init(parseTest(
+          path/filename & ".ssz_snappy", SSZ, T.Forky(lcDataFork)))
+      else: raiseAssert $consensusFork & " does not support LC data"
+  else:
+    withConsensusFork(consensusFork):
+      T.init(parseTest(
+        path/filename & ".ssz_snappy", SSZ, T.Forky(consensusFork)))
+
+proc loadSteps(
+    path: string,
+    fork_digests: ForkDigests
+): seq[TestStep] {.raises: [
+    IOError, KeyError, ValueError, YamlConstructionError, YamlParserError].} =
+  template loadForked[T](t: typedesc[T], s: JsonNode): T =
+    loadForked(t, s, path, fork_digests)
+
+  let stepsYAML = os_ops.readFile(path/"steps.yaml")
+  let steps = yaml.loadToJson(stepsYAML)
+
+  result = @[]
+  for step in steps[0]:
+    if step.hasKey"new_block":
+      let s = step["new_block"]
+      result.add TestStep(
+        kind: TestStepKind.NewBlock,
+        blck: ForkedSignedBeaconBlock.loadForked(s))
+    elif step.hasKey"new_head":
+      let
+        s = step["new_head"]
+        checks = s["checks"]
+      result.add TestStep(
+        kind: TestStepKind.NewHead,
+        headBlockRoot: Eth2Digest.fromHex(s["head_block_root"].getStr()),
+        checks: NewHeadChecks(
+          latestFinalizedCheckpoint: Checkpoint(
+            epoch: Epoch(
+              checks["latest_finalized_checkpoint"]["epoch"].getInt()),
+            root: Eth2Digest.fromHex(
+              checks["latest_finalized_checkpoint"]["root"].getStr())),
+          bootstraps: checks["bootstraps"].foldl((block:
+            check: not a.hasKeyOrPut(
+              Eth2Digest.fromHex(b["block_root"].getStr()),
+              ForkedLightClientBootstrap.loadForked(b{"bootstrap"}))
+            a), newTable[Eth2Digest, ForkedLightClientBootstrap]())[],
+          bestUpdates: checks["best_updates"].foldl((block:
+            check: not a.hasKeyOrPut(
+              SyncCommitteePeriod(b["period"].getInt()),
+              ForkedLightClientUpdate.loadForked(b{"update"}))
+            a), newTable[SyncCommitteePeriod, ForkedLightClientUpdate]())[],
+          latestFinalityUpdate: ForkedLightClientFinalityUpdate
+            .loadForked(checks{"latest_finality_update"}),
+          latestOptimisticUpdate: ForkedLightClientOptimisticUpdate
+            .loadForked(checks{"latest_optimistic_update"})))
+    else:
+      raiseAssert "Unknown test step: " & $step
+
+proc runTest(suiteName, path: string, consensusFork: static ConsensusFork) =
+  let relativePathComponent = path.relativeTestPathComponent()
+  test "Light client - Data collection - " & relativePathComponent:
+    let (cfg, unknowns) = readRuntimeConfig(path/"config.yaml")
+    doAssert unknowns.len == 0
+
+    let
+      initial_state = loadForkedState(
+        path/"initial_state.ssz_snappy", consensusFork)
+      db = BeaconChainDB.new("", cfg = cfg, inMemory = true)
+    defer: db.close()
+    ChainDAGRef.preInit(db, initial_state[])
+
+    let
+      validatorMonitor = newClone(ValidatorMonitor.init(false, false))
+      dag = ChainDAGRef.init(cfg, db, validatorMonitor, {},
+        lcDataConfig = LightClientDataConfig(
+          serve: true, importMode: LightClientDataImportMode.Full))
+      rng = HmacDrbgContext.new()
+      taskpool = TaskPool.new()
+    var
+      verifier = BatchVerifier.init(rng, taskpool)
+      quarantine = newClone(Quarantine.init())
+
+    let steps = loadSteps(path, dag.forkDigests[])
+    for i, step in steps:
+      case step.kind
+      of TestStepKind.NewBlock:
+        checkpoint $i & " new_block: " & $shortLog(step.blck.toBlockId())
+        let added = withBlck(step.blck):
+          const nilCallback = (consensusFork.OnBlockAddedCallback)(nil)
+          dag.addHeadBlock(verifier, forkyBlck, nilCallback)
+        check: added.isOk()
+      of TestStepKind.NewHead:
+        let blck = dag.getBlockRef(step.headBlockRoot)
+        check blck.isSome
+        checkpoint $i & " new_head: " & $shortLog(blck.get.bid)
+        dag.updateHead(blck.get, quarantine[], knownValidators = [])
+        if dag.needStateCachesAndForkChoicePruning():
+          dag.pruneStateCachesDAG()
+        check:
+          step.checks.latestFinalizedCheckpoint.epoch ==
+            dag.finalizedHead.slot.epoch
+          step.checks.latestFinalizedCheckpoint.root == (
+            if dag.finalizedHead.blck.slot != GENESIS_SLOT:
+              dag.finalizedHead.blck.root
+            else:
+              ZERO_HASH)
+          step.checks.bootstraps.pairs().toSeq().allIt:
+            dag.getLightClientBootstrap(it[0]) == it[1]
+          step.checks.bestUpdates.pairs().toSeq().allIt:
+            dag.getLightClientUpdateForPeriod(it[0]) == it[1]
+          step.checks.latestFinalityUpdate ==
+            dag.getLightClientFinalityUpdate()
+          step.checks.latestOptimisticUpdate ==
+            dag.getLightClientOptimisticUpdate()
+
+suite "EF - Light client - Data collection" & preset():
+  const presetPath = SszTestsDir/const_preset
+  for kind, path in walkDir(presetPath, relative = true, checkDir = true):
+    let testsPath =
+      presetPath/path/"light_client"/"data_collection"/"pyspec_tests"
+    if kind != pcDir or not dirExists(testsPath):
+      continue
+    let consensusFork = forkForPathComponent(path).valueOr:
+      let relativePathComponent = path.relativeTestPathComponent()
+      test "Light client - Data collection - " & relativePathComponent:
+        skip()
+      continue
+    for kind, path in walkDir(testsPath, relative = true, checkDir = true):
+      withConsensusFork(consensusFork):
+        runTest(suiteName, testsPath/path, consensusFork)