Implement Altair API (#2514)

* Add getEpochSyncCommittees

* Move URL prefixes to route definition

* Multiplex types in v2 REST API routes

* Add all altair API routes

* Update API tests routes

* Clarify untilEpoch TODO

* Fix route name

* Remove check from gossip handler

* Register new routes to fastify

packages/beacon-state-transition/src/allForks/util/epochContext.ts

@@ -86,6 +86,7 @@ export function createEpochContext(
     nextShuffling,
     currSyncCommitteeIndexes,
     nextSyncCommitteeIndexes,
+    currSyncComitteeValidatorIndexMap: computeSyncComitteeMap(currSyncCommitteeIndexes),
   });
 }
 
@@ -138,6 +139,26 @@ export function computeProposers(
   return proposers;
 }
 
+/**
+ * Compute all index in sync committee for all validatorIndexes in `syncCommitteeIndexes`.
+ * Helps reduce work necessary to verify a validatorIndex belongs in a sync committee and which.
+ */
+export function computeSyncComitteeMap(syncCommitteeIndexes: ValidatorIndex[]): SyncComitteeValidatorIndexMap {
+  const map = new Map<ValidatorIndex, number[]>();
+
+  for (let i = 0, len = syncCommitteeIndexes.length; i < len; i++) {
+    const validatorIndex = syncCommitteeIndexes[i];
+    let indexes = map.get(validatorIndex);
+    if (!indexes) {
+      indexes = [];
+      map.set(validatorIndex, indexes);
+    }
+    indexes.push(i);
+  }
+
+  return map;
+}
+
 /**
  * Called to re-use information, such as the shuffling of the next epoch, after transitioning into a
  * new epoch.
@@ -167,16 +188,19 @@ export function rotateEpochs(
     const nextPeriodEpoch = currEpoch + epochCtx.config.params.EPOCHS_PER_SYNC_COMMITTEE_PERIOD;
     epochCtx.currSyncCommitteeIndexes = epochCtx.nextSyncCommitteeIndexes;
     epochCtx.nextSyncCommitteeIndexes = getSyncCommitteeIndices(epochCtx.config, state, nextPeriodEpoch);
+    epochCtx.currSyncComitteeValidatorIndexMap = computeSyncComitteeMap(epochCtx.currSyncCommitteeIndexes);
   }
 
   // If crossing through the altair fork the caches will be empty, fill them up
   if (currEpoch === epochCtx.config.params.ALTAIR_FORK_EPOCH) {
     const nextPeriodEpoch = currEpoch + epochCtx.config.params.EPOCHS_PER_SYNC_COMMITTEE_PERIOD;
     epochCtx.currSyncCommitteeIndexes = getSyncCommitteeIndices(epochCtx.config, state, currEpoch);
     epochCtx.nextSyncCommitteeIndexes = getSyncCommitteeIndices(epochCtx.config, state, nextPeriodEpoch);
+    epochCtx.currSyncComitteeValidatorIndexMap = computeSyncComitteeMap(epochCtx.currSyncCommitteeIndexes);
   }
 }
 
+type SyncComitteeValidatorIndexMap = Map<ValidatorIndex, number[]>;
 interface IEpochContextData {
   config: IBeaconConfig;
   pubkey2index: PubkeyIndexMap;
@@ -187,6 +211,7 @@ interface IEpochContextData {
   nextShuffling: IEpochShuffling;
   currSyncCommitteeIndexes: ValidatorIndex[];
   nextSyncCommitteeIndexes: ValidatorIndex[];
+  currSyncComitteeValidatorIndexMap: SyncComitteeValidatorIndexMap;
 }
 
 /**
@@ -209,14 +234,20 @@ export class EpochContext {
   currentShuffling: IEpochShuffling;
   nextShuffling: IEpochShuffling;
   /**
-   * Updates every
-   * Memory cost: 1024 Number integers
+   * Update freq: every ~ 27h.
+   * Memory cost: 1024 Number integers.
    */
   currSyncCommitteeIndexes: ValidatorIndex[];
   /**
-   * Memory cost: 1024 Number integers
+   * Update freq: every ~ 27h.
+   * Memory cost: 1024 Number integers.
    */
   nextSyncCommitteeIndexes: ValidatorIndex[];
+  /**
+   * Update freq: every ~ 27h.
+   * Memory cost: Map of Number -> Number with 1024 entries.
+   */
+  currSyncComitteeValidatorIndexMap: SyncComitteeValidatorIndexMap;
   config: IBeaconConfig;
 
   constructor(data: IEpochContextData) {
@@ -229,6 +260,7 @@ export class EpochContext {
     this.nextShuffling = data.nextShuffling;
     this.currSyncCommitteeIndexes = data.currSyncCommitteeIndexes;
     this.nextSyncCommitteeIndexes = data.nextSyncCommitteeIndexes;
+    this.currSyncComitteeValidatorIndexMap = data.currSyncComitteeValidatorIndexMap;
   }
 
   /**

packages/lodestar/src/api/impl/beacon/blocks/index.ts

@@ -1,4 +1,4 @@
-import {Root, phase0, Slot} from "@chainsafe/lodestar-types";
+import {Root, phase0, allForks, Slot} from "@chainsafe/lodestar-types";
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
 
 import {IBeaconChain} from "../../../../chain";
@@ -103,7 +103,7 @@ export class BeaconBlockApi implements IBeaconBlocksApi {
     return toBeaconHeaderResponse(this.config, block, true);
   }
 
-  async getBlock(blockId: BlockId): Promise<phase0.SignedBeaconBlock> {
+  async getBlock(blockId: BlockId): Promise<allForks.SignedBeaconBlock> {
     return await resolveBlockId(this.chain.forkChoice, this.db, blockId);
   }
 

packages/lodestar/src/api/impl/beacon/blocks/interface.ts

@@ -1,9 +1,9 @@
-import {Root, phase0, Slot} from "@chainsafe/lodestar-types";
+import {Root, phase0, allForks, Slot} from "@chainsafe/lodestar-types";
 
 export interface IBeaconBlocksApi {
   getBlockHeaders(filters: Partial<{slot: Slot; parentRoot: Root}>): Promise<phase0.SignedBeaconHeaderResponse[]>;
   getBlockHeader(blockId: BlockId): Promise<phase0.SignedBeaconHeaderResponse>;
-  getBlock(blockId: BlockId): Promise<phase0.SignedBeaconBlock>;
+  getBlock(blockId: BlockId): Promise<allForks.SignedBeaconBlock>;
   publishBlock(block: phase0.SignedBeaconBlock): Promise<void>;
 }
 

packages/lodestar/src/api/impl/beacon/blocks/utils.ts

@@ -1,4 +1,4 @@
-import {phase0} from "@chainsafe/lodestar-types";
+import {phase0, allForks} from "@chainsafe/lodestar-types";
 import {blockToHeader} from "@chainsafe/lodestar-beacon-state-transition";
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
 import {IForkChoice} from "@chainsafe/lodestar-fork-choice";
@@ -27,7 +27,7 @@ export async function resolveBlockId(
   forkChoice: IForkChoice,
   db: IBeaconDb,
   blockId: BlockId
-): Promise<phase0.SignedBeaconBlock> {
+): Promise<allForks.SignedBeaconBlock> {
   const block = await resolveBlockIdOrNull(forkChoice, db, blockId);
   if (!block) {
     throw new ApiError(404, `No block found for id '${blockId}'`);
@@ -40,7 +40,7 @@ async function resolveBlockIdOrNull(
   forkChoice: IForkChoice,
   db: IBeaconDb,
   blockId: BlockId
-): Promise<phase0.SignedBeaconBlock | null> {
+): Promise<allForks.SignedBeaconBlock | null> {
   blockId = blockId.toLowerCase();
   if (blockId === "head") {
     const head = forkChoice.getHead();

packages/lodestar/src/api/impl/beacon/pool/interface.ts

@@ -1,4 +1,4 @@
-import {CommitteeIndex, phase0, Slot} from "@chainsafe/lodestar-types";
+import {altair, CommitteeIndex, phase0, Slot} from "@chainsafe/lodestar-types";
 
 export interface IAttestationFilters {
   slot: Slot;
@@ -7,11 +7,12 @@ export interface IAttestationFilters {
 
 export interface IBeaconPoolApi {
   getAttestations(filters?: Partial<IAttestationFilters>): Promise<phase0.Attestation[]>;
-  submitAttestations(attestations: phase0.Attestation[]): Promise<void>;
   getAttesterSlashings(): Promise<phase0.AttesterSlashing[]>;
-  submitAttesterSlashing(slashing: phase0.AttesterSlashing): Promise<void>;
   getProposerSlashings(): Promise<phase0.ProposerSlashing[]>;
-  submitProposerSlashing(slashing: phase0.ProposerSlashing): Promise<void>;
   getVoluntaryExits(): Promise<phase0.SignedVoluntaryExit[]>;
+  submitAttestations(attestations: phase0.Attestation[]): Promise<void>;
+  submitAttesterSlashing(slashing: phase0.AttesterSlashing): Promise<void>;
+  submitProposerSlashing(slashing: phase0.ProposerSlashing): Promise<void>;
   submitVoluntaryExit(exit: phase0.SignedVoluntaryExit): Promise<void>;
+  submitSyncCommitteeSignatures(signatures: altair.SyncCommitteeSignature[]): Promise<void>;
 }

packages/lodestar/src/api/impl/beacon/pool/pool.ts

@@ -1,18 +1,20 @@
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
+import {altair, Epoch, phase0} from "@chainsafe/lodestar-types";
+import {computeEpochAtSlot, allForks} from "@chainsafe/lodestar-beacon-state-transition";
+import {SYNC_COMMITTEE_SUBNET_COUNT} from "@chainsafe/lodestar-params";
 import {IAttestationJob, IBeaconChain} from "../../../../chain";
 import {AttestationError, AttestationErrorCode} from "../../../../chain/errors";
 import {validateGossipAttestation} from "../../../../chain/validation";
 import {validateGossipAttesterSlashing} from "../../../../chain/validation/attesterSlashing";
 import {validateGossipProposerSlashing} from "../../../../chain/validation/proposerSlashing";
 import {validateGossipVoluntaryExit} from "../../../../chain/validation/voluntaryExit";
+import {validateSyncCommitteeSigOnly} from "../../../../chain/validation/syncCommittee";
 import {IBeaconDb} from "../../../../db";
 import {INetwork} from "../../../../network";
 import {IBeaconSync} from "../../../../sync";
 import {IApiOptions} from "../../../options";
 import {IApiModules} from "../../interface";
 import {IAttestationFilters, IBeaconPoolApi} from "./interface";
-import {phase0} from "@chainsafe/lodestar-types";
-import {allForks} from "@chainsafe/lodestar-beacon-state-transition";
 
 export class BeaconPoolApi implements IBeaconPoolApi {
   private readonly config: IBeaconConfig;
@@ -41,6 +43,18 @@ export class BeaconPoolApi implements IBeaconPoolApi {
     });
   }
 
+  async getAttesterSlashings(): Promise<phase0.AttesterSlashing[]> {
+    return this.db.attesterSlashing.values();
+  }
+
+  async getProposerSlashings(): Promise<phase0.ProposerSlashing[]> {
+    return this.db.proposerSlashing.values();
+  }
+
+  async getVoluntaryExits(): Promise<phase0.SignedVoluntaryExit[]> {
+    return this.db.voluntaryExit.values();
+  }
+
   async submitAttestations(attestations: phase0.Attestation[]): Promise<void> {
     for (const attestation of attestations) {
       const attestationJob = {
@@ -66,30 +80,65 @@ export class BeaconPoolApi implements IBeaconPoolApi {
     }
   }
 
-  async getAttesterSlashings(): Promise<phase0.AttesterSlashing[]> {
-    return this.db.attesterSlashing.values();
-  }
-
   async submitAttesterSlashing(slashing: phase0.AttesterSlashing): Promise<void> {
     await validateGossipAttesterSlashing(this.config, this.chain, this.db, slashing);
     await Promise.all([this.network.gossip.publishAttesterSlashing(slashing), this.db.attesterSlashing.add(slashing)]);
   }
 
-  async getProposerSlashings(): Promise<phase0.ProposerSlashing[]> {
-    return this.db.proposerSlashing.values();
-  }
-
   async submitProposerSlashing(slashing: phase0.ProposerSlashing): Promise<void> {
     await validateGossipProposerSlashing(this.config, this.chain, this.db, slashing);
     await Promise.all([this.network.gossip.publishProposerSlashing(slashing), this.db.proposerSlashing.add(slashing)]);
   }
 
-  async getVoluntaryExits(): Promise<phase0.SignedVoluntaryExit[]> {
-    return this.db.voluntaryExit.values();
-  }
-
   async submitVoluntaryExit(exit: phase0.SignedVoluntaryExit): Promise<void> {
     await validateGossipVoluntaryExit(this.config, this.chain, this.db, exit);
     await Promise.all([this.network.gossip.publishVoluntaryExit(exit), this.db.voluntaryExit.add(exit)]);
   }
+
+  /**
+   * POST `/eth/v1/beacon/pool/sync_committees`
+   *
+   * Submits sync committee signature objects to the node.
+   * Sync committee signatures are not present in phase0, but are required for Altair networks.
+   * If a sync committee signature is validated successfully the node MUST publish that sync committee signature on all applicable subnets.
+   * If one or more sync committee signatures fail validation the node MUST return a 400 error with details of which sync committee signatures have failed, and why.
+   *
+   * https://github.com/ethereum/eth2.0-APIs/pull/135
+   */
+  async submitSyncCommitteeSignatures(signatures: altair.SyncCommitteeSignature[]): Promise<void> {
+    // Fetch states for all epochs of the `signatures`
+    const epochs = new Set<Epoch>();
+    for (const signature of signatures) {
+      epochs.add(computeEpochAtSlot(this.config, signature.slot));
+    }
+
+    // TODO: Fetch states at signature epochs
+    const state = await this.chain.getHeadStateAtCurrentEpoch();
+
+    // TODO: Cache this value
+    const SYNC_COMMITTEE_SUBNET_SIZE = Math.floor(this.config.params.SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT);
+
+    await Promise.all(
+      signatures.map(async (signature) => {
+        const indexesInCommittee = state.currSyncComitteeValidatorIndexMap.get(signature.validatorIndex);
+        if (indexesInCommittee === undefined || indexesInCommittee.length === 0) {
+          return; // Not a sync committee member
+        }
+
+        // Verify signature only, all other data is very likely to be correct, since the `signature` object is created by this node.
+        // Worst case if `signature` is not valid, gossip peers will drop it and slightly downscore us.
+        await validateSyncCommitteeSigOnly(this.chain, state, signature);
+
+        await Promise.all(
+          indexesInCommittee.map(async (indexInCommittee) => {
+            // Sync committee subnet members are just sequential in the order they appear in SyncCommitteeIndexes array
+            const subnet = Math.floor(indexInCommittee / SYNC_COMMITTEE_SUBNET_SIZE);
+            const indexInSubCommittee = indexInCommittee % SYNC_COMMITTEE_SUBNET_SIZE;
+            this.db.syncCommittee.add(subnet, signature, indexInSubCommittee);
+            await this.network.gossip.publishSyncCommitteeSignature(signature, subnet);
+          })
+        );
+      })
+    );
+  }
 }

packages/lodestar/src/api/impl/beacon/state/interface.ts

@@ -1,5 +1,14 @@
-import {allForks, BLSPubkey, CommitteeIndex, Epoch, Root, Slot, ValidatorIndex} from "@chainsafe/lodestar-types";
-import {phase0} from "@chainsafe/lodestar-beacon-state-transition";
+import {
+  phase0,
+  altair,
+  allForks,
+  BLSPubkey,
+  CommitteeIndex,
+  Epoch,
+  Root,
+  Slot,
+  ValidatorIndex,
+} from "@chainsafe/lodestar-types";
 
 export interface IBeaconStateApi {
   getStateRoot(stateId: StateId): Promise<Root>;
@@ -12,6 +21,7 @@ export interface IBeaconStateApi {
     indices?: (BLSPubkey | ValidatorIndex)[]
   ): Promise<phase0.ValidatorBalance[]>;
   getStateCommittees(stateId: StateId, filters?: ICommitteesFilters): Promise<phase0.BeaconCommitteeResponse[]>;
+  getEpochSyncCommittees(stateId: StateId, epoch?: Epoch): Promise<altair.SyncCommitteeByValidatorIndices>;
   getFork(stateId: StateId): Promise<phase0.Fork>;
 }
 

packages/lodestar/src/api/impl/beacon/state/state.ts

@@ -1,5 +1,5 @@
 import {IBeaconConfig} from "@chainsafe/lodestar-config";
-import {Root, phase0, allForks, BLSPubkey} from "@chainsafe/lodestar-types";
+import {Root, phase0, allForks, BLSPubkey, Epoch, altair} from "@chainsafe/lodestar-types";
 import {List, readonlyValues} from "@chainsafe/ssz";
 import {computeEpochAtSlot, getCurrentEpoch} from "@chainsafe/lodestar-beacon-state-transition";
 import {IBeaconChain} from "../../../../chain/interface";
@@ -12,6 +12,7 @@ import {IBeaconStateApi, ICommitteesFilters, IValidatorFilters, StateId} from ".
 import {
   filterStateValidatorsByStatuses,
   getEpochBeaconCommittees,
+  getSyncCommittees,
   getValidatorStatus,
   resolveStateId,
   toValidatorResponse,
@@ -133,9 +134,8 @@ export class BeaconStateApi implements IBeaconStateApi {
   async getStateCommittees(stateId: StateId, filters?: ICommitteesFilters): Promise<phase0.BeaconCommitteeResponse[]> {
     const state = await resolveStateId(this.config, this.chain, this.db, stateId);
 
-    const committes: phase0.ValidatorIndex[][][] = getEpochBeaconCommittees(
+    const committes = getEpochBeaconCommittees(
       this.config,
-      this.chain,
       state,
       filters?.epoch ?? computeEpochAtSlot(this.config, state.slot)
     );
@@ -158,6 +158,28 @@ export class BeaconStateApi implements IBeaconStateApi {
     });
   }
 
+  /**
+   * Retrieves the sync committees for the given state.
+   * @param epoch Fetch sync committees for the given epoch. If not present then the sync committees for the epoch of the state will be obtained.
+   */
+  async getEpochSyncCommittees(stateId: StateId, epoch?: Epoch): Promise<altair.SyncCommitteeByValidatorIndices> {
+    // TODO: Should pick a state with the provided epoch too
+    const state = (await resolveStateId(this.config, this.chain, this.db, stateId)) as altair.BeaconState;
+
+    // TODO: If possible compute the syncCommittees in advance of the fork and expose them here.
+    // So the validators can prepare and potentially attest the first block. Not critical tho, it's very unlikely
+    const stateEpoch = computeEpochAtSlot(this.config, state.slot);
+    if (stateEpoch < this.config.params.ALTAIR_FORK_EPOCH) {
+      throw new ApiError(400, "Requested state before ALTAIR_FORK_EPOCH");
+    }
+
+    return {
+      validators: getSyncCommittees(this.config, state, epoch ?? stateEpoch),
+      // TODO: This is not used by the validator and will be deprecated soon
+      validatorAggregates: [],
+    };
+  }
+
   async getState(stateId: StateId): Promise<allForks.BeaconState> {
     return await resolveStateId(this.config, this.chain, this.db, stateId);
   }