feat: add EVM indexer (#274)

* feat: add EVM indexer

To make indexers extractable it was separated into Indexer class
which instance consumer initiaties. This allows better separation
(no need for pulling deps for all networks) - everything currently
is exported under either evm or starknet object, it can be extracted
later (we might need to extract some common things first into other
package). We also have specific types for writers for each network.

This could also be useful in the future if we put multiple APIs
in single instance of checkpoint, it could accept indexers instead
of just single indexer.

* chore: fetch events from at most 10 blocks at a time

* fix: update network ID

* refactor: only fetch targeted events

* feat: add simple range adjustment logic

* fix: use BaseIndexer instead of EvmIndexer in Checkpoint

package.json

@@ -19,6 +19,11 @@
   },
   "prettier": "@snapshot-labs/prettier-config",
   "dependencies": {
+    "@ethersproject/abi": "^5.7.0",
+    "@ethersproject/address": "^5.7.0",
+    "@ethersproject/keccak256": "^5.7.0",
+    "@ethersproject/providers": "^5.7.2",
+    "@ethersproject/strings": "^5.7.0",
     "@graphql-tools/schema": "^8.5.1",
     "bluebird": "^3.7.2",
     "connection-string": "^4.3.5",

src/checkpoint.ts

@@ -6,7 +6,7 @@ import { Pool as PgPool } from 'pg';
 import getGraphQL, { CheckpointsGraphQLObject, MetadataGraphQLObject } from './graphql';
 import { GqlEntityController } from './graphql/controller';
 import { CheckpointRecord, CheckpointsStore, MetadataId } from './stores/checkpoints';
-import { BaseProvider, StarknetProvider, BlockNotFoundError } from './providers';
+import { BaseIndexer, BlockNotFoundError } from './providers';
 import { createLogger, Logger, LogLevel } from './utils/logger';
 import { getConfigChecksum, getContractsFromConfig } from './utils/checkpoint';
 import { extendSchema } from './utils/graphql';
@@ -15,41 +15,38 @@ import { AsyncMySqlPool, createMySqlPool } from './mysql';
 import { createPgPool } from './pg';
 import { checkpointConfigSchema } from './schemas';
 import { register } from './register';
-import {
-  ContractSourceConfig,
-  CheckpointConfig,
-  CheckpointOptions,
-  CheckpointWriters,
-  TemplateSource
-} from './types';
-
-const BLOCK_PRELOAD = 1000;
+import { ContractSourceConfig, CheckpointConfig, CheckpointOptions, TemplateSource } from './types';
+
+const BLOCK_PRELOAD_START_RANGE = 1000;
+const BLOCK_RELOAD_MIN_RANGE = 10;
+const BLOCK_PRELOAD_STEP = 100;
+const BLOCK_PRELOAD_TARGET = 10;
 const BLOCK_PRELOAD_OFFSET = 50;
 const DEFAULT_FETCH_INTERVAL = 2000;
 
 export default class Checkpoint {
   public config: CheckpointConfig;
-  public writer: CheckpointWriters;
   public opts?: CheckpointOptions;
   public schema: string;
 
   private readonly entityController: GqlEntityController;
   private readonly log: Logger;
-  private readonly networkProvider: BaseProvider;
+  private readonly indexer: BaseIndexer;
 
   private dbConnection: string;
   private knex: Knex;
   private mysqlPool?: AsyncMySqlPool;
   private pgPool?: PgPool;
   private checkpointsStore?: CheckpointsStore;
   private activeTemplates: TemplateSource[] = [];
+  private preloadStep: number = BLOCK_PRELOAD_START_RANGE;
   private preloadedBlocks: number[] = [];
   private preloadEndBlock = 0;
   private cpBlocksCache: number[] | null;
 
   constructor(
     config: CheckpointConfig,
-    writer: CheckpointWriters,
+    indexer: BaseIndexer,
     schema: string,
     opts?: CheckpointOptions
   ) {
@@ -59,12 +56,9 @@ export default class Checkpoint {
     }
 
     this.config = config;
-    this.writer = writer;
     this.opts = opts;
     this.schema = extendSchema(schema);
 
-    this.validateConfig();
-
     this.entityController = new GqlEntityController(this.schema, config);
 
     this.log = createLogger({
@@ -79,8 +73,14 @@ export default class Checkpoint {
         : {})
     });
 
-    const NetworkProvider = opts?.NetworkProvider || StarknetProvider;
-    this.networkProvider = new NetworkProvider({ instance: this, log: this.log, abis: opts?.abis });
+    this.indexer = indexer;
+    this.indexer.init({
+      instance: this,
+      log: this.log,
+      abis: opts?.abis
+    });
+
+    this.validateConfig();
 
     this.cpBlocksCache = [];
 
@@ -139,7 +139,7 @@ export default class Checkpoint {
   }
 
   public get sourceContracts() {
-    return this.networkProvider.formatAddresses(getContractsFromConfig(this.config));
+    return this.indexer.getProvider().formatAddresses(getContractsFromConfig(this.config));
   }
 
   public getCurrentSources(blockNumber: number) {
@@ -159,7 +159,7 @@ export default class Checkpoint {
     this.log.debug('starting');
 
     await this.validateStore();
-    await this.networkProvider.init();
+    await this.indexer.getProvider().init();
 
     const templateSources = await this.store.getTemplateSources();
     await Promise.all(
@@ -177,7 +177,7 @@ export default class Checkpoint {
 
     const blockNum = await this.getStartBlockNum();
     this.preloadEndBlock =
-      (await this.networkProvider.getLatestBlockNumber()) - BLOCK_PRELOAD_OFFSET;
+      (await this.indexer.getProvider().getLatestBlockNumber()) - BLOCK_PRELOAD_OFFSET;
 
     return await this.next(blockNum);
   }
@@ -332,8 +332,14 @@ export default class Checkpoint {
     let currentBlock = blockNum;
 
     while (currentBlock <= this.preloadEndBlock) {
-      const endBlock = Math.min(currentBlock + BLOCK_PRELOAD, this.preloadEndBlock);
-      const checkpoints = await this.networkProvider.getCheckpointsRange(currentBlock, endBlock);
+      const endBlock = Math.min(currentBlock + this.preloadStep, this.preloadEndBlock);
+      const checkpoints = await this.indexer
+        .getProvider()
+        .getCheckpointsRange(currentBlock, endBlock);
+
+      const increase =
+        checkpoints.length > BLOCK_PRELOAD_TARGET ? -BLOCK_PRELOAD_STEP : +BLOCK_PRELOAD_STEP;
+      this.preloadStep = Math.max(BLOCK_RELOAD_MIN_RANGE, this.preloadStep + increase);
 
       if (checkpoints.length > 0) {
         this.preloadedBlocks = checkpoints.map(cp => cp.blockNumber).sort();
@@ -363,7 +369,7 @@ export default class Checkpoint {
 
     try {
       const initialSources = this.getCurrentSources(blockNum);
-      const nextBlockNumber = await this.networkProvider.processBlock(blockNum);
+      const nextBlockNumber = await this.indexer.getProvider().processBlock(blockNum);
       const sources = this.getCurrentSources(nextBlockNumber);
 
       if (initialSources.length !== sources.length) {
@@ -375,7 +381,7 @@ export default class Checkpoint {
       if (err instanceof BlockNotFoundError) {
         if (this.config.optimistic_indexing) {
           try {
-            await this.networkProvider.processPool(blockNum);
+            await this.indexer.getProvider().processPool(blockNum);
           } catch (err) {
             this.log.error({ blockNumber: blockNum, err }, 'error occurred during pool processing');
           }
@@ -480,7 +486,9 @@ export default class Checkpoint {
     ];
 
     const missingAbis = usedAbis.filter(abi => !this.opts?.abis?.[abi]);
-    const missingWriters = usedWriters.filter(writer => !this.writer[writer.fn]);
+    const missingWriters = usedWriters.filter(
+      writer => !this.indexer.getHandlers().includes(writer.fn)
+    );
 
     if (missingAbis.length > 0) {
       throw new Error(
@@ -498,7 +506,7 @@ export default class Checkpoint {
   }
 
   private async validateStore() {
-    const networkIdentifier = await this.networkProvider.getNetworkIdentifier();
+    const networkIdentifier = await this.indexer.getProvider().getNetworkIdentifier();
     const configChecksum = getConfigChecksum(this.config);
 
     const storedNetworkIdentifier = await this.store.getMetadata(MetadataId.NetworkIdentifier);

src/index.ts

@@ -3,6 +3,7 @@ export { LogLevel } from './utils/logger';
 export { AsyncMySqlPool } from './mysql';
 export { createGetLoader } from './graphql';
 export { Model } from './orm';
+export * from './providers';
 export * from './types';
 
 export default Checkpoint;

src/providers/base.ts

@@ -3,10 +3,9 @@ import Checkpoint from '../checkpoint';
 import { CheckpointRecord } from '../stores/checkpoints';
 import { Logger } from '../utils/logger';
 import { AsyncMySqlPool } from '../mysql';
-import { CheckpointConfig, CheckpointWriters, ContractSourceConfig } from '../types';
+import { CheckpointConfig, ContractSourceConfig } from '../types';
 
-type Instance = {
-  writer: CheckpointWriters;
+export type Instance = {
   config: CheckpointConfig;
   getCurrentSources(blockNumber: number): ContractSourceConfig[];
   setLastIndexedBlock(blockNum: number);
@@ -76,7 +75,28 @@ export class BaseProvider {
 
   async getCheckpointsRange(fromBlock: number, toBlock: number): Promise<CheckpointRecord[]> {
     throw new Error(
-      `getEventsRange method was not defined when fetching events from ${fromBlock} to ${toBlock}`
+      `getCheckpointsRange method was not defined when fetching events from ${fromBlock} to ${toBlock}`
     );
   }
 }
+
+export class BaseIndexer {
+  protected provider?: BaseProvider;
+
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  init({ instance, log, abis }: { instance: Instance; log: Logger; abis?: Record<string, any> }) {
+    throw new Error('init method was not defined');
+  }
+
+  public getProvider() {
+    if (!this.provider) {
+      throw new Error('Provider not initialized');
+    }
+
+    return this.provider;
+  }
+
+  public getHandlers(): string[] {
+    throw new Error('getHandlers method was not defined');
+  }
+}

src/providers/evm/index.ts

@@ -0,0 +1,3 @@
+export { EvmProvider } from './provider';
+export { EvmIndexer } from './indexer';
+export * from './types';

src/providers/evm/indexer.ts

@@ -0,0 +1,21 @@
+import { Logger } from '../../utils/logger';
+import { Instance, BaseIndexer } from '../base';
+import { EvmProvider } from './provider';
+import { Writer } from './types';
+
+export class EvmIndexer extends BaseIndexer {
+  private writers: Record<string, Writer>;
+
+  constructor(writers: Record<string, Writer>) {
+    super();
+    this.writers = writers;
+  }
+
+  init({ instance, log, abis }: { instance: Instance; log: Logger; abis?: Record<string, any> }) {
+    this.provider = new EvmProvider({ instance, log, abis, writers: this.writers });
+  }
+
+  public getHandlers(): string[] {
+    return Object.keys(this.writers);
+  }
+}