feat: store cache-state data in database

config/default.json5

@@ -10,13 +10,6 @@
     path: null
   },
 
-  conf: {
-    name: 'config',
-
-    // Specifies if configuration using Conf() should be persisted between restarts
-    persist: true
-  },
-
   rates: {
     enabled: true,
     provider: 'coingecko',
@@ -142,6 +135,11 @@
         // If to use polling strategy, if false then listening is used.
         polling: true
       }
+    },
+
+    // RNS Registrar contract settings used for RNS precache
+    registrar: {
+      startingBlock: "genesis"
     }
   }
 }

package.json

@@ -55,7 +55,7 @@
     "@oclif/command": "^1.5.19",
     "@oclif/config": "^1.14.0",
     "@oclif/errors": "^1.2.2",
-    "@oclif/parser": "^3.8.4",
+    "@oclif/parser": "^3.8.5",
     "@oclif/plugin-help": "^3.0.1",
     "@rsksmart/rif-marketplace-nfts": "~0.0.2",
     "@rsksmart/rif-marketplace-storage": "^0.1.0-dev.0",
@@ -67,32 +67,32 @@
     "coingecko-api": "^1.0.10",
     "colors": "^1.4.0",
     "compression": "^1.7.4",
-    "conf": "^6.2.4",
     "config": "^3.3.1",
     "cors": "^2.8.5",
     "feathers-hooks-common": "^5.0.3",
     "feathers-sequelize": "^6.1.0",
     "helmet": "^3.22.0",
     "listr": "^0.14.3",
-    "pg": "^8.0.3",
+    "pg": "^8.2.1",
     "pg-hstore": "^2.3.3",
     "reflect-metadata": "^0.1.13",
-    "sequelize": "^5.21.7",
+    "sequelize": "^5.21.10",
+    "sequelize-store": "^0.3.1",
     "sequelize-typescript": "^1.1.0",
     "sql-formatter": "^2.3.3",
     "sqlite3": "^4.2.0",
-    "web3-core-subscriptions": "^1.2.6",
-    "web3-eth": "^1.2.6",
-    "web3-eth-contract": "^1.2.6",
-    "web3-utils": "^1.2.6",
+    "web3-core-subscriptions": "^1.2.8",
+    "web3-eth": "^1.2.8",
+    "web3-eth-contract": "^1.2.8",
+    "web3-utils": "^1.2.8",
     "winston": "^3.2.1",
     "winston-transport": "^4.3.0"
   },
   "devDependencies": {
-    "@fluffy-spoon/substitute": "^1.133.0",
+    "@fluffy-spoon/substitute": "^1.140.0",
     "@oclif/dev-cli": "^1.22.2",
-    "@oclif/test": "^1.2.5",
-    "@types/bluebird": "^3.5.29",
+    "@oclif/test": "^1.2.6",
+    "@types/bluebird": "^3.5.32",
     "@types/chai": "^4.2.11",
     "@types/chai-as-promised": "^7.1.2",
     "@types/compression": "^1.0.1",
@@ -103,10 +103,10 @@
     "@types/helmet": "0.0.47",
     "@types/listr": "^0.14.2",
     "@types/mocha": "^7.0.2",
-    "@types/node": "^14.0.1",
+    "@types/node": "^14.0.4",
     "@types/node-fetch": "^2.5.7",
     "@types/serve-favicon": "^2.5.0",
-    "@types/sinon": "^9.0.0",
+    "@types/sinon": "^9.0.3",
     "@types/sql-formatter": "^2.3.0",
     "@types/validator": "^13.0.0",
     "bignumber.js": "^9.0.0",

src/app.ts

@@ -10,7 +10,7 @@ import { Application } from './definitions'
 import { loggingFactory } from './logger'
 import sequelize from './sequelize'
 import blockchain from './blockchain'
-import { configure as confConfigure } from './conf'
+import { configureStore } from './store'
 
 import storage from './storage'
 import rates from './rates'
@@ -51,7 +51,7 @@ export function appFactory (): Application {
   // Custom general services
   app.configure(sequelize)
   app.configure(blockchain)
-  app.configure(confConfigure)
+  app.configure(configureStore)
 
   /**********************************************************/
   // Configure each services

src/blockchain/events.ts

@@ -5,12 +5,12 @@ import { EventEmitter } from 'events'
 import { NotImplemented } from '@feathersjs/errors'
 import { Op } from 'sequelize'
 import { Sema } from 'async-sema'
+import { getObject as getStore } from 'sequelize-store'
 
-import { asyncFilter, scopeStore } from '../utils'
-import { confFactory } from '../conf'
+import { asyncFilter } from '../utils'
 import { loggingFactory } from '../logger'
 import Event, { EventInterface } from './event.model'
-import { Store, Logger } from '../definitions'
+import { Logger } from '../definitions'
 
 // Constant number that defines default interval of all polling mechanisms.
 const DEFAULT_POLLING_INTERVAL = 5000
@@ -24,6 +24,10 @@ export interface PollingOptions {
   pollingInterval?: number
 }
 
+export interface BlockTrackerStore {
+  [PROCESSED_BLOCK_KEY]?: number
+}
+
 export enum EventsEmitterStrategy {
   POLLING = 1,
   LISTENING,
@@ -43,7 +47,7 @@ export interface EventsEmitterOptions {
   startingBlock?: number | string
 
   // Defines BlockTracker or its configuration
-  blockTracker?: BlockTracker | { store?: Store, keyPrefix?: string }
+  blockTracker?: BlockTracker | { store?: BlockTrackerStore, keyPrefix?: string }
 
   // Defines the NewBlockEmitter or its configuration
   newBlockEmitter?: EventEmitter | PollingOptions
@@ -57,21 +61,18 @@ export interface EventsEmitterOptions {
  * of the server.
  */
 export class BlockTracker {
-  private store: Store
-  private lastProcessedBlock: number
+  private readonly store: BlockTrackerStore
 
-  constructor (store: Store) {
+  constructor (store: BlockTrackerStore) {
     this.store = store
-    this.lastProcessedBlock = this.store.get(PROCESSED_BLOCK_KEY)
   }
 
   setLastProcessedBlock (block: number): void {
-    this.lastProcessedBlock = block
-    this.store.set(PROCESSED_BLOCK_KEY, block)
+    this.store[PROCESSED_BLOCK_KEY] = block
   }
 
   getLastProcessedBlock (): number | undefined {
-    return this.lastProcessedBlock
+    return this.store[PROCESSED_BLOCK_KEY]
   }
 }
 
@@ -225,16 +226,11 @@ export abstract class BaseEventsEmitter extends AutoStartStopEventEmitter {
       if (options.blockTracker instanceof BlockTracker) {
         this.blockTracker = options.blockTracker
       } else {
-        let confStore = options.blockTracker.store || confFactory()
-
-        if (options.blockTracker.keyPrefix) {
-          confStore = scopeStore(confStore, options.blockTracker.keyPrefix)
-        }
-
+        const confStore = options.blockTracker.store || getStore(options.blockTracker.keyPrefix ? options.blockTracker.keyPrefix : undefined)
         this.blockTracker = new BlockTracker(confStore)
       }
     } else {
-      this.blockTracker = new BlockTracker(confFactory())
+      this.blockTracker = new BlockTracker(getStore())
     }
 
     if (options?.newBlockEmitter) {

src/blockchain/utils.ts

@@ -2,29 +2,22 @@ import { AbiItem } from 'web3-utils'
 import Eth from 'web3-eth'
 import { EventEmitter } from 'events'
 import config from 'config'
+import { getObject } from 'sequelize-store'
 
 import { loggingFactory } from '../logger'
 import eventsEmitterFactory, { BlockTracker, EventsEmitterOptions, PollingOptions } from './events'
-import { confFactory } from '../conf'
-import { scopeStore } from '../utils'
-import { Store } from '../definitions'
 
 function getBlockTracker (keyPrefix?: string): BlockTracker {
-  let confStore: Store = confFactory()
-
-  if (keyPrefix) {
-    confStore = scopeStore(confStore, keyPrefix)
-  }
-
-  return new BlockTracker(confStore)
+  const store = getObject(keyPrefix)
+  return new BlockTracker(store)
 }
 
 export async function getBlockDate (eth: Eth, blockNumber: number): Promise<Date> {
   return new Date(((await eth.getBlock(blockNumber)).timestamp as number) * 1000)
 }
 
 export function isServiceInitialized (serviceName: string): boolean {
-  const blockTracker = getBlockTracker(serviceName)
+  const blockTracker = getBlockTracker(`${serviceName}.`)
   return blockTracker.getLastProcessedBlock() !== undefined
 }
 
@@ -43,7 +36,7 @@ export function getEventsEmitterForService (serviceName: string, eth: Eth, contr
     {
       newBlockEmitter: newBlockEmitterOptions,
       loggerBaseName: serviceName,
-      blockTracker: { keyPrefix: serviceName }
+      blockTracker: { keyPrefix: `${serviceName}.` }
     } as EventsEmitterOptions
   )
 

src/cli/precache.ts

@@ -2,6 +2,7 @@ import { services, SupportedServices } from '../app'
 import Listr from 'listr'
 import { sequelizeFactory } from '../sequelize'
 import { BaseCLICommand, capitalizeFirstLetter, validateServices } from '../utils'
+import { initStore } from '../store'
 
 export default class PreCache extends BaseCLICommand {
   static get description () {
@@ -36,7 +37,10 @@ ${formattedServices}`
     }
 
     // Init database connection
-    sequelizeFactory()
+    const sequelize = sequelizeFactory()
+
+    // Init Store
+    await initStore(sequelize)
 
     const tasksDefinition = servicesToPreCache.map(
       serviceName => {

src/cli/purge.ts

@@ -3,6 +3,7 @@ import Listr from 'listr'
 import { sequelizeFactory } from '../sequelize'
 import { BaseCLICommand, capitalizeFirstLetter, validateServices } from '../utils'
 import Event from '../blockchain/event.model'
+import { initStore } from '../store'
 
 export default class Purge extends BaseCLICommand {
   static get description () {
@@ -37,7 +38,10 @@ ${formattedServices}`
     }
 
     // Init database connection
-    sequelizeFactory()
+    const sequelize = sequelizeFactory()
+
+    // Init Store
+    await initStore(sequelize)
 
     const tasksDefinition = servicesToPurge.map(
       serviceName => {

src/definitions.ts

@@ -20,11 +20,6 @@ interface ServiceTypes {
 // The application instance type that will be used everywhere else
 export type Application = ExpressFeathers<ServiceTypes>;
 
-export interface Store {
-  get (key: string): any
-  set (key: string, value: any): void
-}
-
 export interface CachedService {
   precache (eth?: Eth): Promise<void>
   purge (): Promise<void>
@@ -95,14 +90,6 @@ export interface Config {
     path?: string
   }
 
-  conf?: {
-    // Name for config to have separate configs for different environments (test, dev, prod)
-    name?: string
-
-    // Specifies if configuration using Conf() should be persisted between restarts
-    persist?: boolean
-  }
-
   blockchain?: {
     // Address to where web3js should connect to. Should be WS endpoint.
     provider?: string

src/rates/providers/coingecko.ts

@@ -1,7 +1,6 @@
-import { FetchedRates, ToSymbols, FromSymbols, SupportedToSymbols, SupportedFromSymbols } from '../../definitions'
+import { FetchedRates, ToSymbols, FromSymbols } from '../../definitions'
 import CoinGecko from 'coingecko-api'
-import { ConfigurationError, RatesProviderError } from '../../errors'
-import config from 'config'
+import { RatesProviderError } from '../../errors'
 
 const SUPPORTED_FROM_SYMBOLS: Record<FromSymbols, string> = {
   rbtc: 'rootstock',

src/rns/index.ts

@@ -3,10 +3,10 @@ import Eth from 'web3-eth'
 import { AbiItem } from 'web3-utils'
 import config from 'config'
 import { EventData } from 'web3-eth-contract'
+import { getObject } from 'sequelize-store'
 
 import { Application, CachedService } from '../definitions'
 import { loggingFactory } from '../logger'
-import { confFactory } from '../conf'
 import { ethFactory } from '../blockchain'
 import { getEventsEmitterForService, isServiceInitialized } from '../blockchain/utils'
 
@@ -22,7 +22,7 @@ import rnsContractAbi from '@rsksmart/rns-rskregistrar/RSKOwnerData.json'
 import rnsReverseContractAbi from '@rsksmart/rns-reverse/NameResolverData.json'
 import auctionRegistrarContractAbi from '@rsksmart/rns-auction-registrar/TokenRegistrarData.json'
 import simplePlacementsContractAbi from '@rsksmart/rif-marketplace-nfts/ERC721SimplePlacementsABI.json'
-import { errorHandler } from '../utils'
+import { errorHandler, waitForReadyApp } from '../utils'
 
 import { processRskOwner, processAuctionRegistrar } from './rns.precache'
 
@@ -83,6 +83,8 @@ const rns: CachedService = {
     }
     logger.info('RNS service: enabled')
 
+    await waitForReadyApp(app)
+
     // Initialize feather's service
     app.use('/rns/v0/:ownerAddress/domains', new RnsService({ Model: Domain }))
     app.use('/rns/v0/:ownerAddress/sold', new RnsService({ Model: SoldDomain }))
@@ -139,9 +141,10 @@ const rns: CachedService = {
     const domainsCount = await Domain.destroy({ where: {}, truncate: true, cascade: true })
     logger.info(`Removed ${offersCount} offers entries, ${soldCount} sold domains and ${domainsCount} domains`)
 
-    confFactory().delete('rns.owner')
-    confFactory().delete('rns.reverse')
-    confFactory().delete('rns.placement')
+    const store = getObject()
+    delete store['rns.placement.lastProcessedBlock']
+    delete store['rns.reverse.lastProcessedBlock']
+    delete store['rns.owner.lastProcessedBlock']
   }
 
 }