add indexing network logic

src/@types/OceanNode.ts

@@ -66,6 +66,7 @@ export interface OceanNodeConfig {
   httpPort: number
   feeStrategy: FeeStrategy
   supportedNetworks?: RPCS
+  indexingNetworks?: RPCS
   c2dClusters: C2DClusterInfo[]
   accountPurgatoryUrl: string
   assetPurgatoryUrl: string

src/index.ts

@@ -101,7 +101,7 @@ if (config.hasP2P) {
   await node.start()
 }
 if (config.hasIndexer && dbconn) {
-  indexer = new OceanIndexer(dbconn, config.supportedNetworks)
+  indexer = new OceanIndexer(dbconn, config.indexingNetworks)
   // if we set this var
   // it also loads initial data (useful for testing, or we might actually want to have a bootstrap list)
   // store and advertise DDOs

src/test/.env.test

@@ -2,6 +2,7 @@ HTTP_API_PORT=8001
 P2P_ipV4BindTcpPort=8000
 PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58
 RPCS='{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}}'
+INDEXER_NETWORKS='[8996]'
 DB_URL=http://localhost:8108/?apiKey=xyz
 IPFS_GATEWAY=https://ipfs.io/
 ARWEAVE_GATEWAY=https://arweave.net/

src/test/integration/compute.test.ts

@@ -108,6 +108,7 @@ describe('Compute', () => {
       buildEnvOverrideConfig(
         [
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS,
@@ -116,6 +117,7 @@ describe('Compute', () => {
         ],
         [
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           'http://localhost:8108/?apiKey=xyz',
           JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']),
@@ -127,7 +129,7 @@ describe('Compute', () => {
     config = await getConfiguration(true)
     dbconn = await new Database(config.dbConfig)
     oceanNode = await OceanNode.getInstance(dbconn)
-    indexer = new OceanIndexer(dbconn, mockSupportedNetworks)
+    indexer = new OceanIndexer(dbconn, config.indexingNetworks)
     oceanNode.addIndexer(indexer)
 
     provider = new JsonRpcProvider('http://127.0.0.1:8545')

src/test/integration/download.test.ts

@@ -68,6 +68,7 @@ describe('Should run a complete node flow.', () => {
       buildEnvOverrideConfig(
         [
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS,
@@ -76,6 +77,7 @@ describe('Should run a complete node flow.', () => {
         ],
         [
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           'http://localhost:8108/?apiKey=xyz',
           JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']),
@@ -88,7 +90,7 @@ describe('Should run a complete node flow.', () => {
     config = await getConfiguration(true) // Force reload the configuration
     database = await new Database(config.dbConfig)
     oceanNode = await OceanNode.getInstance(database)
-    indexer = new OceanIndexer(database, mockSupportedNetworks)
+    indexer = new OceanIndexer(database, config.indexingNetworks)
     oceanNode.addIndexer(indexer)
 
     let network = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID)

src/test/integration/encryptDecryptDDO.test.ts

@@ -81,13 +81,15 @@ describe('Should encrypt and decrypt DDO', () => {
         [
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.ADDRESS_FILE
         ],
         [
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           JSON.stringify([publisherAddress]),
           'http://localhost:8108/?apiKey=xyz',
           `${homedir}/.ocean/ocean-contracts/artifacts/address.json`

src/test/integration/indexer.test.ts

@@ -98,12 +98,14 @@ describe('Indexer stores a new metadata events and orders.', () => {
       buildEnvOverrideConfig(
         [
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.ADDRESS_FILE
         ],
         [
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           dbConfig.url,
           `${homedir}/.ocean/ocean-contracts/artifacts/address.json`

src/test/integration/operationsDashboard.test.ts

@@ -79,6 +79,7 @@ describe('Should test admin operations', () => {
       buildEnvOverrideConfig(
         [
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS,
@@ -87,6 +88,7 @@ describe('Should test admin operations', () => {
         ],
         [
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           'http://localhost:8108/?apiKey=xyz',
           JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']),
@@ -99,7 +101,7 @@ describe('Should test admin operations', () => {
     config = await getConfiguration(true) // Force reload the configuration
     dbconn = await new Database(config.dbConfig)
     oceanNode = await OceanNode.getInstance(dbconn)
-    indexer = new OceanIndexer(dbconn, mockSupportedNetworks)
+    indexer = new OceanIndexer(dbconn, config.indexingNetworks)
     oceanNode.addIndexer(indexer)
   })
 

src/test/integration/transactionValidation.test.ts

@@ -50,13 +50,15 @@ describe('validateOrderTransaction Function with Orders', () => {
       buildEnvOverrideConfig(
         [
           ENVIRONMENT_VARIABLES.RPCS,
+          ENVIRONMENT_VARIABLES.INDEXER_NETWORKS,
           ENVIRONMENT_VARIABLES.PRIVATE_KEY,
           ENVIRONMENT_VARIABLES.DB_URL,
           ENVIRONMENT_VARIABLES.AUTHORIZED_DECRYPTERS,
           ENVIRONMENT_VARIABLES.ADDRESS_FILE
         ],
         [
           JSON.stringify(mockSupportedNetworks),
+          JSON.stringify([8996]),
           '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
           'http://localhost:8108/?apiKey=xyz',
           JSON.stringify(['0xe2DD09d719Da89e5a3D0F2549c7E24566e947260']),
@@ -68,7 +70,7 @@ describe('validateOrderTransaction Function with Orders', () => {
     config = await getConfiguration(true) // Force reload the configuration
     const dbconn = await new Database(config.dbConfig)
     oceanNode = await OceanNode.getInstance(dbconn)
-    indexer = new OceanIndexer(dbconn, mockSupportedNetworks)
+    indexer = new OceanIndexer(dbconn, config.indexingNetworks)
     oceanNode.addIndexer(indexer)
 
     let network = getOceanArtifactsAdressesByChainId(DEVELOPMENT_CHAIN_ID)

src/test/unit/ocean.test.ts

@@ -22,21 +22,23 @@ describe('Status command tests', async () => {
       ENVIRONMENT_VARIABLES.PRIVATE_KEY,
       ENVIRONMENT_VARIABLES.IPFS_GATEWAY,
       ENVIRONMENT_VARIABLES.ARWEAVE_GATEWAY,
-      ENVIRONMENT_VARIABLES.RPCS
+      ENVIRONMENT_VARIABLES.RPCS,
+      ENVIRONMENT_VARIABLES.INDEXER_NETWORKS
     ],
     [
       '0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58',
       'https://ipfs.io/',
       'https://arweave.net/',
-      '{ "1": "https://rpc.eth.gateway.fm", "137": "https://polygon.meowrpc.com" }'
+      '{ "1": "https://rpc.eth.gateway.fm", "137": "https://polygon.meowrpc.com" }',
+      JSON.stringify([1, 137])
     ]
   )
   envOverrides = await setupEnvironment(null, envOverrides)
   // because of this
   const config = await getConfiguration(true)
   const db = await new Database(config.dbConfig)
   const oceanP2P = new OceanP2P(config, db)
-  const oceanIndexer = new OceanIndexer(db, config.supportedNetworks)
+  const oceanIndexer = new OceanIndexer(db, config.indexingNetworks)
   const oceanProvider = new OceanProvider(db)
   const oceanNode = OceanNode.getInstance(db, oceanP2P)
 
@@ -63,7 +65,7 @@ describe('Status command tests', async () => {
   })
   it('Ocean Indexer should be initialized correctly', () => {
     oceanNode.addIndexer(oceanIndexer)
-    expect(oceanNode.getIndexer().getSupportedNetworks()).to.eql(config.supportedNetworks)
+    expect(oceanNode.getIndexer().getSupportedNetworks()).to.eql(config.indexingNetworks)
     expect(oceanNode.getIndexer().getDatabase()).to.eql(db)
   })
   it('Ocean Provider should be initialized correctly', () => {

src/utils/config.ts

@@ -95,6 +95,49 @@ function getSupportedChains(): RPCS | null {
   return supportedNetworks
 }
 
+function getIndexingNetworks(supportedNetworks: RPCS): RPCS | null {
+  const indexerNetworksEnv = process.env.INDEXER_NETWORKS
+  if (!indexerNetworksEnv) {
+    CONFIG_LOGGER.logMessageWithEmoji(
+      'INDEXER_NETWORKS is not defined, running Indexer with all supported networks defined in RPCS env variable ...',
+      true,
+      GENERIC_EMOJIS.EMOJI_CROSS_MARK,
+      LOG_LEVELS_STR.LEVEL_ERROR
+    )
+    return supportedNetworks
+  }
+  try {
+    const indexerNetworks: number[] = JSON.parse(indexerNetworksEnv)
+
+    if (indexerNetworks.length === 0) {
+      CONFIG_LOGGER.logMessageWithEmoji(
+        'INDEXER_NETWORKS is an empty array, Running node without the Indexer component...',
+        true,
+        GENERIC_EMOJIS.EMOJI_CROSS_MARK,
+        LOG_LEVELS_STR.LEVEL_ERROR
+      )
+      return null
+    }
+
+    // Use reduce to filter supportedNetworks
+    const filteredNetworks = indexerNetworks.reduce((acc: RPCS, chainId) => {
+      if (supportedNetworks[chainId]) {
+        acc[chainId] = supportedNetworks[chainId]
+      }
+      return acc
+    }, {})
+
+    return filteredNetworks
+  } catch (e) {
+    CONFIG_LOGGER.logMessageWithEmoji(
+      'Missing or Invalid INDEXER_NETWORKS env variable format,running Indexer with all supported networks defined in RPCS env variable ...',
+      true,
+      GENERIC_EMOJIS.EMOJI_CROSS_MARK,
+      LOG_LEVELS_STR.LEVEL_ERROR
+    )
+    return supportedNetworks
+  }
+}
 // valid decrypthers
 function getAuthorizedDecrypters(isStartup?: boolean): string[] {
   return readAddressListFromEnvVariable(
@@ -418,6 +461,9 @@ async function getEnvConfig(isStartup?: boolean): Promise<OceanNodeConfig> {
   }
 
   const supportedNetworks = getSupportedChains()
+  const indexingNetworks = supportedNetworks
+    ? getIndexingNetworks(supportedNetworks)
+    : null
   // Notes: we need to have this config on the class and use always that, otherwise we're processing
   // all this info every time we call getConfig(), and also loggin too much
 
@@ -439,7 +485,7 @@ async function getEnvConfig(isStartup?: boolean): Promise<OceanNodeConfig> {
     allowedValidators: getAllowedValidators(isStartup),
     keys,
     // Only enable indexer if we have a DB_URL and supportedNetworks
-    hasIndexer: !!(!!getEnvValue(process.env.DB_URL, '') && !!supportedNetworks),
+    hasIndexer: !!(!!getEnvValue(process.env.DB_URL, '') && !!indexingNetworks),
     hasHttp: interfaces.includes('HTTP'),
     hasP2P: interfaces.includes('P2P'),
     p2pConfig: {
@@ -503,6 +549,7 @@ async function getEnvConfig(isStartup?: boolean): Promise<OceanNodeConfig> {
       url: getEnvValue(process.env.DB_URL, '')
     },
     supportedNetworks,
+    indexingNetworks,
     feeStrategy: getOceanNodeFees(supportedNetworks, isStartup),
     c2dClusters: getC2DClusterEnvironment(isStartup),
     accountPurgatoryUrl: getEnvValue(process.env.ACCOUNT_PURGATORY_URL, ''),

src/utils/constants.ts

@@ -160,6 +160,11 @@ export const ENVIRONMENT_VARIABLES: Record<any, EnvVariable> = {
     required: false
   },
   RPCS: { name: 'RPCS', value: process.env.RPCS, required: false },
+  INDEXER_NETWORKS: {
+    name: 'INDEXER_NETWORKS',
+    value: process.env.INDEXER_NETWORKS,
+    required: false
+  },
   DB_URL: { name: 'DB_URL', value: process.env.DB_URL, required: false },
   // these 2 bellow will change in the future (not required, just remove functionality)
   IPFS_GATEWAY: {