Merge branch 'develop' into issue-563-dashboard-no-queue-indexer

env.md

@@ -25,6 +25,9 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3`
 - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10`
 - `LOG_LEVEL`: Define the default log level. Example: `debug`
+- `LOG_CONSOLE`: Write logs to the console. Default is `false`, but becomes `true` if neither `LOG_FILES` or `LOG_DB` are set.
+- `LOG_FILES`: Write logs to files. Default is `false`
+- `LOG_DB`: Write logs to noSQL database. Default is `false`
 
 ## HTTP
 

src/components/database/index.ts

@@ -6,7 +6,7 @@ import {
   LOG_LEVELS_STR,
   configureCustomDBTransport,
   GENERIC_EMOJIS,
-  isDevelopmentEnvironment
+  USE_DB_TRANSPORT
 } from '../../utils/logging/Logger.js'
 import { DATABASE_LOGGER } from '../../utils/logging/common.js'
 import { validateObject } from '../core/utils/validateDdoHandler.js'
@@ -1013,11 +1013,11 @@ export class Database {
     // add this DB transport too
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
-    if (!isDevelopmentEnvironment()) {
+    if (USE_DB_TRANSPORT()) {
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
-        '"NODE_ENV" is set to "development". This means logs will be saved to console and file(s) only.'
+        'Property "LOG_DB" is set to "false". This means logs will NOT be saved to database!'
       )
     }
     return (async (): Promise<Database> => {

src/components/storage/index.ts

@@ -14,6 +14,7 @@ import urlJoin from 'url-join'
 import { encrypt as encryptData, decrypt as decryptData } from '../../utils/crypt.js'
 import { Readable } from 'stream'
 import { getConfiguration } from '../../utils/index.js'
+import { CORE_LOGGER } from '../../utils/logging/common.js'
 
 export abstract class Storage {
   private file: UrlFileObject | IpfsFileObject | ArweaveFileObject
@@ -89,7 +90,7 @@ export abstract class Storage {
         response.push(fileInfo)
       }
     } catch (error) {
-      console.log(error)
+      CORE_LOGGER.error(error)
     }
     return response
   }

src/test/.env.test

@@ -2,10 +2,11 @@ HTTP_API_PORT=8001
 P2P_ipV4BindTcpPort=8000
 PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58
 RPCS='{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}}'
-DB_URL=http://localhost:8108?apiKey=xyz
+DB_URL=http://localhost:8108/?apiKey=xyz
 IPFS_GATEWAY=https://ipfs.io/
 ARWEAVE_GATEWAY=https://arweave.net/
 NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474a7ff
 NODE2_PRIVATE_KEY=0x3634cc4a3d2694a1186a7ce545f149e022eea103cc254d18d08675104bb4b5ac
 INDEXER_INTERVAL=9000
 ADDRESS_FILE=${HOME}/.ocean/ocean-contracts/artifacts/address.json
+LOG_LEVEL=debug

src/test/integration/logs.test.ts

@@ -7,28 +7,44 @@ import {
   configureCustomDBTransport,
   getCustomLoggerForModule
 } from '../../utils/logging/Logger.js'
+import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js'
+import {
+  buildEnvOverrideConfig,
+  OverrideEnvConfig,
+  setupEnvironment,
+  tearDownEnvironment
+} from '../utils/utils.js'
+
+let previousConfiguration: OverrideEnvConfig[]
 
 describe('LogDatabase CRUD', () => {
   let database: Database
   let logger: CustomNodeLogger
   const logEntry = {
     timestamp: Date.now(),
-    level: 'info',
+    level: 'debug',
     message: `Test log message ${Date.now()}`,
     moduleName: 'testModule-1',
     meta: 'Test meta information'
   }
   let logId: string // Variable to store the ID of the created log entry
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
     database = await new Database(dbConfig)
     // Initialize logger with the custom transport that writes to the LogDatabase
-    logger = getCustomLoggerForModule(LOGGER_MODULE_NAMES.HTTP, LOG_LEVELS_STR.LEVEL_INFO)
+    logger = getCustomLoggerForModule(
+      LOGGER_MODULE_NAMES.HTTP,
+      LOG_LEVELS_STR.LEVEL_DEBUG
+    )
     // normally this is only added on production environments
-    configureCustomDBTransport(database, logger)
+    logger = configureCustomDBTransport(database, logger)
   })
 
   it('insert log', async () => {
@@ -56,24 +72,24 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log event is triggered', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `NEW Test log message ${Date.now()}`
     }
     // Trigger a log event which should be saved in the database
     logger.log(newLogEntry.level, newLogEntry.message)
-
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
-    // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    // Retrieve the latest log entries
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    logs = logs.filter((log) => log.message === newLogEntry.message)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 1))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
@@ -82,7 +98,7 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log.logMessage is called', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `logMessage: Test log message ${Date.now()}`,
       moduleName: 'testModule-3',
       meta: 'Test meta information'
@@ -94,14 +110,15 @@ describe('LogDatabase CRUD', () => {
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
+    logs = logs.filter((log) => log.message === newLogEntry.message)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 2))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
@@ -110,7 +127,7 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log.logMessageWithEmoji is called', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `logMessageWithEmoji: Test log message ${Date.now()}`,
       moduleName: 'testModule-4',
       meta: 'Test meta information'
@@ -122,18 +139,23 @@ describe('LogDatabase CRUD', () => {
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
+    logs = logs.filter((log) => log.message.includes(newLogEntry.message))
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 3))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     assert(logs?.[0].message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 
 describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
@@ -143,6 +165,11 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
   const endTime = new Date() // now
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
+    )
+
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -163,7 +190,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
   })
 
   it('should retrieve logs with a specific level', async () => {
-    const level = 'info'
+    const level = 'debug'
     const logs = await database.logs.retrieveMultipleLogs(
       startTime,
       endTime,
@@ -176,7 +203,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
 
   it('should retrieve logs with both a specific moduleName and level', async () => {
     const moduleName = 'testModule-1'
-    const level = 'info'
+    const level = 'debug'
     const logs = await database.logs.retrieveMultipleLogs(
       startTime,
       endTime,
@@ -203,7 +230,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
     let database: Database
     const logEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: 'Test log message for single deletion',
       moduleName: 'testModule-2',
       meta: 'Test meta information for single deletion'
@@ -288,34 +315,42 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
     const elapsedTimeInMs = endPerfTime[0] * 1000 + endPerfTime[1] / 1e6
     expect(elapsedTimeInMs).to.be.below(1000) // threshold
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 
 describe('LogDatabase deleteOldLogs', () => {
   let database: Database
-  const logEntry = {
+  const oldLogEntry = {
     timestamp: new Date().getTime() - 31 * 24 * 60 * 60 * 1000, // 31 days ago
-    level: 'info',
+    level: 'debug',
     message: 'Old log message for deletion test',
     moduleName: 'testModule-1',
     meta: 'Test meta information'
   }
   const recentLogEntry = {
     timestamp: new Date().getTime(), // current time
-    level: 'info',
+    level: 'debug',
     message: 'Recent log message not for deletion',
     moduleName: 'testModule-1',
     meta: 'Test meta information'
   }
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
     database = await new Database(dbConfig)
   })
 
   it('should insert an old log and a recent log', async () => {
-    const oldLogResult = await database.logs.insertLog(logEntry)
+    const oldLogResult = await database.logs.insertLog(oldLogEntry)
     expect(oldLogResult).to.include.keys(
       'id',
       'timestamp',
@@ -337,26 +372,39 @@ describe('LogDatabase deleteOldLogs', () => {
   })
 
   it('should delete logs older than 30 days', async () => {
-    await database.logs.deleteOldLogs()
+    const deleted = await database.logs.deleteOldLogs()
+    assert(deleted > 0, 'could not delete old logs')
 
     // Adjust the time window to ensure we don't catch the newly inserted log
-    const startTime = new Date(logEntry.timestamp)
-    const endTime = new Date()
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    let startTime = new Date(oldLogEntry.timestamp)
+    let endTime = new Date()
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
 
     // Check that the old log is not present, but the recent one is
-    const oldLogPresent = logs?.some((log) => log.message === logEntry.message)
-    const recentLogPresent = logs?.some((log) => log.message === recentLogEntry.message)
-
+    const oldLogPresent = logs?.some((log) => log.message === oldLogEntry.message)
     assert(oldLogPresent === false, 'Old logs are still present')
+
+    // since we have many logs going to DB by default, we need to re-frame the timestamp to grab it
+    startTime = new Date(recentLogEntry.timestamp - 1000)
+    endTime = new Date(recentLogEntry.timestamp + 1000)
+    logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    const recentLogPresent = logs?.some((log) => log.message === recentLogEntry.message)
     assert(recentLogPresent === true, 'Recent logs are not present')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 describe('LogDatabase retrieveMultipleLogs with pagination', () => {
   let database: Database
   const logCount = 10 // Total number of logs to insert and also the limit for logs per page
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -422,4 +470,8 @@ describe('LogDatabase retrieveMultipleLogs with pagination', () => {
     )
     assert.isEmpty(logs, 'Expected logs to be empty')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })