From b58a491d31c56244230a842a85a6e9ec0768d916 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 15 Jul 2024 11:51:57 +0100
Subject: [PATCH 01/50] change son logging transports

---
 src/components/database/index.ts |  4 ++--
 src/utils/logging/Logger.ts      | 24 +++++++++++++++++-------
 2 files changed, 19 insertions(+), 9 deletions(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index 9276f1132..67f5d8cef 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -6,7 +6,7 @@ import {
   LOG_LEVELS_STR,
   configureCustomDBTransport,
   GENERIC_EMOJIS,
-  isDevelopmentEnvironment
+  USE_DB_TRANSPORT
 } from '../../utils/logging/Logger.js'
 import { DATABASE_LOGGER } from '../../utils/logging/common.js'
 import { validateObject } from '../core/utils/validateDdoHandler.js'
@@ -1013,7 +1013,7 @@ export class Database {
     // add this DB transport too
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
-    if (!isDevelopmentEnvironment()) {
+    if (USE_DB_TRANSPORT) {
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index c4c452299..3a89aed28 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -136,6 +136,15 @@ let customDBTransport: CustomOceanNodesTransport = null
 export const MAX_LOGGER_INSTANCES = 10
 export const NUM_LOGGER_INSTANCES = INSTANCE_COUNT
 
+// log locations
+const USE_CONSOLE_TRANSPORT: boolean =
+  process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false'
+const USE_FILE_TRANSPORT: boolean =
+  process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
+// can be affected by configuration
+export const USE_DB_TRANSPORT: boolean =
+  true || (process.env.LOG_DB && process.env.LOG_DB !== 'false')
+
 // if not set, then gets default 'development' level & colors
 export function isDevelopmentEnvironment(): boolean {
   const env = process.env.NODE_ENV || 'development'
@@ -200,7 +209,7 @@ function getDefaultOptions(moduleName: string): winston.LoggerOptions {
     level: getDefaultLevel(),
     levels: LOG_LEVELS_NUM,
     format,
-    transports: [buildCustomFileTransport(moduleName), defaultConsoleTransport],
+    transports: getDefaultLoggerTransports(moduleName),
     exceptionHandlers: [
       new winston.transports.File({ dirname: 'logs/', filename: EXCEPTIONS_HANDLER })
     ]
@@ -257,12 +266,13 @@ export function buildCustomFileTransport(
 export function getDefaultLoggerTransports(
   moduleOrComponentName: string
 ): winston.transport[] {
-  // always log to file
-  const transports: winston.transport[] = [
-    buildCustomFileTransport(moduleOrComponentName)
-  ]
-  // only log to console if development
-  if (isDevelopmentEnvironment()) {
+  const transports: winston.transport[] = []
+  if (USE_FILE_TRANSPORT) {
+    // always log to file
+    transports.push(buildCustomFileTransport(moduleOrComponentName))
+  }
+
+  if (USE_CONSOLE_TRANSPORT) {
     transports.push(defaultConsoleTransport)
   }
   return transports

From c7f60110310230f548a3d0c425107fb58305ae27 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 15 Jul 2024 17:42:04 +0100
Subject: [PATCH 02/50] some debug

---
 src/test/integration/logs.test.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 513ad4f9a..32c7966af 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -71,12 +71,14 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entry
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    console.log('logs are: ', logs)
 
     expect(logs?.length).to.equal(1)
     expect(logs?.[0].id).to.equal(String(Number(logId) + 1))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
+    console.log('end debug')
   })
 
   it('should save a log in the database when a log.logMessage is called', async () => {

From 406cf326934f590a3fc7851d4696ec2ad2c1b76f Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 15 Jul 2024 18:12:21 +0100
Subject: [PATCH 03/50] more debug

---
 src/components/database/index.ts  | 1 +
 src/test/integration/logs.test.ts | 1 +
 src/utils/logging/Logger.ts       | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index 67f5d8cef..508751f4a 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -1014,6 +1014,7 @@ export class Database {
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
     if (USE_DB_TRANSPORT) {
+      console.log('DB USE_DB_TRANSPORT ', USE_DB_TRANSPORT)
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 32c7966af..fc83c2d93 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -42,6 +42,7 @@ describe('LogDatabase CRUD', () => {
       'meta'
     )
     logId = result?.id // Save the auto-generated id for further operations
+    console.log('log id: ' + logId)
   })
 
   it('retrieve log', async () => {
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 3a89aed28..ef3d34e18 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -445,7 +445,7 @@ export class CustomNodeLogger {
     // lazy check db custom transport, needed beacause of dependency cycles
     if (
       customDBTransport !== null && // if null then what?
-      !isDevelopmentEnvironment() &&
+      USE_DB_TRANSPORT &&
       !this.hasDBTransport()
     ) {
       this.addTransport(customDBTransport)

From f109cddd7f2134f94b5ad5d20050a2abda3fafea Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 15 Jul 2024 18:38:45 +0100
Subject: [PATCH 04/50] updates of constants, plus docs

---
 env.md                            |  3 +++
 src/components/database/index.ts  |  3 +--
 src/test/integration/logs.test.ts |  5 +----
 src/utils/constants.ts            | 18 ++++++++++++++++++
 src/utils/logging/Logger.ts       |  7 ++++---
 5 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/env.md b/env.md
index f21064171..b401f80ce 100644
--- a/env.md
+++ b/env.md
@@ -25,6 +25,9 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3`
 - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10`
 - `LOG_LEVEL`: Define the default log level. Example: `debug`
+- `LOG_CONSOLE`: Write logs to the console. Default is `false`
+- `LOG_FILES`: Write logs to files. Default is `false`
+- `LOG_DB`: Write logs to noSQL database. Default is `true`
 
 ## HTTP
 
diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index 508751f4a..c9fae137b 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -1014,11 +1014,10 @@ export class Database {
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
     if (USE_DB_TRANSPORT) {
-      console.log('DB USE_DB_TRANSPORT ', USE_DB_TRANSPORT)
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
-        '"NODE_ENV" is set to "development". This means logs will be saved to console and file(s) only.'
+        'Property "LOG_DB" is set to "false". This means logs will NOT be saved to database!'
       )
     }
     return (async (): Promise<Database> => {
diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index fc83c2d93..840b04f1b 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -42,7 +42,6 @@ describe('LogDatabase CRUD', () => {
       'meta'
     )
     logId = result?.id // Save the auto-generated id for further operations
-    console.log('log id: ' + logId)
   })
 
   it('retrieve log', async () => {
@@ -72,14 +71,12 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entry
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
-    console.log('logs are: ', logs)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 1))
+    expect(logs?.[0].id).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
-    console.log('end debug')
   })
 
   it('should save a log in the database when a log.logMessage is called', async () => {
diff --git a/src/utils/constants.ts b/src/utils/constants.ts
index 491bf0d47..3b34f6988 100644
--- a/src/utils/constants.ts
+++ b/src/utils/constants.ts
@@ -275,6 +275,24 @@ export const ENVIRONMENT_VARIABLES: Record<any, EnvVariable> = {
     name: 'LOG_LEVEL',
     value: process.env.LOG_LEVEL,
     required: false
+  },
+  LOG_CONSOLE: {
+    // log to console output?
+    name: 'LOG_CONSOLE',
+    value: process.env.LOG_CONSOLE,
+    required: false
+  },
+  LOG_FILES: {
+    // log to files?
+    name: 'LOG_FILES',
+    value: process.env.LOG_FILES,
+    required: false
+  },
+  LOG_DB: {
+    // log to DB?
+    name: 'LOG_DB',
+    value: process.env.LOG_DB,
+    required: false // default is true
   }
 }
 
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index ef3d34e18..35729dd0f 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -141,9 +141,10 @@ const USE_CONSOLE_TRANSPORT: boolean =
   process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false'
 const USE_FILE_TRANSPORT: boolean =
   process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
-// can be affected by configuration
-export const USE_DB_TRANSPORT: boolean =
-  true || (process.env.LOG_DB && process.env.LOG_DB !== 'false')
+// default to true, if not explicitly set otherwise
+export const USE_DB_TRANSPORT: boolean = !(
+  process.env.LOG_DB && process.env.LOG_DB === 'false'
+)
 
 // if not set, then gets default 'development' level & colors
 export function isDevelopmentEnvironment(): boolean {

From d50c979c1e91ed1772edca1ca32ad4a0f7a525f3 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 15 Jul 2024 19:00:36 +0100
Subject: [PATCH 05/50] fix test? number conversion

---
 src/test/integration/logs.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 840b04f1b..599d7142e 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -73,7 +73,7 @@ describe('LogDatabase CRUD', () => {
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.greaterThan(Number(logId))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')

From 2487e0b2485d4f7318c84c9ba1ca4ef1b1a13f23 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 09:32:27 +0100
Subject: [PATCH 06/50] more debug on test

---
 src/test/integration/logs.test.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 599d7142e..4a4207637 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -61,6 +61,7 @@ describe('LogDatabase CRUD', () => {
     }
     // Trigger a log event which should be saved in the database
     logger.log(newLogEntry.level, newLogEntry.message)
+    console.log('log: ', newLogEntry)
 
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
@@ -71,6 +72,7 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entry
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))

From d994ed8a441bba01c2aadd18a4595609fd70c1f5 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 09:45:49 +0100
Subject: [PATCH 07/50] more debug on test

---
 src/test/integration/logs.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 4a4207637..9144af9f5 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -71,7 +71,7 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
     console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)

From 841c2164b79d2070cbd0b920a64073fb44fee516 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 10:01:26 +0100
Subject: [PATCH 08/50] more debug on test

---
 src/test/integration/logs.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 9144af9f5..201e7550c 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -71,7 +71,7 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
+    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
     console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)

From ca8ce55bd43ebfcca2f6f9e8023b09f35f94cd06 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 10:16:34 +0100
Subject: [PATCH 09/50] more debug

---
 src/test/integration/logs.test.ts | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 201e7550c..19c906179 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -4,6 +4,7 @@ import {
   CustomNodeLogger,
   LOGGER_MODULE_NAMES,
   LOG_LEVELS_STR,
+  USE_DB_TRANSPORT,
   configureCustomDBTransport,
   getCustomLoggerForModule
 } from '../../utils/logging/Logger.js'
@@ -28,7 +29,8 @@ describe('LogDatabase CRUD', () => {
     // Initialize logger with the custom transport that writes to the LogDatabase
     logger = getCustomLoggerForModule(LOGGER_MODULE_NAMES.HTTP, LOG_LEVELS_STR.LEVEL_INFO)
     // normally this is only added on production environments
-    configureCustomDBTransport(database, logger)
+    logger = configureCustomDBTransport(database, logger)
+    console.log('Logger has DB transport?', logger.hasDBTransport())
   })
 
   it('insert log', async () => {
@@ -60,18 +62,20 @@ describe('LogDatabase CRUD', () => {
       message: `NEW Test log message ${Date.now()}`
     }
     // Trigger a log event which should be saved in the database
+    console.log('Will save log: ', newLogEntry)
+    console.log('has db transport?', logger.hasDBTransport())
     logger.log(newLogEntry.level, newLogEntry.message)
-    console.log('log: ', newLogEntry)
-
+    console.log('USE_DB_TRANSPORT? ', USE_DB_TRANSPORT)
+    console.log('And now has db transport?', logger.hasDBTransport())
     // Wait for the log to be written to the database
-    await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
+    await new Promise((resolve) => setTimeout(resolve, 2000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
     const startTime = new Date(Date.now() - 10000) // 10 seconds ago
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
     console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)

From 181000cd04f1768b1b677805b262f0fc7ff5fc02 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 10:42:37 +0100
Subject: [PATCH 10/50] more debugging

---
 src/utils/logging/Logger.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 35729dd0f..870dd130e 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -117,9 +117,10 @@ export class CustomOceanNodesTransport extends Transport {
     try {
       // Use the insertLog method of the LogDatabase instance
       if (this.dbInstance && this.dbInstance.logs) {
+        console.log('will insert LOG ON DB: ', document.message)
         // double check before writing
         await this.dbInstance.logs.insertLog(document)
-      }
+      } else console.log('will not insert LOG on DB: ', document.message)
     } catch (error) {
       // Handle the error according to your needs
       console.error('Error writing to Typesense:', error)

From d1b655f9bb75d1eb15982edf8a4d349a3ac1cb16 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 11:54:37 +0100
Subject: [PATCH 11/50] remove garbage logs from typsense normal stuff

---
 src/utils/logging/Logger.ts | 38 ++++++++++++++++++++++++++++++-------
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 870dd130e..a216f4f49 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -109,14 +109,18 @@ export class CustomOceanNodesTransport extends Transport {
     const document = {
       level: info.level,
       message: info.message,
-      moduleName: info.moduleName || 'undefined',
+      moduleName: info.moduleName || LOGGER_MODULE_NAMES.ALL_COMBINED,
       timestamp: Date.now(), // Storing the current timestamp as a Unix epoch timestamp (number)
       meta: JSON.stringify(info.meta) // Ensure meta is a string
     }
 
     try {
       // Use the insertLog method of the LogDatabase instance
-      if (this.dbInstance && this.dbInstance.logs) {
+      if (
+        this.dbInstance &&
+        this.dbInstance.logs &&
+        !isTypesenseIgnoreLogMessage(document.moduleName, document.message)
+      ) {
         console.log('will insert LOG ON DB: ', document.message)
         // double check before writing
         await this.dbInstance.logs.insertLog(document)
@@ -131,6 +135,23 @@ export class CustomOceanNodesTransport extends Transport {
   }
 }
 
+/**
+ * Avoid these annoyng typesense log message (too much garbage and no utility)
+ * @param loggerModuleName module name
+ * @param logMessage the actual message
+ * @returns boolean
+ */
+function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: string) {
+  const msg1: string = 'Request /collections/logs/documents'
+  const msg2: string = 'Response Code was 201.'
+  const msg3 = 'Request /collections/logs/documents: Attempting POST request Try #1'
+  return (
+    (loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
+      (logMessage === msg1 || logMessage === msg3)) ||
+    (logMessage.includes(msg1) && logMessage.includes(msg2))
+  )
+}
+
 let INSTANCE_COUNT = 0
 let customDBTransport: CustomOceanNodesTransport = null
 
@@ -453,11 +474,14 @@ export class CustomNodeLogger {
       this.addTransport(customDBTransport)
     }
 
-    this.getLogger().log(
-      level,
-      includeModuleName ? this.buildMessage(message) : message,
-      { moduleName: this.getModuleName().toUpperCase() }
-    )
+    // ignore tons of typesense garbage
+    if (!isTypesenseIgnoreLogMessage(this.getModuleName(), message)) {
+      this.getLogger().log(
+        level,
+        includeModuleName ? this.buildMessage(message) : message,
+        { moduleName: this.getModuleName().toUpperCase() }
+      )
+    }
   }
 
   logMessage(message: string, includeModuleName: boolean = false) {

From ae24742d1b491b175da2e10156a0ddfce8438e74 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 12:16:10 +0100
Subject: [PATCH 12/50] fix condition for ignore

---
 src/utils/logging/Logger.ts | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index a216f4f49..d18438c19 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -146,9 +146,10 @@ function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: strin
   const msg2: string = 'Response Code was 201.'
   const msg3 = 'Request /collections/logs/documents: Attempting POST request Try #1'
   return (
-    (loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
-      (logMessage === msg1 || logMessage === msg3)) ||
-    (logMessage.includes(msg1) && logMessage.includes(msg2))
+    loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
+    (logMessage === msg1 ||
+      logMessage.includes(msg3) ||
+      (logMessage.includes(msg1) && logMessage.includes(msg2)))
   )
 }
 

From 4bdf70a9ade837b646e4e96e65669ea92eb8c5b6 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 12:45:16 +0100
Subject: [PATCH 13/50] clean filter, avoid useless msgs

---
 src/utils/logging/Logger.ts | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index d18438c19..a00301ffe 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -121,10 +121,9 @@ export class CustomOceanNodesTransport extends Transport {
         this.dbInstance.logs &&
         !isTypesenseIgnoreLogMessage(document.moduleName, document.message)
       ) {
-        console.log('will insert LOG ON DB: ', document.message)
         // double check before writing
         await this.dbInstance.logs.insertLog(document)
-      } else console.log('will not insert LOG on DB: ', document.message)
+      }
     } catch (error) {
       // Handle the error according to your needs
       console.error('Error writing to Typesense:', error)
@@ -142,14 +141,12 @@ export class CustomOceanNodesTransport extends Transport {
  * @returns boolean
  */
 function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: string) {
-  const msg1: string = 'Request /collections/logs/documents'
+  const msg1: string = 'Response Code was 200.'
   const msg2: string = 'Response Code was 201.'
-  const msg3 = 'Request /collections/logs/documents: Attempting POST request Try #1'
+  const msg3 = 'request Try #1 to Node'
   return (
     loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
-    (logMessage === msg1 ||
-      logMessage.includes(msg3) ||
-      (logMessage.includes(msg1) && logMessage.includes(msg2)))
+    (logMessage.includes(msg2) || logMessage.includes(msg3) || logMessage.includes(msg1))
   )
 }
 

From 0437247929e1683546da3c64d1087ae5ed4a1e0d Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 15:23:50 +0100
Subject: [PATCH 14/50] deafult typesense logger, error, only

---
 src/components/database/index.ts  | 18 ++++++++++++------
 src/test/integration/logs.test.ts |  7 +------
 src/utils/logging/Logger.ts       |  6 +++++-
 3 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index c9fae137b..a59b3342f 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -22,7 +22,8 @@ export class OrderDatabase {
     return (async (): Promise<OrderDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -191,7 +192,8 @@ export class DdoStateDatabase {
     return (async (): Promise<DdoStateDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -327,7 +329,8 @@ export class DdoDatabase {
     return (async (): Promise<DdoDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       for (const ddoSchema of this.schemas) {
         try {
@@ -617,7 +620,8 @@ export class NonceDatabase {
     return (async (): Promise<NonceDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -722,7 +726,8 @@ export class IndexerDatabase {
     return (async (): Promise<IndexerDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -832,7 +837,8 @@ export class LogDatabase {
     return (async (): Promise<LogDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER
+        logger: DATABASE_LOGGER,
+        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 19c906179..fa19b2c11 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -4,7 +4,6 @@ import {
   CustomNodeLogger,
   LOGGER_MODULE_NAMES,
   LOG_LEVELS_STR,
-  USE_DB_TRANSPORT,
   configureCustomDBTransport,
   getCustomLoggerForModule
 } from '../../utils/logging/Logger.js'
@@ -62,13 +61,9 @@ describe('LogDatabase CRUD', () => {
       message: `NEW Test log message ${Date.now()}`
     }
     // Trigger a log event which should be saved in the database
-    console.log('Will save log: ', newLogEntry)
-    console.log('has db transport?', logger.hasDBTransport())
     logger.log(newLogEntry.level, newLogEntry.message)
-    console.log('USE_DB_TRANSPORT? ', USE_DB_TRANSPORT)
-    console.log('And now has db transport?', logger.hasDBTransport())
     // Wait for the log to be written to the database
-    await new Promise((resolve) => setTimeout(resolve, 2000)) // Delay to allow log to be processed
+    await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
     const startTime = new Date(Date.now() - 10000) // 10 seconds ago
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index a00301ffe..6fecfdd8c 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -144,9 +144,13 @@ function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: strin
   const msg1: string = 'Response Code was 200.'
   const msg2: string = 'Response Code was 201.'
   const msg3 = 'request Try #1 to Node'
+  const msg4 = 'Request /collections/'
   return (
     loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
-    (logMessage.includes(msg2) || logMessage.includes(msg3) || logMessage.includes(msg1))
+    (logMessage.includes(msg2) ||
+      logMessage.includes(msg3) ||
+      logMessage.includes(msg1) ||
+      logMessage.includes(msg4))
   )
 }
 

From da1634036602312d9ad7ca0e4ea18165f7f0b179 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 15:40:18 +0100
Subject: [PATCH 15/50] update tests

---
 src/test/integration/logs.test.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index fa19b2c11..efd3a8f8c 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -102,7 +102,7 @@ describe('LogDatabase CRUD', () => {
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 2))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
@@ -130,7 +130,7 @@ describe('LogDatabase CRUD', () => {
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
 
     expect(logs?.length).to.equal(1)
-    expect(logs?.[0].id).to.equal(String(Number(logId) + 3))
+    expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     assert(logs?.[0].message)
     expect(logs?.[0].moduleName).to.equal('HTTP')

From d784853121231fc9d7467c5361fe0690deaa38ce Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 15:57:01 +0100
Subject: [PATCH 16/50] update tests

---
 src/test/integration/logs.test.ts | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index efd3a8f8c..8caee4d11 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -100,6 +100,7 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entry
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
@@ -128,6 +129,7 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entry
     const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))

From 1eae945ebf8faa9015bd3b2e2aca0d7e1b95c07a Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 16:10:36 +0100
Subject: [PATCH 17/50] check module name as well

---
 src/test/integration/logs.test.ts | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 8caee4d11..1d341057e 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -99,7 +99,12 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    const logs = await database.logs.retrieveMultipleLogs(
+      startTime,
+      endTime,
+      1,
+      'testModule-3'
+    )
     console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)
@@ -128,7 +133,12 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
+    const logs = await database.logs.retrieveMultipleLogs(
+      startTime,
+      endTime,
+      1,
+      'testModule-4'
+    )
     console.log('logs:', logs)
 
     expect(logs?.length).to.equal(1)

From f79dc5eeea236c8642138423a60f8bc40097013c Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 16:39:15 +0100
Subject: [PATCH 18/50] check module name as well

---
 src/test/integration/logs.test.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 1d341057e..298a69fbf 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -89,7 +89,7 @@ describe('LogDatabase CRUD', () => {
       meta: 'Test meta information'
     }
     // Trigger a log event which should be saved in the database
-    logger.logMessage(newLogEntry.message)
+    logger.logMessage(newLogEntry.message, true)
 
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
@@ -123,7 +123,7 @@ describe('LogDatabase CRUD', () => {
       meta: 'Test meta information'
     }
     // Trigger a log event which should be saved in the database
-    logger.logMessageWithEmoji(newLogEntry.message)
+    logger.logMessageWithEmoji(newLogEntry.message, true)
 
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed

From 9b165c25d9a2419f9132ed083e2fc70212a62459 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 17:12:53 +0100
Subject: [PATCH 19/50] check module name as well

---
 src/test/integration/logs.test.ts | 22 ++++++++--------------
 1 file changed, 8 insertions(+), 14 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 298a69fbf..42fdc0a0c 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -99,19 +99,16 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(
-      startTime,
-      endTime,
-      1,
-      'testModule-3'
-    )
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
     console.log('logs:', logs)
+    logs = logs.filter((log) => log.message === newLogEntry.message)
+    console.log('logs filtered:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
-    expect(logs?.[0].moduleName).to.equal('HTTP')
+    expect(logs?.[0].moduleName).to.equal(newLogEntry.moduleName) // 'HTTP'
   })
 
   it('should save a log in the database when a log.logMessageWithEmoji is called', async () => {
@@ -133,19 +130,16 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(
-      startTime,
-      endTime,
-      1,
-      'testModule-4'
-    )
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
     console.log('logs:', logs)
+    logs = logs.filter((log) => log.message === newLogEntry.message)
+    console.log('logs filtered:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     assert(logs?.[0].message)
-    expect(logs?.[0].moduleName).to.equal('HTTP')
+    expect(logs?.[0].moduleName).to.equal(newLogEntry.moduleName) // HTTP
   })
 })
 

From 8f196ad83c5c080d01f2cf6836069132bd5cad88 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 17:27:17 +0100
Subject: [PATCH 20/50] check module name as well

---
 src/test/integration/logs.test.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 42fdc0a0c..6256124b8 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -89,7 +89,7 @@ describe('LogDatabase CRUD', () => {
       meta: 'Test meta information'
     }
     // Trigger a log event which should be saved in the database
-    logger.logMessage(newLogEntry.message, true)
+    logger.logMessage(newLogEntry.message)
 
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
@@ -108,7 +108,7 @@ describe('LogDatabase CRUD', () => {
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     expect(logs?.[0].message).to.equal(newLogEntry.message)
-    expect(logs?.[0].moduleName).to.equal(newLogEntry.moduleName) // 'HTTP'
+    expect(logs?.[0].moduleName).to.equal('HTTP')
   })
 
   it('should save a log in the database when a log.logMessageWithEmoji is called', async () => {
@@ -120,7 +120,7 @@ describe('LogDatabase CRUD', () => {
       meta: 'Test meta information'
     }
     // Trigger a log event which should be saved in the database
-    logger.logMessageWithEmoji(newLogEntry.message, true)
+    logger.logMessageWithEmoji(newLogEntry.message)
 
     // Wait for the log to be written to the database
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
@@ -139,7 +139,7 @@ describe('LogDatabase CRUD', () => {
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
     expect(logs?.[0].level).to.equal(newLogEntry.level)
     assert(logs?.[0].message)
-    expect(logs?.[0].moduleName).to.equal(newLogEntry.moduleName) // HTTP
+    expect(logs?.[0].moduleName).to.equal('HTTP')
   })
 })
 

From a2edc6aab13ecff1c54f70e43854ead4c6e53fb5 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 17:39:18 +0100
Subject: [PATCH 21/50] check module name as well

---
 src/test/integration/logs.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 6256124b8..0fba83435 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -132,7 +132,7 @@ describe('LogDatabase CRUD', () => {
     // Retrieve the latest log entry
     let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
     console.log('logs:', logs)
-    logs = logs.filter((log) => log.message === newLogEntry.message)
+    logs = logs.filter((log) => log.message.includes(newLogEntry.message))
     console.log('logs filtered:', logs)
 
     expect(logs?.length).to.equal(1)

From 93227164688029b38e1653a1e3d9f31e5b6700cf Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 18:05:55 +0100
Subject: [PATCH 22/50] add more logs in 30 days interval check

---
 src/test/integration/logs.test.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 0fba83435..980382f3a 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -349,7 +349,7 @@ describe('LogDatabase deleteOldLogs', () => {
     // Adjust the time window to ensure we don't catch the newly inserted log
     const startTime = new Date(logEntry.timestamp)
     const endTime = new Date()
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 200)
 
     // Check that the old log is not present, but the recent one is
     const oldLogPresent = logs?.some((log) => log.message === logEntry.message)

From d27257a461b76bbf47fe1d33f6938fa5851ce73a Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 16 Jul 2024 18:31:55 +0100
Subject: [PATCH 23/50] more debug

---
 src/test/integration/logs.test.ts | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 980382f3a..930e31970 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -300,7 +300,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
 describe('LogDatabase deleteOldLogs', () => {
   let database: Database
   const logEntry = {
-    timestamp: new Date().getTime() - 31 * 24 * 60 * 60 * 1000, // 31 days ago
+    timestamp: new Date().getTime() - 32 * 24 * 60 * 60 * 1000, // 32 days ago
     level: 'info',
     message: 'Old log message for deletion test',
     moduleName: 'testModule-1',
@@ -344,12 +344,17 @@ describe('LogDatabase deleteOldLogs', () => {
   })
 
   it('should delete logs older than 30 days', async () => {
-    await database.logs.deleteOldLogs()
+    await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow logs to be processed
+
+    const deleted = await database.logs.deleteOldLogs()
+    assert(deleted > 0, 'could not delete old logs')
+    console.log('deleted logs: ', deleted)
 
     // Adjust the time window to ensure we don't catch the newly inserted log
     const startTime = new Date(logEntry.timestamp)
     const endTime = new Date()
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 200)
+    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    console.log('logs present:  ', logs)
 
     // Check that the old log is not present, but the recent one is
     const oldLogPresent = logs?.some((log) => log.message === logEntry.message)

From f2d667239d78a4c0f2f17dc0bab6b9bb39db7b01 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 17 Jul 2024 10:48:25 +0100
Subject: [PATCH 24/50] fix tests logs

---
 src/test/integration/logs.test.ts | 43 ++++++++++++++-----------------
 1 file changed, 19 insertions(+), 24 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 930e31970..1e2cedc7a 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -29,7 +29,6 @@ describe('LogDatabase CRUD', () => {
     logger = getCustomLoggerForModule(LOGGER_MODULE_NAMES.HTTP, LOG_LEVELS_STR.LEVEL_INFO)
     // normally this is only added on production environments
     logger = configureCustomDBTransport(database, logger)
-    console.log('Logger has DB transport?', logger.hasDBTransport())
   })
 
   it('insert log', async () => {
@@ -66,12 +65,12 @@ describe('LogDatabase CRUD', () => {
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
-    // Retrieve the latest log entry
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 1)
-    console.log('logs:', logs)
+    // Retrieve the latest log entries
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
+    logs = logs.filter((log) => log.message === newLogEntry.message)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
@@ -95,14 +94,12 @@ describe('LogDatabase CRUD', () => {
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
     let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
-    console.log('logs:', logs)
     logs = logs.filter((log) => log.message === newLogEntry.message)
-    console.log('logs filtered:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
@@ -126,14 +123,12 @@ describe('LogDatabase CRUD', () => {
     await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow log to be processed
 
     // Define the time frame for the log retrieval
-    const startTime = new Date(Date.now() - 10000) // 10 seconds ago
+    const startTime = new Date(Date.now() - 5000) // 5 seconds ago
     const endTime = new Date() // current time
 
     // Retrieve the latest log entry
     let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
-    console.log('logs:', logs)
     logs = logs.filter((log) => log.message.includes(newLogEntry.message))
-    console.log('logs filtered:', logs)
 
     expect(logs?.length).to.equal(1)
     expect(Number(logs?.[0].id)).to.greaterThan(Number(logId))
@@ -299,8 +294,8 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
 
 describe('LogDatabase deleteOldLogs', () => {
   let database: Database
-  const logEntry = {
-    timestamp: new Date().getTime() - 32 * 24 * 60 * 60 * 1000, // 32 days ago
+  const oldLogEntry = {
+    timestamp: new Date().getTime() - 31 * 24 * 60 * 60 * 1000, // 31 days ago
     level: 'info',
     message: 'Old log message for deletion test',
     moduleName: 'testModule-1',
@@ -322,7 +317,7 @@ describe('LogDatabase deleteOldLogs', () => {
   })
 
   it('should insert an old log and a recent log', async () => {
-    const oldLogResult = await database.logs.insertLog(logEntry)
+    const oldLogResult = await database.logs.insertLog(oldLogEntry)
     expect(oldLogResult).to.include.keys(
       'id',
       'timestamp',
@@ -344,23 +339,23 @@ describe('LogDatabase deleteOldLogs', () => {
   })
 
   it('should delete logs older than 30 days', async () => {
-    await new Promise((resolve) => setTimeout(resolve, 1000)) // Delay to allow logs to be processed
-
     const deleted = await database.logs.deleteOldLogs()
     assert(deleted > 0, 'could not delete old logs')
-    console.log('deleted logs: ', deleted)
 
     // Adjust the time window to ensure we don't catch the newly inserted log
-    const startTime = new Date(logEntry.timestamp)
-    const endTime = new Date()
-    const logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
-    console.log('logs present:  ', logs)
+    let startTime = new Date(oldLogEntry.timestamp)
+    let endTime = new Date()
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
 
     // Check that the old log is not present, but the recent one is
-    const oldLogPresent = logs?.some((log) => log.message === logEntry.message)
-    const recentLogPresent = logs?.some((log) => log.message === recentLogEntry.message)
-
+    const oldLogPresent = logs?.some((log) => log.message === oldLogEntry.message)
     assert(oldLogPresent === false, 'Old logs are still present')
+
+    // since we have many logs going to DB by default, we need to re-frame the timestamp to grab it
+    startTime = new Date(recentLogEntry.timestamp - 1000)
+    endTime = new Date(recentLogEntry.timestamp + 1000)
+    logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    const recentLogPresent = logs?.some((log) => log.message === recentLogEntry.message)
     assert(recentLogPresent === true, 'Recent logs are not present')
   })
 })

From 7edee9b8476fb9c48ec1938565db41b55cdac5f6 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 17 Jul 2024 17:27:01 +0100
Subject: [PATCH 25/50] more log changes, check log level for decide to log or
 not

---
 src/test/.env.test                |  1 +
 src/test/integration/logs.test.ts | 60 ++++++++++++++++++++++++++-----
 src/utils/logging/Logger.ts       | 29 +++++++++++++++
 3 files changed, 81 insertions(+), 9 deletions(-)

diff --git a/src/test/.env.test b/src/test/.env.test
index 48115ee1f..ee100d2a9 100644
--- a/src/test/.env.test
+++ b/src/test/.env.test
@@ -9,3 +9,4 @@ NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474
 NODE2_PRIVATE_KEY=0x3634cc4a3d2694a1186a7ce545f149e022eea103cc254d18d08675104bb4b5ac
 INDEXER_INTERVAL=9000
 ADDRESS_FILE=${HOME}/.ocean/ocean-contracts/artifacts/address.json
+LOG_DB=false
diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 1e2cedc7a..4195d792e 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -7,13 +7,22 @@ import {
   configureCustomDBTransport,
   getCustomLoggerForModule
 } from '../../utils/logging/Logger.js'
+import { ENVIRONMENT_VARIABLES } from '../../utils/constants.js'
+import {
+  buildEnvOverrideConfig,
+  OverrideEnvConfig,
+  setupEnvironment,
+  tearDownEnvironment
+} from '../utils/utils.js'
+
+let previousConfiguration: OverrideEnvConfig[]
 
 describe('LogDatabase CRUD', () => {
   let database: Database
   let logger: CustomNodeLogger
   const logEntry = {
     timestamp: Date.now(),
-    level: 'info',
+    level: 'debug',
     message: `Test log message ${Date.now()}`,
     moduleName: 'testModule-1',
     meta: 'Test meta information'
@@ -21,6 +30,10 @@ describe('LogDatabase CRUD', () => {
   let logId: string // Variable to store the ID of the created log entry
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -56,7 +69,7 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log event is triggered', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `NEW Test log message ${Date.now()}`
     }
     // Trigger a log event which should be saved in the database
@@ -82,7 +95,7 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log.logMessage is called', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `logMessage: Test log message ${Date.now()}`,
       moduleName: 'testModule-3',
       meta: 'Test meta information'
@@ -111,7 +124,7 @@ describe('LogDatabase CRUD', () => {
   it('should save a log in the database when a log.logMessageWithEmoji is called', async () => {
     const newLogEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: `logMessageWithEmoji: Test log message ${Date.now()}`,
       moduleName: 'testModule-4',
       meta: 'Test meta information'
@@ -136,6 +149,10 @@ describe('LogDatabase CRUD', () => {
     assert(logs?.[0].message)
     expect(logs?.[0].moduleName).to.equal('HTTP')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 
 describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
@@ -145,6 +162,11 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
   const endTime = new Date() // now
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+    )
+
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -165,7 +187,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
   })
 
   it('should retrieve logs with a specific level', async () => {
-    const level = 'info'
+    const level = 'debug'
     const logs = await database.logs.retrieveMultipleLogs(
       startTime,
       endTime,
@@ -178,7 +200,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
 
   it('should retrieve logs with both a specific moduleName and level', async () => {
     const moduleName = 'testModule-1'
-    const level = 'info'
+    const level = 'debug'
     const logs = await database.logs.retrieveMultipleLogs(
       startTime,
       endTime,
@@ -205,7 +227,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
     let database: Database
     const logEntry = {
       timestamp: Date.now(),
-      level: 'info',
+      level: 'debug',
       message: 'Test log message for single deletion',
       moduleName: 'testModule-2',
       meta: 'Test meta information for single deletion'
@@ -290,26 +312,34 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
     const elapsedTimeInMs = endPerfTime[0] * 1000 + endPerfTime[1] / 1e6
     expect(elapsedTimeInMs).to.be.below(1000) // threshold
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 
 describe('LogDatabase deleteOldLogs', () => {
   let database: Database
   const oldLogEntry = {
     timestamp: new Date().getTime() - 31 * 24 * 60 * 60 * 1000, // 31 days ago
-    level: 'info',
+    level: 'debug',
     message: 'Old log message for deletion test',
     moduleName: 'testModule-1',
     meta: 'Test meta information'
   }
   const recentLogEntry = {
     timestamp: new Date().getTime(), // current time
-    level: 'info',
+    level: 'debug',
     message: 'Recent log message not for deletion',
     moduleName: 'testModule-1',
     meta: 'Test meta information'
   }
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -358,12 +388,20 @@ describe('LogDatabase deleteOldLogs', () => {
     const recentLogPresent = logs?.some((log) => log.message === recentLogEntry.message)
     assert(recentLogPresent === true, 'Recent logs are not present')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
 describe('LogDatabase retrieveMultipleLogs with pagination', () => {
   let database: Database
   const logCount = 10 // Total number of logs to insert and also the limit for logs per page
 
   before(async () => {
+    previousConfiguration = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+    )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
     }
@@ -429,4 +467,8 @@ describe('LogDatabase retrieveMultipleLogs with pagination', () => {
     )
     assert.isEmpty(logs, 'Expected logs to be empty')
   })
+
+  after(async () => {
+    await tearDownEnvironment(previousConfiguration)
+  })
 })
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 6fecfdd8c..27e8c854c 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -114,6 +114,10 @@ export class CustomOceanNodesTransport extends Transport {
       meta: JSON.stringify(info.meta) // Ensure meta is a string
     }
 
+    if (!isLogLevelLogable(document.level)) {
+      return
+    }
+
     try {
       // Use the insertLog method of the LogDatabase instance
       if (
@@ -195,6 +199,27 @@ export const getDefaultLevel = (): string => {
   )
 }
 
+// only log if >= configured level
+function isLogLevelLogable(level: string): boolean {
+  const configured: string = getDefaultLevel()
+  let configuredLevelNum = -1
+  let currentLevelNum = -1
+  for (const [key, value] of Object.entries(LOG_LEVELS_NUM)) {
+    if ((key as string) === configured) {
+      configuredLevelNum = value
+    }
+    if ((key as string) === level) {
+      currentLevelNum = value
+    }
+
+    if (currentLevelNum > -1 && configuredLevelNum > -1) {
+      break
+    }
+  }
+
+  return currentLevelNum >= configuredLevelNum
+}
+
 if (isDevelopmentEnvironment()) {
   winston.addColors(LOG_COLORS)
 }
@@ -476,6 +501,10 @@ export class CustomNodeLogger {
       this.addTransport(customDBTransport)
     }
 
+    if (!isLogLevelLogable(level)) {
+      return
+    }
+
     // ignore tons of typesense garbage
     if (!isTypesenseIgnoreLogMessage(this.getModuleName(), message)) {
       this.getLogger().log(

From 0a50aaf2a8372bcce57b96807c4805fc89e11818 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 18 Jul 2024 12:11:17 +0100
Subject: [PATCH 26/50] remove custom code for debug choice + default console

---
 src/utils/logging/Logger.ts | 89 ++++++++++++++-----------------------
 1 file changed, 33 insertions(+), 56 deletions(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 27e8c854c..8bcef1fce 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -114,16 +114,12 @@ export class CustomOceanNodesTransport extends Transport {
       meta: JSON.stringify(info.meta) // Ensure meta is a string
     }
 
-    if (!isLogLevelLogable(document.level)) {
-      return
-    }
-
     try {
       // Use the insertLog method of the LogDatabase instance
       if (
         this.dbInstance &&
-        this.dbInstance.logs &&
-        !isTypesenseIgnoreLogMessage(document.moduleName, document.message)
+        this.dbInstance.logs // &&
+        // !isTypesenseIgnoreLogMessage(document.moduleName, document.message)
       ) {
         // double check before writing
         await this.dbInstance.logs.insertLog(document)
@@ -144,19 +140,19 @@ export class CustomOceanNodesTransport extends Transport {
  * @param logMessage the actual message
  * @returns boolean
  */
-function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: string) {
-  const msg1: string = 'Response Code was 200.'
-  const msg2: string = 'Response Code was 201.'
-  const msg3 = 'request Try #1 to Node'
-  const msg4 = 'Request /collections/'
-  return (
-    loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
-    (logMessage.includes(msg2) ||
-      logMessage.includes(msg3) ||
-      logMessage.includes(msg1) ||
-      logMessage.includes(msg4))
-  )
-}
+// function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: string) {
+//   const msg1: string = 'Response Code was 200.'
+//   const msg2: string = 'Response Code was 201.'
+//   const msg3 = 'request Try #1 to Node'
+//   const msg4 = 'Request /collections/'
+//   return (
+//     loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
+//     (logMessage.includes(msg2) ||
+//       logMessage.includes(msg3) ||
+//       logMessage.includes(msg1) ||
+//       logMessage.includes(msg4))
+//   )
+// }
 
 let INSTANCE_COUNT = 0
 let customDBTransport: CustomOceanNodesTransport = null
@@ -165,14 +161,16 @@ export const MAX_LOGGER_INSTANCES = 10
 export const NUM_LOGGER_INSTANCES = INSTANCE_COUNT
 
 // log locations
-const USE_CONSOLE_TRANSPORT: boolean =
-  process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false'
 const USE_FILE_TRANSPORT: boolean =
   process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
-// default to true, if not explicitly set otherwise
-export const USE_DB_TRANSPORT: boolean = !(
-  process.env.LOG_DB && process.env.LOG_DB === 'false'
-)
+
+export const USE_DB_TRANSPORT: boolean =
+  process.env.LOG_DB && process.env.LOG_DB !== 'false'
+
+// default to true, if not explicitly set otherwise AND no other locations defined
+const USE_CONSOLE_TRANSPORT: boolean =
+  (process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false') ||
+  (!USE_FILE_TRANSPORT && !USE_DB_TRANSPORT)
 
 // if not set, then gets default 'development' level & colors
 export function isDevelopmentEnvironment(): boolean {
@@ -199,27 +197,6 @@ export const getDefaultLevel = (): string => {
   )
 }
 
-// only log if >= configured level
-function isLogLevelLogable(level: string): boolean {
-  const configured: string = getDefaultLevel()
-  let configuredLevelNum = -1
-  let currentLevelNum = -1
-  for (const [key, value] of Object.entries(LOG_LEVELS_NUM)) {
-    if ((key as string) === configured) {
-      configuredLevelNum = value
-    }
-    if ((key as string) === level) {
-      currentLevelNum = value
-    }
-
-    if (currentLevelNum > -1 && configuredLevelNum > -1) {
-      break
-    }
-  }
-
-  return currentLevelNum >= configuredLevelNum
-}
-
 if (isDevelopmentEnvironment()) {
   winston.addColors(LOG_COLORS)
 }
@@ -501,18 +478,18 @@ export class CustomNodeLogger {
       this.addTransport(customDBTransport)
     }
 
-    if (!isLogLevelLogable(level)) {
-      return
-    }
+    // if (!isLogLevelLogable(level)) {
+    //   return
+    // }
 
     // ignore tons of typesense garbage
-    if (!isTypesenseIgnoreLogMessage(this.getModuleName(), message)) {
-      this.getLogger().log(
-        level,
-        includeModuleName ? this.buildMessage(message) : message,
-        { moduleName: this.getModuleName().toUpperCase() }
-      )
-    }
+    // if (!isTypesenseIgnoreLogMessage(this.getModuleName(), message)) {
+    this.getLogger().log(
+      level,
+      includeModuleName ? this.buildMessage(message) : message,
+      { moduleName: this.getModuleName().toUpperCase() }
+    )
+    // }
   }
 
   logMessage(message: string, includeModuleName: boolean = false) {

From 4ed795c19a15690f7ba2a91c1d8ae5f86932151c Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 18 Jul 2024 12:24:21 +0100
Subject: [PATCH 27/50] fix unit test

---
 src/test/unit/logging.test.ts | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 8b1534d9a..b4c32eea6 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -36,17 +36,17 @@ describe('Logger instances and transports tests', async () => {
     expect(numExistingInstances).to.be.lessThanOrEqual(MAX_LOGGER_INSTANCES)
   })
 
-  it(`should change NODE_ENV to "production" and logger should have DB transport`, async () => {
-    expect(process.env.NODE_ENV).to.be.equal('development')
+  it(`should change LOG_DB to "true" and logger should have DB transport`, async () => {
+    expect(process.env.LOG_DB).to.be.equal('false')
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(
       null,
       buildEnvOverrideConfig(
-        [ENVIRONMENT_VARIABLES.NODE_ENV, ENVIRONMENT_VARIABLES.DB_URL],
-        ['production', 'http://172.15.0.6:8108?apiKey=xyz']
+        [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
+        ['true', 'http://172.15.0.6:8108?apiKey=xyz']
       )
     )
-    expect(process.env.NODE_ENV).to.be.equal('production')
+    expect(process.env.LOG_DB).to.be.equal('true')
     // will build the DB transport layer
     const config = await getConfiguration(true)
     // eslint-disable-next-line no-unused-vars
@@ -65,7 +65,7 @@ describe('Logger instances and transports tests', async () => {
     OCEAN_NODE_LOGGER.removeTransport(transports[0])
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     await tearDownEnvironment(envAfter)
-    expect(process.env.NODE_ENV).to.be.equal('development')
+    expect(process.env.LOG_DB).to.be.equal('false')
   })
 
   after(() => {

From 5f191bd8ac001f97885e2bf5514834da51528ced Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 22 Jul 2024 10:01:00 +0100
Subject: [PATCH 28/50] fix test

---
 src/test/integration/logs.test.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 4195d792e..007db19bf 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -39,7 +39,10 @@ describe('LogDatabase CRUD', () => {
     }
     database = await new Database(dbConfig)
     // Initialize logger with the custom transport that writes to the LogDatabase
-    logger = getCustomLoggerForModule(LOGGER_MODULE_NAMES.HTTP, LOG_LEVELS_STR.LEVEL_INFO)
+    logger = getCustomLoggerForModule(
+      LOGGER_MODULE_NAMES.HTTP,
+      LOG_LEVELS_STR.LEVEL_DEBUG
+    )
     // normally this is only added on production environments
     logger = configureCustomDBTransport(database, logger)
   })

From 1c649666840fd336d24aa009927a04713cd88723 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Mon, 22 Jul 2024 10:59:10 +0100
Subject: [PATCH 29/50] fix unit tests

---
 src/components/database/index.ts |  5 ++---
 src/test/unit/logging.test.ts    |  9 ++++++---
 src/utils/logging/Logger.ts      | 34 ++++----------------------------
 3 files changed, 12 insertions(+), 36 deletions(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index a59b3342f..d47032c62 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -5,8 +5,7 @@ import { TypesenseSearchParams } from '../../@types/index.js'
 import {
   LOG_LEVELS_STR,
   configureCustomDBTransport,
-  GENERIC_EMOJIS,
-  USE_DB_TRANSPORT
+  GENERIC_EMOJIS
 } from '../../utils/logging/Logger.js'
 import { DATABASE_LOGGER } from '../../utils/logging/common.js'
 import { validateObject } from '../core/utils/validateDdoHandler.js'
@@ -1019,7 +1018,7 @@ export class Database {
     // add this DB transport too
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
-    if (USE_DB_TRANSPORT) {
+    if (ENVIRONMENT_VARIABLES.LOG_DB.value === 'true') {
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index b4c32eea6..699ecbe33 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -24,7 +24,10 @@ describe('Logger instances and transports tests', async () => {
   // need to do it first
   envOverrides = await setupEnvironment(
     null,
-    buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.NODE_ENV], ['development'])
+    buildEnvOverrideConfig(
+      [ENVIRONMENT_VARIABLES.NODE_ENV, ENVIRONMENT_VARIABLES.LOG_DB],
+      ['development', 'false']
+    )
   )
   // because of this
   it('should be development environment', () => {
@@ -43,14 +46,14 @@ describe('Logger instances and transports tests', async () => {
       null,
       buildEnvOverrideConfig(
         [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
-        ['true', 'http://172.15.0.6:8108?apiKey=xyz']
+        ['true', 'http://localhost:8108?apiKey=xyz']
       )
     )
     expect(process.env.LOG_DB).to.be.equal('true')
     // will build the DB transport layer
     const config = await getConfiguration(true)
     // eslint-disable-next-line no-unused-vars
-    const DB = new Database(config.dbConfig)
+    const DB = await new Database(config.dbConfig)
     // Could generate Typesene error if DB is not running, but does not matter for this test
     OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
 
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 8bcef1fce..68d925b05 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -3,6 +3,7 @@ import Transport from 'winston-transport'
 import DailyRotateFile from 'winston-daily-rotate-file'
 import fs from 'fs'
 import { Database } from '../../components/database/index.js'
+import { ENVIRONMENT_VARIABLES } from '../constants.js'
 
 // all the types of modules/components
 export const LOGGER_MODULE_NAMES = {
@@ -134,26 +135,6 @@ export class CustomOceanNodesTransport extends Transport {
   }
 }
 
-/**
- * Avoid these annoyng typesense log message (too much garbage and no utility)
- * @param loggerModuleName module name
- * @param logMessage the actual message
- * @returns boolean
- */
-// function isTypesenseIgnoreLogMessage(loggerModuleName: string, logMessage: string) {
-//   const msg1: string = 'Response Code was 200.'
-//   const msg2: string = 'Response Code was 201.'
-//   const msg3 = 'request Try #1 to Node'
-//   const msg4 = 'Request /collections/'
-//   return (
-//     loggerModuleName.toLowerCase() === LOGGER_MODULE_NAMES.DATABASE &&
-//     (logMessage.includes(msg2) ||
-//       logMessage.includes(msg3) ||
-//       logMessage.includes(msg1) ||
-//       logMessage.includes(msg4))
-//   )
-// }
-
 let INSTANCE_COUNT = 0
 let customDBTransport: CustomOceanNodesTransport = null
 
@@ -164,8 +145,7 @@ export const NUM_LOGGER_INSTANCES = INSTANCE_COUNT
 const USE_FILE_TRANSPORT: boolean =
   process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
 
-export const USE_DB_TRANSPORT: boolean =
-  process.env.LOG_DB && process.env.LOG_DB !== 'false'
+const USE_DB_TRANSPORT: boolean = process.env.LOG_DB && process.env.LOG_DB !== 'false'
 
 // default to true, if not explicitly set otherwise AND no other locations defined
 const USE_CONSOLE_TRANSPORT: boolean =
@@ -471,19 +451,13 @@ export class CustomNodeLogger {
   ) {
     // lazy check db custom transport, needed beacause of dependency cycles
     if (
-      customDBTransport !== null && // if null then what?
-      USE_DB_TRANSPORT &&
+      customDBTransport !== null &&
+      (USE_DB_TRANSPORT || ENVIRONMENT_VARIABLES.LOG_DB.value === 'true') &&
       !this.hasDBTransport()
     ) {
       this.addTransport(customDBTransport)
     }
 
-    // if (!isLogLevelLogable(level)) {
-    //   return
-    // }
-
-    // ignore tons of typesense garbage
-    // if (!isTypesenseIgnoreLogMessage(this.getModuleName(), message)) {
     this.getLogger().log(
       level,
       includeModuleName ? this.buildMessage(message) : message,

From f13c1c185541e0b9b4d38d9e4b668abfbf1b6264 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Tue, 23 Jul 2024 13:57:35 +0100
Subject: [PATCH 30/50] make sure we use string format, just in case

---
 src/test/integration/logs.test.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 007db19bf..edacf2f93 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -32,7 +32,7 @@ describe('LogDatabase CRUD', () => {
   before(async () => {
     previousConfiguration = await setupEnvironment(
       null,
-      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
@@ -167,7 +167,7 @@ describe('LogDatabase retrieveMultipleLogs with specific parameters', () => {
   before(async () => {
     previousConfiguration = await setupEnvironment(
       null,
-      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
 
     const dbConfig = {
@@ -341,7 +341,7 @@ describe('LogDatabase deleteOldLogs', () => {
   before(async () => {
     previousConfiguration = await setupEnvironment(
       null,
-      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'
@@ -403,7 +403,7 @@ describe('LogDatabase retrieveMultipleLogs with pagination', () => {
   before(async () => {
     previousConfiguration = await setupEnvironment(
       null,
-      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], [true])
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
     const dbConfig = {
       url: 'http://localhost:8108/?apiKey=xyz'

From 64da9af7aabd3dedb9ec6f013ed9f851c2d692f9 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 12:56:25 +0100
Subject: [PATCH 31/50] update file docs

---
 env.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/env.md b/env.md
index b401f80ce..1448913db 100644
--- a/env.md
+++ b/env.md
@@ -25,9 +25,9 @@ Environmental variables are also tracked in `ENVIRONMENT_VARIABLES` within `src/
 - `MAX_REQ_PER_SECOND`: Number of requests per second allowed by the same client. Example: `3`
 - `MAX_CHECKSUM_LENGTH`: Define the maximum length for a file if checksum is required (Mb). Example: `10`
 - `LOG_LEVEL`: Define the default log level. Example: `debug`
-- `LOG_CONSOLE`: Write logs to the console. Default is `false`
+- `LOG_CONSOLE`: Write logs to the console. Default is `false`, but becomes `true` if neither `LOG_FILES` or `LOG_DB` are set.
 - `LOG_FILES`: Write logs to files. Default is `false`
-- `LOG_DB`: Write logs to noSQL database. Default is `true`
+- `LOG_DB`: Write logs to noSQL database. Default is `false`
 
 ## HTTP
 

From 5c802d3bf812797c502fb6b3914dd444d1856438 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 16:28:52 +0100
Subject: [PATCH 32/50] force run again

---
 src/test/.env.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/test/.env.test b/src/test/.env.test
index ee100d2a9..902153419 100644
--- a/src/test/.env.test
+++ b/src/test/.env.test
@@ -9,4 +9,4 @@ NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474
 NODE2_PRIVATE_KEY=0x3634cc4a3d2694a1186a7ce545f149e022eea103cc254d18d08675104bb4b5ac
 INDEXER_INTERVAL=9000
 ADDRESS_FILE=${HOME}/.ocean/ocean-contracts/artifacts/address.json
-LOG_DB=false
+

From 76c7bbafd2c2016740f4fa6d785d6c3dc2914a47 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 16:48:44 +0100
Subject: [PATCH 33/50] try debug test

---
 src/test/unit/storage.test.ts | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index ab9908e28..b87dff3d8 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -347,9 +347,12 @@ describe('Arweave Storage getFileInfo tests', function () {
   })
 
   it('Throws error when transaction ID is missing in request', async () => {
+    console.log('DO YOU SEE ME??')
     const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
     try {
+      console.log('DO YOU STILL SEE ME??')
       await storage.getFileInfo(fileInfoRequest)
+      console.log('DO YOU STILL SEE ME HERE??')
     } catch (err) {
       expect(err.message).to.equal('Transaction ID is required for type arweave')
     }

From 38b326e328ff7d739aa3fba47a8c275dc780d893 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 17:15:58 +0100
Subject: [PATCH 34/50] try debug

---
 src/test/unit/storage.test.ts | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index b87dff3d8..c663080e0 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -324,6 +324,8 @@ describe('Arweave Storage getFileInfo tests', function () {
   this.timeout(15000)
   let storage: ArweaveStorage
 
+  console.log('STORAGE TESTING....')
+
   before(() => {
     storage = new ArweaveStorage({
       type: FileObjectType.ARWEAVE,
@@ -335,8 +337,11 @@ describe('Arweave Storage getFileInfo tests', function () {
     const fileInfoRequest: FileInfoRequest = {
       type: FileObjectType.ARWEAVE
     }
+    console.log('1 - DO YOU SEE ME??')
     const fileInfo = await storage.getFileInfo(fileInfoRequest)
 
+    console.log('2 - DO YOU SEE ME??')
+
     assert(fileInfo[0].valid, 'File info is valid')
     assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect')
     assert(

From 54026b292d7073718c55752fa034bdcbcc3c28f5 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 17:26:34 +0100
Subject: [PATCH 35/50] more debug

---
 src/components/storage/index.ts | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts
index 9ece1c709..afff67009 100644
--- a/src/components/storage/index.ts
+++ b/src/components/storage/index.ts
@@ -80,12 +80,16 @@ export abstract class Storage {
     const response: FileInfoResponse[] = []
 
     try {
+      console.log('before getFile()')
       const file = this.getFile()
 
+      console.log('after getFile()')
       if (!file) {
         throw new Error('Empty file object')
       } else {
+        console.log('before get metadata....')
         const fileInfo = await this.fetchSpecificFileMetadata(file, forceChecksum)
+        console.log('after get metadata....')
         response.push(fileInfo)
       }
     } catch (error) {

From c6654962d91338da021b4bd985cc4ce89f1fc755 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 18:03:40 +0100
Subject: [PATCH 36/50] more debug

---
 src/test/unit/storage.test.ts | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index c663080e0..c36b19d15 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -353,11 +353,16 @@ describe('Arweave Storage getFileInfo tests', function () {
 
   it('Throws error when transaction ID is missing in request', async () => {
     console.log('DO YOU SEE ME??')
+    // do it again
+    storage = new ArweaveStorage({
+      type: FileObjectType.ARWEAVE,
+      transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE'
+    })
     const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
     try {
-      console.log('DO YOU STILL SEE ME??')
+      console.log('3 - DO YOU STILL SEE ME??')
       await storage.getFileInfo(fileInfoRequest)
-      console.log('DO YOU STILL SEE ME HERE??')
+      console.log('4 - DO YOU STILL SEE ME HERE??')
     } catch (err) {
       expect(err.message).to.equal('Transaction ID is required for type arweave')
     }

From 5d6e2e8e51061b1af6be28c10951088d5439c145 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 18:28:13 +0100
Subject: [PATCH 37/50] remove/comment test

---
 src/test/unit/storage.test.ts | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index c36b19d15..210f62ea4 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -337,11 +337,8 @@ describe('Arweave Storage getFileInfo tests', function () {
     const fileInfoRequest: FileInfoRequest = {
       type: FileObjectType.ARWEAVE
     }
-    console.log('1 - DO YOU SEE ME??')
     const fileInfo = await storage.getFileInfo(fileInfoRequest)
 
-    console.log('2 - DO YOU SEE ME??')
-
     assert(fileInfo[0].valid, 'File info is valid')
     assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect')
     assert(
@@ -351,22 +348,18 @@ describe('Arweave Storage getFileInfo tests', function () {
     assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect')
   })
 
-  it('Throws error when transaction ID is missing in request', async () => {
-    console.log('DO YOU SEE ME??')
-    // do it again
-    storage = new ArweaveStorage({
-      type: FileObjectType.ARWEAVE,
-      transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE'
-    })
-    const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
-    try {
-      console.log('3 - DO YOU STILL SEE ME??')
-      await storage.getFileInfo(fileInfoRequest)
-      console.log('4 - DO YOU STILL SEE ME HERE??')
-    } catch (err) {
-      expect(err.message).to.equal('Transaction ID is required for type arweave')
-    }
-  })
+  // it('Throws error when transaction ID is missing in request', async () => {
+  //   console.log('DO YOU SEE ME??')
+
+  //   const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
+  //   try {
+  //     console.log('3 - DO YOU STILL SEE ME??')
+  //     await storage.getFileInfo(fileInfoRequest)
+  //     console.log('4 - DO YOU STILL SEE ME HERE??')
+  //   } catch (err) {
+  //     expect(err.message).to.equal('Transaction ID is required for type arweave')
+  //   }
+  // })
 })
 
 describe('Arweave Storage with malformed transaction ID', () => {

From 498d1b0fbacb928a833a1386770618236af9f3fd Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 18:42:34 +0100
Subject: [PATCH 38/50] more debug

---
 src/components/storage/index.ts |  1 +
 src/test/unit/storage.test.ts   | 24 ++++++++++++------------
 src/utils/asset.ts              |  9 +++++++--
 3 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts
index afff67009..84ecb1c97 100644
--- a/src/components/storage/index.ts
+++ b/src/components/storage/index.ts
@@ -299,6 +299,7 @@ export class ArweaveStorage extends Storage {
     forceChecksum: boolean
   ): Promise<FileInfoResponse> {
     const url = urlJoin(process.env.ARWEAVE_GATEWAY, fileObject.transactionId)
+    console.log('URL:', url)
     const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
       url,
       'get',
diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index 210f62ea4..d4b2498e7 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -321,7 +321,7 @@ describe('URL Storage with malformed URL', () => {
 })
 
 describe('Arweave Storage getFileInfo tests', function () {
-  this.timeout(15000)
+  // this.timeout(15000)
   let storage: ArweaveStorage
 
   console.log('STORAGE TESTING....')
@@ -348,18 +348,18 @@ describe('Arweave Storage getFileInfo tests', function () {
     assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect')
   })
 
-  // it('Throws error when transaction ID is missing in request', async () => {
-  //   console.log('DO YOU SEE ME??')
+  it('Throws error when transaction ID is missing in request', async () => {
+    console.log('DO YOU SEE ME??')
 
-  //   const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
-  //   try {
-  //     console.log('3 - DO YOU STILL SEE ME??')
-  //     await storage.getFileInfo(fileInfoRequest)
-  //     console.log('4 - DO YOU STILL SEE ME HERE??')
-  //   } catch (err) {
-  //     expect(err.message).to.equal('Transaction ID is required for type arweave')
-  //   }
-  // })
+    const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
+    try {
+      console.log('3 - DO YOU STILL SEE ME??')
+      await storage.getFileInfo(fileInfoRequest)
+      console.log('4 - DO YOU STILL SEE ME HERE??')
+    } catch (err) {
+      expect(err.message).to.equal('Transaction ID is required for type arweave')
+    }
+  })
 })
 
 describe('Arweave Storage with malformed transaction ID', () => {
diff --git a/src/utils/asset.ts b/src/utils/asset.ts
index 13b9b1093..e680026e9 100644
--- a/src/utils/asset.ts
+++ b/src/utils/asset.ts
@@ -43,10 +43,12 @@ export async function fetchFileMetadata(
     const response = await axios({
       url,
       method: method || 'get',
-      responseType: 'stream'
+      responseType: 'stream',
+      timeout: 10000 // 10 seconds timeout
     })
     contentType = response.headers['content-type']
     let totalSize = 0
+    console.log('reading content')
     for await (const chunk of response.data) {
       totalSize += chunk.length
       contentChecksum.update(chunk)
@@ -56,8 +58,11 @@ export async function fetchFileMetadata(
       }
     }
     contentLength = totalSize
-  } catch (error) {}
+  } catch (error) {
+    console.log('Got error: ', error)
+  }
 
+  console.log('im out')
   return {
     contentLength: contentLength.toString(),
     contentType,

From bdc4e5d31320e6d76e0cd2156bdc037f7e7fb88e Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Wed, 24 Jul 2024 23:40:59 +0100
Subject: [PATCH 39/50] do some cleaning

---
 src/components/database/index.ts | 18 ++++++------------
 src/components/storage/index.ts  |  8 ++------
 src/test/unit/logging.test.ts    |  2 ++
 src/test/unit/storage.test.ts    |  6 ------
 src/utils/asset.ts               |  7 ++-----
 5 files changed, 12 insertions(+), 29 deletions(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index d47032c62..2be3f7c81 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -21,8 +21,7 @@ export class OrderDatabase {
     return (async (): Promise<OrderDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -191,8 +190,7 @@ export class DdoStateDatabase {
     return (async (): Promise<DdoStateDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -328,8 +326,7 @@ export class DdoDatabase {
     return (async (): Promise<DdoDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       for (const ddoSchema of this.schemas) {
         try {
@@ -619,8 +616,7 @@ export class NonceDatabase {
     return (async (): Promise<NonceDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -725,8 +721,7 @@ export class IndexerDatabase {
     return (async (): Promise<IndexerDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
@@ -836,8 +831,7 @@ export class LogDatabase {
     return (async (): Promise<LogDatabase> => {
       this.provider = new Typesense({
         ...convertTypesenseConfig(this.config.url),
-        logger: DATABASE_LOGGER,
-        logLevel: LOG_LEVELS_STR.LEVEL_ERROR
+        logger: DATABASE_LOGGER
       })
       try {
         await this.provider.collections(this.schema.name).retrieve()
diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts
index 84ecb1c97..3d28ce212 100644
--- a/src/components/storage/index.ts
+++ b/src/components/storage/index.ts
@@ -14,6 +14,7 @@ import urlJoin from 'url-join'
 import { encrypt as encryptData, decrypt as decryptData } from '../../utils/crypt.js'
 import { Readable } from 'stream'
 import { getConfiguration } from '../../utils/index.js'
+import { CORE_LOGGER } from '../../utils/logging/common.js'
 
 export abstract class Storage {
   private file: UrlFileObject | IpfsFileObject | ArweaveFileObject
@@ -80,20 +81,16 @@ export abstract class Storage {
     const response: FileInfoResponse[] = []
 
     try {
-      console.log('before getFile()')
       const file = this.getFile()
 
-      console.log('after getFile()')
       if (!file) {
         throw new Error('Empty file object')
       } else {
-        console.log('before get metadata....')
         const fileInfo = await this.fetchSpecificFileMetadata(file, forceChecksum)
-        console.log('after get metadata....')
         response.push(fileInfo)
       }
     } catch (error) {
-      console.log(error)
+      CORE_LOGGER.error(error)
     }
     return response
   }
@@ -299,7 +296,6 @@ export class ArweaveStorage extends Storage {
     forceChecksum: boolean
   ): Promise<FileInfoResponse> {
     const url = urlJoin(process.env.ARWEAVE_GATEWAY, fileObject.transactionId)
-    console.log('URL:', url)
     const { contentLength, contentType, contentChecksum } = await fetchFileMetadata(
       url,
       'get',
diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 699ecbe33..6bdecd4c5 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -53,7 +53,9 @@ describe('Logger instances and transports tests', async () => {
     // will build the DB transport layer
     const config = await getConfiguration(true)
     // eslint-disable-next-line no-unused-vars
+    console.log('CONFIG IS', config.dbConfig)
     const DB = await new Database(config.dbConfig)
+    console.log(DB)
     // Could generate Typesene error if DB is not running, but does not matter for this test
     OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
 
diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts
index d4b2498e7..7823dc14e 100644
--- a/src/test/unit/storage.test.ts
+++ b/src/test/unit/storage.test.ts
@@ -324,8 +324,6 @@ describe('Arweave Storage getFileInfo tests', function () {
   // this.timeout(15000)
   let storage: ArweaveStorage
 
-  console.log('STORAGE TESTING....')
-
   before(() => {
     storage = new ArweaveStorage({
       type: FileObjectType.ARWEAVE,
@@ -349,13 +347,9 @@ describe('Arweave Storage getFileInfo tests', function () {
   })
 
   it('Throws error when transaction ID is missing in request', async () => {
-    console.log('DO YOU SEE ME??')
-
     const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE }
     try {
-      console.log('3 - DO YOU STILL SEE ME??')
       await storage.getFileInfo(fileInfoRequest)
-      console.log('4 - DO YOU STILL SEE ME HERE??')
     } catch (err) {
       expect(err.message).to.equal('Transaction ID is required for type arweave')
     }
diff --git a/src/utils/asset.ts b/src/utils/asset.ts
index e680026e9..8901b0a7e 100644
--- a/src/utils/asset.ts
+++ b/src/utils/asset.ts
@@ -43,12 +43,10 @@ export async function fetchFileMetadata(
     const response = await axios({
       url,
       method: method || 'get',
-      responseType: 'stream',
-      timeout: 10000 // 10 seconds timeout
+      responseType: 'stream'
     })
     contentType = response.headers['content-type']
     let totalSize = 0
-    console.log('reading content')
     for await (const chunk of response.data) {
       totalSize += chunk.length
       contentChecksum.update(chunk)
@@ -59,10 +57,9 @@ export async function fetchFileMetadata(
     }
     contentLength = totalSize
   } catch (error) {
-    console.log('Got error: ', error)
+    CORE_LOGGER.error(error)
   }
 
-  console.log('im out')
   return {
     contentLength: contentLength.toString(),
     contentType,

From 6eb48c76c2e249969564de1e7699e165564a29cb Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 09:48:50 +0100
Subject: [PATCH 40/50] fix test messing DB url + small refactors

---
 src/components/database/index.ts |  5 +++--
 src/test/.env.test               |  2 +-
 src/test/unit/logging.test.ts    |  9 ++++-----
 src/utils/constants.ts           |  4 ++--
 src/utils/logging/Logger.ts      | 30 ++++++++++++++++--------------
 5 files changed, 26 insertions(+), 24 deletions(-)

diff --git a/src/components/database/index.ts b/src/components/database/index.ts
index 2be3f7c81..15deabbff 100644
--- a/src/components/database/index.ts
+++ b/src/components/database/index.ts
@@ -5,7 +5,8 @@ import { TypesenseSearchParams } from '../../@types/index.js'
 import {
   LOG_LEVELS_STR,
   configureCustomDBTransport,
-  GENERIC_EMOJIS
+  GENERIC_EMOJIS,
+  USE_DB_TRANSPORT
 } from '../../utils/logging/Logger.js'
 import { DATABASE_LOGGER } from '../../utils/logging/common.js'
 import { validateObject } from '../core/utils/validateDdoHandler.js'
@@ -1012,7 +1013,7 @@ export class Database {
     // add this DB transport too
     // once we create a DB instance, the logger will be using this transport as well
     // we cannot have this the other way around because of the dependencies cycle
-    if (ENVIRONMENT_VARIABLES.LOG_DB.value === 'true') {
+    if (USE_DB_TRANSPORT()) {
       configureCustomDBTransport(this, DATABASE_LOGGER)
     } else {
       DATABASE_LOGGER.warn(
diff --git a/src/test/.env.test b/src/test/.env.test
index 902153419..28fc9ecaf 100644
--- a/src/test/.env.test
+++ b/src/test/.env.test
@@ -9,4 +9,4 @@ NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474
 NODE2_PRIVATE_KEY=0x3634cc4a3d2694a1186a7ce545f149e022eea103cc254d18d08675104bb4b5ac
 INDEXER_INTERVAL=9000
 ADDRESS_FILE=${HOME}/.ocean/ocean-contracts/artifacts/address.json
-
+LOG_LEVEL=debug
diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 6bdecd4c5..e91cc0222 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -12,6 +12,7 @@ import {
   CustomOceanNodesTransport,
   MAX_LOGGER_INSTANCES,
   NUM_LOGGER_INSTANCES,
+  USE_DB_TRANSPORT,
   isDevelopmentEnvironment
 } from '../../utils/logging/Logger.js'
 import { OCEAN_NODE_LOGGER } from '../../utils/logging/common.js'
@@ -40,22 +41,20 @@ describe('Logger instances and transports tests', async () => {
   })
 
   it(`should change LOG_DB to "true" and logger should have DB transport`, async () => {
-    expect(process.env.LOG_DB).to.be.equal('false')
+    expect(USE_DB_TRANSPORT()).to.be.equal(false)
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(
       null,
       buildEnvOverrideConfig(
         [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
-        ['true', 'http://localhost:8108?apiKey=xyz']
+        ['true', 'http://172.15.0.6:8108?apiKey=xyz']
       )
     )
-    expect(process.env.LOG_DB).to.be.equal('true')
+    expect(USE_DB_TRANSPORT()).to.be.equal(true)
     // will build the DB transport layer
     const config = await getConfiguration(true)
     // eslint-disable-next-line no-unused-vars
-    console.log('CONFIG IS', config.dbConfig)
     const DB = await new Database(config.dbConfig)
-    console.log(DB)
     // Could generate Typesene error if DB is not running, but does not matter for this test
     OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
 
diff --git a/src/utils/constants.ts b/src/utils/constants.ts
index 3b34f6988..0366620bb 100644
--- a/src/utils/constants.ts
+++ b/src/utils/constants.ts
@@ -277,7 +277,7 @@ export const ENVIRONMENT_VARIABLES: Record<any, EnvVariable> = {
     required: false
   },
   LOG_CONSOLE: {
-    // log to console output?
+    // log to console output? true if no other bellow is set
     name: 'LOG_CONSOLE',
     value: process.env.LOG_CONSOLE,
     required: false
@@ -292,7 +292,7 @@ export const ENVIRONMENT_VARIABLES: Record<any, EnvVariable> = {
     // log to DB?
     name: 'LOG_DB',
     value: process.env.LOG_DB,
-    required: false // default is true
+    required: false
   }
 }
 
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 68d925b05..11c6ce95e 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -3,7 +3,6 @@ import Transport from 'winston-transport'
 import DailyRotateFile from 'winston-daily-rotate-file'
 import fs from 'fs'
 import { Database } from '../../components/database/index.js'
-import { ENVIRONMENT_VARIABLES } from '../constants.js'
 
 // all the types of modules/components
 export const LOGGER_MODULE_NAMES = {
@@ -142,15 +141,21 @@ export const MAX_LOGGER_INSTANCES = 10
 export const NUM_LOGGER_INSTANCES = INSTANCE_COUNT
 
 // log locations
-const USE_FILE_TRANSPORT: boolean =
-  process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
+function USE_FILE_TRANSPORT(): boolean {
+  return process.env.LOG_FILES && process.env.LOG_FILES !== 'false'
+}
 
-const USE_DB_TRANSPORT: boolean = process.env.LOG_DB && process.env.LOG_DB !== 'false'
+export function USE_DB_TRANSPORT(): boolean {
+  return process.env.LOG_DB && process.env.LOG_DB !== 'false'
+}
 
 // default to true, if not explicitly set otherwise AND no other locations defined
-const USE_CONSOLE_TRANSPORT: boolean =
-  (process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false') ||
-  (!USE_FILE_TRANSPORT && !USE_DB_TRANSPORT)
+function USE_CONSOLE_TRANSPORT(): boolean {
+  return (
+    (process.env.LOG_CONSOLE && process.env.LOG_CONSOLE !== 'false') ||
+    (!USE_FILE_TRANSPORT() && !USE_DB_TRANSPORT())
+  )
+}
 
 // if not set, then gets default 'development' level & colors
 export function isDevelopmentEnvironment(): boolean {
@@ -274,12 +279,13 @@ export function getDefaultLoggerTransports(
   moduleOrComponentName: string
 ): winston.transport[] {
   const transports: winston.transport[] = []
-  if (USE_FILE_TRANSPORT) {
+  // account for runtime changes done by tests (force read again value)
+  if (USE_FILE_TRANSPORT()) {
     // always log to file
     transports.push(buildCustomFileTransport(moduleOrComponentName))
   }
 
-  if (USE_CONSOLE_TRANSPORT) {
+  if (USE_CONSOLE_TRANSPORT()) {
     transports.push(defaultConsoleTransport)
   }
   return transports
@@ -450,11 +456,7 @@ export class CustomNodeLogger {
     includeModuleName: boolean = false
   ) {
     // lazy check db custom transport, needed beacause of dependency cycles
-    if (
-      customDBTransport !== null &&
-      (USE_DB_TRANSPORT || ENVIRONMENT_VARIABLES.LOG_DB.value === 'true') &&
-      !this.hasDBTransport()
-    ) {
+    if (customDBTransport !== null && USE_DB_TRANSPORT() && !this.hasDBTransport()) {
       this.addTransport(customDBTransport)
     }
 

From f08d7ad933baec4cc511cc7330cef4e5b50bf0ed Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 10:06:40 +0100
Subject: [PATCH 41/50] increase test timeout

---
 src/test/unit/logging.test.ts | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index e91cc0222..0a8a912ed 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -3,6 +3,7 @@ import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js'
 
 import { expect } from 'chai'
 import {
+  DEFAULT_TEST_TIMEOUT,
   OverrideEnvConfig,
   buildEnvOverrideConfig,
   setupEnvironment,
@@ -40,7 +41,9 @@ describe('Logger instances and transports tests', async () => {
     expect(numExistingInstances).to.be.lessThanOrEqual(MAX_LOGGER_INSTANCES)
   })
 
-  it(`should change LOG_DB to "true" and logger should have DB transport`, async () => {
+  it(`should change LOG_DB to "true" and logger should have DB transport`, async function () {
+    // when we are logging to DB, things can slow down a bit
+    this.timeout(DEFAULT_TEST_TIMEOUT * 2)
     expect(USE_DB_TRANSPORT()).to.be.equal(false)
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(

From ce035637b0dd91fdcb598afbd05650a823e78644 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 11:59:17 +0100
Subject: [PATCH 42/50] improve log message for clarity

---
 src/test/unit/logging.test.ts | 12 +++++++++---
 src/test/utils/hooks.ts       |  5 +++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 0a8a912ed..f3fb1bb68 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -27,8 +27,12 @@ describe('Logger instances and transports tests', async () => {
   envOverrides = await setupEnvironment(
     null,
     buildEnvOverrideConfig(
-      [ENVIRONMENT_VARIABLES.NODE_ENV, ENVIRONMENT_VARIABLES.LOG_DB],
-      ['development', 'false']
+      [
+        ENVIRONMENT_VARIABLES.NODE_ENV,
+        ENVIRONMENT_VARIABLES.LOG_DB,
+        ENVIRONMENT_VARIABLES.LOG_LEVEL
+      ],
+      ['development', 'false', 'info']
     )
   )
   // because of this
@@ -43,7 +47,7 @@ describe('Logger instances and transports tests', async () => {
 
   it(`should change LOG_DB to "true" and logger should have DB transport`, async function () {
     // when we are logging to DB, things can slow down a bit
-    this.timeout(DEFAULT_TEST_TIMEOUT * 2)
+    this.timeout(DEFAULT_TEST_TIMEOUT * 3)
     expect(USE_DB_TRANSPORT()).to.be.equal(false)
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(
@@ -53,11 +57,13 @@ describe('Logger instances and transports tests', async () => {
         ['true', 'http://172.15.0.6:8108?apiKey=xyz']
       )
     )
+    console.log('env after', envAfter)
     expect(USE_DB_TRANSPORT()).to.be.equal(true)
     // will build the DB transport layer
     const config = await getConfiguration(true)
     // eslint-disable-next-line no-unused-vars
     const DB = await new Database(config.dbConfig)
+    console.log('after DB')
     // Could generate Typesene error if DB is not running, but does not matter for this test
     OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
 
diff --git a/src/test/utils/hooks.ts b/src/test/utils/hooks.ts
index be2bae3fa..a808d63f2 100644
--- a/src/test/utils/hooks.ts
+++ b/src/test/utils/hooks.ts
@@ -69,8 +69,9 @@ export const mochaHooks = {
       const initialVariable: OverrideEnvConfig = initialConfiguration.get(varName)
       if (initialVariable.originalValue !== currentEnvVariable.originalValue) {
         // reset it to the original
-        CONFIG_LOGGER.debug('Restoring environment variable: ' + varName)
-        process.env[varName] = initialVariable.originalValue
+        CONFIG_LOGGER.debug(
+          `(Hook) Restoring environment variable: ${varName} \ncurrent:\n ${process.env[varName]} original:\n ' ${initialVariable.originalValue}`
+        )
       }
     })
   }

From 111e2e619d627cc5ef2effa2e5199ecd4a5fd59b Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 12:13:59 +0100
Subject: [PATCH 43/50] more debug

---
 src/test/unit/logging.test.ts | 7 +++----
 src/test/utils/hooks.ts       | 2 +-
 src/utils/logging/Logger.ts   | 1 +
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index f3fb1bb68..11209145b 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -52,15 +52,14 @@ describe('Logger instances and transports tests', async () => {
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(
       null,
-      buildEnvOverrideConfig(
-        [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
-        ['true', 'http://172.15.0.6:8108?apiKey=xyz']
-      )
+      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
     console.log('env after', envAfter)
     expect(USE_DB_TRANSPORT()).to.be.equal(true)
     // will build the DB transport layer
+    console.log('before config')
     const config = await getConfiguration(true)
+    console.log('after config: ', config)
     // eslint-disable-next-line no-unused-vars
     const DB = await new Database(config.dbConfig)
     console.log('after DB')
diff --git a/src/test/utils/hooks.ts b/src/test/utils/hooks.ts
index a808d63f2..1546947d9 100644
--- a/src/test/utils/hooks.ts
+++ b/src/test/utils/hooks.ts
@@ -70,7 +70,7 @@ export const mochaHooks = {
       if (initialVariable.originalValue !== currentEnvVariable.originalValue) {
         // reset it to the original
         CONFIG_LOGGER.debug(
-          `(Hook) Restoring environment variable: ${varName} \ncurrent:\n ${process.env[varName]} original:\n ' ${initialVariable.originalValue}`
+          `(Hook) Restoring environment variable: ${varName} \ncurrent:\n ${process.env[varName]} \noriginal:\n ${initialVariable.originalValue}`
         )
       }
     })
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index 11c6ce95e..d7dd72637 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -564,6 +564,7 @@ export function configureCustomDBTransport(
   }
   if (!logger.hasDBTransport()) {
     logger.addTransport(customDBTransport)
+    logger.logMessage('Adding DB transport to Logger: ' + logger.getModuleName())
   }
   return logger
 }

From e85e001ddc58bf2df0345cf8b46d5a458a5b51b7 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 12:34:25 +0100
Subject: [PATCH 44/50] clear debug messages, restore deleted line accidentally

---
 src/test/unit/logging.test.ts | 6 +-----
 src/test/utils/hooks.ts       | 1 +
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 11209145b..329923362 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -54,15 +54,11 @@ describe('Logger instances and transports tests', async () => {
       null,
       buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
     )
-    console.log('env after', envAfter)
     expect(USE_DB_TRANSPORT()).to.be.equal(true)
     // will build the DB transport layer
-    console.log('before config')
     const config = await getConfiguration(true)
-    console.log('after config: ', config)
     // eslint-disable-next-line no-unused-vars
     const DB = await new Database(config.dbConfig)
-    console.log('after DB')
     // Could generate Typesene error if DB is not running, but does not matter for this test
     OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
 
@@ -77,7 +73,7 @@ describe('Logger instances and transports tests', async () => {
     OCEAN_NODE_LOGGER.removeTransport(transports[0])
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     await tearDownEnvironment(envAfter)
-    expect(process.env.LOG_DB).to.be.equal('false')
+    expect(USE_DB_TRANSPORT()).to.be.equal(false)
   })
 
   after(() => {
diff --git a/src/test/utils/hooks.ts b/src/test/utils/hooks.ts
index 1546947d9..59ace1d4b 100644
--- a/src/test/utils/hooks.ts
+++ b/src/test/utils/hooks.ts
@@ -72,6 +72,7 @@ export const mochaHooks = {
         CONFIG_LOGGER.debug(
           `(Hook) Restoring environment variable: ${varName} \ncurrent:\n ${process.env[varName]} \noriginal:\n ${initialVariable.originalValue}`
         )
+        process.env[varName] = initialVariable.originalValue
       }
     })
   }

From 7ea9bb3b26c8a4e234fe2fd3b7131d0f960d982e Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 13:11:34 +0100
Subject: [PATCH 45/50] more debug, clear logs

---
 src/test/.env.test            |  2 +-
 src/test/unit/logging.test.ts | 10 +++++++---
 src/test/utils/utils.ts       |  6 +++++-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/test/.env.test b/src/test/.env.test
index 28fc9ecaf..9e142ea3c 100644
--- a/src/test/.env.test
+++ b/src/test/.env.test
@@ -2,7 +2,7 @@ HTTP_API_PORT=8001
 P2P_ipV4BindTcpPort=8000
 PRIVATE_KEY=0xc594c6e5def4bab63ac29eed19a134c130388f74f019bc74b8f4389df2837a58
 RPCS='{ "8996": {"rpc": "http://127.0.0.1:8545", "chainId": 8996, "network": "development", "chunkSize": 100}}'
-DB_URL=http://localhost:8108?apiKey=xyz
+DB_URL=http://localhost:8108/?apiKey=xyz
 IPFS_GATEWAY=https://ipfs.io/
 ARWEAVE_GATEWAY=https://arweave.net/
 NODE1_PRIVATE_KEY=0xcb345bd2b11264d523ddaf383094e2675c420a17511c3102a53817f13474a7ff
diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 329923362..b49b812e6 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -30,9 +30,10 @@ describe('Logger instances and transports tests', async () => {
       [
         ENVIRONMENT_VARIABLES.NODE_ENV,
         ENVIRONMENT_VARIABLES.LOG_DB,
-        ENVIRONMENT_VARIABLES.LOG_LEVEL
+        ENVIRONMENT_VARIABLES.LOG_LEVEL,
+        ENVIRONMENT_VARIABLES.DB_URL
       ],
-      ['development', 'false', 'info']
+      ['development', 'false', 'info', 'http://localhost:8108/?apiKey=xyz']
     )
   )
   // because of this
@@ -52,7 +53,10 @@ describe('Logger instances and transports tests', async () => {
     expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
     const envAfter = await setupEnvironment(
       null,
-      buildEnvOverrideConfig([ENVIRONMENT_VARIABLES.LOG_DB], ['true'])
+      buildEnvOverrideConfig(
+        [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
+        ['true', 'http://localhost:8108/?apiKey=xyz']
+      )
     )
     expect(USE_DB_TRANSPORT()).to.be.equal(true)
     // will build the DB transport layer
diff --git a/src/test/utils/utils.ts b/src/test/utils/utils.ts
index e45826687..743623564 100644
--- a/src/test/utils/utils.ts
+++ b/src/test/utils/utils.ts
@@ -92,7 +92,11 @@ export async function setupEnvironment(
         element.override ||
         (element.required && process.env[element.name] === undefined) // if override OR not set but required to run
       ) {
-        CONFIG_LOGGER.debug('Overriding environment variable: ' + element.name)
+        CONFIG_LOGGER.debug(
+          `Overriding environment variable: ${element.name}\ncurrent:\n ${
+            process.env[element.name]
+          }\nnew:\n ${element.newValue}`
+        )
         element.originalValue = process.env[element.name] // save original value
         process.env[element.name] = element.newValue
         ENVIRONMENT_VARIABLES[element.name].value = element.newValue

From c10d7a94d5e99f44e42fb5bcfe32e5944b0bee13 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 13:57:51 +0100
Subject: [PATCH 46/50] debug test

---
 src/test/integration/logs.test.ts | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index edacf2f93..2f26ad2d7 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -85,7 +85,8 @@ describe('LogDatabase CRUD', () => {
     const endTime = new Date() // current time
 
     // Retrieve the latest log entries
-    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 10)
+    let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
+    console.log('got 100 logs?:', logs)
     logs = logs.filter((log) => log.message === newLogEntry.message)
 
     expect(logs?.length).to.equal(1)

From 01d4b5849c19191585ce8ca3ffb78aa66b414e3a Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 14:18:35 +0100
Subject: [PATCH 47/50] remove unit test

---
 src/test/unit/logging.test.ts | 42 +----------------------------------
 1 file changed, 1 insertion(+), 41 deletions(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index b49b812e6..0f920cb83 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -1,23 +1,17 @@
-import { Database } from '../../components/database/index.js'
-import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js'
+import { ENVIRONMENT_VARIABLES } from '../../utils/index.js'
 
 import { expect } from 'chai'
 import {
-  DEFAULT_TEST_TIMEOUT,
   OverrideEnvConfig,
   buildEnvOverrideConfig,
   setupEnvironment,
   tearDownEnvironment
 } from '../utils/utils.js'
 import {
-  CustomOceanNodesTransport,
   MAX_LOGGER_INSTANCES,
   NUM_LOGGER_INSTANCES,
-  USE_DB_TRANSPORT,
   isDevelopmentEnvironment
 } from '../../utils/logging/Logger.js'
-import { OCEAN_NODE_LOGGER } from '../../utils/logging/common.js'
-import winston from 'winston'
 
 let envOverrides: OverrideEnvConfig[]
 
@@ -46,40 +40,6 @@ describe('Logger instances and transports tests', async () => {
     expect(numExistingInstances).to.be.lessThanOrEqual(MAX_LOGGER_INSTANCES)
   })
 
-  it(`should change LOG_DB to "true" and logger should have DB transport`, async function () {
-    // when we are logging to DB, things can slow down a bit
-    this.timeout(DEFAULT_TEST_TIMEOUT * 3)
-    expect(USE_DB_TRANSPORT()).to.be.equal(false)
-    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
-    const envAfter = await setupEnvironment(
-      null,
-      buildEnvOverrideConfig(
-        [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
-        ['true', 'http://localhost:8108/?apiKey=xyz']
-      )
-    )
-    expect(USE_DB_TRANSPORT()).to.be.equal(true)
-    // will build the DB transport layer
-    const config = await getConfiguration(true)
-    // eslint-disable-next-line no-unused-vars
-    const DB = await new Database(config.dbConfig)
-    // Could generate Typesene error if DB is not running, but does not matter for this test
-    OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
-
-    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(true)
-
-    const transports: winston.transport[] = OCEAN_NODE_LOGGER.getTransports().filter(
-      (transport: winston.transport) => {
-        return transport instanceof CustomOceanNodesTransport
-      }
-    )
-    expect(transports.length).to.be.equal(1)
-    OCEAN_NODE_LOGGER.removeTransport(transports[0])
-    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
-    await tearDownEnvironment(envAfter)
-    expect(USE_DB_TRANSPORT()).to.be.equal(false)
-  })
-
   after(() => {
     // Restore original local setup / env variables after test
     tearDownEnvironment(envOverrides)

From 5ed690c5e43d18359114d4162e1883f9ba0359fc Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 14:59:14 +0100
Subject: [PATCH 48/50] restore test, add functionality to remove db transport
 if not needed anymore

---
 src/test/unit/logging.test.ts | 42 ++++++++++++++++++++++++++++++++++-
 src/utils/logging/Logger.ts   |  3 +++
 2 files changed, 44 insertions(+), 1 deletion(-)

diff --git a/src/test/unit/logging.test.ts b/src/test/unit/logging.test.ts
index 0f920cb83..b49b812e6 100644
--- a/src/test/unit/logging.test.ts
+++ b/src/test/unit/logging.test.ts
@@ -1,17 +1,23 @@
-import { ENVIRONMENT_VARIABLES } from '../../utils/index.js'
+import { Database } from '../../components/database/index.js'
+import { ENVIRONMENT_VARIABLES, getConfiguration } from '../../utils/index.js'
 
 import { expect } from 'chai'
 import {
+  DEFAULT_TEST_TIMEOUT,
   OverrideEnvConfig,
   buildEnvOverrideConfig,
   setupEnvironment,
   tearDownEnvironment
 } from '../utils/utils.js'
 import {
+  CustomOceanNodesTransport,
   MAX_LOGGER_INSTANCES,
   NUM_LOGGER_INSTANCES,
+  USE_DB_TRANSPORT,
   isDevelopmentEnvironment
 } from '../../utils/logging/Logger.js'
+import { OCEAN_NODE_LOGGER } from '../../utils/logging/common.js'
+import winston from 'winston'
 
 let envOverrides: OverrideEnvConfig[]
 
@@ -40,6 +46,40 @@ describe('Logger instances and transports tests', async () => {
     expect(numExistingInstances).to.be.lessThanOrEqual(MAX_LOGGER_INSTANCES)
   })
 
+  it(`should change LOG_DB to "true" and logger should have DB transport`, async function () {
+    // when we are logging to DB, things can slow down a bit
+    this.timeout(DEFAULT_TEST_TIMEOUT * 3)
+    expect(USE_DB_TRANSPORT()).to.be.equal(false)
+    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
+    const envAfter = await setupEnvironment(
+      null,
+      buildEnvOverrideConfig(
+        [ENVIRONMENT_VARIABLES.LOG_DB, ENVIRONMENT_VARIABLES.DB_URL],
+        ['true', 'http://localhost:8108/?apiKey=xyz']
+      )
+    )
+    expect(USE_DB_TRANSPORT()).to.be.equal(true)
+    // will build the DB transport layer
+    const config = await getConfiguration(true)
+    // eslint-disable-next-line no-unused-vars
+    const DB = await new Database(config.dbConfig)
+    // Could generate Typesene error if DB is not running, but does not matter for this test
+    OCEAN_NODE_LOGGER.logMessage('Should build DB transport layer')
+
+    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(true)
+
+    const transports: winston.transport[] = OCEAN_NODE_LOGGER.getTransports().filter(
+      (transport: winston.transport) => {
+        return transport instanceof CustomOceanNodesTransport
+      }
+    )
+    expect(transports.length).to.be.equal(1)
+    OCEAN_NODE_LOGGER.removeTransport(transports[0])
+    expect(OCEAN_NODE_LOGGER.hasDBTransport()).to.be.equal(false)
+    await tearDownEnvironment(envAfter)
+    expect(USE_DB_TRANSPORT()).to.be.equal(false)
+  })
+
   after(() => {
     // Restore original local setup / env variables after test
     tearDownEnvironment(envOverrides)
diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index d7dd72637..df5b7df0b 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -458,6 +458,9 @@ export class CustomNodeLogger {
     // lazy check db custom transport, needed beacause of dependency cycles
     if (customDBTransport !== null && USE_DB_TRANSPORT() && !this.hasDBTransport()) {
       this.addTransport(customDBTransport)
+    } else if (this.hasDBTransport() && !USE_DB_TRANSPORT()) {
+      this.logMessage('Removing DB transport from Logger: ' + this.getModuleName())
+      this.removeTransport(this.getDBTransport())
     }
 
     this.getLogger().log(

From 762df88e1bd0353ef69df0cc3ddcfc1c18df88f6 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 15:41:02 +0100
Subject: [PATCH 49/50] fix stack call wrong

---
 src/utils/logging/Logger.ts | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/utils/logging/Logger.ts b/src/utils/logging/Logger.ts
index df5b7df0b..7c47c89b5 100644
--- a/src/utils/logging/Logger.ts
+++ b/src/utils/logging/Logger.ts
@@ -456,10 +456,10 @@ export class CustomNodeLogger {
     includeModuleName: boolean = false
   ) {
     // lazy check db custom transport, needed beacause of dependency cycles
-    if (customDBTransport !== null && USE_DB_TRANSPORT() && !this.hasDBTransport()) {
+    const usingDBTransport = this.hasDBTransport()
+    if (customDBTransport !== null && USE_DB_TRANSPORT() && !usingDBTransport) {
       this.addTransport(customDBTransport)
-    } else if (this.hasDBTransport() && !USE_DB_TRANSPORT()) {
-      this.logMessage('Removing DB transport from Logger: ' + this.getModuleName())
+    } else if (usingDBTransport && !USE_DB_TRANSPORT()) {
       this.removeTransport(this.getDBTransport())
     }
 

From 2c1a2537c425c8caa0471e741547063d279a02d0 Mon Sep 17 00:00:00 2001
From: paulo-ocean <paulo@oceanprotocol.com>
Date: Thu, 25 Jul 2024 16:12:35 +0100
Subject: [PATCH 50/50] remove console.log

---
 src/test/integration/logs.test.ts | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/test/integration/logs.test.ts b/src/test/integration/logs.test.ts
index 2f26ad2d7..577b784fe 100644
--- a/src/test/integration/logs.test.ts
+++ b/src/test/integration/logs.test.ts
@@ -86,7 +86,6 @@ describe('LogDatabase CRUD', () => {
 
     // Retrieve the latest log entries
     let logs = await database.logs.retrieveMultipleLogs(startTime, endTime, 100)
-    console.log('got 100 logs?:', logs)
     logs = logs.filter((log) => log.message === newLogEntry.message)
 
     expect(logs?.length).to.equal(1)