Skip to content

Commit

Permalink
Merge pull request #723 from oceanprotocol/issue-721-nonce-sqllite
Browse files Browse the repository at this point in the history 
nonce db sql lite
  • Loading branch information
@paulo-ocean
paulo-ocean authored Oct 21, 2024
2 parents 824b608 + 0365aa7 commit 8a65610
Show file tree
Hide file tree
Showing 10 changed files with 183 additions and 303 deletions.
8 changes: 4 additions & 4 deletions src/components/core/utils/nonceHandler.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ export async function getNonce(
): Promise<P2PCommandResponse> {
// get nonce from db
try {
const nonce = await db.retrieve(address)
if (nonce !== null) {
return getDefaultResponse(nonce.nonce)
const nonceResponse = await db.retrieve(address)
if (nonceResponse && nonceResponse.nonce !== null) {
return getDefaultResponse(nonceResponse.nonce)
}
// // did not found anything, try add it and return default
const setFirst = await db.create(address, 0)
Expand Down Expand Up @@ -107,7 +107,7 @@ export async function checkNonce(
// get nonce from db
let previousNonce = 0 // if none exists
const existingNonce = await db.retrieve(consumer)
if (existingNonce !== null) {
if (existingNonce && existingNonce.nonce !== null) {
previousNonce = existingNonce.nonce
}
// check if bigger than previous stored one and validate signature
Expand Down
8 changes: 4 additions & 4 deletions src/components/database/DatabaseFactory.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import {
ElasticsearchDdoStateDatabase,
ElasticsearchIndexerDatabase,
ElasticsearchLogDatabase,
ElasticsearchNonceDatabase,
ElasticsearchOrderDatabase
} from './ElasticSearchDatabase.js'
import { typesenseSchemas } from './TypesenseSchemas.js'
Expand All @@ -21,7 +20,6 @@ import {
TypesenseDdoStateDatabase,
TypesenseIndexerDatabase,
TypesenseLogDatabase,
TypesenseNonceDatabase,
TypesenseOrderDatabase
} from './TypenseDatabase.js'
import { elasticSchemas } from './ElasticSchemas.js'
Expand All @@ -32,11 +30,13 @@ import { TypesenseMetadataQuery } from './TypesenseMetadataQuery.js'
import { IMetadataQuery } from '../../@types/DDO/IMetadataQuery.js'
import { ElasticSearchMetadataQuery } from './ElasticSearchMetadataQuery.js'
import { DB_TYPES } from '../../utils/index.js'
import { SQLLiteNonceDatabase } from './SQLLiteNonceDatabase.js'

export class DatabaseFactory {
private static databaseMap = {
elasticsearch: {
nonce: (config: OceanNodeDBConfig) => new ElasticsearchNonceDatabase(config),
nonce: (config: OceanNodeDBConfig) =>
new SQLLiteNonceDatabase(config, typesenseSchemas.nonceSchemas),
ddo: (config: OceanNodeDBConfig) =>
new ElasticsearchDdoDatabase(config, elasticSchemas.ddoSchemas),
indexer: (config: OceanNodeDBConfig) => new ElasticsearchIndexerDatabase(config),
Expand All @@ -49,7 +49,7 @@ export class DatabaseFactory {
},
typesense: {
nonce: (config: OceanNodeDBConfig) =>
new TypesenseNonceDatabase(config, typesenseSchemas.nonceSchemas),
new SQLLiteNonceDatabase(config, typesenseSchemas.nonceSchemas),
ddo: (config: OceanNodeDBConfig) =>
new TypesenseDdoDatabase(config, typesenseSchemas.ddoSchemas),
indexer: (config: OceanNodeDBConfig) =>
Expand Down
177 changes: 40 additions & 137 deletions src/components/database/ElasticSearchDatabase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import {
AbstractDdoStateDatabase,
AbstractIndexerDatabase,
AbstractLogDatabase,
AbstractNonceDatabase,
AbstractOrderDatabase
} from './BaseDatabase.js'
import { createElasticsearchClient } from './ElasticsearchConfigHelper.js'
Expand All @@ -14,132 +13,6 @@ import { DATABASE_LOGGER } from '../../utils/logging/common.js'
import { GENERIC_EMOJIS, LOG_LEVELS_STR } from '../../utils/logging/Logger.js'
import { validateObject } from '../core/utils/validateDdoHandler.js'

export class ElasticsearchNonceDatabase extends AbstractNonceDatabase {
private client: Client
private index: string

constructor(config: OceanNodeDBConfig) {
super(config)
this.client = new Client({ node: config.url })
this.index = 'nonce'
this.initializeIndex()
}

private async initializeIndex() {
try {
const indexExists = await this.client.indices.exists({ index: this.index })
if (!indexExists) {
await this.client.indices.create({
index: this.index,
body: {
mappings: {
properties: {
id: { type: 'keyword' },
nonce: { type: 'integer' }
}
}
}
})
}
} catch (e) {
DATABASE_LOGGER.error(e.message)
}
}

async create(address: string, nonce: number) {
try {
await this.client.index({
index: this.index,
id: address,
body: { nonce },
refresh: 'wait_for'
})
return { id: address, nonce }
} catch (error) {
const errorMsg = `Error when creating new nonce entry ${nonce} for address ${address}: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
errorMsg,
true,
GENERIC_EMOJIS.EMOJI_CROSS_MARK,
LOG_LEVELS_STR.LEVEL_ERROR
)
return null
}
}

async retrieve(address: string) {
try {
const result = await this.client.get({
index: this.index,
id: address
})
return result._source
} catch (error) {
const errorMsg = `Error when retrieving nonce entry for address ${address}: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
errorMsg,
true,
GENERIC_EMOJIS.EMOJI_CROSS_MARK,
LOG_LEVELS_STR.LEVEL_ERROR
)
return null
}
}

async update(address: string, nonce: number) {
try {
const exists = await this.client.exists({
index: this.index,
id: address
})

if (exists) {
await this.client.update({
index: this.index,
id: address,
body: {
doc: { nonce }
},
refresh: 'wait_for'
})
} else {
await this.create(address, nonce)
}

return { id: address, nonce }
} catch (error) {
const errorMsg = `Error when updating nonce entry ${nonce} for address ${address}: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
errorMsg,
true,
GENERIC_EMOJIS.EMOJI_CROSS_MARK,
LOG_LEVELS_STR.LEVEL_ERROR
)
return null
}
}

async delete(address: string) {
try {
await this.client.delete({
index: this.index,
id: address,
refresh: 'wait_for'
})
return { id: address }
} catch (error) {
const errorMsg = `Error when deleting nonce entry for address ${address}: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
errorMsg,
true,
GENERIC_EMOJIS.EMOJI_CROSS_MARK,
LOG_LEVELS_STR.LEVEL_ERROR
)
return null
}
}
}

export class ElasticsearchIndexerDatabase extends AbstractIndexerDatabase {
private client: Client
private index: string
Expand Down Expand Up @@ -361,7 +234,9 @@ export class ElasticsearchDdoStateDatabase extends AbstractDdoStateDatabase {
}
}
})
return result.hits.hits.map((hit: any) => hit._source)
return result.hits.hits.map((hit: any) => {
return normalizeDocumentId(hit._source, hit._id)
})
} catch (error) {
const errorMsg = `Error when searching by query ${JSON.stringify(query)}: ${
error.message
Expand Down Expand Up @@ -467,7 +342,9 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase {
}
}
const result = await this.provider.search(searchParams)
return result.hits.hits.map((hit: any) => hit._source)
return result.hits.hits.map((hit: any) => {
return normalizeDocumentId(hit._source, hit._id)
})
} catch (error) {
const errorMsg =
`Error when searching order entry by query ${JSON.stringify(query)}: ` +
Expand Down Expand Up @@ -521,7 +398,7 @@ export class ElasticsearchOrderDatabase extends AbstractOrderDatabase {
index: this.getSchema().index,
id: orderId
})
return result._source
return normalizeDocumentId(result._source, result._id)
} catch (error) {
const errorMsg = `Error when retrieving order ${orderId}: ` + error.message
DATABASE_LOGGER.logMessageWithEmoji(errorMsg, true, LOG_LEVELS_STR.LEVEL_ERROR)
Expand Down Expand Up @@ -652,7 +529,10 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase {
}
})
if (response.hits?.hits.length > 0) {
results.push(response)
const nomalizedResponse = response.hits.hits.map((hit: any) => {
return normalizeDocumentId(hit._source, hit._id)
})
results.push(nomalizedResponse)
}
} catch (error) {
const schemaErrorMsg = `Error for schema ${query.index}: ${error.message}`
Expand All @@ -675,7 +555,10 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase {
}
})
if (response.hits?.hits.length > 0) {
results.push(response)
const nomalizedResponse = response.hits.hits.map((hit: any) => {
return normalizeDocumentId(hit._source, hit._id)
})
results.push(nomalizedResponse)
}
} catch (error) {
const schemaErrorMsg = `Error for schema ${schema.index}: ${error.message}`
Expand Down Expand Up @@ -779,7 +662,7 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase {
// make sure we do not have different responses 4 between DBs
// do the same thing on other methods
if (response._id === ddo.id) {
response.id = response._id
return normalizeDocumentId(response, response._id)
}
return response
} else {
Expand All @@ -792,7 +675,7 @@ export class ElasticsearchDdoDatabase extends AbstractDdoDatabase {
await this.delete(ddo.id)
const response = await this.create(ddo)
if (response._id === ddo.id) {
response.id = response._id
return normalizeDocumentId(response, response._id)
}
return response
}
Expand Down Expand Up @@ -925,7 +808,7 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase {
})
// uniformize result response (we need an id for the retrieveLog API)
if (result._id) {
logEntry.id = result._id
return normalizeDocumentId(logEntry, result._id)
}
return logEntry
} catch (error) {
Expand All @@ -946,7 +829,7 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase {
index: this.index,
id
})
return result._source
return normalizeDocumentId(result._source, result._id)
} catch (error) {
const errorMsg = `Error when retrieving log entry: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
Expand Down Expand Up @@ -1005,7 +888,13 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase {
from
})

return result.hits.hits.map((hit: any) => hit._source)
console.log('logs results:', result)
console.log('logs results hits:', result.hits)
console.log('logs results hits hits:', result.hits.hits)

return result.hits.hits.map((hit: any) => {
return normalizeDocumentId(hit._source, hit._id)
})
} catch (error) {
const errorMsg = `Error when retrieving multiple log entries: ${error.message}`
DATABASE_LOGGER.logMessageWithEmoji(
Expand Down Expand Up @@ -1085,3 +974,17 @@ export class ElasticsearchLogDatabase extends AbstractLogDatabase {
}
}
}

/**
* Make DB agnostic APIs. The response should be similar, no matter what DB engine is used
* Normalizes the document responses to match same kind of typesense ones
* @param dbResult response from DB
* @param _id id of the element
* @returns result object with id property
*/
export function normalizeDocumentId(dbResult: any, _id?: any): any {
if (_id && !dbResult.id) {
dbResult.id = _id
}
return dbResult
}
Loading

0 comments on commit 8a65610

Please sign in to comment.