Skip to content

Commit

Permalink
Modify get entities query to avoid unnecessary join (#872)
Browse files Browse the repository at this point in the history
  • Loading branch information
@agusaldasoro
agusaldasoro authored Feb 1, 2022
1 parent 67fd548 commit 9343911
Show file tree
Hide file tree
Showing 11 changed files with 111 additions and 528 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,3 +12,4 @@ tmpbin
linked-peer-package
**/tsconfig.tsbuildinfo
storage_*/**
serverStorage.json
11 changes: 4 additions & 7 deletions content/src/controller/Controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,7 +268,7 @@ export class Controller {
// don't replace this until the denylist is implemented outside of the service
const { deployments } = await this.components.deployer.getDeployments({
fields: [DeploymentField.AUDIT_INFO],
filters: { entityIds: [entityId], entityTypes: [type] }
filters: { entityIds: [entityId], entityTypes: [type], includeOverwrittenInfo: true }
})

if (deployments.length > 0) {
Expand Down Expand Up @@ -355,7 +355,8 @@ export class Controller {
entityTypes: entityTypes as EntityType[] | undefined,
from: fromFilter,
to: toFilter,
includeAuthChain
includeAuthChain,
includeOverwrittenInfo: includeAuthChain
}

const {
Expand Down Expand Up @@ -418,17 +419,14 @@ export class Controller {
async getDeployments(req: express.Request, res: express.Response) {
// Method: GET
// Path: /deployments
// Query String: ?from={timestamp}&toLocalTimestamp={timestamp}&entityType={entityType}&entityId={entityId}&onlyCurrentlyPointed={boolean}&deployedBy={ethAddress}
// Query String: ?from={timestamp}&toLocalTimestamp={timestamp}&entityType={entityType}&entityId={entityId}&onlyCurrentlyPointed={boolean}

const stringEntityTypes = this.asArray<string>(req.query.entityType as string | string[])
const entityTypes: (EntityType | undefined)[] | undefined = stringEntityTypes
? stringEntityTypes.map((type) => this.parseEntityType(type))
: undefined
const entityIds: EntityId[] | undefined = this.asArray<EntityId>(req.query.entityId)
const onlyCurrentlyPointed: boolean | undefined = this.asBoolean(req.query.onlyCurrentlyPointed)
const deployedBy: EthAddress[] | undefined = this.asArray<EthAddress>(req.query.deployedBy)?.map((p) =>
p.toLowerCase()
)
const pointers: Pointer[] | undefined = this.asArray<Pointer>(req.query.pointer)?.map((p) => p.toLowerCase())
const offset: number | undefined = this.asInt(req.query.offset)
const limit: number | undefined = this.asInt(req.query.limit)
Expand Down Expand Up @@ -508,7 +506,6 @@ export class Controller {
pointers,
entityTypes: entityTypes as EntityType[],
entityIds,
deployedBy,
onlyCurrentlyPointed,
from: fromFilter,
to: toFilter
Expand Down
143 changes: 83 additions & 60 deletions content/src/logic/database-queries/deployments-queries.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,28 +81,92 @@ export function getHistoricalDeploymentsQuery(
const timestampField: string = sorting.field
const order: string = sorting.order

const query = SQL`
SELECT
dep1.id,
dep1.entity_type,
dep1.entity_id,
dep1.entity_pointers,
date_part('epoch', dep1.entity_timestamp) * 1000 AS entity_timestamp,
dep1.entity_metadata,
dep1.deployer_address,
dep1.version,
dep1.auth_chain,
date_part('epoch', dep1.local_timestamp) * 1000 AS local_timestamp,
dep2.entity_id AS overwritten_by
FROM deployments AS dep1
LEFT JOIN deployments AS dep2 ON dep1.deleter_deployment = dep2.id`
// Generate the select according the info needed
let query: SQLStatement
if (!!filters?.includeOverwrittenInfo) {
query = SQL`
SELECT
dep1.id,
dep1.entity_type,
dep1.entity_id,
dep1.entity_pointers,
date_part('epoch', dep1.entity_timestamp) * 1000 AS entity_timestamp,
dep1.entity_metadata,
dep1.deployer_address,
dep1.version,
dep1.auth_chain,
date_part('epoch', dep1.local_timestamp) * 1000 AS local_timestamp,
dep2.entity_id AS overwritten_by
FROM deployments AS dep1
LEFT JOIN deployments AS dep2 ON dep1.deleter_deployment = dep2.id`
} else {
query = SQL`
SELECT
dep1.id,
dep1.entity_type,
dep1.entity_id,
dep1.entity_pointers,
date_part('epoch', dep1.entity_timestamp) * 1000 AS entity_timestamp,
dep1.entity_metadata,
dep1.deployer_address,
dep1.version,
dep1.auth_chain,
date_part('epoch', dep1.local_timestamp) * 1000 AS local_timestamp
FROM deployments AS dep1`
}

const whereClause: SQLStatement[] = []
// Configure sort and order
configureSortWhereClause(order, timestampField, filters, lastId, whereClause)

if (filters?.entityTypes && filters.entityTypes.length > 0) {
const entityTypes = filters.entityTypes
whereClause.push(SQL`dep1.entity_type = ANY (${entityTypes})`)
}

if (filters?.entityIds && filters.entityIds.length > 0) {
const entityIds = filters.entityIds
whereClause.push(SQL`dep1.entity_id = ANY (${entityIds})`)
}

if (filters?.onlyCurrentlyPointed) {
whereClause.push(SQL`dep1.deleter_deployment IS NULL`)
}

if (filters?.pointers && filters.pointers.length > 0) {
const pointers = filters.pointers.map((p) => p.toLowerCase())
whereClause.push(SQL`dep1.entity_pointers && ${pointers}`)
}

/** The lastId is a field that we only want to compare with when paginating.
* If the filter specifies a timestamp value that it's repeated among many deployments,
* then to know where the page should start we will use the lastId.
*/
let where = SQL``
if (whereClause.length > 0) {
where = SQL` WHERE `.append(whereClause[0])
for (const condition of whereClause.slice(1)) {
where = where.append(' AND ').append(condition)
}
}

query.append(where)
query
.append(` ORDER BY dep1.`)
.append(pg.Client.prototype.escapeIdentifier(timestampField))
.append(` ${order}, LOWER(dep1.entity_id) ${order} `) // raw values need to be strings not sql templates
.append(SQL`LIMIT ${limit} OFFSET ${offset}`)

return query
}

/** The lastId is a field that we only want to compare with when paginating.
* If the filter specifies a timestamp value that it's repeated among many deployments,
* then to know where the page should start we will use the lastId.
*/
function configureSortWhereClause(
order: string,
timestampField: string,
filters: DeploymentFilters | undefined,
lastId: string | undefined,
whereClause: SQLStatement[]
) {
const pageBorder: string =
(order === SortingOrder.ASCENDING ? 'from' : 'to') +
(timestampField === SortingField.ENTITY_TIMESTAMP ? 'EntityTimestamp' : 'LocalTimestamp')
Expand Down Expand Up @@ -140,47 +204,6 @@ export function getHistoricalDeploymentsQuery(
whereClause.push(SQL`dep1.entity_timestamp <= to_timestamp(${toEntityTimestamp} / 1000.0)`)
}
}

if (filters?.deployedBy && filters.deployedBy.length > 0) {
const deployedBy = filters.deployedBy.map((deployedBy) => deployedBy.toLocaleLowerCase())
whereClause.push(SQL`LOWER(dep1.deployer_address) = ANY (${deployedBy})`)
}

if (filters?.entityTypes && filters.entityTypes.length > 0) {
const entityTypes = filters.entityTypes
whereClause.push(SQL`dep1.entity_type = ANY (${entityTypes})`)
}

if (filters?.entityIds && filters.entityIds.length > 0) {
const entityIds = filters.entityIds
whereClause.push(SQL`dep1.entity_id = ANY (${entityIds})`)
}

if (filters?.onlyCurrentlyPointed) {
whereClause.push(SQL`dep1.deleter_deployment IS NULL`)
}

if (filters?.pointers && filters.pointers.length > 0) {
const pointers = filters.pointers.map((p) => p.toLowerCase())
whereClause.push(SQL`dep1.entity_pointers && ${pointers}`)
}

let where = SQL``
if (whereClause.length > 0) {
where = SQL` WHERE `.append(whereClause[0])
for (const condition of whereClause.slice(1)) {
where = where.append(' AND ').append(condition)
}
}

query.append(where)
query
.append(` ORDER BY dep1.`)
.append(pg.Client.prototype.escapeIdentifier(timestampField))
.append(` ${order}, LOWER(dep1.entity_id) ${order} `) // raw values need to be strings not sql templates
.append(SQL`LIMIT ${limit} OFFSET ${offset}`)

return query
}

export async function getHistoricalDeployments(
Expand Down
151 changes: 1 addition & 150 deletions content/src/repository/extensions/DeploymentsRepository.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,4 @@
import {
AuditInfo,
DeploymentFilters,
DeploymentSorting,
Entity,
EntityId,
EntityType,
Pointer,
SortingField,
SortingOrder,
Timestamp
} from 'dcl-catalyst-commons'
import { AuditInfo, Entity, EntityId, EntityType, Pointer, Timestamp } from 'dcl-catalyst-commons'
import { AuthChain, Authenticator } from 'dcl-crypto'
import { Database } from '../../repository/Database'

Expand Down Expand Up @@ -52,144 +41,6 @@ export class DeploymentsRepository {
return new Map(entries)
}

getHistoricalDeployments(
offset: number,
limit: number,
filters?: DeploymentFilters,
sortBy?: DeploymentSorting,
lastId?: string
) {
const sorting = Object.assign({ field: SortingField.LOCAL_TIMESTAMP, order: SortingOrder.DESCENDING }, sortBy)
return this.getDeploymentsBy(sorting.field, sorting.order, offset, limit, filters, lastId)
}

private getDeploymentsBy(
timestampField: string,
order: string,
offset: number,
limit: number,
filters?: DeploymentFilters,
lastId?: string
) {
let query = `
SELECT
dep1.id,
dep1.entity_type,
dep1.entity_id,
dep1.entity_pointers,
date_part('epoch', dep1.entity_timestamp) * 1000 AS entity_timestamp,
dep1.entity_metadata,
dep1.deployer_address,
dep1.version,
dep1.auth_chain,
date_part('epoch', dep1.local_timestamp) * 1000 AS local_timestamp,
dep2.entity_id AS overwritten_by
FROM deployments AS dep1
LEFT JOIN deployments AS dep2 ON dep1.deleter_deployment = dep2.id`

const whereClause: string[] = []

const values: any = {
limit,
offset
}

if (lastId) {
values.lastId = lastId
}

/** The lastId is a field that we only want to compare with when paginating.
* If the filter specifies a timestamp value that it's repeated among many deployments,
* then to know where the page should start we will use the lastId.
*/
const pageBorder: string =
(order === SortingOrder.ASCENDING ? 'from' : 'to') +
(timestampField === SortingField.ENTITY_TIMESTAMP ? 'EntityTimestamp' : 'LocalTimestamp')

if (filters?.from && timestampField == SortingField.LOCAL_TIMESTAMP) {
values.fromLocalTimestamp = filters.from
if (pageBorder == 'fromLocalTimestamp' && lastId) {
whereClause.push(this.createOrClause('local_timestamp', '>', 'fromLocalTimestamp'))
} else {
whereClause.push(`dep1.local_timestamp >= to_timestamp($(fromLocalTimestamp) / 1000.0)`)
}
}
if (filters?.to && timestampField == SortingField.LOCAL_TIMESTAMP) {
values.toLocalTimestamp = filters.to
if (pageBorder == 'toLocalTimestamp' && lastId) {
whereClause.push(this.createOrClause('local_timestamp', '<', 'toLocalTimestamp'))
} else {
whereClause.push(`dep1.local_timestamp <= to_timestamp($(toLocalTimestamp) / 1000.0)`)
}
}

if (filters?.from && timestampField == SortingField.ENTITY_TIMESTAMP) {
values.fromEntityTimestamp = filters.from
if (pageBorder == 'fromEntityTimestamp' && lastId) {
whereClause.push(this.createOrClause('entity_timestamp', '>', 'fromEntityTimestamp'))
} else {
whereClause.push(`dep1.entity_timestamp >= to_timestamp($(fromEntityTimestamp) / 1000.0)`)
}
}
if (filters?.to && timestampField == SortingField.ENTITY_TIMESTAMP) {
values.toEntityTimestamp = filters.to
if (pageBorder == 'toEntityTimestamp' && lastId) {
whereClause.push(this.createOrClause('entity_timestamp', '<', 'toEntityTimestamp'))
} else {
whereClause.push(`dep1.entity_timestamp <= to_timestamp($(toEntityTimestamp) / 1000.0)`)
}
}

if (filters?.deployedBy && filters.deployedBy.length > 0) {
values.deployedBy = filters.deployedBy.map((deployedBy) => deployedBy.toLocaleLowerCase())
whereClause.push(`LOWER(dep1.deployer_address) IN ($(deployedBy:list))`)
}

if (filters?.entityTypes && filters.entityTypes.length > 0) {
values.entityTypes = filters.entityTypes
whereClause.push(`dep1.entity_type IN ($(entityTypes:list))`)
}

if (filters?.entityIds && filters.entityIds.length > 0) {
values.entityIds = filters.entityIds
whereClause.push(`dep1.entity_id IN ($(entityIds:list))`)
}

if (filters?.onlyCurrentlyPointed) {
whereClause.push(`dep1.deleter_deployment IS NULL`)
}

if (filters?.pointers && filters.pointers.length > 0) {
values.pointers = filters.pointers.map((p) => p.toLowerCase())
whereClause.push(`dep1.entity_pointers && ARRAY[$(pointers:list)]`)
}

const where = whereClause.length > 0 ? ' WHERE ' + whereClause.join(' AND ') : ''

query += where
query += ` ORDER BY dep1.${timestampField} ${order}, LOWER(dep1.entity_id) ${order} LIMIT $(limit) OFFSET $(offset)`

return this.db.map(query, values, (row) => ({
deploymentId: row.id,
entityType: row.entity_type,
entityId: row.entity_id,
pointers: row.entity_pointers,
entityTimestamp: row.entity_timestamp,
metadata: row.entity_metadata ? row.entity_metadata.v : undefined,
deployerAddress: row.deployer_address,
version: row.version,
authChain: row.auth_chain,
localTimestamp: row.local_timestamp,
overwrittenBy: row.overwritten_by ?? undefined
}))
}

private createOrClause(timestampField: string, compare: string, timestampFilter: string): string {
const equalWithEntityIdComparison = `(LOWER(dep1.entity_id) ${compare} LOWER($(lastId)) AND dep1.${timestampField} = to_timestamp($(${timestampFilter}) / 1000.0))`
const timestampComparison = `(dep1.${timestampField} ${compare} to_timestamp($(${timestampFilter}) / 1000.0))`
return `(${equalWithEntityIdComparison} OR ${timestampComparison})`
}

deploymentsSince(entityType: EntityType, timestamp: Timestamp): Promise<number> {
return this.db.one(
`SELECT COUNT(*) AS count ` +
Expand Down
4 changes: 3 additions & 1 deletion content/src/service/ServiceImpl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,9 @@ export class ServiceImpl implements MetaverseContentService {
}

async getEntitiesByIds(ids: EntityId[]): Promise<Entity[]> {
const deployments = await getDeployments(this.components, { filters: { entityIds: ids } })
const deployments = await getDeployments(this.components, {
filters: { entityIds: ids, onlyCurrentlyPointed: true }
})
return this.mapDeploymentsToEntities(deployments)
}

Expand Down
Loading

0 comments on commit 9343911

Please sign in to comment.