From 5505d354ecae6e52c751b3b634752fd56d24642f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rafael=20C=C3=A1rdenas?= Date: Fri, 12 Jan 2024 08:04:56 -0600 Subject: [PATCH] fix: optimize re-org queries and indexes (#1821) --- .../1705013096459_update-re-org-indexes.js | 91 +++++++++++++++++++ src/datastore/pg-write-store.ts | 60 ++++-------- src/tests/datastore-tests.ts | 2 +- src/tests/other-tests.ts | 2 +- src/tests/search-tests.ts | 4 +- 5 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 migrations/1705013096459_update-re-org-indexes.js diff --git a/migrations/1705013096459_update-re-org-indexes.js b/migrations/1705013096459_update-re-org-indexes.js new file mode 100644 index 0000000000..64d8ec649d --- /dev/null +++ b/migrations/1705013096459_update-re-org-indexes.js @@ -0,0 +1,91 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = pgm => { + pgm.dropIndex('txs', 'index_block_hash'); + pgm.createIndex('txs', ['index_block_hash', 'canonical']); + + pgm.dropIndex('miner_rewards', 'index_block_hash'); + pgm.createIndex('miner_rewards', ['index_block_hash', 'canonical']); + + pgm.dropIndex('stx_lock_events', 'index_block_hash'); + pgm.createIndex('stx_lock_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('stx_events', 'index_block_hash'); + pgm.createIndex('stx_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('ft_events', 'index_block_hash'); + pgm.createIndex('ft_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('nft_events', 'index_block_hash'); + pgm.createIndex('nft_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('pox2_events', 'index_block_hash'); + pgm.createIndex('pox2_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('pox3_events', 'index_block_hash'); + pgm.createIndex('pox3_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('pox4_events', 'index_block_hash'); + pgm.createIndex('pox4_events', ['index_block_hash', 'canonical']); + + pgm.dropIndex('contract_logs', 'index_block_hash'); + pgm.createIndex('contract_logs', ['index_block_hash', 'canonical']); + + pgm.dropIndex('smart_contracts', 'index_block_hash'); + pgm.createIndex('smart_contracts', ['index_block_hash', 'canonical']); + + pgm.dropIndex('names', 'index_block_hash'); + pgm.createIndex('names', ['index_block_hash', 'canonical']); + + pgm.dropIndex('namespaces', 'index_block_hash'); + pgm.createIndex('namespaces', ['index_block_hash', 'canonical']); + + pgm.dropIndex('subdomains', 'index_block_hash'); + pgm.createIndex('subdomains', ['index_block_hash', 'canonical']); +}; + +exports.down = pgm => { + pgm.dropIndex('txs', ['index_block_hash', 'canonical']); + pgm.createIndex('txs', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('miner_rewards', ['index_block_hash', 'canonical']); + pgm.createIndex('miner_rewards', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('stx_lock_events', ['index_block_hash', 'canonical']); + pgm.createIndex('stx_lock_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('stx_events', ['index_block_hash', 'canonical']); + pgm.createIndex('stx_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('ft_events', ['index_block_hash', 'canonical']); + pgm.createIndex('ft_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('nft_events', ['index_block_hash', 'canonical']); + pgm.createIndex('nft_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('pox2_events', ['index_block_hash', 'canonical']); + pgm.createIndex('pox2_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('pox3_events', ['index_block_hash', 'canonical']); + pgm.createIndex('pox3_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('pox4_events', ['index_block_hash', 'canonical']); + pgm.createIndex('pox4_events', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('contract_logs', ['index_block_hash', 'canonical']); + pgm.createIndex('contract_logs', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('smart_contracts', ['index_block_hash', 'canonical']); + pgm.createIndex('smart_contracts', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('names', ['index_block_hash', 'canonical']); + pgm.createIndex('names', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('namespaces', ['index_block_hash', 'canonical']); + pgm.createIndex('namespaces', 'index_block_hash', { method: 'hash' }); + + pgm.dropIndex('subdomains', ['index_block_hash', 'canonical']); + pgm.createIndex('subdomains', 'index_block_hash', { method: 'hash' }); +}; diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index bee49b519d..0de9723d16 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -1134,29 +1134,6 @@ export class PgWriteStore extends PgStore { }); } - async updateStxEvent(sql: PgSqlClient, tx: DbTx, event: DbStxEvent) { - const values: StxEventInsertValues = { - event_index: event.event_index, - tx_id: event.tx_id, - tx_index: event.tx_index, - block_height: event.block_height, - index_block_hash: tx.index_block_hash, - parent_index_block_hash: tx.parent_index_block_hash, - microblock_hash: tx.microblock_hash, - microblock_sequence: tx.microblock_sequence, - microblock_canonical: tx.microblock_canonical, - canonical: event.canonical, - asset_event_type_id: event.asset_event_type_id, - sender: event.sender ?? null, - recipient: event.recipient ?? null, - amount: event.amount, - memo: event.memo ?? null, - }; - await sql` - INSERT INTO stx_events ${sql(values)} - `; - } - async updateFtEvents(sql: PgSqlClient, entries: { tx: DbTx; ftEvents: DbFtEvent[] }[]) { const values: FtEventInsertValues[] = []; for (const { tx, ftEvents } of entries) { @@ -1438,8 +1415,10 @@ export class PgWriteStore extends PgStore { acceptedMicroblocks: string[]; orphanedMicroblocks: string[]; }> { - // Find the parent microblock if this anchor block points to one. If not, perform a sanity check for expected block headers in this case: - // > Anchored blocks that do not have parent microblock streams will have their parent microblock header hashes set to all 0's, and the parent microblock sequence number set to 0. + // Find the parent microblock if this anchor block points to one. If not, perform a sanity check + // for expected block headers in this case: Anchored blocks that do not have parent microblock + // streams will have their parent microblock header hashes set to all 0's, and the parent + // microblock sequence number set to 0. let acceptedMicroblockTip: DbMicroblock | undefined; if (BigInt(blockData.parentMicroblockHash) === 0n) { if (blockData.parentMicroblockSequence !== 0) { @@ -2521,29 +2500,25 @@ export class PgWriteStore extends PgStore { canonical: boolean, updatedEntities: ReOrgUpdatedEntities ): Promise<{ txsMarkedCanonical: string[]; txsMarkedNonCanonical: string[] }> { - const txResult = await sql` + const txResult = await sql<{ tx_id: string }[]>` UPDATE txs SET canonical = ${canonical} WHERE index_block_hash = ${indexBlockHash} AND canonical != ${canonical} - RETURNING ${sql(TX_COLUMNS)} + RETURNING tx_id `; - const txIds = txResult.map(row => parseTxQueryResult(row)); + const txIds = txResult.map(row => row.tx_id); if (canonical) { - updatedEntities.markedCanonical.txs += txResult.length; + updatedEntities.markedCanonical.txs += txResult.count; } else { - updatedEntities.markedNonCanonical.txs += txResult.length; + updatedEntities.markedNonCanonical.txs += txResult.count; } - for (const txId of txIds) { - logger.debug(`Marked tx as ${canonical ? 'canonical' : 'non-canonical'}: ${txId.tx_id}`); - } - if (txIds.length) { + if (txResult.count) await sql` UPDATE principal_stx_txs SET canonical = ${canonical} - WHERE tx_id IN ${sql(txIds.map(tx => tx.tx_id))} + WHERE tx_id IN ${sql(txIds)} AND index_block_hash = ${indexBlockHash} AND canonical != ${canonical} `; - } const minerRewardResults = await sql` UPDATE miner_rewards @@ -2599,10 +2574,11 @@ export class PgWriteStore extends PgStore { } else { updatedEntities.markedNonCanonical.nftEvents += nftResult.count; } - await this.updateNftCustodyFromReOrg(sql, { - index_block_hash: indexBlockHash, - microblocks: [], - }); + if (nftResult.count) + await this.updateNftCustodyFromReOrg(sql, { + index_block_hash: indexBlockHash, + microblocks: [], + }); const pox2Result = await sql` UPDATE pox2_events @@ -2693,8 +2669,8 @@ export class PgWriteStore extends PgStore { } return { - txsMarkedCanonical: canonical ? txIds.map(t => t.tx_id) : [], - txsMarkedNonCanonical: canonical ? [] : txIds.map(t => t.tx_id), + txsMarkedCanonical: canonical ? txIds : [], + txsMarkedNonCanonical: canonical ? [] : txIds, }; } diff --git a/src/tests/datastore-tests.ts b/src/tests/datastore-tests.ts index 0458162c43..9faac56268 100644 --- a/src/tests/datastore-tests.ts +++ b/src/tests/datastore-tests.ts @@ -167,7 +167,7 @@ describe('postgres datastore', () => { createStxEvent('addrA', 'addrC', 35), ]; for (const event of events) { - await db.updateStxEvent(client, tx, event); + await db.updateStxEvents(client, [{ tx, stxEvents: [event] }]); } const createStxLockEvent = ( diff --git a/src/tests/other-tests.ts b/src/tests/other-tests.ts index d64bf877de..19489d4d47 100644 --- a/src/tests/other-tests.ts +++ b/src/tests/other-tests.ts @@ -157,7 +157,7 @@ describe('other tests', () => { event_type: DbEventTypeId.StxAsset, amount: 10_000_000_000_000n, }; - await db.updateStxEvent(client, tx, stxBurnEvent1); + await db.updateStxEvents(client, [{ tx, stxEvents: [stxBurnEvent1] }]); const expectedTotalStx2 = stxMintEvent1.amount + stxMintEvent2.amount - stxBurnEvent1.amount; const result2 = await supertest(api.server).get(`/extended/v1/stx_supply`); expect(result2.status).toBe(200); diff --git a/src/tests/search-tests.ts b/src/tests/search-tests.ts index c7806dfff3..ada372a422 100644 --- a/src/tests/search-tests.ts +++ b/src/tests/search-tests.ts @@ -717,7 +717,7 @@ describe('search tests', () => { recipient: addr3, sender: 'none', }; - await db.updateStxEvent(client, stxTx1, stxEvent1); + await db.updateStxEvents(client, [{ tx: stxTx1, stxEvents: [stxEvent1] }]); // test address as a stx event recipient const searchResult3 = await supertest(api.server).get(`/extended/v1/search/${addr3}`); @@ -745,7 +745,7 @@ describe('search tests', () => { sender: addr4, }; - await db.updateStxEvent(client, stxTx1, stxEvent2); + await db.updateStxEvents(client, [{ tx: stxTx1, stxEvents: [stxEvent2] }]); // test address as a stx event sender const searchResult4 = await supertest(api.server).get(`/extended/v1/search/${addr4}`);