From 1211db4faf2fb96a4ea2c30cbb864d48bcbc80f2 Mon Sep 17 00:00:00 2001 From: Iain Collins Date: Sat, 9 Nov 2024 02:25:06 +0000 Subject: [PATCH] Update indexes on trade db to improve query performance --- lib/db/trade-db.js | 32 +++++++++++++++++++++++++------- package.json | 2 +- scripts/optimize.js | 18 +++++++++--------- 3 files changed, 35 insertions(+), 17 deletions(-) diff --git a/lib/db/trade-db.js b/lib/db/trade-db.js index d137673..27378e7 100644 --- a/lib/db/trade-db.js +++ b/lib/db/trade-db.js @@ -41,15 +41,33 @@ function ensureTables () { function ensureIndexes () { getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_commodityName_collate ON commodities (commodityName COLLATE NOCASE)') + + // Having a compound index for name and day seems to improve query times noticeably on slower hardware + // Note: Commodity names have been normalized during ingestion so it's ok to be case sensitive + getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_commodityName_updatedAtDay ON commodities (commodityName, updatedAtDay)') + + // Case sensitive on system name and station name can still be useful for supporting searches getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stationName_collate ON commodities (stationName COLLATE NOCASE)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_systemName_collate ON commodities (systemName COLLATE NOCASE)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_fleetCarrier ON commodities (fleetCarrier)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_buyPrice ON commodities (buyPrice)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_sellPrice ON commodities (sellPrice)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_demand ON commodities (demand)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stock ON commodities (stock)') + getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_systemName_collate ON commodities (systemName COLLATE NOCASE)') + + // Market ID is a useful index to have getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_marketId ON commodities (marketId)') - getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_updatedAtDay ON commodities (updatedAtDay)') + + // Based on testing, these indexes can probably be removed without impacting performance + // 2024-11-09 Leaving these definitions in just in case I want to roll back, but will delete them if all goes well + // + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_fleetCarrier ON commodities (fleetCarrier)') + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_buyPrice ON commodities (buyPrice)') + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_sellPrice ON commodities (sellPrice)') + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_demand ON commodities (demand)') + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stock ON commodities (stock)') + // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_updatedAtDay ON commodities (updatedAtDay)') + getDatabase().exec('DROP INDEX IF EXISTS commodities_fleetCarrier') + getDatabase().exec('DROP INDEX IF EXISTS commodities_buyPrice') + getDatabase().exec('DROP INDEX IF EXISTS commodities_sellPrice') + getDatabase().exec('DROP INDEX IF EXISTS commodities_demand') + getDatabase().exec('DROP INDEX IF EXISTS commodities_stock') + getDatabase().exec('DROP INDEX IF EXISTS commodities_updatedAtDay') } module.exports = { diff --git a/package.json b/package.json index 8c4bfa6..9782bab 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ardent-collector", - "version": "1.56.0", + "version": "1.57.0", "description": "Ardent Collector saves data submitted to EDDN", "main": "index.js", "scripts": { diff --git a/scripts/optimize.js b/scripts/optimize.js index d002aae..5a7a99d 100644 --- a/scripts/optimize.js +++ b/scripts/optimize.js @@ -6,12 +6,12 @@ const { TRADE_DATA_MAX_AGE_DAYS } = require('../lib/consts') // It is superior at optimization than relying on the optimize command, but in // practice it is slow and can only be used when a database is not otherwise in // use. It is typically faster (and involves less downtime) to create a new -// backup (which will use VACCUM INTO, which is faster) and then restore from -// that backup, as the backup created in VACCUM INTO will be fully optimized. -const FULL_VACCUM = false +// backup (which will use VACUUM INTO, which is faster) and then restore from +// that backup, as the backup created in VACUUM INTO will be fully optimized. +const FULL_VACUUM = false console.time('Optimize locationsDb') -if (FULL_VACCUM === true) locationsDb.exec('VACUUM') +if (FULL_VACUUM === true) locationsDb.exec('VACUUM') locationsDb.pragma('wal_checkpoint(TRUNCATE)') locationsDb.pragma('optimize') locationsDb.pragma('analysis_limit=0') @@ -20,7 +20,7 @@ locationsDb.close() console.timeEnd('Optimize locationsDb') console.time('Optimize stationsDb') -if (FULL_VACCUM === true) stationsDb.exec('VACUUM') +if (FULL_VACUUM === true) stationsDb.exec('VACUUM') stationsDb.pragma('wal_checkpoint(TRUNCATE)') stationsDb.pragma('optimize') stationsDb.pragma('analysis_limit=0') @@ -29,12 +29,12 @@ stationsDb.close() console.timeEnd('Optimize stationsDb') console.time('Optimize tradeDb') -// Purge old trade data +// Delete old trade data // tradeDb.exec(` // DELETE FROM commodities WHERE updatedAt <= '${getISOTimestamp(`-${TRADE_DATA_MAX_AGE_DAYS}`)}' // `) -// Always vacuum the trade database after deleting old data to shrink it's size -// tradeDb.exec('VACUUM') +// The trade database should be vacuumed periodically to allow it to shrink in size as old data is deleted +if (FULL_VACUUM === true) tradeDb.exec('VACUUM') tradeDb.pragma('wal_checkpoint(TRUNCATE)') tradeDb.pragma('optimize') tradeDb.pragma('analysis_limit=0') @@ -43,7 +43,7 @@ tradeDb.close() console.timeEnd('Optimize tradeDb') console.time('Optimize systemsDb') -if (FULL_VACCUM === true) systemsDb.exec('VACUUM') +if (FULL_VACUUM === true) systemsDb.exec('VACUUM') systemsDb.pragma('wal_checkpoint(TRUNCATE)') systemsDb.pragma('optimize') systemsDb.pragma('analysis_limit=0')