Update indexes on trade db to improve query performance

lib/db/trade-db.js

@@ -41,15 +41,33 @@ function ensureTables () {
 
 function ensureIndexes () {
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_commodityName_collate ON commodities (commodityName COLLATE NOCASE)')
+
+  // Having a compound index for name and day seems to improve query times noticeably on slower hardware
+  // Note: Commodity names have been normalized during ingestion so it's ok to be case sensitive
+  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_commodityName_updatedAtDay ON commodities (commodityName, updatedAtDay)')
+
+  // Case sensitive on system name and station name can still be useful for supporting searches
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stationName_collate ON commodities (stationName COLLATE NOCASE)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_systemName_collate ON commodities (systemName COLLATE NOCASE)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_fleetCarrier ON commodities (fleetCarrier)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_buyPrice ON commodities (buyPrice)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_sellPrice ON commodities (sellPrice)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_demand ON commodities (demand)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stock ON commodities (stock)')
+  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_systemName_collate ON commodities (systemName COLLATE NOCASE)')  
+
+  // Market ID is a useful index to have
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_marketId ON commodities (marketId)')
-  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_updatedAtDay ON commodities (updatedAtDay)')
+
+  // Based on testing, these indexes can probably be removed without impacting performance
+  // 2024-11-09 Leaving these definitions in just in case I want to roll back, but will delete them if all goes well
+  //
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_fleetCarrier ON commodities (fleetCarrier)')
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_buyPrice ON commodities (buyPrice)')
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_sellPrice ON commodities (sellPrice)')
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_demand ON commodities (demand)')
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stock ON commodities (stock)')
+  // getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_updatedAtDay ON commodities (updatedAtDay)')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_fleetCarrier')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_buyPrice')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_sellPrice')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_demand')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_stock')
+  getDatabase().exec('DROP INDEX IF EXISTS commodities_updatedAtDay')
 }
 
 module.exports = {

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ardent-collector",
-  "version": "1.56.0",
+  "version": "1.57.0",
   "description": "Ardent Collector saves data submitted to EDDN",
   "main": "index.js",
   "scripts": {

scripts/optimize.js

@@ -6,12 +6,12 @@ const { TRADE_DATA_MAX_AGE_DAYS } = require('../lib/consts')
 // It is superior at optimization than relying on the optimize command, but in
 // practice it is slow and can only be used when a database is not otherwise in
 // use. It is typically faster (and involves less downtime) to create a new
-// backup (which will use VACCUM INTO, which is faster) and then restore from
-// that backup, as the backup created in VACCUM INTO will be fully optimized.
-const FULL_VACCUM = false
+// backup (which will use VACUUM INTO, which is faster) and then restore from
+// that backup, as the backup created in VACUUM INTO will be fully optimized.
+const FULL_VACUUM = false
 
 console.time('Optimize locationsDb')
-if (FULL_VACCUM === true) locationsDb.exec('VACUUM')
+if (FULL_VACUUM === true) locationsDb.exec('VACUUM')
 locationsDb.pragma('wal_checkpoint(TRUNCATE)')
 locationsDb.pragma('optimize')
 locationsDb.pragma('analysis_limit=0')
@@ -20,7 +20,7 @@ locationsDb.close()
 console.timeEnd('Optimize locationsDb')
 
 console.time('Optimize stationsDb')
-if (FULL_VACCUM === true) stationsDb.exec('VACUUM')
+if (FULL_VACUUM === true) stationsDb.exec('VACUUM')
 stationsDb.pragma('wal_checkpoint(TRUNCATE)')
 stationsDb.pragma('optimize')
 stationsDb.pragma('analysis_limit=0')
@@ -29,12 +29,12 @@ stationsDb.close()
 console.timeEnd('Optimize stationsDb')
 
 console.time('Optimize tradeDb')
-// Purge old trade data
+// Delete old trade data
 // tradeDb.exec(`
 //   DELETE FROM commodities WHERE updatedAt <= '${getISOTimestamp(`-${TRADE_DATA_MAX_AGE_DAYS}`)}'
 // `)
-// Always vacuum the trade database after deleting old data to shrink it's size
-// tradeDb.exec('VACUUM')
+// The trade database should be vacuumed periodically to allow it to shrink in size as old data is deleted
+if (FULL_VACUUM === true) tradeDb.exec('VACUUM')
 tradeDb.pragma('wal_checkpoint(TRUNCATE)')
 tradeDb.pragma('optimize')
 tradeDb.pragma('analysis_limit=0')
@@ -43,7 +43,7 @@ tradeDb.close()
 console.timeEnd('Optimize tradeDb')
 
 console.time('Optimize systemsDb')
-if (FULL_VACCUM === true) systemsDb.exec('VACUUM')
+if (FULL_VACUUM === true) systemsDb.exec('VACUUM')
 systemsDb.pragma('wal_checkpoint(TRUNCATE)')
 systemsDb.pragma('optimize')
 systemsDb.pragma('analysis_limit=0')