Add indexed date column to improve commodity query performance

lib/consts.js

@@ -31,6 +31,8 @@ const ARDENT_TRADE_DB = path.join(ARDENT_DATA_DIR, '/trade.db')
 const SYSTEM_GRID_SIZE = 100 // In light years
 const SYSTEM_SECTOR_HASH_LENGTH = 8 // Enough to minimise sector ID collisions
 
+const TRADE_DATA_MAX_AGE_DAYS = 365
+
 module.exports = {
   EDDN_SERVER,
   ARDENT_COLLECTOR_LOCAL_PORT,
@@ -45,5 +47,6 @@ module.exports = {
   ARDENT_STATIONS_DB,
   ARDENT_TRADE_DB,
   SYSTEM_GRID_SIZE,
-  SYSTEM_SECTOR_HASH_LENGTH
+  SYSTEM_SECTOR_HASH_LENGTH,
+  TRADE_DATA_MAX_AGE_DAYS
 }

lib/db/trade-db.js

@@ -33,7 +33,8 @@ function ensureTables () {
       stock INT,
       stockBracket INT,
       statusFlags TEXT,
-      updatedAt TEXT
+      updatedAt TEXT,
+      updatedAtDay TEXT
     )
   `)
 }
@@ -48,6 +49,7 @@ function ensureIndexes () {
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_demand ON commodities (demand)')
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_stock ON commodities (stock)')
   getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_marketId ON commodities (marketId)')
+  getDatabase().exec('CREATE INDEX IF NOT EXISTS commodities_updatedAtDay ON commodities (updatedAtDay)')
 }
 
 module.exports = {

lib/event-handlers/commodity-event.js

@@ -46,7 +46,8 @@ const insertOrReplaceCommodity = tradeDb.prepare(`
     stock,
     stockBracket,
     statusFlags,
-    updatedAt
+    updatedAt,
+    updatedAtDay
   ) VALUES (
     @commodityId,
     @commodityName,
@@ -65,7 +66,8 @@ const insertOrReplaceCommodity = tradeDb.prepare(`
     @stock,
     @stockBracket,
     @statusFlags,
-    @updatedAt
+    @updatedAt,
+    @updatedAtDay
   )
 `)
 
@@ -75,6 +77,7 @@ module.exports = (payload) => {
   const isFleetCarrier = stationName.match('^[A-Z0-9]{3}-[A-Z0-9]{3}$')
   const marketId = payload.message.marketId
   const updatedAt = new Date(payload.message.timestamp).toISOString()
+  const updatedAtDay = new Date(payload.message.timestamp).toISOString().split('T')[0]
   const system = selectSystemByName.get({ systemName })
 
   if (isFleetCarrier) {
@@ -93,7 +96,8 @@ module.exports = (payload) => {
         systemX: system?.systemX ?? null,
         systemY: system?.systemY ?? null,
         systemZ: system?.systemZ ?? null,
-        updatedAt
+        updatedAt,
+        updatedAtDay
       })
     } else {
       // If carrier not in database add it (even if we don't know services yet)
@@ -106,7 +110,8 @@ module.exports = (payload) => {
         systemX: system?.systemX ?? null,
         systemY: system?.systemY ?? null,
         systemZ: system?.systemZ ?? null,
-        updatedAt
+        updatedAt,
+        updatedAtDay
       })
     }
   } else {
@@ -143,7 +148,8 @@ module.exports = (payload) => {
       stock: commodity.stock,
       stockBracket: commodity.stockBracket,
       statusFlags: commodity?.statusFlags?.join(', ') ?? null,
-      updatedAt
+      updatedAt,
+      updatedAtDay
     })
   }
 }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ardent-collector",
-  "version": "1.55.1",
+  "version": "1.56.0",
   "description": "Ardent Collector saves data submitted to EDDN",
   "main": "index.js",
   "scripts": {

scripts/optimize.js

@@ -1,5 +1,6 @@
 const { systemsDb, locationsDb, stationsDb, tradeDb } = require('../lib/db')
-
+const { getISOTimestamp } = require('../lib/utils/dates')
+const { TRADE_DATA_MAX_AGE_DAYS } = require('../lib/consts')
 // Using 'VACUUM' can be very slow and use up to 2x the disk space when running.
 //
 // It is superior at optimization than relying on the optimize command, but in
@@ -28,7 +29,12 @@ stationsDb.close()
 console.timeEnd('Optimize stationsDb')
 
 console.time('Optimize tradeDb')
-if (FULL_VACCUM === true) tradeDb.exec('VACUUM')
+// Purge old trade data
+// tradeDb.exec(`
+//   DELETE FROM commodities WHERE updatedAt <= '${getISOTimestamp(`-${TRADE_DATA_MAX_AGE_DAYS}`)}'
+// `)
+// Always vacuum the trade database after deleting old data to shrink it's size
+// tradeDb.exec('VACUUM')
 tradeDb.pragma('wal_checkpoint(TRUNCATE)')
 tradeDb.pragma('optimize')
 tradeDb.pragma('analysis_limit=0')