wip(perf): backend completed fixes

tabarra · May 23, 2024 · 16b21bf · 16b21bf
1 parent e5b1108
commit 16b21bf
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 55 deletions.
diff --git a/core/components/PerformanceCollector/index.ts b/core/components/PerformanceCollector/index.ts
@@ -8,36 +8,15 @@ import type { LogNodeHeapEventType, SSFileType, SSLogDataType, SSLogType, SSPerf
 import { diffPerfs, fetchFxsMemory, fetchRawPerfData, perfCountsToHist } from './perfUtils';
 import { optimizeStatsLog } from './statsLogOptimizer';
 import { convars } from '@core/globalData';
-import { ValuesType } from 'utility-types';
-import bytes from 'bytes';
+import { ZodError } from 'zod';
+import { PERF_DATA_BUCKET_COUNT, PERF_DATA_INITIAL_RESOLUTION, PERF_DATA_MIN_TICKS } from './statsConfigs';
 const console = consoleFactory(modulename);
 
 
 //Consts
-const minutesMs = 60 * 1000;
-const hoursMs = 60 * minutesMs;
 const megabyte = 1024 * 1024;
-
-
-/**
- * Configs
- */
 const STATS_DATA_FILE_VERSION = 1;
 const STATS_DATA_FILE_NAME = 'statsData.json';
-export const PERF_DATA_BUCKET_COUNT = 15;
-export const PERF_DATA_MIN_TICKS = 2000; //less than that and the data is not reliable
-export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs;
-export const STATS_RESOLUTION_TABLE = [
-    //00~12h =  5min = 12/h = 144 snaps
-    //12~24h = 15min =  4/h =  48 snaps
-    //24~96h = 30min =  2/h = 144 snaps
-    { maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION },
-    { maxAge: 24 * hoursMs, resolution: 15 * minutesMs },
-    { maxAge: 96 * hoursMs, resolution: 30 * minutesMs },
-];
-export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins)
-export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const;
-export type PerfDataThreadNamesType = ValuesType<typeof PERF_DATA_THREAD_NAMES>;
 
 
 /**
@@ -78,6 +57,13 @@ export default class PerformanceCollector {
     resetPerfState() {
         this.lastPerfCounts = undefined;
         this.lastPerfSaved = undefined;
+    }
+
+
+    /**
+     * Reset the last perf data except boundaries
+     */
+    resetMemoryState() {
         this.lastNodeMemory = undefined;
         this.lastFxsMemory = undefined;
     }
@@ -88,6 +74,7 @@ export default class PerformanceCollector {
      */
     logServerBoot(bootTime: number) {
         this.resetPerfState();
+        this.resetMemoryState();
         //If last log is a boot, remove it as the server didn't really start 
         // otherwise it would have lived long enough to have stats logged
         if (this.statsLog.length && this.statsLog.at(-1)!.type === 'svBoot') {
@@ -98,6 +85,7 @@ export default class PerformanceCollector {
             type: 'svBoot',
             bootTime,
         });
+        this.saveStatsHistory();
     }
 
 
@@ -106,6 +94,7 @@ export default class PerformanceCollector {
      */
     logServerClose(reason: string) {
         this.resetPerfState();
+        this.resetMemoryState();
         if (this.statsLog.length) {
             if (this.statsLog.at(-1)!.type === 'svClose') {
                 //If last log is a close, skip saving a new one
@@ -121,6 +110,7 @@ export default class PerformanceCollector {
             type: 'svClose',
             reason,
         });
+        this.saveStatsHistory();
     }
 
 
@@ -244,16 +234,10 @@ export default class PerformanceCollector {
             perf: perfHistToSave,
         };
         this.statsLog.push(currSnapshot);
-        await optimizeStatsLog(this.statsLog);
+        console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`);
 
         //Save perf series do file
-        const savePerfData: SSFileType = {
-            version: STATS_DATA_FILE_VERSION,
-            lastPerfBoundaries: this.lastPerfBoundaries,
-            log: this.statsLog,
-        };
-        await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData));
-        console.verbose.ok(`Collected performance snapshot #${this.statsLog.length}`);
+        await this.saveStatsHistory();
     }
 
 
@@ -265,15 +249,38 @@ export default class PerformanceCollector {
             const rawFileData = await fsp.readFile(this.statsDataPath, 'utf8');
             const fileData = JSON.parse(rawFileData);
             if (fileData?.version !== STATS_DATA_FILE_VERSION) throw new Error('invalid version');
-            const statsData = await SSFileSchema.parseAsync(fileData);
+            const statsData = SSFileSchema.parse(fileData);
             this.lastPerfBoundaries = statsData.lastPerfBoundaries;
             this.statsLog = statsData.log;
             this.resetPerfState();
             console.verbose.debug(`Loaded ${this.statsLog.length} performance snapshots from cache`);
-            optimizeStatsLog(this.statsLog);
+            await optimizeStatsLog(this.statsLog);
+        } catch (error) {
+            if (error instanceof ZodError) {
+                console.warn(`Failed to load ${STATS_DATA_FILE_NAME} due to invalid data.`);
+                console.warn('Since this is not a critical file, it will be reset.');
+            } else {
+                console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
+                console.warn('Since this is not a critical file, it will be reset.');
+            }
+        }
+    }
+
+
+    /**
+     * Saves the stats database/cache/history
+     */
+    async saveStatsHistory() {
+        try {
+            await optimizeStatsLog(this.statsLog);
+            const savePerfData: SSFileType = {
+                version: STATS_DATA_FILE_VERSION,
+                lastPerfBoundaries: this.lastPerfBoundaries,
+                log: this.statsLog,
+            };
+            await fsp.writeFile(this.statsDataPath, JSON.stringify(savePerfData));
         } catch (error) {
-            console.warn(`Failed to load ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
-            console.warn('Since this is not a critical file, it will be reset.');
+            console.warn(`Failed to save ${STATS_DATA_FILE_NAME} with message: ${(error as Error).message}`);
         }
     }
 

diff --git a/core/components/PerformanceCollector/perfParser.ts b/core/components/PerformanceCollector/perfParser.ts
@@ -1,4 +1,4 @@
-import { PERF_DATA_BUCKET_COUNT } from "./index";
+import { PERF_DATA_BUCKET_COUNT } from "./statsConfigs";
 import { isValidPerfThreadName, type SSPerfBoundariesType, type SSPerfCountsType } from "./perfSchemas";
 
 

diff --git a/core/components/PerformanceCollector/perfSchemas.ts b/core/components/PerformanceCollector/perfSchemas.ts
@@ -1,5 +1,5 @@
 import * as z from 'zod';
-import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './index';
+import { PERF_DATA_BUCKET_COUNT, PERF_DATA_THREAD_NAMES, PerfDataThreadNamesType } from './statsConfigs';
 import { ValuesType } from 'utility-types';
 
 
@@ -74,7 +74,7 @@ export const SSLogSvCloseSchema = z.object({
 
 export const SSFileSchema = z.object({
     version: z.literal(1),
-    lastPerfBoundaries: SSPerfBoundariesSchema,
+    lastPerfBoundaries: SSPerfBoundariesSchema.optional(),
     log: z.array(z.union([SSLogDataSchema, SSLogSvBootSchema, SSLogSvCloseSchema])),
 });
 

diff --git a/core/components/PerformanceCollector/perfUtils.ts b/core/components/PerformanceCollector/perfUtils.ts
@@ -3,7 +3,7 @@ import type { SSPerfCountsType, SSPerfHistType } from "./perfSchemas";
 import got from '@core/extras/got.js';
 import { parseRawPerf } from './perfParser';
 import { getProcessesData } from '@core/webroutes/diagnostics/diagnosticsFuncs';
-import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './index';
+import { PERF_DATA_BUCKET_COUNT, PerfDataThreadNamesType } from './statsConfigs';
 
 
 //Consts
@@ -30,24 +30,22 @@ const perfDataRawThreadsTemplate: SSPerfCountsType = {
  * Compares a perf snapshot with the one that came before
  */
 export const diffPerfs = (newPerf: SSPerfCountsType, oldPerf?: SSPerfCountsType) => {
-    if (!oldPerf) {
-        oldPerf = cloneDeep(perfDataRawThreadsTemplate);
-    }
+    const basePerf = oldPerf ?? cloneDeep(perfDataRawThreadsTemplate);
     return {
         svSync: {
-            count: newPerf.svSync.count - oldPerf.svSync.count,
-            // sum: newPerf.svSync.sum - oldPerf.svSync.sum,
-            buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - oldPerf.svSync.buckets[i]),
+            count: newPerf.svSync.count - basePerf.svSync.count,
+            // sum: newPerf.svSync.sum - basePerf.svSync.sum,
+            buckets: newPerf.svSync.buckets.map((bucket, i) => bucket - basePerf.svSync.buckets[i]),
         },
         svNetwork: {
-            count: newPerf.svNetwork.count - oldPerf.svNetwork.count,
-            // sum: newPerf.svNetwork.sum - oldPerf.svNetwork.sum,
-            buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - oldPerf.svNetwork.buckets[i]),
+            count: newPerf.svNetwork.count - basePerf.svNetwork.count,
+            // sum: newPerf.svNetwork.sum - basePerf.svNetwork.sum,
+            buckets: newPerf.svNetwork.buckets.map((bucket, i) => bucket - basePerf.svNetwork.buckets[i]),
         },
         svMain: {
-            count: newPerf.svMain.count - oldPerf.svMain.count,
-            // sum: newPerf.svMain.sum - oldPerf.svMain.sum,
-            buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - oldPerf.svMain.buckets[i]),
+            count: newPerf.svMain.count - basePerf.svMain.count,
+            // sum: newPerf.svMain.sum - basePerf.svMain.sum,
+            buckets: newPerf.svMain.buckets.map((bucket, i) => bucket - basePerf.svMain.buckets[i]),
         },
     };
 };
@@ -78,7 +76,6 @@ export const perfCountsToHist = (threads: SSPerfCountsType) => {
             return (bucketValue - prevBucketValue) / tData.count;
         });
     }
-
     return currPerfFreqs;
 }
 
@@ -98,9 +95,9 @@ export const fetchRawPerfData = async (fxServerHost: string) => {
 export const fetchFxsMemory = async () => {
     const allProcsData = await getProcessesData();
     if (!allProcsData) return;
-    
+
     const fxProcData = allProcsData.find((proc) => proc.name === 'FXServer');
     if (!fxProcData) return;
 
-    return fxProcData.memory;
+    return parseFloat((fxProcData.memory).toFixed(2));
 }
diff --git a/core/components/PerformanceCollector/statsConfigs.ts b/core/components/PerformanceCollector/statsConfigs.ts
@@ -0,0 +1,22 @@
+import { ValuesType } from "utility-types";
+
+
+/**
+ * Configs
+ */
+const minutesMs = 60 * 1000;
+const hoursMs = 60 * minutesMs;
+export const PERF_DATA_BUCKET_COUNT = 15;
+export const PERF_DATA_MIN_TICKS = 1000; //less than that and the data is not reliable
+export const PERF_DATA_INITIAL_RESOLUTION = 5 * minutesMs;
+export const STATS_RESOLUTION_TABLE = [
+    //00~12h =  5min = 12/h = 144 snaps
+    //12~24h = 15min =  4/h =  48 snaps
+    //24~96h = 30min =  2/h = 144 snaps
+    { maxAge: 12 * hoursMs, resolution: PERF_DATA_INITIAL_RESOLUTION },
+    { maxAge: 24 * hoursMs, resolution: 15 * minutesMs },
+    { maxAge: 96 * hoursMs, resolution: 30 * minutesMs },
+];
+export const STATS_LOG_SIZE_LIMIT = 720; //144+48+144 (max data snaps) + 384 (1 reboot every 30 mins)
+export const PERF_DATA_THREAD_NAMES = ['svNetwork', 'svSync', 'svMain'] as const;
+export type PerfDataThreadNamesType = ValuesType<typeof PERF_DATA_THREAD_NAMES>;
diff --git a/core/components/PerformanceCollector/statsLogOptimizer.ts b/core/components/PerformanceCollector/statsLogOptimizer.ts
@@ -1,4 +1,4 @@
-import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./index";
+import { STATS_LOG_SIZE_LIMIT, STATS_RESOLUTION_TABLE } from "./statsConfigs";
 import type { SSLogType } from "./perfSchemas";
 
 //Consts