Skip to content

Commit

Permalink
chore: fixed diag. report + small fixes
Browse files Browse the repository at this point in the history
  • Loading branch information
tabarra committed Oct 3, 2024
1 parent d01028a commit b2ee806
Show file tree
Hide file tree
Showing 10 changed files with 59 additions and 25 deletions.
33 changes: 22 additions & 11 deletions core/components/HealthMonitor/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ export default class HealthMonitor {
this.lastRefreshStatus = currTimestamp;

//Get elapsed times & process status
const anySuccessfulHealthCheck = (this.lastSuccessfulHealthCheck !== null);
const elapsedHealthCheck = currTimestamp - this.lastSuccessfulHealthCheck;
const healthCheckFailed = (elapsedHealthCheck > this.hardConfigs.healthCheck.failThreshold);
const anySuccessfulHeartBeat = (this.lastSuccessfulFD3HeartBeat !== null || this.lastSuccessfulHTTPHeartBeat !== null);
Expand Down Expand Up @@ -299,10 +300,9 @@ export default class HealthMonitor {
}

//Check if already over the limit
if (
elapsedHealthCheck > this.hardConfigs.healthCheck.failLimit
|| elapsedHeartBeat > this.hardConfigs.heartBeat.failLimit
) {
const healthCheckOverLimit = (elapsedHealthCheck > this.hardConfigs.healthCheck.failLimit);
const heartBeatOverLimit = (elapsedHeartBeat > this.hardConfigs.heartBeat.failLimit);
if (healthCheckOverLimit || heartBeatOverLimit) {
if (anySuccessfulHeartBeat === false) {
if (starting.startingElapsedSecs !== null) {
//Resource didn't finish starting (if res boot still active)
Expand All @@ -326,18 +326,29 @@ export default class HealthMonitor {
timesPrefix,
);
}
} else if (elapsedHealthCheck > this.hardConfigs.healthCheck.failLimit) {
//FIXME: se der hang tanto HB quanto HC, ele ainda sim cai nesse caso
globals.statsManager.txRuntime.registerFxserverRestart('healthCheck');
} else if (anySuccessfulHealthCheck === false) {
//HB started, but HC didn't
this.restartFXServer(
'server partial hang detected',
globals.translator.t('restarter.hang_detected'),
`server failed to start within time limit - resources running but HTTP endpoint unreachable`,
globals.translator.t('restarter.start_timeout'),
timesPrefix,
);
} else {
globals.statsManager.txRuntime.registerFxserverRestart('heartBeat');
//Both threads started, but now at least one stopepd
let issueMsg, issueSrc;
if (healthCheckFailed && heartBeatFailed) {
issueMsg = 'server full hang detected';
issueSrc = 'both';
} else if (healthCheckFailed) {
issueMsg = 'server http hang detected';
issueSrc = 'healthCheck';
} else {
issueMsg = 'server resources hang detected';
issueSrc = 'heartBeat';
}
globals.statsManager.txRuntime.registerFxserverRestart(issueSrc);
this.restartFXServer(
'server hang detected',
issueMsg,
globals.translator.t('restarter.hang_detected'),
timesPrefix,
);
Expand Down
1 change: 1 addition & 0 deletions core/components/StatsManager/txRuntime/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ export default class TxRuntimeStatsManager {
close: 0,
heartBeat: 0,
healthCheck: 0,
both: 0,
},
};

Expand Down
15 changes: 15 additions & 0 deletions core/webroutes/diagnostics/diagnosticsFuncs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import si from 'systeminformation';
import consoleFactory from '@extras/console';
import TxAdmin from '@core/txAdmin';
import { parseFxserverVersion } from '@extras/fxsVersionParser';
import { getHeapStatistics } from 'node:v8';
import bytes from 'bytes';
const console = consoleFactory(modulename);


Expand Down Expand Up @@ -279,6 +281,7 @@ export const getTxAdminData = async (txAdmin: TxAdmin) => {
const whitelistCheckTime = txAdmin.statsManager.txRuntime.whitelistCheckTime.resultSummary();
const playersTableSearchTime = txAdmin.statsManager.txRuntime.playersTableSearchTime.resultSummary();
const historyTableSearchTime = txAdmin.statsManager.txRuntime.historyTableSearchTime.resultSummary();
const memoryUsage = getHeapStatistics();

return {
//Stats
Expand All @@ -287,6 +290,7 @@ export const getTxAdminData = async (txAdmin: TxAdmin) => {
close: txAdmin.statsManager.txRuntime.monitorStats.restartReasons.close,
heartBeat: txAdmin.statsManager.txRuntime.monitorStats.restartReasons.heartBeat,
healthCheck: txAdmin.statsManager.txRuntime.monitorStats.restartReasons.healthCheck,
both: txAdmin.statsManager.txRuntime.monitorStats.restartReasons.both,
},
hbFD3Fails: txAdmin.statsManager.txRuntime.monitorStats.healthIssues.fd3,
hbHTTPFails: txAdmin.statsManager.txRuntime.monitorStats.healthIssues.http,
Expand All @@ -306,5 +310,16 @@ export const getTxAdminData = async (txAdmin: TxAdmin) => {
fxServerHost: (txAdmin.fxRunner.fxServerHost)
? txAdmin.fxRunner.fxServerHost
: '--',

//Usage stuff
memoryUsage: {
heap_used: bytes(memoryUsage.used_heap_size),
heap_limit: bytes(memoryUsage.heap_size_limit),
heap_pct: (memoryUsage.heap_size_limit > 0)
? (memoryUsage.used_heap_size / memoryUsage.heap_size_limit * 100).toFixed(2)
: 0,
physical: bytes(memoryUsage.total_physical_size),
peak_malloced: bytes(memoryUsage.peak_malloced_memory),
},
};
}
4 changes: 2 additions & 2 deletions core/webroutes/diagnostics/sendReport.ts
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,14 @@ export default async function SendDiagnosticsReport(ctx: AuthedCtx) {
const reportData = {
$schemaVersion: 2,
$txVersion: txEnv.txAdminVersion,
$fxVersion: txEnv.fxServerVersion, //TODO: update_txdiagnostics
$fxVersion: txEnv.fxServerVersion,
diagnostics,
txSystemLog,
txActionLog,
serverLog,
fxserverLog,
envVars,
perfSvMain, //TODO: update_txdiagnostics
perfSvMain,
dbStats,
settings,
adminList,
Expand Down
5 changes: 2 additions & 3 deletions docs/dev_notes.md
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ Legend:
- from `SvRuntimeStatsManager` and `PlayerDropStatsManager` + persistent cache
- [x] fix: doing /tx <disconnected id> shows empty modal
- [x] fix the spam of `[tx:WebServer:AuthMws] Invalid session auth: admin_not_found`
- [ ] fix: `server partial hang detected` should not be the error for `(HB:0HC:--)`
- [ ] fix txDiagnostics (and add tx v8 heap data to it)
- [x] fix: `server partial hang detected` should not be the error for `(HB:0HC:--)`
- [x] fix txDiagnostics (and add tx v8 heap data to it)

## Chores + refactor + boring stuff
- [x] remove /legacy/dashboard route + handler
Expand All @@ -150,7 +150,6 @@ Legend:
- [!] check tx on node 22
- [!] update packages
- [!] check cicd stuff on testing repo before release
- [!] uncommited nui controls stuff
- [ ] update wouter and add search/filters state to URL of the players/history pages
- [ ] fix(core): a `EMFILE: too many open files` error on windows will cause the `admins.json` to reset
- [ref](/core/components/AdminVault/index.js#L289)
Expand Down
1 change: 0 additions & 1 deletion nui/src/hooks/useNuiListenersService.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import {
ServerCtx,
useSetServerCtx,
} from "../state/server.state";
import { fetchWebPipe } from "../utils/fetchWebPipe";

// Passive Message Event Listeners & Handlers for global state
export const useNuiListenerService = () => {
Expand Down
4 changes: 4 additions & 0 deletions resource/menu/client/cl_webpipe.lua
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ local staticCacheData = {}

-- catching all NUI requests for https://monitor/WebPipe/
RegisterRawNuiCallback('WebPipe', function(req, cb)
if not menuIsAccessible or not isMenuVisible then
return txPrint('^1NUI request received while the menu is not accessible or visible.')
end

local path = req.path
local headers = req.headers
local body = req.body
Expand Down
6 changes: 3 additions & 3 deletions resource/menu/server/sv_webpipe.lua
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,11 @@ if not TX_SERVER_MODE then return end
if not TX_MENU_ENABLED then return end

if TX_LUACOMHOST == "invalid" or TX_LUACOMTOKEN == "invalid" then
log('^1API Host or Pipe Token ConVars not found. Do not start this resource if not using txAdmin.')
txPrint('^1API Host or Pipe Token ConVars not found. Do not start this resource if not using txAdmin.')
return
end
if TX_LUACOMTOKEN == "removed" then
log('^1Please do not restart the monitor resource.')
txPrint('^1Please do not restart the monitor resource.')
return
end

Expand Down Expand Up @@ -93,7 +93,7 @@ RegisterNetEvent('txsv:webpipe:req', function(callbackId, method, path, headers,
headers['X-TxAdmin-Token'] = TX_LUACOMTOKEN
headers['X-TxAdmin-Identifiers'] = table.concat(GetPlayerIdentifiers(s), ',')


debugPrint(("^3WebPipe[^5%d^0:^1%d^3]^0 ^4>>^0 ^6%s^0"):format(s, callbackId, url))
debugPrint(("^3WebPipe[^5%d^0:^1%d^3]^0 ^4>>^0 ^6Headers: %s^0"):format(s, callbackId, json.encode(headers)))

Expand Down
4 changes: 2 additions & 2 deletions resource/sv_admins.lua
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@
if not TX_SERVER_MODE then return end

if TX_LUACOMHOST == "invalid" or TX_LUACOMTOKEN == "invalid" then
log('^1API Host or Pipe Token ConVars not found. Do not start this resource if not using txAdmin.')
txPrint('^1API Host or Pipe Token ConVars not found. Do not start this resource if not using txAdmin.')
return
end
if TX_LUACOMTOKEN == "removed" then
log('^1Please do not restart the monitor resource.')
txPrint('^1Please do not restart the monitor resource.')
return
end

Expand Down
11 changes: 8 additions & 3 deletions web/main/diagnostics.ejs
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,8 @@
<strong>Monitor Restarts:</strong>
<code>CL <%= txadmin.monitorRestarts.close %></code> /
<code>HB <%= txadmin.monitorRestarts.heartBeat %></code> /
<code>HC <%= txadmin.monitorRestarts.healthCheck %></code> <br>
<code>HC <%= txadmin.monitorRestarts.healthCheck %></code> /
<code>BO <%= txadmin.monitorRestarts.both %></code> <br>
<strong>HB Fails:</strong>
<code>HTTP <%= txadmin.hbHTTPFails %></code> / <code>FD3 <%= txadmin.hbFD3Fails %></code> <br>
<strong>Logs Storage Size:</strong> <code><%= txadmin.logStorageSize %></code> <br>
Expand All @@ -54,9 +55,13 @@
└─ Server: <code><%= txadmin.loggerStatusServer %></code> <br>
<strong>Performance Times (cnt / q5 / q25 / q50 / q75 / q95):</strong> <br>
├─ BanCheck: <code><%= txadmin.banCheckTime %></code> <br>
─ WhitelistCheck: <code><%= txadmin.whitelistCheckTime %></code> <br>
─ PlayersTable: <code><%= txadmin.playersTableSearchTime %></code> <br>
─ WhitelistCheck: <code><%= txadmin.whitelistCheckTime %></code> <br>
─ PlayersTable: <code><%= txadmin.playersTableSearchTime %></code> <br>
└─ HistoryTable: <code><%= txadmin.historyTableSearchTime %></code> <br>
<strong>Memory:</strong> <br>
├─ Heap: <code><%= txadmin.memoryUsage.heap_used %> / <%= txadmin.memoryUsage.heap_limit %> (<%= txadmin.memoryUsage.heap_pct %>%)</code> <br>
├─ Physical: <code><%= txadmin.memoryUsage.physical %></code> <br>
└─ Peak. Alloc.: <code><%= txadmin.memoryUsage.peak_malloced %></code> <br>
</div>
</div>

Expand Down

0 comments on commit b2ee806

Please sign in to comment.