diff --git a/package.json b/package.json index 148ce48..047dfcb 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "preinstall": "npx only-allow pnpm", "start": "vite", "test": "vitest --silent", + "test-logs": "vitest", "coverage": "vitest run --coverage --silent", "build": "vite build && cp analog.sh dist", "preview": "vite preview" diff --git a/src/analyzer/gridService.ts b/src/analyzer/gridService.ts index 6ffbf5b..b4e4883 100644 --- a/src/analyzer/gridService.ts +++ b/src/analyzer/gridService.ts @@ -15,7 +15,13 @@ function defaultCols(): ColDef[] { cellRenderer: JSONCellRenderer, flex: 2, }, - { field: "timestamp", width: 270, sortable: true }, + { + field: "timestamp", + width: 270, + sortable: true, + sort: "asc", + sortingOrder: ["asc", "desc"], + }, ]; } diff --git a/src/analyzer/useViewModel.test.tsx b/src/analyzer/useViewModel.test.tsx index e3c7eb2..b0bcc92 100644 --- a/src/analyzer/useViewModel.test.tsx +++ b/src/analyzer/useViewModel.test.tsx @@ -110,7 +110,7 @@ describe("useViewModel", () => { startTime: "2023-10-20T01:00:00.000Z", endTime: "2023-10-20T11:00:00.000Z", errorsOnly: true, - msgs: ["msg", "test"], + logs: [comparer.last().logs[0], comparer.last().logs[1]], regex: "^tes.*our$", }; @@ -161,7 +161,7 @@ describe("useViewModel", () => { .mockReturnValueOnce(30); filters = { - msgs: [], + logs: [], } as any; }); diff --git a/src/analyzer/useViewModel.tsx b/src/analyzer/useViewModel.tsx index 1c564e8..51af8dd 100644 --- a/src/analyzer/useViewModel.tsx +++ b/src/analyzer/useViewModel.tsx @@ -32,7 +32,11 @@ function useViewModel() { prevJumps = []; nextJumps = []; - const filteredLogs: JSONLogs = comparer.last().logs.filter((log) => { + let filteredLogs: JSONLogs = filtersData.logs.length + ? filtersData.logs + : comparer.last().logs; + + filteredLogs = filteredLogs.filter((log) => { let keep = true; if (keep && filtersData.startTime) { @@ -51,12 +55,6 @@ function useViewModel() { ); } - if (keep && filtersData.msgs.length) { - keep = filtersData.msgs.some((msg) => - log[Processor.logKeys.msg].startsWith(msg) - ); - } - if (keep) { const id = log[Processor.logKeys.id]; const time = new Date(log[Processor.logKeys.timestamp]); diff --git a/src/components/filters/index.tsx b/src/components/filters/index.tsx index 1aabf9d..bda61e0 100644 --- a/src/components/filters/index.tsx +++ b/src/components/filters/index.tsx @@ -10,7 +10,7 @@ import { } from "@suid/material"; import useViewModel, { type FiltersProps } from "./useViewModel"; import AgGridSolid, { type AgGridSolidRef } from "ag-grid-solid"; -import type { RowClassParams, RowStyle } from "ag-grid-community"; +import type { GridOptions } from "ag-grid-community"; import { GroupedMsg } from "../../models/processor"; import { Show } from "solid-js"; import comparer from "../../models/comparer"; @@ -31,11 +31,48 @@ function Filters(props: FiltersProps) { handleResetClick, } = useViewModel(props); - function getRowStyle( - params: RowClassParams - ): RowStyle | undefined { - return params.data?.hasErrors ? { background: "#FFBFBF" } : undefined; - } + const commonGridOptions: GridOptions = { + enableCellTextSelection: true, + columnDefs: [ + { + field: "msg", + flex: 2, + checkboxSelection: true, + filter: "agTextColumnFilter", + }, + { + headerName: "count", + flex: 0.5, + sortable: true, + valueGetter: (params) => params.data?.logs.length, + }, + ], + rowSelection: "multiple", + suppressRowClickSelection: true, + onSelectionChanged: handleLogsSelectionChanged, + getRowStyle: (params) => + params.data?.hasErrors ? { background: "#FFBFBF" } : undefined, + }; + + const topLogsGridOptions: GridOptions = { + ...commonGridOptions, + rowData: topLogs(), + }; + + const addedLogsGridOptions: GridOptions = { + ...commonGridOptions, + rowData: addedMsgs(), + }; + + const removedLogsGridOptions: GridOptions = { + ...commonGridOptions, + rowData: removedMsgs(), + columnDefs: [ + { ...commonGridOptions.columnDefs![0], checkboxSelection: undefined }, + { ...commonGridOptions.columnDefs![1] }, + ], + rowSelection: undefined, + }; function handleEnterKey(e: KeyboardEvent) { if (e.key === "Enter") { @@ -100,24 +137,7 @@ function Filters(props: FiltersProps) { {topLogs().length ? " : " + topLogs().length.toLocaleString() : ""}
- +
@@ -130,24 +150,7 @@ function Filters(props: FiltersProps) { : ""}
- +
@@ -158,21 +161,7 @@ function Filters(props: FiltersProps) { : ""}
- +
diff --git a/src/components/filters/useViewModel.test.tsx b/src/components/filters/useViewModel.test.tsx index 778e753..ed4ceb7 100644 --- a/src/components/filters/useViewModel.test.tsx +++ b/src/components/filters/useViewModel.test.tsx @@ -1,22 +1,24 @@ import { createRoot } from "solid-js"; -import useViewModel, { FiltersProps } from "./useViewModel"; +import useViewModel from "./useViewModel"; +import type { FiltersData, FiltersProps } from "./useViewModel"; import comparer from "../../models/comparer"; import Processor from "../../models/processor"; +import type { GroupedMsg } from "../../models/processor"; describe("useViewModel", () => { - const topLogs = [ + const topLogs: GroupedMsg[] = [ { - count: 3, + logs: [{}, {}, {}], hasErrors: false, msg: "grp1", }, { - count: 1, + logs: [{}], hasErrors: true, msg: "grp2", }, { - count: 2, + logs: [{}, {}], hasErrors: true, msg: "grp3", }, @@ -25,22 +27,22 @@ describe("useViewModel", () => { comparer.removed = [topLogs[0], topLogs[2]]; comparer.added = [ { - count: 5, + logs: [{}, {}, {}, {}, {}], hasErrors: false, msg: "grp11", }, { - count: 2, + logs: [{}, {}], hasErrors: true, msg: "grp22", }, ]; - const defaultFilters = { + const defaultFilters: FiltersData = { startTime: "", endTime: "", regex: "", - msgs: [], + logs: [], errorsOnly: false, }; @@ -76,7 +78,7 @@ describe("useViewModel", () => { test("handleFiltersChange", () => { createRoot((dispose) => { const vm = useViewModel(props); - vm.setFilters("msgs", ["some log"]); + vm.setFilters("logs", [{}]); vm.handleFiltersChange(); expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith( vm.filters @@ -101,7 +103,7 @@ describe("useViewModel", () => { startTime: "some start time", endTime: "some end time", regex: "some regex", - msgs: ["log1", "log2"], + logs: [{}, {}], errorsOnly: true, })); expect(vm.filters.regex, "regex").toEqual("some regex"); @@ -123,8 +125,8 @@ describe("useViewModel", () => { test("handleLogsSelectionChanged", () => { createRoot((dispose) => { - const rows = [{ msg: "log1" }, { msg: "log2" }]; - const msgs = rows.map((r) => r.msg); + const rows = [{ logs: [{}, {}] }, { logs: [{}] }]; + const logs = rows.flatMap((r) => r.logs); const selectionEvent = { api: { getSelectedRows: () => rows, @@ -134,10 +136,10 @@ describe("useViewModel", () => { const vm = useViewModel(props); vm.handleLogsSelectionChanged(selectionEvent as any); - expect(vm.filters.msgs, "filters.msgs").toEqual(msgs); + expect(vm.filters.logs, "filters.msgs").toEqual(logs); expect(props.onFiltersChange, "onFiltersChange").toBeCalledWith({ ...defaultFilters, - msgs, + logs, }); dispose(); diff --git a/src/components/filters/useViewModel.tsx b/src/components/filters/useViewModel.tsx index e3d06d4..bc3a3e4 100644 --- a/src/components/filters/useViewModel.tsx +++ b/src/components/filters/useViewModel.tsx @@ -1,6 +1,6 @@ import { createSignal } from "solid-js"; import { createStore } from "solid-js/store"; -import type { GroupedMsg } from "../../models/processor"; +import type { GroupedMsg, JSONLogs } from "../../models/processor"; import { SelectionChangedEvent } from "ag-grid-community"; import { type AgGridSolidRef } from "ag-grid-solid"; import comparer from "../../models/comparer"; @@ -13,7 +13,7 @@ interface FiltersData { startTime: string; endTime: string; regex: string; - msgs: string[]; + logs: JSONLogs; errorsOnly: boolean; } @@ -22,7 +22,7 @@ function defaultFilters(): FiltersData { startTime: "", endTime: "", regex: "", - msgs: [], + logs: [], errorsOnly: false, }; } @@ -53,8 +53,8 @@ function useViewModel(props: FiltersProps) { function handleLogsSelectionChanged(e: SelectionChangedEvent) { setFilters( - "msgs", - e.api.getSelectedRows().map((n) => n.msg) + "logs", + e.api.getSelectedRows().flatMap((n) => n.logs) ); handleFiltersChange(); } diff --git a/src/models/comparer.test.ts b/src/models/comparer.test.ts index 5296725..40a9eda 100644 --- a/src/models/comparer.test.ts +++ b/src/models/comparer.test.ts @@ -12,9 +12,9 @@ describe("addProcessor", () => { const processor1 = new Processor(); processor1.topLogsMap = new Map([ - ["grp1", { count: 2, hasErrors: false, msg: "grp1" }], - ["grp2", { count: 6, hasErrors: false, msg: "grp2" }], - ["grp3", { count: 4, hasErrors: false, msg: "grp3" }], + ["grp1", { logs: [], hasErrors: false, msg: "grp1" }], + ["grp2", { logs: [], hasErrors: false, msg: "grp2" }], + ["grp3", { logs: [], hasErrors: false, msg: "grp3" }], ]); test("1st processor", () => { @@ -29,9 +29,9 @@ describe("addProcessor", () => { const processor2 = new Processor(); processor2.topLogsMap = new Map([ - ["grp11", { count: 7, hasErrors: false, msg: "grp11" }], + ["grp11", { logs: [], hasErrors: false, msg: "grp11" }], ["grp2", processor1.topLogsMap.get("grp2")!], - ["grp44", { count: 3, hasErrors: false, msg: "grp44" }], + ["grp44", { logs: [], hasErrors: false, msg: "grp44" }], ["grp3", processor1.topLogsMap.get("grp3")!], ]); diff --git a/src/models/processor.test.ts b/src/models/processor.test.ts index a8c273b..07e3deb 100644 --- a/src/models/processor.test.ts +++ b/src/models/processor.test.ts @@ -1,3 +1,4 @@ +import stringsUtils from "../utils/strings"; import Processor, { type GroupedMsg } from "./processor"; describe("isErrorLog", () => { @@ -37,6 +38,8 @@ describe("isErrorLog", () => { }); describe("init", () => { + const cutOffLen = Processor["msgCutOffLen"]; + it("init", async () => { const log1 = { [Processor.logKeys.error]: "some error", @@ -52,7 +55,7 @@ describe("init", () => { const log2String = getJSONString(log2); const log3 = { [Processor.logKeys.level]: "info", - [Processor.logKeys.msg]: "msg".repeat(20) + "group 1", + [Processor.logKeys.msg]: "abc ".repeat(cutOffLen) + "group 1", parentKey1: { childKey1: 11, childKey2: 12, @@ -61,18 +64,18 @@ describe("init", () => { const log3String = getJSONString(log3); const log4 = { [Processor.logKeys.level]: "error", - [Processor.logKeys.msg]: "msg".repeat(20) + "group 1", + [Processor.logKeys.msg]: "abc ".repeat(cutOffLen) + "group 1", parentKey1: "k1", }; const log4String = getJSONString(log4); const log5 = { [Processor.logKeys.level]: "info", - [Processor.logKeys.msg]: "abc".repeat(20) + "group 2", + [Processor.logKeys.msg]: "qwe ".repeat(cutOffLen) + "group 2", }; const log5String = getJSONString(log5); const log6 = { [Processor.logKeys.level]: "info", - [Processor.logKeys.msg]: "abc".repeat(20) + "group 2", + [Processor.logKeys.msg]: "qwe ".repeat(cutOffLen) + "group 2", }; const log6String = getJSONString(log6); @@ -116,27 +119,56 @@ describe("init", () => { expect(processor.logs, "logs").toEqual(expectedLogs); function getCutOffMsg(log: any) { - return log[Processor.logKeys.msg].substring(0, Processor["msgCutOffLen"]); + return stringsUtils + .cleanText(log[Processor.logKeys.msg]) + .substring(0, cutOffLen) + .trim(); } const expectedTopLogsMap = new Map([ [ getCutOffMsg(log1), - { msg: getCutOffMsg(log1), count: 1, hasErrors: true }, + { + msg: getCutOffMsg(log1), + logs: [expectedLogs[0]] as any, + hasErrors: true, + }, ], [ getCutOffMsg(log2), - { msg: getCutOffMsg(log2), count: 1, hasErrors: true }, + { + msg: getCutOffMsg(log2), + logs: [expectedLogs[1]] as any, + hasErrors: true, + }, ], [ getCutOffMsg(log3), - { msg: getCutOffMsg(log3), count: 2, hasErrors: true }, + { + msg: getCutOffMsg(log3), + logs: [expectedLogs[2], expectedLogs[3]] as any, + hasErrors: true, + }, ], [ getCutOffMsg(log5), - { msg: getCutOffMsg(log5), count: 2, hasErrors: false }, + { + msg: getCutOffMsg(log5), + logs: [expectedLogs[4], expectedLogs[5]] as any, + hasErrors: false, + }, ], ]); - expect(processor.topLogsMap, "topLogsMap").toEqual(expectedTopLogsMap); + expect(processor.topLogsMap.size, "topLogsMap.size").toEqual( + expectedTopLogsMap.size + ); + + for (const [k, v] of expectedTopLogsMap) { + const topLog = processor.topLogsMap.get(k); + expect(topLog, "topLog").toBeTruthy(); + expect(topLog?.hasErrors, "topLogsMap.hasErrors").toEqual(v.hasErrors); + expect(topLog?.logs, "topLogsMap.logs").toEqual(v.logs); + } + expect(processor.topLogs, "topLogs").toEqual( [...expectedTopLogsMap.values()].sort(Processor.sortComparerFn) ); diff --git a/src/models/processor.ts b/src/models/processor.ts index d6f1886..c0ff47e 100644 --- a/src/models/processor.ts +++ b/src/models/processor.ts @@ -1,11 +1,12 @@ import objectsUtils from "../utils/objects"; +import stringsUtils from "../utils/strings"; type JSONLog = Record; type JSONLogs = JSONLog[]; interface GroupedMsg { msg: string; - count: number; + logs: JSONLogs; hasErrors: boolean; } @@ -21,7 +22,7 @@ class Processor { keys: string[] = []; static readonly sortComparerFn = (a: GroupedMsg, b: GroupedMsg) => - b.count - a.count; + b.logs.length - a.logs.length; private static readonly msgCutOffLen = 80; static readonly logKeys = { @@ -76,21 +77,22 @@ class Processor { } private initTopLogsMap(log: JSONLog) { - const cutOffMsg = log[Processor.logKeys.msg].substring( - 0, - Processor.msgCutOffLen - ); - - if (!this.topLogsMap.has(cutOffMsg)) { - this.topLogsMap.set(cutOffMsg, { - count: 0, + const msg = log[Processor.logKeys.msg]; + const cleanMsg = stringsUtils + .cleanText(msg) + .substring(0, Processor.msgCutOffLen) + .trim(); + + if (!this.topLogsMap.has(cleanMsg)) { + this.topLogsMap.set(cleanMsg, { + msg: cleanMsg, hasErrors: false, - msg: cutOffMsg, + logs: [], }); } - const topLog = this.topLogsMap.get(cutOffMsg)!; - topLog.count++; + const topLog = this.topLogsMap.get(cleanMsg)!; + topLog.logs.push(log); if (!topLog.hasErrors && Processor.isErrorLog(log)) { topLog.hasErrors = true; } diff --git a/src/utils/strings.test.ts b/src/utils/strings.test.ts index 0d96d73..110f01e 100644 --- a/src/utils/strings.test.ts +++ b/src/utils/strings.test.ts @@ -1,6 +1,6 @@ import stringsUtils from "./strings"; -it.each` +test.each` text | pattern | expected ${"test regex pattern"} | ${"re.*pa"} | ${true} ${"test regex pattern"} | ${"qwe.*qwe"} | ${false} @@ -10,3 +10,12 @@ it.each` expect(stringsUtils.regexMatch(text, pattern)).toBe(expected); } ); + +test("cleanText", () => { + const expected = + "symbols xyz josief long hyphenated words durations timestamps URLs valid string text hex value of numbers CAPs OKAY z a an the long word works"; + const text = + "symbols xyz:34=== (josief) long hyphenated words lksjdf23sj-qwe-321-asd durations (5.342ms)(5.342s) timestamps 2023-08-10 19:25:41.543 +05:30 URLs http://www.google.com/nested valid string text - hex value of 0xc000e9b6c0 numbers 23423 CAPs OKAY z a an the long word qwertyuiopasdfghjklwewewas works https://www.google.com"; + + expect(stringsUtils.cleanText(text), "cleanText").toEqual(expected); +}); diff --git a/src/utils/strings.ts b/src/utils/strings.ts index 33165f9..5c89925 100644 --- a/src/utils/strings.ts +++ b/src/utils/strings.ts @@ -1,9 +1,26 @@ +const regURL = /(https?|ftp):\/\/[^\s/$.?#].[^\s]*/g; +const regWordGreaterThan20 = /\b[\w-]{21,}\b/g; +const regHex = /0[xX][0-9a-fA-F]+/g; +const regNonWordOrNumberOrDuration = /\W|\d\s?m?s?m?/g; + function regexMatch(text: string, pattern: string): boolean { return new RegExp(pattern, "i").test(text); } +function cleanText(text: string): string { + return text + .replace(regURL, " ") + .replace(regWordGreaterThan20, " ") + .replace(regHex, " ") + .replace(regNonWordOrNumberOrDuration, " ") + .split(" ") + .filter((x) => x.trim()) + .join(" "); +} + const stringsUtils = { regexMatch, + cleanText, }; export default stringsUtils;