Skip to content

Commit

Permalink
feat: Add async library to make performance improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@erdkse
erdkse committed Oct 20, 2021
1 parent 4fc447a commit 02beb97
Show file tree
Hide file tree
Showing 6 changed files with 199 additions and 143 deletions.
41 changes: 32 additions & 9 deletions package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"@testing-library/jest-dom": "5.14.1",
"@testing-library/react": "12.0.0",
"@testing-library/user-event": "13.2.1",
"@types/async": "^3.2.8",
"@types/dagre": "0.7.46",
"@types/electron-devtools-installer": "2.2.0",
"@types/jest": "27.0.1",
Expand Down Expand Up @@ -79,6 +80,7 @@
"@trodi/electron-splashscreen": "1.0.2",
"ajv": "6.12.6",
"antd": "4.16.13",
"async": "^3.2.1",
"chokidar": "3.5.2",
"dagre": "0.8.5",
"dayjs": "1.10.7",
Expand Down
66 changes: 40 additions & 26 deletions src/redux/reducers/main.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ import {AlertType} from '@models/alert';
import {getResourceKindHandler} from '@src/kindhandlers';
import {getFileStats} from '@utils/files';
import electronStore from '@utils/electronStore';
import {parallelLimit} from 'async';

import initialState from '../initialState';
import {clearResourceSelections, highlightChildrenResources, updateSelectionAndHighlights} from '../services/selection';
import {
Expand Down Expand Up @@ -144,17 +146,21 @@ export const mainSlice = createSlice({
) => {
let filePaths: Array<string> = action.payload.paths;
const appConfig = action.payload.appConfig;
filePaths.forEach((filePath: string) => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
if (getFileStats(filePath)?.isDirectory() === false) {
log.info(`added file ${filePath} already exists - updating`);
reloadFile(filePath, fileEntry, state);
parallelLimit(
filePaths.map(filePath => () => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
if (getFileStats(filePath)?.isDirectory() === false) {
log.info(`added file ${filePath} already exists - updating`);
reloadFile(filePath, fileEntry, state);
}
} else {
addPath(filePath, state, appConfig);
}
} else {
addPath(filePath, state, appConfig);
}
});
}),
20,
(err, results) => {}
);
},
/**
* called by the file monitor when a file is changed in the file system
Expand All @@ -178,14 +184,18 @@ export const mainSlice = createSlice({
) => {
let filePaths = action.payload.paths;
const appConfig = action.payload.appConfig;
filePaths.forEach((filePath: string) => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
reloadFile(filePath, fileEntry, state);
} else {
addPath(filePath, state, appConfig);
}
});
parallelLimit(
filePaths.map(filePath => () => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
reloadFile(filePath, fileEntry, state);
} else {
addPath(filePath, state, appConfig);
}
}),
20,
() => {}
);
},
/**
* called by the file monitor when a path is removed from the file system
Expand All @@ -204,14 +214,18 @@ export const mainSlice = createSlice({
*/
multiplePathsRemoved: (state: Draft<AppState>, action: PayloadAction<Array<string>>) => {
let filePaths: Array<string> = action.payload;
filePaths.forEach((filePath: string) => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
removePath(filePath, state, fileEntry);
} else {
log.warn(`removed file ${filePath} not known - ignoring..`);
}
});
parallelLimit(
filePaths.map(filePath => () => {
let fileEntry = getFileEntryForAbsolutePath(filePath, state.fileMap);
if (fileEntry) {
removePath(filePath, state, fileEntry);
} else {
log.warn(`removed file ${filePath} not known - ignoring..`);
}
}),
20,
() => {}
);
},
/**
* updates the content of the specified path to the specified value
Expand Down
143 changes: 77 additions & 66 deletions src/redux/services/fileEntry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import {clearResourceSelections, updateSelectionAndHighlights} from '@redux/serv
import {HelmChart, HelmValuesFile} from '@models/helm';
import {v4 as uuidv4} from 'uuid';
import {getFileStats} from '@utils/files';
import {parallelLimit} from 'async';

import {extractK8sResources, reprocessResources} from './resource';

type PathRemovalSideEffect = {
Expand Down Expand Up @@ -69,43 +71,47 @@ function processHelmChartFolder(
valueFileIds: [],
};

files.forEach(file => {
const filePath = path.join(folder, file);
const fileEntryPath = filePath.substr(rootFolder.length);
const fileEntry = createFileEntry(fileEntryPath);
parallelLimit(
files.map(file => () => {
const filePath = path.join(folder, file);
const fileEntryPath = filePath.substr(rootFolder.length);
const fileEntry = createFileEntry(fileEntryPath);

if (fileIsExcluded(appConfig, fileEntry)) {
fileEntry.isExcluded = true;
} else if (getFileStats(filePath)?.isDirectory()) {
if (depth === appConfig.folderReadsMaxDepth) {
log.warn(`[readFiles]: Ignored ${filePath} because max depth was reached.`);
} else {
fileEntry.children = readFiles(
filePath,
appConfig,
resourceMap,
fileMap,
helmChartMap,
helmValuesMap,
depth + 1
);
if (fileIsExcluded(appConfig, fileEntry)) {
fileEntry.isExcluded = true;
} else if (getFileStats(filePath)?.isDirectory()) {
if (depth === appConfig.folderReadsMaxDepth) {
log.warn(`[readFiles]: Ignored ${filePath} because max depth was reached.`);
} else {
fileEntry.children = readFiles(
filePath,
appConfig,
resourceMap,
fileMap,
helmChartMap,
helmValuesMap,
depth + 1
);
}
} else if (micromatch.isMatch(file, '*values*.yaml')) {
const helmValues: HelmValuesFile = {
id: uuidv4(),
filePath: fileEntryPath,
name: file,
isSelected: false,
helmChartId: helmChart.id,
};

helmValuesMap[helmValues.id] = helmValues;
helmChart.valueFileIds.push(helmValues.id);
}
} else if (micromatch.isMatch(file, '*values*.yaml')) {
const helmValues: HelmValuesFile = {
id: uuidv4(),
filePath: fileEntryPath,
name: file,
isSelected: false,
helmChartId: helmChart.id,
};

helmValuesMap[helmValues.id] = helmValues;
helmChart.valueFileIds.push(helmValues.id);
}

fileMap[fileEntry.filePath] = fileEntry;
result.push(fileEntry.name);
});
fileMap[fileEntry.filePath] = fileEntry;
result.push(fileEntry.name);
}),
20,
() => {}
);

helmChartMap[helmChart.id] = helmChart;
}
Expand Down Expand Up @@ -151,40 +157,44 @@ export function readFiles(
depth
);
} else {
files.forEach(file => {
const filePath = path.join(folder, file);
const fileEntryPath = filePath.substr(rootFolder.length);
const fileEntry = createFileEntry(fileEntryPath);

if (fileIsExcluded(appConfig, fileEntry)) {
fileEntry.isExcluded = true;
} else if (getFileStats(filePath)?.isDirectory()) {
if (depth === appConfig.folderReadsMaxDepth) {
log.warn(`[readFiles]: Ignored ${filePath} because max depth was reached.`);
} else {
fileEntry.children = readFiles(
filePath,
appConfig,
resourceMap,
fileMap,
helmChartMap,
helmValuesMap,
depth + 1
);
parallelLimit(
files.map(file => () => {
const filePath = path.join(folder, file);
const fileEntryPath = filePath.substr(rootFolder.length);
const fileEntry = createFileEntry(fileEntryPath);

if (fileIsExcluded(appConfig, fileEntry)) {
fileEntry.isExcluded = true;
} else if (getFileStats(filePath)?.isDirectory()) {
if (depth === appConfig.folderReadsMaxDepth) {
log.warn(`[readFiles]: Ignored ${filePath} because max depth was reached.`);
} else {
fileEntry.children = readFiles(
filePath,
appConfig,
resourceMap,
fileMap,
helmChartMap,
helmValuesMap,
depth + 1
);
}
} else if (appConfig.fileIncludes.some(e => micromatch.isMatch(fileEntry.name, e))) {
try {
extractK8sResourcesFromFile(filePath, fileMap).forEach(resource => {
resourceMap[resource.id] = resource;
});
} catch (e) {
log.warn(`Failed to parse yaml in file ${fileEntry.name}; ${e}`);
}
}
} else if (appConfig.fileIncludes.some(e => micromatch.isMatch(fileEntry.name, e))) {
try {
extractK8sResourcesFromFile(filePath, fileMap).forEach(resource => {
resourceMap[resource.id] = resource;
});
} catch (e) {
log.warn(`Failed to parse yaml in file ${fileEntry.name}; ${e}`);
}
}

fileMap[fileEntry.filePath] = fileEntry;
result.push(fileEntry.name);
});
fileMap[fileEntry.filePath] = fileEntry;
result.push(fileEntry.name);
}),
20,
() => {}
);
}

return result;
Expand Down Expand Up @@ -259,6 +269,7 @@ export function extractK8sResourcesFromFile(filePath: string, fileMap: FileMapTy
export function getAllFileEntriesForPath(filePath: string, fileMap: FileMapType) {
let parent = fileMap[ROOT_FILE_ENTRY];
const result: FileEntry[] = [];

filePath.split(path.sep).forEach(pathSegment => {
if (parent.children?.includes(pathSegment)) {
const child = fileMap[getChildFilePath(pathSegment, parent, fileMap)];
Expand Down
Loading

0 comments on commit 02beb97

Please sign in to comment.