From 18e28cefe69bba4e916b3e2bdde68fd1144e2650 Mon Sep 17 00:00:00 2001 From: Jose Fabio Date: Tue, 22 Aug 2023 21:19:59 -0600 Subject: [PATCH] Delete all unused functions and configs --- src/DuplicatedFilesCleaner.ts | 63 ------------------------ src/copyData.ts | 4 +- src/getInfo.ts | 5 +- src/run/deleteUnusedFiles.ts | 30 ----------- src/run/moveFilesToStorage.ts | 54 -------------------- src/run/run.ts | 18 ------- src/run/substituteFiles.ts | 93 ----------------------------------- types/constants.type.ts | 3 -- utils/getFilesOfNodes.ts | 18 ++----- utils/getStorageFiles.ts | 38 -------------- 10 files changed, 6 insertions(+), 320 deletions(-) delete mode 100644 src/run/deleteUnusedFiles.ts delete mode 100644 src/run/moveFilesToStorage.ts delete mode 100644 src/run/run.ts delete mode 100644 src/run/substituteFiles.ts delete mode 100644 utils/getStorageFiles.ts diff --git a/src/DuplicatedFilesCleaner.ts b/src/DuplicatedFilesCleaner.ts index 5472da9..59b0dce 100644 --- a/src/DuplicatedFilesCleaner.ts +++ b/src/DuplicatedFilesCleaner.ts @@ -1,47 +1,24 @@ import move from "./move.ts"; -import run from "./run/run.ts"; -import { join } from "../deps.ts"; import getInfo from "./getInfo.ts"; import copyData from "./copyData.ts"; import Constants from "../types/constants.type.ts"; -import substituteFiles from "./run/substituteFiles.ts"; import getFilesOfNodes from "../utils/getFilesOfNodes.ts"; -import getStorageFiles from "../utils/getStorageFiles.ts"; -import deleteUnusedFiles from "./run/deleteUnusedFiles.ts"; -import moveFilesToStorage from "./run/moveFilesToStorage.ts"; export default class DuplicatedFilesCleaner { homePath: string; - storageFolder: string; dockerIndexes: number[]; - filesToStripIfOnline: number; - filesToStripIfOffline: number; minFilesToConsiderShard: number; - // They are calculated only once when they are called and then cached. - #homeStoragePath: string | undefined = undefined; - constructor({ homePath, dockerIndexes, - storageFolder, - filesToStripIfOnline, - filesToStripIfOffline, minFilesToConsiderShard = 30, }: Omit) { this.homePath = homePath; this.dockerIndexes = dockerIndexes; - this.storageFolder = storageFolder; - this.filesToStripIfOnline = filesToStripIfOnline; - this.filesToStripIfOffline = filesToStripIfOffline; this.minFilesToConsiderShard = minFilesToConsiderShard; } - get homeStoragePath() { - if (this.#homeStoragePath) return this.#homeStoragePath; - return (this.#homeStoragePath = join(this.homePath, this.storageFolder)); - } - /** * Get information of nodes * @param nodes The nodes to get the info from. If not provided, it will get the info from all nodes. @@ -75,49 +52,9 @@ export default class DuplicatedFilesCleaner { * @param logProgressBar Whether to log a progress bar. Defaults to false. */ declare copyData: OmitThisParameter; - - /** Runs the whole process of deleting duplicated files. */ - declare run: OmitThisParameter; - - /** - * Move the files of the nodes to the storage folder using hard links. No changes are made to the nodes. - * @param useCachedStorageFiles If true, it will use the cached storage files. Default is false. - * @param useCachedFilesOfNodes If true, it will use the cached files of nodes. Default is false. - * @param shards The shards to substitute. Default: beacon. - */ - declare moveFilesToStorage: OmitThisParameter; - - /** - * Get the files in the storage directory. - * @param useCache Use the cache if it exists. Default is false. - * @param shards The shards to get the files from. Default: all shards present in instructions. - * @returns The files in the storage directory. - */ - declare getStorageFiles: OmitThisParameter; - - /** - * Substitute the files in the nodes with the files in the storage directory. - * @param useCachedStorageFiles If true, it will use the cached storage files. Default is false. - * @param useCachedFilesOfNodes If true, it will use the cached files of nodes. Default is false. - * @param useCachedDockersStatuses If true, it will use the cached docker statuses. Default is false. - * @param shards The shards to substitute. Default: beacon. - */ - declare substituteFiles: OmitThisParameter; - - /** - * Delete unused files in the storage folder. - * @param useCachedStorageFiles If true, it will use the cached storage files. Default is false. - * @param shards The shards to substitute. Default: beacon. - */ - declare deleteUnusedFiles: OmitThisParameter; } DuplicatedFilesCleaner.prototype.getInfo = getInfo; DuplicatedFilesCleaner.prototype.getFilesOfNodes = getFilesOfNodes; DuplicatedFilesCleaner.prototype.move = move; DuplicatedFilesCleaner.prototype.copyData = copyData; -DuplicatedFilesCleaner.prototype.run = run; -DuplicatedFilesCleaner.prototype.moveFilesToStorage = moveFilesToStorage; -DuplicatedFilesCleaner.prototype.getStorageFiles = getStorageFiles; -DuplicatedFilesCleaner.prototype.substituteFiles = substituteFiles; -DuplicatedFilesCleaner.prototype.deleteUnusedFiles = deleteUnusedFiles; diff --git a/src/copyData.ts b/src/copyData.ts index 62f0089..6eff670 100644 --- a/src/copyData.ts +++ b/src/copyData.ts @@ -32,12 +32,12 @@ export default async function copyData( const allLdbFiles = getFiles(fromShardPath); - const ldbFiles = allLdbFiles.slice(this.filesToStripIfOffline >= 0 ? this.filesToStripIfOffline : 0); + const ldbFiles = allLdbFiles; const otherFiles = [ // the files that are not ldb files ...[...Deno.readDirSync(fromShardPath)].filter((file) => !file.name.endsWith(".ldb")), // the files that were sliced from allLdbFiles - ...allLdbFiles.slice(0, this.filesToStripIfOffline >= 0 ? this.filesToStripIfOffline : 0), + ...allLdbFiles, ]; console.log("Emptying the destination directory"); diff --git a/src/getInfo.ts b/src/getInfo.ts index a711c31..aa0e7f3 100644 --- a/src/getInfo.ts +++ b/src/getInfo.ts @@ -10,10 +10,7 @@ export default async function getInfo( ) { const dockerStatus = await dockerPs(nodes instanceof Set ? [...nodes] : nodes); - const filesOfNodes = await this.getFilesOfNodes({ - nodes, - strip: false, - }); + const filesOfNodes = this.getFilesOfNodes({ nodes }); const nodesInfo: Record = {}; diff --git a/src/run/deleteUnusedFiles.ts b/src/run/deleteUnusedFiles.ts deleted file mode 100644 index ff35f61..0000000 --- a/src/run/deleteUnusedFiles.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { join } from "../../deps.ts"; -import { ShardsNames } from "../../mod.ts"; -import DuplicatedFilesCleaner from "../DuplicatedFilesCleaner.ts"; - -interface DeleteUnusedFilesOptions { - useCachedStorageFiles?: boolean; - shards?: ShardsNames[]; -} - -export default async function deleteUnusedFiles( - this: DuplicatedFilesCleaner, - { useCachedStorageFiles = false, shards = ["beacon"] }: DeleteUnusedFilesOptions = {} -) { - const storageFiles = this.getStorageFiles({ useCache: useCachedStorageFiles }); - - for (const shardName of shards) { - const shardStoragePath = join(this.homeStoragePath, shardName); - - console.group(`Deleting unused files in ${shardStoragePath}`); - - const promises: Promise[] = []; - - for (const file of storageFiles[shardName]) - if (file.used === 0) promises.push(Deno.remove(join(shardStoragePath, file.name)).catch(() => {})); - - await Promise.all(promises); - console.log(promises.length); - console.groupEnd(); - } -} diff --git a/src/run/moveFilesToStorage.ts b/src/run/moveFilesToStorage.ts deleted file mode 100644 index 9ea93e6..0000000 --- a/src/run/moveFilesToStorage.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { join } from "../../deps.ts"; -import { ShardsNames } from "../../mod.ts"; -import DuplicatedFilesCleaner from "../DuplicatedFilesCleaner.ts"; - -interface MoveFilesToStorageOptions { - useCachedStorageFiles?: boolean; - useCachedFilesOfNodes?: boolean; - shards?: ShardsNames[]; -} - -export default async function moveFilesToStorage( - this: DuplicatedFilesCleaner, - { - useCachedStorageFiles = false, - useCachedFilesOfNodes = false, - shards = ["beacon"], - }: MoveFilesToStorageOptions = {} -) { - const storageFiles = this.getStorageFiles({ useCache: useCachedStorageFiles }); - const filesOfNodes = await this.getFilesOfNodes({ useCache: useCachedFilesOfNodes, strip: true }); - - for (const shardName of shards) { - const shardStorageFiles = storageFiles[shardName]; - const shardStoragePath = join(this.homeStoragePath, shardName); - - console.group(shardName); - const numberOfFiles = shardStorageFiles.length; - console.log("Number of files:", numberOfFiles); - - for (const node of this.dockerIndexes) { - console.log(`Prossesing node ${node}`); - - const shardPath = join(this.homePath, `/node_data_${node}/mainnet/block/${shardName}`); - - await Promise.all( - filesOfNodes[shardName][node].map(async (file) => { - try { - // Create the hard link in the storage directory. - await Deno.link(join(shardPath, file.name), join(shardStoragePath, file.name)); - shardStorageFiles.push({ ...file, used: 0 }); - } catch { - // The file already exists. - } - }) - ); - } - - // Sort the files from high to low. - storageFiles[shardName] = storageFiles[shardName].sort((a, b) => b.number - a.number); - - console.log("Net change of files:", storageFiles[shardName].length - numberOfFiles); - console.groupEnd(); - } -} diff --git a/src/run/run.ts b/src/run/run.ts deleted file mode 100644 index c11cdfd..0000000 --- a/src/run/run.ts +++ /dev/null @@ -1,18 +0,0 @@ -import DuplicatedFilesCleaner from "../DuplicatedFilesCleaner.ts"; - -/** Runs the whole process of deleting duplicated files. */ -export default async function run(this: DuplicatedFilesCleaner) { - try { - // Move the new files to the storage directory. - await this.moveFilesToStorage(); - console.log(); - - // Substitute files in nodes with the ones in storage. - await this.substituteFiles({ useCachedFilesOfNodes: true, useCachedStorageFiles: true }); - - // Delete the files that are not used. - await this.deleteUnusedFiles({ useCachedStorageFiles: true }); - } catch (e) { - console.error(e); - } -} diff --git a/src/run/substituteFiles.ts b/src/run/substituteFiles.ts deleted file mode 100644 index a55f99e..0000000 --- a/src/run/substituteFiles.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { join } from "../../deps.ts"; -import { ShardsNames } from "../../mod.ts"; -import { dockerPs } from "../../utils/commands.ts"; -import DuplicatedFilesCleaner from "../DuplicatedFilesCleaner.ts"; - -interface SubstituteFilesOptions { - useCachedStorageFiles?: boolean; - useCachedFilesOfNodes?: boolean; - useCachedDockersStatuses?: boolean; - shards?: ShardsNames[]; -} - -export default async function substituteFiles( - this: DuplicatedFilesCleaner, - { - shards = ["beacon"], - useCachedStorageFiles = false, - useCachedFilesOfNodes = false, - useCachedDockersStatuses = false, - }: SubstituteFilesOptions = {} -) { - const storageFiles = this.getStorageFiles({ useCache: useCachedStorageFiles }); - const filesOfNodes = await this.getFilesOfNodes({ useCache: useCachedFilesOfNodes }); - const dockerStatuses = await dockerPs(this.dockerIndexes, useCachedDockersStatuses); - - for (const shardName of shards) { - console.group(`Substituting files in ${shardName}`); - - const shardStoragePath = join(this.homeStoragePath, shardName); - - for (const node of this.dockerIndexes) { - console.group("Prossesing node", node); - - const shardStorageFiles = storageFiles[shardName]; - const defaultStorageFile = shardStorageFiles[0]; - const shardPath = join(this.homePath, `/node_data_${node}/mainnet/block/${shardName}`); - - let filesProcessed = 0; - - // if filesToStripIfOnline is negative, only deal with offline nodes - if (this.filesToStripIfOnline < 0 && dockerStatuses[node].running) - console.log("Skipping node", node, "because it is online and filesToStripIfOffline is negative."); - else - await Promise.all( - filesOfNodes[shardName][node].map(async (file) => { - // If the file is not in the storage directory, skip it. - // Because it is sorted from high to low, it will return if the files are already lower. - let storageFile = defaultStorageFile; - for (storageFile of shardStorageFiles) { - if (storageFile.number === file.number) break; - if (storageFile.number < file.number) return; - } - - const from = join(shardStoragePath, file.name); - const to = join(shardPath, file.name); - - try { - await Deno.remove(to); - await Deno.link(from, to); - } catch { - // - } - storageFile.used++; - filesProcessed++; - }) - ); - - console.log("Files processed:", filesProcessed, "\n"); - console.groupEnd(); - } - - console.log( - "Total files removed:", - Object.values(storageFiles).reduce( - (total, files) => - total + - files.reduce( - (subTotal, file) => - file.used <= 0 - ? // if the file isn't used at least twice, it was really not removed. - subTotal - : // The first file is not removed, only linked, so it is not counted. - subTotal + file.used - 1, - 0 - ), - 0 - ), - "\n" - ); - - console.groupEnd(); - } -} diff --git a/types/constants.type.ts b/types/constants.type.ts index 80e6b74..0e01836 100644 --- a/types/constants.type.ts +++ b/types/constants.type.ts @@ -3,10 +3,7 @@ export type ValidatorPublicKeys = Record; export default interface Constants { homePath: string; fileSystem?: string; - storageFolder: string; dockerIndexes: number[]; - filesToStripIfOnline: number; - filesToStripIfOffline: number; minFilesToConsiderShard?: number; validatorPublicKeys?: ValidatorPublicKeys; } diff --git a/utils/getFilesOfNodes.ts b/utils/getFilesOfNodes.ts index 4e0b064..b7e98cc 100644 --- a/utils/getFilesOfNodes.ts +++ b/utils/getFilesOfNodes.ts @@ -1,5 +1,4 @@ import { join } from "../deps.ts"; -import { dockerPs } from "../utils/commands.ts"; import getFiles, { LDBFile } from "../utils/getFiles.ts"; import { shardsNames, ShardsNames } from "../types/shards.type.ts"; import DuplicatedFilesCleaner from "../src/DuplicatedFilesCleaner.ts"; @@ -7,31 +6,20 @@ import DuplicatedFilesCleaner from "../src/DuplicatedFilesCleaner.ts"; let cached = false; const filesOfNodes = {} as Record>; -export default async function getFilesOfNodes( +export default function getFilesOfNodes( this: DuplicatedFilesCleaner, - { strip = true, useCache = false, nodes = this.dockerIndexes as Set | (number | string)[] } = {} + { useCache = false, nodes = this.dockerIndexes as Set | (number | string)[] } = {} ) { if (cached && useCache) return filesOfNodes; - const dockerStatuses = await dockerPs(this.dockerIndexes, useCache); - const newInstructions = shardsNames.map((shardName) => ({ shardName, nodes: nodes })); for (const { shardName, nodes } of newInstructions) { filesOfNodes[shardName] = {}; for (const node of nodes) { - const stripValue = strip - ? // strip only if the node is online and filesToStripIfOnline is positive - dockerStatuses[node].running && this.filesToStripIfOnline >= 0 - ? this.filesToStripIfOnline - : // or if the node is offline and filesToStripIfOffline is positive - !dockerStatuses[node].running && this.filesToStripIfOffline >= 0 - ? this.filesToStripIfOffline - : 0 - : 0; filesOfNodes[shardName][node] = getFiles( join(this.homePath, `/node_data_${node}/mainnet/block/${shardName}`) - ).slice(stripValue); + ); } } diff --git a/utils/getStorageFiles.ts b/utils/getStorageFiles.ts deleted file mode 100644 index efcce5b..0000000 --- a/utils/getStorageFiles.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { join } from "../deps.ts"; -import { shardsNames } from "../mod.ts"; -import getFiles, { LDBFile } from "../utils/getFiles.ts"; -import DuplicatedFilesCleaner from "../src/DuplicatedFilesCleaner.ts"; - -export type StorageFile = LDBFile & { used: number }; -export type StorageFiles = Record; - -let cached = false; -const storageFilesCache: StorageFiles = {}; - -export default function getStorageFiles( - this: DuplicatedFilesCleaner, - { shards = shardsNames, useCache = false } = {} -): StorageFiles { - if (cached && useCache) return storageFilesCache; - - for (const shardName of shards) { - const shardStoragePath = join(this.homeStoragePath, shardName); - - // If it doesn't exist, create it. - try { - if (Deno.statSync(shardStoragePath).isDirectory === false) { - // Delete the file and create the directory. - Deno.removeSync(shardStoragePath); - throw new Error(`${shardStoragePath} is not a directory. Deleted it.`); - } - } catch (e) { - console.error("Handled error:", e.message); - Deno.mkdirSync(shardStoragePath, { recursive: true }); - } - - storageFilesCache[shardName] = getFiles(shardStoragePath).map((file) => ({ ...file, used: 0 })); - } - - cached = true; - return storageFilesCache; -}