From 882765275383cad6cac5f8888b6655ba6312c291 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Sun, 2 Jun 2024 12:52:48 +0300 Subject: [PATCH 1/2] Add functions for checksums and streams --- index.ts | 3 ++ package.json | 2 + src/utils/checksumUtils.spec.ts | 48 ++++++++++++++++++++++ src/utils/checksumUtils.ts | 33 ++++++++++++++++ src/utils/streamUtils.spec.ts | 66 +++++++++++++++++++++++++++++++ src/utils/streamUtils.ts | 70 +++++++++++++++++++++++++++++++++ 6 files changed, 222 insertions(+) create mode 100644 src/utils/checksumUtils.spec.ts create mode 100644 src/utils/checksumUtils.ts create mode 100644 src/utils/streamUtils.spec.ts create mode 100644 src/utils/streamUtils.ts diff --git a/index.ts b/index.ts index 5dba839..1872695 100644 --- a/index.ts +++ b/index.ts @@ -111,3 +111,6 @@ export { export { waitAndRetry } from './src/utils/waitUtils' export * from './src/observability/observabilityTypes' + +export * from './src/utils/checksumUtils' +export * from './src/utils/streamUtils' diff --git a/package.json b/package.json index c5fcdf9..7a6667f 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ }, "devDependencies": { "@types/node": "^20.12.8", + "@types/tmp": "^0.2.6", "@typescript-eslint/eslint-plugin": "^7.8.0", "@typescript-eslint/parser": "^7.8.0", "@vitest/coverage-v8": "1.6.0", @@ -51,6 +52,7 @@ "eslint-plugin-import": "^2.29.1", "eslint-plugin-vitest": "0.4.1", "prettier": "^3.2.5", + "tmp": "^0.2.3", "typescript": "^5.4.5", "vitest": "1.6.0" } diff --git a/src/utils/checksumUtils.spec.ts b/src/utils/checksumUtils.spec.ts new file mode 100644 index 0000000..6370f76 --- /dev/null +++ b/src/utils/checksumUtils.spec.ts @@ -0,0 +1,48 @@ +import { Readable } from 'node:stream' + +import { + generateChecksumForBufferOrString, + generateChecksumForObject, + generateChecksumForReadable, +} from './checksumUtils' + +const testObject = { + someField: 123, + someOtherField: 'ferfref', + nestedField: { + level2: { + level3: { + level4: { + value: 'some string', + }, + }, + }, + }, +} + +describe('checksumUtils', () => { + describe('generateChecksumForBufferOrString', () => { + it('generates checksum', () => { + const checksum = generateChecksumForBufferOrString(Buffer.from('some test string value')) + + expect(checksum).toBe('bfd5bbd64f6b83abe1d7d3b06221eb3a') + }) + }) + + describe('generateChecksumForObject', () => { + it('generates checksum', () => { + const checksum = generateChecksumForObject(testObject) + + expect(checksum).toBe('9d15391c6fea84d122e0b22f7b9eb90f') + }) + }) + + describe('generateChecksumForReadable', () => { + it('generates checksum', async () => { + const readable = Readable.from(JSON.stringify(testObject)) + const checksum = await generateChecksumForReadable(readable) + + expect(checksum).toBe('9d15391c6fea84d122e0b22f7b9eb90f') + }) + }) +}) diff --git a/src/utils/checksumUtils.ts b/src/utils/checksumUtils.ts new file mode 100644 index 0000000..d7d6086 --- /dev/null +++ b/src/utils/checksumUtils.ts @@ -0,0 +1,33 @@ +import type { BinaryLike } from 'node:crypto' +import { createHash } from 'node:crypto' +import type { Readable } from 'node:stream' + +const HASH_ALGORITHM = 'md5' + +export function generateChecksumForBufferOrString(data: BinaryLike): string { + return createHash(HASH_ALGORITHM).update(data).digest('hex') +} + +export function generateChecksumForObject(object: object): string { + const objectAsString = JSON.stringify(object) + return generateChecksumForBufferOrString(objectAsString) +} + +export function generateChecksumForReadable(readable: Readable): Promise { + return new Promise((resolve, reject) => { + const hashCreator = createHash(HASH_ALGORITHM) + readable.on('data', function (data) { + // eslint-disable-next-line @typescript-eslint/no-unsafe-argument + hashCreator.update(Buffer.from(data)) + }) + + readable.on('end', function () { + const hash = hashCreator.digest('hex') + resolve(hash) + }) + readable.on('error', function (err) { + /* c8 ignore next 1 */ + reject(err) + }) + }) +} diff --git a/src/utils/streamUtils.spec.ts b/src/utils/streamUtils.spec.ts new file mode 100644 index 0000000..298a8e4 --- /dev/null +++ b/src/utils/streamUtils.spec.ts @@ -0,0 +1,66 @@ +import { Readable } from 'node:stream' + +import tmp from 'tmp' +import { expect } from 'vitest' + +import { generateChecksumForReadable } from './checksumUtils' +import { FsReadableProvider } from './streamUtils' + +const testObject = { + someField: 123, + someOtherField: 'ferfref', + nestedField: { + level2: { + level3: { + level4: { + value: 'some string', + }, + }, + }, + }, +} + +describe('streamUtils', () => { + describe('persistReadableToFs', () => { + it('can create extra readables after persisting', async () => { + const sourceReadable = Readable.from(JSON.stringify(testObject)) + const targetFile = tmp.tmpNameSync() + + const provider = await FsReadableProvider.persistReadableToFs({ + sourceReadable, + targetFile: targetFile, + }) + + const newReadable = await provider.createStream() + const newReadableChecksum = await generateChecksumForReadable(newReadable) + expect(newReadableChecksum).toBe('9d15391c6fea84d122e0b22f7b9eb90f') + + const newReadable2 = await provider.createStream() + const newReadableChecksum2 = await generateChecksumForReadable(newReadable2) + expect(newReadableChecksum2).toBe('9d15391c6fea84d122e0b22f7b9eb90f') + }) + }) + + describe('destroy', () => { + it('deletes the file', async () => { + expect.assertions(3) + const sourceReadable = Readable.from(JSON.stringify(testObject)) + const targetFile = tmp.tmpNameSync() + + const provider = await FsReadableProvider.persistReadableToFs({ + sourceReadable, + targetFile: targetFile, + }) + + expect(await provider.fileExists()).toBe(true) + await provider.destroy() + expect(await provider.fileExists()).toBe(false) + + try { + await provider.createStream() + } catch (err) { + expect((err as Error).message).toMatch(/was already deleted/) + } + }) + }) +}) diff --git a/src/utils/streamUtils.ts b/src/utils/streamUtils.ts new file mode 100644 index 0000000..cd10553 --- /dev/null +++ b/src/utils/streamUtils.ts @@ -0,0 +1,70 @@ +import { F_OK } from 'node:constants' +import { createWriteStream, createReadStream, access } from 'node:fs' +import { unlink } from 'node:fs/promises' +import type { Readable } from 'node:stream' +import { pipeline } from 'node:stream' + +export type ReadableProvider = { + createStream(): Readable + destroy(): Promise +} + +export type PersistToFsOptions = { + targetFile: string + sourceReadable: Readable +} + +export type FsReadableProviderOptions = { + storageFile: string +} + +export class FsReadableProvider implements ReadableProvider { + public readonly storageFile: string + constructor(options: FsReadableProviderOptions) { + this.storageFile = options.storageFile + } + + fileExists(): Promise { + return new Promise((resolve) => { + access(this.storageFile, F_OK, (err) => { + return resolve(!err) + }) + }) + } + + async createStream(): Promise { + if (!(await this.fileExists())) { + throw new Error(`File ${this.storageFile} was already deleted.`) + } + + return createReadStream(this.storageFile) + } + + destroy(): Promise { + return unlink(this.storageFile) + } + + protected async persist(sourceReadable: Readable): Promise { + const writable = createWriteStream(this.storageFile) + return new Promise((resolve, reject) => { + pipeline(sourceReadable, writable, (err) => { + if (err) { + /* c8 ignore next 1 */ + reject(err) + } else { + resolve() + } + }) + }) + } + + public static async persistReadableToFs( + options: PersistToFsOptions, + ): Promise { + const provider = new FsReadableProvider({ + storageFile: options.targetFile, + }) + await provider.persist(options.sourceReadable) + return provider + } +} From 4577ceb36aa076da9db3eb8df3d50b9636590059 Mon Sep 17 00:00:00 2001 From: Igor Savin Date: Mon, 30 Sep 2024 13:37:34 +0300 Subject: [PATCH 2/2] Fix monorepo logger conf type --- src/logging/loggerConfigResolver.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging/loggerConfigResolver.ts b/src/logging/loggerConfigResolver.ts index 4c03cc2..6f78b2c 100644 --- a/src/logging/loggerConfigResolver.ts +++ b/src/logging/loggerConfigResolver.ts @@ -15,7 +15,7 @@ export type MonorepoAppLoggerConfig = AppLoggerConfig & { } /* c8 ignore next 8 */ -export function resolveMonorepoLogger(appConfig: AppLoggerConfig): Logger { +export function resolveMonorepoLogger(appConfig: MonorepoAppLoggerConfig): Logger { if (appConfig.nodeEnv !== 'development') { return resolveLogger(appConfig) }