From 4685125f7ff4de6ab43d2163ce51dd19bce4ea16 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 18:02:13 +0100 Subject: [PATCH 01/12] feat: Update rxjs --- package.json | 28 +- src/Etl.ts | 232 +++++----- src/extractors/JsonExtractor.ts | 34 +- src/interfaces/Extractor.ts | 6 +- src/interfaces/GeneralTransformer.ts | 6 +- src/interfaces/Loader.ts | 4 +- src/interfaces/Transformer.ts | 4 +- src/loaders/ConsoleLoader.ts | 12 +- src/transformers/MapTransformer.ts | 16 +- src/transformers/MatchMergeTransformer.ts | 54 +-- test/Etl.spec.ts | 528 ++++++++++++---------- test/JsonExtractor.spec.ts | 99 ++-- test/MapTransformer.spec.ts | 35 +- test/MatchMergeTransformer.spec.ts | 40 +- 14 files changed, 592 insertions(+), 506 deletions(-) diff --git a/package.json b/package.json index cf1d5c7..c8fe8f5 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "typescript" ], "engines": { - "node": ">=6" + "node": ">=16" }, "repository": { "type": "git", @@ -30,20 +30,20 @@ "author": "Christoph Bühler ", "license": "MIT", "devDependencies": { - "@smartive/tslint-config": "^2.0.0", - "@types/jest": "^22.0.1", - "del-cli": "^1.1.0", - "jest": "^22.1.1", - "semantic-release": "^12.2.2", - "ts-jest": "^22.0.1", - "tslint": "^5.9.1", - "tsutils": "^2.18.0", - "typedoc": "^0.10.0", - "typescript": "^2.6.2" + "@smartive/tslint-config": "^7.0.1", + "@types/jest": "^29.2.4", + "del-cli": "^5.0.0", + "jest": "^29.3.1", + "semantic-release": "^19.0.5", + "ts-jest": "^29.0.3", + "tslint": "^5.20.1", + "tsutils": "^3.21.0", + "typedoc": "^0.23.23", + "typescript": "^4.9.4" }, "dependencies": { - "@types/node": "^9.3.0", - "rxjs": "^5.5.6", - "tslib": "^1.8.1" + "@types/node": "^18.11.17", + "rxjs": "^7.8.0", + "tslib": "^2.4.1" } } diff --git a/src/Etl.ts b/src/Etl.ts index 47b9637..80892e3 100644 --- a/src/Etl.ts +++ b/src/Etl.ts @@ -1,15 +1,15 @@ -import { Observable } from 'rxjs'; +import { EMPTY, merge, mergeMap, Observable, throwError } from "rxjs"; -import { Extractor } from './interfaces/Extractor'; -import { GeneralTransformer } from './interfaces/GeneralTransformer'; -import { Loader } from './interfaces/Loader'; -import { Transformer } from './interfaces/Transformer'; -import { MapTransformer } from './transformers/MapTransformer'; +import { Extractor } from "./interfaces/Extractor"; +import { GeneralTransformer } from "./interfaces/GeneralTransformer"; +import { Loader } from "./interfaces/Loader"; +import { Transformer } from "./interfaces/Transformer"; +import { MapTransformer } from "./transformers/MapTransformer"; export enum EtlState { - Running, - Stopped, - Error, + Running, + Stopped, + Error, } /** @@ -19,105 +19,121 @@ export enum EtlState { * This processor is modular, you can find other implemented loaders and extractors in the README */ export class Etl { - private _extractors: Extractor[] = []; - private _generalTransformers: GeneralTransformer[] = []; - private _transformers: Transformer[] = []; - private _loaders: Loader[] = []; - private _state: EtlState = EtlState.Stopped; - private _context: any = null; - - public constructor(context?: any) { - this.setContext(context); + private _extractors: Extractor[] = []; + private _generalTransformers: GeneralTransformer[] = []; + private _transformers: Transformer[] = []; + private _loaders: Loader[] = []; + private _state: EtlState = EtlState.Stopped; + private _context: any = null; + + public constructor(context?: any) { + this.setContext(context); + } + + public get extractors(): Extractor[] { + return this._extractors; + } + + public get generalTransformers(): GeneralTransformer[] { + return this._generalTransformers; + } + + public get transformers(): Transformer[] { + return this._transformers; + } + + public get loaders(): Loader[] { + return this._loaders; + } + + public get state(): EtlState { + return this._state; + } + + public setContext(context: any): this { + if (this._state !== EtlState.Stopped) { + this._state = EtlState.Error; + throw new Error("Tried to set context on invalid state."); } - - public get extractors(): Extractor[] { - return this._extractors; - } - - public get generalTransformers(): GeneralTransformer[] { - return this._generalTransformers; - } - - public get transformers(): Transformer[] { - return this._transformers; - } - - public get loaders(): Loader[] { - return this._loaders; - } - - public get state(): EtlState { - return this._state; - } - - public setContext(context: any): this { - if (this._state !== EtlState.Stopped) { - this._state = EtlState.Error; - throw new Error('Tried to set context on invalid state.'); - } - this._context = context; - return this; - } - - public addExtractor(extract: Extractor): Etl { - this._extractors.push(extract); - return this; - } - - public addGeneralTransformer(transformer: GeneralTransformer): Etl { - this._generalTransformers.push(transformer); - return this; - } - - public addTransformer(transformer: Transformer): Etl { - this.addGeneralTransformer(new MapTransformer(transformer)); - this._transformers.push(transformer); - return this; - } - - public addLoader(loader: Loader): Etl { - this._loaders.push(loader); - return this; - } - - /** - * Starts the etl process. First, all extractors are run in parallel and deliver their results into an observable. - * Once the buffer gets a result, it transfers all objects through the transformers (one by one). - * After that, the transformed results are run through all loaders in parallel. - * - * @returns {Observable} Observable that completes when the process is finished, - * during the "next" process step you get update on how many are processed yet. - * Throws when any step produces an error. - */ - public start(observable: Observable = Observable.empty()): Observable { - this._state = EtlState.Running; - - const o: Observable = Observable - .merge(observable, ...this._extractors.map(extractor => extractor.read(this._context))); - - return this._generalTransformers - .reduce((observable, transformer) => transformer.process(observable, this._context), o) - .flatMap(object => Observable.merge(...this._loaders.map(loader => loader.write(object, this._context)))) - .do( - () => { }, - (err) => { - this._state = EtlState.Error; - return Observable.throw(err); - }, - () => { - this._state = EtlState.Stopped; - }, - ); - } - - /** - * Resets the whole Etl object. Deletes all modifiers and resets the state. - */ - public reset(): void { - this._extractors = []; - this._transformers = []; - this._loaders = []; + this._context = context; + return this; + } + + public addExtractor(extract: Extractor): Etl { + this._extractors.push(extract); + return this; + } + + public addGeneralTransformer(transformer: GeneralTransformer): Etl { + this._generalTransformers.push(transformer); + return this; + } + + public addTransformer(transformer: Transformer): Etl { + this.addGeneralTransformer(new MapTransformer(transformer)); + this._transformers.push(transformer); + return this; + } + + public addLoader(loader: Loader): Etl { + this._loaders.push(loader); + return this; + } + + /** + * Starts the etl process. First, all extractors are run in parallel and deliver their results into an observable. + * Once the buffer gets a result, it transfers all objects through the transformers (one by one). + * After that, the transformed results are run through all loaders in parallel. + * + * @returns {Observable} Observable that completes when the process is finished, + * during the "next" process step you get update on how many are processed yet. + * Throws when any step produces an error. + */ + public start(observable: Observable = EMPTY): Observable { + this._state = EtlState.Running; + + const o: Observable = merge( + observable, + ...this._extractors.map((extractor) => extractor.read(this._context)) + ); + + const pipe = this._generalTransformers + .reduce( + (observable, transformer) => + transformer.process(observable, this._context), + o + ) + .pipe( + mergeMap((object) => + merge( + ...this._loaders.map((loader) => + loader.write(object, this._context) + ) + ) + ) + ); + + pipe.subscribe({ + error: (err) => { + this._state = EtlState.Error; + return throwError(() => err); + }, + complete: () => { this._state = EtlState.Stopped; - this._context = null; - } + }, + }); + + return pipe; + } + + /** + * Resets the whole Etl object. Deletes all modifiers and resets the state. + */ + public reset(): void { + this._extractors = []; + this._transformers = []; + this._loaders = []; + this._state = EtlState.Stopped; + this._context = null; + } } diff --git a/src/extractors/JsonExtractor.ts b/src/extractors/JsonExtractor.ts index 5d9ba44..5a3e956 100644 --- a/src/extractors/JsonExtractor.ts +++ b/src/extractors/JsonExtractor.ts @@ -1,27 +1,27 @@ -import { resolve } from 'path'; -import { Observable } from 'rxjs'; +import { resolve } from "path"; +import { from, Observable, throwError } from "rxjs"; -import { Extractor } from '../interfaces/Extractor'; +import { Extractor } from "../interfaces/Extractor"; /** * Extractor that reads a JSON file at a given filepath. The path is resolved relatively to the running tasks root dir. */ export class JsonExtractor implements Extractor { - private filePath: string; + private filePath: string; - constructor(filePath: string) { - this.filePath = resolve(process.cwd(), filePath); - } + constructor(filePath: string) { + this.filePath = resolve(process.cwd(), filePath); + } - public read(): Observable { - try { - const content = require(this.filePath); - if (!(content instanceof Array) && content.constructor !== Array) { - return Observable.from([content]); - } - return Observable.from(content); - } catch (e) { - return Observable.throw(e); - } + public read(): Observable { + try { + const content = require(this.filePath); + if (!(content instanceof Array) && content.constructor !== Array) { + return from([content]); + } + return from(content); + } catch (e) { + return throwError(() => e); } + } } diff --git a/src/interfaces/Extractor.ts b/src/interfaces/Extractor.ts index 52da085..321cd74 100644 --- a/src/interfaces/Extractor.ts +++ b/src/interfaces/Extractor.ts @@ -1,11 +1,11 @@ -import { Observable } from 'rxjs'; +import { Observable } from "rxjs"; /** * Extractor interface. Only provides "read()" method that returns an observable with the result. - * + * * @export * @interface Extractor */ export interface Extractor { - read(context?: any): Observable; + read(context?: any): Observable; } diff --git a/src/interfaces/GeneralTransformer.ts b/src/interfaces/GeneralTransformer.ts index 2f91ac5..e50776c 100644 --- a/src/interfaces/GeneralTransformer.ts +++ b/src/interfaces/GeneralTransformer.ts @@ -1,12 +1,12 @@ -import { Observable } from 'rxjs'; +import { Observable } from "rxjs"; /** * GeneralTransformer interface. Provides a "process(observable)" method that processes an observable. * Represents a stage in the ETL pipeline. - * + * * @export * @interface GeneralTransformer */ export interface GeneralTransformer { - process(observable: Observable, context?: any): Observable; + process(observable: Observable, context?: any): Observable; } diff --git a/src/interfaces/Loader.ts b/src/interfaces/Loader.ts index 7f838b1..b31cd8e 100644 --- a/src/interfaces/Loader.ts +++ b/src/interfaces/Loader.ts @@ -1,8 +1,8 @@ -import { Observable } from 'rxjs'; +import { Observable } from "rxjs"; /** * Loader interface. Provides ".write(obj)" method that returns an observable with the loaded value. */ export interface Loader { - write(object: any, context?: any): Observable; + write(object: any, context?: any): Observable; } diff --git a/src/interfaces/Transformer.ts b/src/interfaces/Transformer.ts index b92bcd0..9a96482 100644 --- a/src/interfaces/Transformer.ts +++ b/src/interfaces/Transformer.ts @@ -1,9 +1,9 @@ -import { Observable } from 'rxjs'; +import { Observable } from "rxjs"; /** * Transformer interface. Only provides ".process(obj)" that returns an Observable with * the new result (array will be flattend). */ export interface Transformer { - process(object: any, context?: any): Observable; + process(object: any, context?: any): Observable; } diff --git a/src/loaders/ConsoleLoader.ts b/src/loaders/ConsoleLoader.ts index 0a0a35b..f08d2e6 100644 --- a/src/loaders/ConsoleLoader.ts +++ b/src/loaders/ConsoleLoader.ts @@ -1,6 +1,6 @@ -import { Observable } from 'rxjs'; +import { Observable, of } from "rxjs"; -import { Loader } from '../interfaces/Loader'; +import { Loader } from "../interfaces/Loader"; /** * Loader that outputs everything to the console. @@ -10,8 +10,8 @@ import { Loader } from '../interfaces/Loader'; * @implements {Loader} */ export class ConsoleLoader implements Loader { - public write(object: any): Observable { - console.log(object); - return Observable.of(object); - } + public write(object: any): Observable { + console.log(object); + return of(object); + } } diff --git a/src/transformers/MapTransformer.ts b/src/transformers/MapTransformer.ts index 6430510..a123381 100644 --- a/src/transformers/MapTransformer.ts +++ b/src/transformers/MapTransformer.ts @@ -1,12 +1,14 @@ -import { Observable } from 'rxjs'; +import { mergeMap, Observable } from "rxjs"; -import { GeneralTransformer } from '../interfaces/GeneralTransformer'; -import { Transformer } from '../interfaces/Transformer'; +import { GeneralTransformer } from "../interfaces/GeneralTransformer"; +import { Transformer } from "../interfaces/Transformer"; export class MapTransformer implements GeneralTransformer { - constructor(private transformer: Transformer) { } + constructor(private transformer: Transformer) {} - process(observable: Observable, context?: any): Observable { - return observable.flatMap(o => this.transformer.process(o, context)); - } + process(observable: Observable, context?: any): Observable { + return observable.pipe( + mergeMap((o) => this.transformer.process(o, context)) + ); + } } diff --git a/src/transformers/MatchMergeTransformer.ts b/src/transformers/MatchMergeTransformer.ts index a69fa63..dfe1e32 100644 --- a/src/transformers/MatchMergeTransformer.ts +++ b/src/transformers/MatchMergeTransformer.ts @@ -1,35 +1,35 @@ -import { Observable } from 'rxjs'; +import { from, mergeMap, Observable, reduce } from "rxjs"; -import { GeneralTransformer } from '../interfaces/GeneralTransformer'; +import { GeneralTransformer } from "../interfaces/GeneralTransformer"; export abstract class MatchMergeTransformer implements GeneralTransformer { + public process(observable: Observable, context?: any): Observable { + const matchMerge = (merged: any[], o2: any) => { + return this.matchMerge(merged, o2, context); + }; + return observable.pipe(reduce(matchMerge, [])).pipe( + mergeMap((merged) => { + return from(merged); + }) + ); + } - public process(observable: Observable, context?: any): Observable { - const matchMerge = (merged: any[], o2: any) => { - return this.matchMerge(merged, o2, context); - }; - return observable.reduce(matchMerge, []).flatMap((merged) => { - return Observable.from(merged); - }); - } - - protected abstract match(o1: any, o2: any, context?: any): boolean; + protected abstract match(o1: any, o2: any, context?: any): boolean; - protected abstract merge(o1: any, o2: any, context?: any): any; + protected abstract merge(o1: any, o2: any, context?: any): any; - private matchMerge(merged: any[], o2: any, context?: any): any[] { - for (let i = 0; i < merged.length; i++) { - if (this.match(merged[i], o2, context)) { - const o1 = merged.splice(i, 1)[0]; - // tslint:disable-next-line - o2 = this.merge(o1, o2, context); - // Try to merge the merged element with the remaining elements, - // starting from the current position - i--; - } - } - merged.push(o2); - return merged; + private matchMerge(merged: any[], o2: any, context?: any): any[] { + for (let i = 0; i < merged.length; i++) { + if (this.match(merged[i], o2, context)) { + const o1 = merged.splice(i, 1)[0]; + // tslint:disable-next-line + o2 = this.merge(o1, o2, context); + // Try to merge the merged element with the remaining elements, + // starting from the current position + i--; + } } - + merged.push(o2); + return merged; + } } diff --git a/test/Etl.spec.ts b/test/Etl.spec.ts index aaccc7e..7a8c7e3 100644 --- a/test/Etl.spec.ts +++ b/test/Etl.spec.ts @@ -1,259 +1,323 @@ -import { Observable } from 'rxjs'; +import { from, of, reduce, throwError } from "rxjs"; -import { Etl, EtlState, Extractor, JsonExtractor, Loader, MatchMergeTransformer, Transformer } from '../src'; +import { + Etl, + EtlState, + Extractor, + JsonExtractor, + Loader, + MatchMergeTransformer, + Transformer, +} from "../src"; -describe('Etl', () => { +describe("Etl", () => { + let etl: Etl; + let extractor: Extractor = new JsonExtractor( + "./test/.testdata/json-extractor.object.json" + ); + let arrayExtractor: Extractor = new JsonExtractor( + "./test/.testdata/json-extractor.array.json" + ); + let matchMergeExtractor: Extractor = new JsonExtractor( + "./test/.testdata/match-merge.json" + ); + let o; + let dummyExtractor: Extractor; + let dummyTransformer: Transformer; + let dummyLoader: Loader; - let etl: Etl; - let extractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.object.json'); - let arrayExtractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.array.json'); - let matchMergeExtractor: Extractor = new JsonExtractor('./test/.testdata/match-merge.json'); - let o; - let dummyExtractor: Extractor; - let dummyTransformer: Transformer; - let dummyLoader: Loader; + beforeEach(() => { + etl = new Etl(); - beforeEach(() => { - etl = new Etl(); + o = { _id: "001" }; - o = {_id: "001"}; + dummyExtractor = { + read: () => of(o), + }; + dummyExtractor.read = jest.fn(dummyExtractor.read); - dummyExtractor = { - read: () => Observable.of(o), - }; - dummyExtractor.read = jest.fn(dummyExtractor.read); + dummyTransformer = { + process: (o) => of(o), + }; + dummyTransformer.process = jest.fn(dummyTransformer.process); - dummyTransformer = { - process: o => Observable.of(o), - }; - dummyTransformer.process = jest.fn(dummyTransformer.process); + dummyLoader = { + write: (o) => of(o), + }; + dummyLoader.write = jest.fn(dummyLoader.write); + }); - dummyLoader = { - write: o => Observable.of(o), - }; - dummyLoader.write = jest.fn(dummyLoader.write); + it("should initialize with correct default params", () => { + expect(etl.state).toBe(EtlState.Stopped); + expect(etl.extractors.length).toBe(0); + expect(etl.transformers.length).toBe(0); + expect(etl.loaders.length).toBe(0); + }); + it("should reset correctly", () => { + etl.addExtractor({ + read: function () { + return of(null); + }, }); - it('should initialize with correct default params', () => { - expect(etl.state).toBe(EtlState.Stopped); - expect(etl.extractors.length).toBe(0); - expect(etl.transformers.length).toBe(0); - expect(etl.loaders.length).toBe(0); - }); + expect(etl.extractors.length).toBe(1); + etl.reset(); + expect(etl.extractors.length).toBe(0); + }); - it('should reset correctly', () => { - etl.addExtractor({ - read: function() { - return null; - } - }); + it("should pass context down the pipeline", (done) => { + const context = 1; + etl = new Etl(context); + etl + .addExtractor(dummyExtractor) + .addTransformer(dummyTransformer) + .addLoader(dummyLoader) + .start() + .subscribe({ + complete: () => { + expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); + expect((dummyTransformer.process as any).mock.calls[0]).toContain( + context + ); + expect((dummyLoader.write as any).mock.calls[0]).toContain(context); + done(); + }, + }); + }); - expect(etl.extractors.length).toBe(1); - etl.reset(); - expect(etl.extractors.length).toBe(0); - }); + it("should pass newly set context down the pipeline", (done) => { + const context = 1; + etl + .addExtractor(dummyExtractor) + .addTransformer(dummyTransformer) + .addLoader(dummyLoader) + .setContext(context) + .start() + .subscribe({ + complete: () => { + expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); + expect((dummyTransformer.process as any).mock.calls[0]).toContain( + context + ); + expect((dummyLoader.write as any).mock.calls[0]).toContain(context); + done(); + }, + }); + }); - it('should pass context down the pipeline', done => { - const context = 1; - etl = new Etl(context); - etl - .addExtractor(dummyExtractor) - .addTransformer(dummyTransformer) - .addLoader(dummyLoader) - .start() - .subscribe(null, null, () => { - expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); - expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); - expect((dummyLoader.write as any).mock.calls[0]).toContain(context); - done(); - }); - }); + it("should process simple object", (done) => { + etl + .addExtractor(extractor) + .addLoader(dummyLoader) + .start() + .subscribe({ + complete: () => { + expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ + foo: "bar", + hello: "world", + }); + done(); + }, + }); + }); - it('should pass newly set context down the pipeline', done => { - const context = 1; - etl - .addExtractor(dummyExtractor) - .addTransformer(dummyTransformer) - .addLoader(dummyLoader) - .setContext(context) - .start() - .subscribe(null, null, () => { - expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); - expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); - expect((dummyLoader.write as any).mock.calls[0]).toContain(context); - done(); - }); - }); + it("should process simple array", (done) => { + etl + .addExtractor(arrayExtractor) + .addLoader(dummyLoader) + .start() + .subscribe({ + complete: () => { + expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ + objId: 1, + name: "foobar", + }); + expect((dummyLoader.write as any).mock.calls[1][0]).toMatchObject({ + objId: 2, + name: "hello world", + }); + expect((dummyLoader.write as any).mock.calls[2][0]).toMatchObject({ + objId: 3, + name: "third test", + }); + done(); + }, + }); + }); - it('should process simple object', done => { - etl - .addExtractor(extractor) - .addLoader(dummyLoader) - .start() - .subscribe(null, null, () => { - expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ foo: 'bar', hello: 'world' }); - done(); - }); - }); + it("should call error on extractor error", (done) => { + etl + .addExtractor({ + read: () => throwError(() => new Error("test")), + }) + .addLoader(dummyLoader) + .start() + .subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error("did not throw")); + }, + }); + }); - it('should process simple array', done => { - etl - .addExtractor(arrayExtractor) - .addLoader(dummyLoader) - .start() - .subscribe(null, null, () => { - expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ objId: 1, name: 'foobar' }); - expect((dummyLoader.write as any).mock.calls[1][0]).toMatchObject({ objId: 2, name: 'hello world' }); - expect((dummyLoader.write as any).mock.calls[2][0]).toMatchObject({ objId: 3, name: 'third test' }); - done(); - }); - }); + it("should call error on loader error", (done) => { + etl + .addExtractor(extractor) + .addLoader({ + write: (o) => throwError(() => new Error("test")), + }) + .start() + .subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error("did not throw")); + }, + }); + }); - it('should call error on extractor error', done => { - etl - .addExtractor({ - read: () => Observable.throw(new Error('test')) - }) - .addLoader(dummyLoader) - .start() - .subscribe(null, () => { - done(); - }, () => { - done(new Error('did not throw')); - }); - }); + it("should call error on transformer error", (done) => { + etl + .addExtractor(extractor) + .addLoader(dummyLoader) + .addTransformer({ + process: (o) => throwError(() => new Error("test")), + }) + .start() + .subscribe({ + error: () => { + done(); + }, + complete: () => { + done(new Error("did not throw")); + }, + }); + }); - it('should call error on loader error', done => { - etl - .addExtractor(extractor) - .addLoader({ - write: o => Observable.throw(new Error('test')) - }) - .start() - .subscribe(null, () => { - done(); - }, () => { - done(new Error('did not throw')); - }); - }); + it("should process simple object with transformer", (done) => { + let spy = jest.fn(); + etl + .addExtractor(extractor) + .addLoader(dummyLoader) + .addTransformer({ + process: (o) => of(o), + }) + .start() + .subscribe({ + next: spy, + error: () => { + done(new Error("did throw")); + }, + complete: () => { + expect(spy.mock.calls.length).toBe(1); + done(); + }, + }); + }); - it('should call error on transformer error', done => { - etl - .addExtractor(extractor) - .addLoader(dummyLoader) - .addTransformer({ - process: o => Observable.throw(new Error('test')) - }) - .start() - .subscribe(null, () => { - done(); - }, () => { - done(new Error('did not throw')); - }); - }); - - it('should process simple object with transformer', done => { - let spy = jest.fn(); - etl - .addExtractor(extractor) - .addLoader(dummyLoader) - .addTransformer({ - process: o => Observable.of(o) - }) - .start() - .subscribe(spy, () => { - done(new Error('did throw')); - }, () => { - expect(spy.mock.calls.length).toBe(1); - done(); - }); - }); - - it('should process simple array with transformer (flat)', done => { - let spy = jest.fn(); - etl - .addExtractor(arrayExtractor) - .addLoader(dummyLoader) - .addTransformer({ - process: o => Observable.from([o, o]) - }) - .start() - .subscribe(spy, () => { - done(new Error('did throw')); - }, () => { - expect(spy.mock.calls.length).toBe(6); - done(); - }); - }); + it("should process simple array with transformer (flat)", (done) => { + let spy = jest.fn(); + etl + .addExtractor(arrayExtractor) + .addLoader(dummyLoader) + .addTransformer({ + process: (o) => from([o, o]), + }) + .start() + .subscribe({ + next: spy, + error: () => { + done(new Error("did throw")); + }, + complete: () => { + expect(spy.mock.calls.length).toBe(6); + done(); + }, + }); + }); - it('should process a general transformer', done => { - let spy = jest.fn(); - etl - .addExtractor(arrayExtractor) - .addLoader(dummyLoader) - .addGeneralTransformer({ - process: o => o.reduce((x, y) => x + y.objId, 0) - }) - .start() - .subscribe(spy, () => { - done(new Error('did throw')); - }, () => { - expect(spy.mock.calls.length).toBe(1); - expect(spy.mock.calls[0][0]).toBe(6); - done(); - }); - }); - - it('should process a match-merge transformer', done => { - let spy = jest.fn(); + it("should process a general transformer", (done) => { + let spy = jest.fn(); + etl + .addExtractor(arrayExtractor) + .addLoader(dummyLoader) + .addGeneralTransformer({ + process: (o) => o.pipe(reduce((x, y) => x + y.objId, 0)), + }) + .start() + .subscribe({ + next: spy, + error: () => { + done(new Error("did throw")); + }, + complete: () => { + expect(spy.mock.calls.length).toBe(1); + expect(spy.mock.calls[0][0]).toBe(6); + done(); + }, + }); + }); - class TestMatchTransformer extends MatchMergeTransformer { - match(o1, o2) { - return o1.location === o2.location; - } + it("should process a match-merge transformer", (done) => { + let spy = jest.fn(); - merge(o1, o2) { - return { - location: o1.location, - things: [...o1.things, ...o2.things] - }; - } - } + class TestMatchTransformer extends MatchMergeTransformer { + match(o1, o2) { + return o1.location === o2.location; + } - etl - .addExtractor(matchMergeExtractor) - .addLoader(dummyLoader) - .addGeneralTransformer(new TestMatchTransformer) - .start() - .subscribe(spy, () => { - done(new Error('did throw')); - }, () => { - expect(spy.mock.calls.length).toBe(2); - expect(spy.mock.calls[0][0]).toMatchObject({ - location: "A", - things: ["a", "c"] - }); - expect(spy.mock.calls[1][0]).toMatchObject({ - location: "B", - things: ["b", "d"] - }); - done(); - }); - }); + merge(o1, o2) { + return { + location: o1.location, + things: [...o1.things, ...o2.things], + }; + } + } - it('should pipe inital observable', done => { - const context = 1; - etl = new Etl(context); - etl - .addTransformer(dummyTransformer) - .addLoader(dummyLoader) - .start(Observable.of('hi')) - .subscribe(null, null, () => { - expect((dummyTransformer.process as any).mock.calls[0]).toContain('hi'); - expect((dummyLoader.write as any).mock.calls[0]).toContain('hi'); - done(); - }); - }); + etl + .addExtractor(matchMergeExtractor) + .addLoader(dummyLoader) + .addGeneralTransformer(new TestMatchTransformer()) + .start() + .subscribe({ + next: spy, + error: () => { + done(new Error("did throw")); + }, + complete: () => { + expect(spy.mock.calls.length).toBe(2); + expect(spy.mock.calls[0][0]).toMatchObject({ + location: "A", + things: ["a", "c"], + }); + expect(spy.mock.calls[1][0]).toMatchObject({ + location: "B", + things: ["b", "d"], + }); + done(); + }, + }); + }); + it("should pipe inital observable", (done) => { + const context = 1; + etl = new Etl(context); + etl + .addTransformer(dummyTransformer) + .addLoader(dummyLoader) + .start(of("hi")) + .subscribe({ + complete: () => { + expect((dummyTransformer.process as any).mock.calls[0]).toContain( + "hi" + ); + expect((dummyLoader.write as any).mock.calls[0]).toContain("hi"); + done(); + }, + }); + }); }); diff --git a/test/JsonExtractor.spec.ts b/test/JsonExtractor.spec.ts index 64e189e..f51f2ba 100644 --- a/test/JsonExtractor.spec.ts +++ b/test/JsonExtractor.spec.ts @@ -1,51 +1,58 @@ -import { join } from 'path'; - -import { JsonExtractor } from '../src'; - -describe('JsonExtractor', () => { - - it('should return an observable', () => { - const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); - expect(ext.read()).toBeInstanceOf(Object); - }); - - it('should get correct path', () => { - const ext = new JsonExtractor('hello'); - const anyExt: any = ext; - const result = join(process.cwd(), 'hello'); - expect(anyExt.filePath).toBe(result); - }); - - it('should receive a json object', (done) => { - const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); - ext.read().subscribe((obj) => { - expect(obj).toMatchObject({ - foo: 'bar', - hello: 'world', - }); - done(); +import { join } from "path"; + +import { JsonExtractor } from "../src"; + +describe("JsonExtractor", () => { + it("should return an observable", () => { + const ext = new JsonExtractor( + "./test/.testdata/json-extractor.object.json" + ); + expect(ext.read()).toBeInstanceOf(Object); + }); + + it("should get correct path", () => { + const ext = new JsonExtractor("hello"); + const anyExt: any = ext; + const result = join(process.cwd(), "hello"); + expect(anyExt.filePath).toBe(result); + }); + + it("should receive a json object", (done) => { + const ext = new JsonExtractor( + "./test/.testdata/json-extractor.object.json" + ); + ext.read().subscribe({ + next: (obj) => { + expect(obj).toMatchObject({ + foo: "bar", + hello: "world", }); + done(); + }, }); - - it('should receive a json array', (done) => { - const ext = new JsonExtractor('./test/.testdata/json-extractor.array.json'); - const spy = jest.fn(); - ext.read().subscribe(spy, null, () => { - expect(spy.mock.calls.length).toBe(3); - done(); - }); + }); + + it("should receive a json array", (done) => { + const ext = new JsonExtractor("./test/.testdata/json-extractor.array.json"); + const spy = jest.fn(); + ext.read().subscribe({ + next: spy, + complete: () => { + expect(spy.mock.calls.length).toBe(3); + done(); + }, }); - - it('should throw on not found file', (done) => { - const ext = new JsonExtractor('404.json'); - ext.read().subscribe( - () => { - done(new Error('did not throw')); - }, - () => { - done(); - }, - ); + }); + + it("should throw on not found file", (done) => { + const ext = new JsonExtractor("404.json"); + ext.read().subscribe({ + next: () => { + done(new Error("did not throw")); + }, + error: () => { + done(); + }, }); - + }); }); diff --git a/test/MapTransformer.spec.ts b/test/MapTransformer.spec.ts index 165dbf4..5e1aa0f 100644 --- a/test/MapTransformer.spec.ts +++ b/test/MapTransformer.spec.ts @@ -1,27 +1,24 @@ -import { Observable } from 'rxjs/Rx'; +import { from, of } from "rxjs"; -import { MapTransformer } from '../src'; +import { MapTransformer } from "../src"; -describe('MapTransformer', () => { +describe("MapTransformer", () => { + it("should return an observable", () => { + const spy = jest.fn(); - it('should return an observable', () => { - const spy = jest.fn(); + const subt = { + process(o) { + return of(o); + }, + }; - const subt = { - process(o) { - return Observable.of(o); - } - } + subt.process = jest.fn(subt.process); - subt.process = jest.fn(subt.process); + const t = new MapTransformer(subt); - const t = new MapTransformer(subt); - - t.process(Observable.from([1]), 2) - .subscribe(spy, null, () => { - expect(spy.mock.calls.length).toBe(1); - expect((subt.process as any).mock.calls.length).toBe(1); - }); + t.process(from([1]), 2).subscribe(spy, null, () => { + expect(spy.mock.calls.length).toBe(1); + expect((subt.process as any).mock.calls.length).toBe(1); }); - + }); }); diff --git a/test/MatchMergeTransformer.spec.ts b/test/MatchMergeTransformer.spec.ts index 06660f4..4917330 100644 --- a/test/MatchMergeTransformer.spec.ts +++ b/test/MatchMergeTransformer.spec.ts @@ -1,28 +1,28 @@ -import { Observable } from 'rxjs/Rx'; +import { Observable } from "rxjs/Rx"; -import { MatchMergeTransformer } from '../src/transformers/MatchMergeTransformer'; +import { MatchMergeTransformer } from "../src/transformers/MatchMergeTransformer"; -describe('MatchMergeTransformer', () => { +describe("MatchMergeTransformer", () => { + it("should return an observable", () => { + const spy = jest.fn(); - it('should return an observable', () => { - const spy = jest.fn(); + class TestMatchMergeTransformer extends MatchMergeTransformer { + match(o1, o2) { + return o1 === o2; + } - class TestMatchMergeTransformer extends MatchMergeTransformer { - match(o1, o2) { - return o1 === o2; - } + merge(o1, o2) { + return o1; + } + } - merge(o1, o2) { - return o1; - } - } + const t = new TestMatchMergeTransformer(); - const t = new TestMatchMergeTransformer(); - - t.process(Observable.from([1, 2, 3, 2, 3])) - .subscribe(spy, null, () => { - expect(spy.mock.calls.length).toBe(3); - }); + t.process(Observable.from([1, 2, 3, 2, 3])).subscribe({ + next: spy, + complete: () => { + expect(spy.mock.calls.length).toBe(3); + }, }); - + }); }); From e1c857baa4f38f3ab86467679cdebb8f18f879d0 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 18:25:42 +0100 Subject: [PATCH 02/12] feat: Update rxjs --- jest.json | 25 ++++++++--------------- package.json | 6 +++--- src/index.ts | 16 +++++++-------- src/transformers/MapTransformer.ts | 2 +- src/transformers/MatchMergeTransformer.ts | 9 ++------ test/Etl.spec.ts | 18 ++++++++-------- test/MatchMergeTransformer.spec.ts | 6 +++--- tslint.json | 12 ----------- 8 files changed, 34 insertions(+), 60 deletions(-) delete mode 100644 tslint.json diff --git a/jest.json b/jest.json index 8e751c3..a33ba5c 100644 --- a/jest.json +++ b/jest.json @@ -1,19 +1,10 @@ { - "collectCoverage": true, - "mapCoverage": true, - "transform": { - "^.+\\.tsx?$": "/node_modules/ts-jest/preprocessor.js" - }, - "testMatch": [ - "**/test/**/*.spec.ts" - ], - "testPathIgnorePatterns": [ - "/node_modules/" - ], - "moduleFileExtensions": [ - "ts", - "tsx", - "js", - "json" - ] + "collectCoverage": true, + "mapCoverage": true, + "transform": { + "^.+\\.tsx?$": "ts-jest" + }, + "testMatch": ["**/test/**/*.spec.ts"], + "testPathIgnorePatterns": ["/node_modules/"], + "moduleFileExtensions": ["ts", "tsx", "js", "json"] } diff --git a/package.json b/package.json index c8fe8f5..b055975 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "clean": "del-cli ./build ./coverage", "build": "npm run clean && tsc -p ./config/tsconfig.build.json", "develop": "npm run clean && tsc -p .", - "lint": "tslint -c ./tslint.json -p ./config/tsconfig.build.json", + "lint": "eslint --max-warnings=-1 './**/*.{ts,tsx}'", "test": "npm run lint && npm run clean && jest -c ./jest.json", "test:watch": "npm run clean && jest -c ./jest.json --watch", "typedoc": "del-cli ./docs && typedoc --ignoreCompilerErrors --out ./docs --mode file --tsconfig ./config/tsconfig.build.json ./src/", @@ -30,13 +30,13 @@ "author": "Christoph Bühler ", "license": "MIT", "devDependencies": { - "@smartive/tslint-config": "^7.0.1", + "@smartive/eslint-config": "3.1.1", "@types/jest": "^29.2.4", "del-cli": "^5.0.0", + "eslint": "8.30.0", "jest": "^29.3.1", "semantic-release": "^19.0.5", "ts-jest": "^29.0.3", - "tslint": "^5.20.1", "tsutils": "^3.21.0", "typedoc": "^0.23.23", "typescript": "^4.9.4" diff --git a/src/index.ts b/src/index.ts index 6091173..7b835ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,8 @@ -export * from './Etl'; -export * from './extractors/JsonExtractor'; -export * from './loaders/ConsoleLoader'; -export * from './transformers/MapTransformer'; -export * from './transformers/MatchMergeTransformer'; -export * from './interfaces/Extractor'; -export * from './interfaces/Transformer'; -export * from './interfaces/Loader'; +export * from "./Etl"; +export * from "./extractors/JsonExtractor"; +export * from "./loaders/ConsoleLoader"; +export * from "./transformers/MapTransformer"; +export * from "./transformers/MatchMergeTransformer"; +export * from "./interfaces/Extractor"; +export * from "./interfaces/Transformer"; +export * from "./interfaces/Loader"; diff --git a/src/transformers/MapTransformer.ts b/src/transformers/MapTransformer.ts index a123381..aa90edc 100644 --- a/src/transformers/MapTransformer.ts +++ b/src/transformers/MapTransformer.ts @@ -6,7 +6,7 @@ import { Transformer } from "../interfaces/Transformer"; export class MapTransformer implements GeneralTransformer { constructor(private transformer: Transformer) {} - process(observable: Observable, context?: any): Observable { + public process(observable: Observable, context?: any): Observable { return observable.pipe( mergeMap((o) => this.transformer.process(o, context)) ); diff --git a/src/transformers/MatchMergeTransformer.ts b/src/transformers/MatchMergeTransformer.ts index dfe1e32..849d7c2 100644 --- a/src/transformers/MatchMergeTransformer.ts +++ b/src/transformers/MatchMergeTransformer.ts @@ -1,4 +1,4 @@ -import { from, mergeMap, Observable, reduce } from "rxjs"; +import { mergeMap, Observable, reduce } from "rxjs"; import { GeneralTransformer } from "../interfaces/GeneralTransformer"; @@ -7,11 +7,7 @@ export abstract class MatchMergeTransformer implements GeneralTransformer { const matchMerge = (merged: any[], o2: any) => { return this.matchMerge(merged, o2, context); }; - return observable.pipe(reduce(matchMerge, [])).pipe( - mergeMap((merged) => { - return from(merged); - }) - ); + return observable.pipe(reduce(matchMerge, [])).pipe(mergeMap((v) => v)); } protected abstract match(o1: any, o2: any, context?: any): boolean; @@ -22,7 +18,6 @@ export abstract class MatchMergeTransformer implements GeneralTransformer { for (let i = 0; i < merged.length; i++) { if (this.match(merged[i], o2, context)) { const o1 = merged.splice(i, 1)[0]; - // tslint:disable-next-line o2 = this.merge(o1, o2, context); // Try to merge the merged element with the remaining elements, // starting from the current position diff --git a/test/Etl.spec.ts b/test/Etl.spec.ts index 7a8c7e3..eff1293 100644 --- a/test/Etl.spec.ts +++ b/test/Etl.spec.ts @@ -12,13 +12,13 @@ import { describe("Etl", () => { let etl: Etl; - let extractor: Extractor = new JsonExtractor( + const extractor: Extractor = new JsonExtractor( "./test/.testdata/json-extractor.object.json" ); - let arrayExtractor: Extractor = new JsonExtractor( + const arrayExtractor: Extractor = new JsonExtractor( "./test/.testdata/json-extractor.array.json" ); - let matchMergeExtractor: Extractor = new JsonExtractor( + const matchMergeExtractor: Extractor = new JsonExtractor( "./test/.testdata/match-merge.json" ); let o; @@ -167,7 +167,7 @@ describe("Etl", () => { etl .addExtractor(extractor) .addLoader({ - write: (o) => throwError(() => new Error("test")), + write: () => throwError(() => new Error("test")), }) .start() .subscribe({ @@ -185,7 +185,7 @@ describe("Etl", () => { .addExtractor(extractor) .addLoader(dummyLoader) .addTransformer({ - process: (o) => throwError(() => new Error("test")), + process: () => throwError(() => new Error("test")), }) .start() .subscribe({ @@ -199,7 +199,7 @@ describe("Etl", () => { }); it("should process simple object with transformer", (done) => { - let spy = jest.fn(); + const spy = jest.fn(); etl .addExtractor(extractor) .addLoader(dummyLoader) @@ -220,7 +220,7 @@ describe("Etl", () => { }); it("should process simple array with transformer (flat)", (done) => { - let spy = jest.fn(); + const spy = jest.fn(); etl .addExtractor(arrayExtractor) .addLoader(dummyLoader) @@ -241,7 +241,7 @@ describe("Etl", () => { }); it("should process a general transformer", (done) => { - let spy = jest.fn(); + const spy = jest.fn(); etl .addExtractor(arrayExtractor) .addLoader(dummyLoader) @@ -263,7 +263,7 @@ describe("Etl", () => { }); it("should process a match-merge transformer", (done) => { - let spy = jest.fn(); + const spy = jest.fn(); class TestMatchTransformer extends MatchMergeTransformer { match(o1, o2) { diff --git a/test/MatchMergeTransformer.spec.ts b/test/MatchMergeTransformer.spec.ts index 4917330..dbbf935 100644 --- a/test/MatchMergeTransformer.spec.ts +++ b/test/MatchMergeTransformer.spec.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs/Rx"; +import { from } from "rxjs"; import { MatchMergeTransformer } from "../src/transformers/MatchMergeTransformer"; @@ -11,14 +11,14 @@ describe("MatchMergeTransformer", () => { return o1 === o2; } - merge(o1, o2) { + merge(o1) { return o1; } } const t = new TestMatchMergeTransformer(); - t.process(Observable.from([1, 2, 3, 2, 3])).subscribe({ + t.process(from([1, 2, 3, 2, 3])).subscribe({ next: spy, complete: () => { expect(spy.mock.calls.length).toBe(3); diff --git a/tslint.json b/tslint.json deleted file mode 100644 index d5c5f71..0000000 --- a/tslint.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "extends": "@smartive/tslint-config", - "rules": { - "ter-indent": [ - true, - 4, - { - "SwitchCase": 1 - } - ] - } -} From 9056bed99d39d8a6cda929de6459358fcd73c0c9 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 21:38:41 +0100 Subject: [PATCH 03/12] feat: Update rxjs --- .eslintrc.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .eslintrc.json diff --git a/.eslintrc.json b/.eslintrc.json new file mode 100644 index 0000000..fc2bc49 --- /dev/null +++ b/.eslintrc.json @@ -0,0 +1,3 @@ +{ + "extends": ["@smartive/eslint-config/react"] +} From 4cbbee7ad8b12923fded9de9d974d54245ed1e86 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 21:42:57 +0100 Subject: [PATCH 04/12] feat: Update rxjs --- .appveyor.yml | 6 ++---- .travis.yml | 16 ++++------------ 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index c4a2f0f..eab5feb 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -4,10 +4,8 @@ skip_branch_with_pr: true environment: matrix: - - nodejs_version: "9" - - nodejs_version: "8" - - nodejs_version: "7" - - nodejs_version: "6" + - nodejs_version: "16" + - nodejs_version: "18" install: - ps: Install-Product node $env:nodejs_version diff --git a/.travis.yml b/.travis.yml index f831765..e65c4e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,23 +12,15 @@ notifications: jobs: include: - stage: test - node_js: '9' + node_js: "16" after_success: - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - stage: test - node_js: '8' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: '7' - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: '6' + node_js: "18" after_success: - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - stage: deploy - node_js: '9' + node_js: "16" script: npm run typedoc deploy: provider: pages @@ -36,6 +28,6 @@ jobs: github_token: $GH_TOKEN local_dir: ./docs - stage: deploy - node_js: '9' + node_js: "16" before_script: npm run build script: npm run semantic-release From 2d1f1de3c5da73f87941f7d5cdefe07a1bf11ce6 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 21:52:53 +0100 Subject: [PATCH 05/12] feat: Update rxjs --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 0c0b898..0f428d6 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Basically instantiate the `Etl` class and add extractors (which pull data from a A basic, hypothetic example could be: "Load data from a JSON array, snake_case all properties and store those objects into a mongoDB." -The package is written in `typescript` but can be used in plain javascript as well. +The package is written in `typescript` but can be used in plain javascript as well ##### A bunch of badges @@ -21,14 +21,14 @@ The package is written in `typescript` but can be used in plain javascript as we ## Usage ```typescript -import {Etl} from 'proc-that'; +import { Etl } from "proc-that"; new Etl() - .addExtractor(/* class that implements Extractor */) - .addTransformer(/* class that implements Transformer */) - .addLoader(/* class that implements Loader */) - .start() - .subscribe(progress, error, success); + .addExtractor(/* class that implements Extractor */) + .addTransformer(/* class that implements Transformer */) + .addLoader(/* class that implements Loader */) + .start() + .subscribe(progress, error, success); ``` After all objects are extracted, transformed and loaded, the `.start()` observable completes and the process is finished. @@ -37,15 +37,15 @@ Below is a list if extractors and loaders that are already implemented. Feel fre ## Extractors - Name | Description | Link ---------------------------------|--------------------------------------------------|------------------------------------------------------- - `proc-that-rest-extractor` | Extract objects from GET requests | https://github.com/smartive/proc-that-rest-extractor +| Name | Description | Link | +| -------------------------- | --------------------------------- | ---------------------------------------------------- | +| `proc-that-rest-extractor` | Extract objects from GET requests | https://github.com/smartive/proc-that-rest-extractor | ## Loaders - Name | Description | Link ---------------------------------|--------------------------------------------------|------------------------------------------------------- - `proc-that-elastic-loader` | Load transformed objects into elasticsearch | https://github.com/smartive/proc-that-elastic-loader +| Name | Description | Link | +| -------------------------- | ------------------------------------------- | ---------------------------------------------------- | +| `proc-that-elastic-loader` | Load transformed objects into elasticsearch | https://github.com/smartive/proc-that-elastic-loader | ## Implement your own From 78a659a99afd5e227531fae354bbf3627ddcedb5 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 22:34:49 +0100 Subject: [PATCH 06/12] feat: Update rxjs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b055975..8d7db85 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,7 @@ "clean": "del-cli ./build ./coverage", "build": "npm run clean && tsc -p ./config/tsconfig.build.json", "develop": "npm run clean && tsc -p .", - "lint": "eslint --max-warnings=-1 './**/*.{ts,tsx}'", + "lint": "eslint --max-warnings=-1", "test": "npm run lint && npm run clean && jest -c ./jest.json", "test:watch": "npm run clean && jest -c ./jest.json --watch", "typedoc": "del-cli ./docs && typedoc --ignoreCompilerErrors --out ./docs --mode file --tsconfig ./config/tsconfig.build.json ./src/", From 90fac2572e3ed1b091b04ed22deb78556e89d1cd Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 22:51:15 +0100 Subject: [PATCH 07/12] feat: Update rxjs --- .appveyor.yml | 17 ----------------- .github/workflows/release.yml | 25 +++++++++++++++++++++++++ .github/workflows/test.yml | 25 +++++++++++++++++++++++++ .travis.yml | 33 --------------------------------- 4 files changed, 50 insertions(+), 50 deletions(-) delete mode 100644 .appveyor.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index eab5feb..0000000 --- a/.appveyor.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: "{build} - {branch}" -skip_tags: true -skip_branch_with_pr: true - -environment: - matrix: - - nodejs_version: "16" - - nodejs_version: "18" - -install: - - ps: Install-Product node $env:nodejs_version - - npm install - -test_script: - - npm test - -build: off diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bbf0ffb --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,25 @@ +name: Release +on: + push: + branches: + - master +jobs: + release: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: "lts/*" + - name: Install dependencies + run: npm ci + - name: Release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npx semantic-release diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..248f655 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,25 @@ +name: Unit Tests + +on: + push: + branches: + - master + pull_request: + branches: [master] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: 'lts/*' + - name: Run Tests + run: | + npm install + npm test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e65c4e4..0000000 --- a/.travis.yml +++ /dev/null @@ -1,33 +0,0 @@ -language: node_js - -stages: - - name: test - if: tag IS blank - - name: deploy - if: branch = master AND type != pull_request - -notifications: - email: false - -jobs: - include: - - stage: test - node_js: "16" - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: test - node_js: "18" - after_success: - - npm install coveralls@^2.11.9 && cat ./coverage/lcov.info | coveralls - - stage: deploy - node_js: "16" - script: npm run typedoc - deploy: - provider: pages - skip_cleanup: true - github_token: $GH_TOKEN - local_dir: ./docs - - stage: deploy - node_js: "16" - before_script: npm run build - script: npm run semantic-release From ad401454fae33c6d3990591cc1684bf244bcdde6 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 22:53:45 +0100 Subject: [PATCH 08/12] feat: Update rxjs --- .github/workflows/test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 248f655..df0e314 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,24 +1,22 @@ name: Unit Tests - on: push: branches: - master pull_request: branches: [master] - jobs: test: runs-on: ubuntu-latest steps: - - name: Checkout + - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - name: Setup Node.js uses: actions/setup-node@v2 with: - node-version: 'lts/*' + node-version: "lts/*" - name: Run Tests run: | npm install From 5c79987483c6392fa8081259ecf269095181ac5c Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 22:56:45 +0100 Subject: [PATCH 09/12] feat: Update rxjs --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df0e314..e484d3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: branches: - master pull_request: - branches: [master] + branches: [master, develop] jobs: test: runs-on: ubuntu-latest From 1193d45174c96d003276dd55e3b7c6fdcaeb898e Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Mon, 19 Dec 2022 22:57:53 +0100 Subject: [PATCH 10/12] feat: Update rxjs --- .github/workflows/test.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e484d3a..3887cf7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,15 +8,15 @@ on: jobs: test: runs-on: ubuntu-latest + strategy: + matrix: + node-version: [16.x, 18.x] steps: - - name: Checkout - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 with: - fetch-depth: 0 - - name: Setup Node.js - uses: actions/setup-node@v2 - with: - node-version: "lts/*" + node-version: ${{ matrix.node-version }} - name: Run Tests run: | npm install From fa4ea41045135fc24c764826e61a6af998735b78 Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Tue, 20 Dec 2022 07:58:22 +0100 Subject: [PATCH 11/12] feat: Fix subscribe ETL --- src/Etl.ts | 27 +++++++++++------------ src/transformers/MatchMergeTransformer.ts | 6 +++-- test/Etl.spec.ts | 2 ++ 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/src/Etl.ts b/src/Etl.ts index 80892e3..b4d5e87 100644 --- a/src/Etl.ts +++ b/src/Etl.ts @@ -1,4 +1,4 @@ -import { EMPTY, merge, mergeMap, Observable, throwError } from "rxjs"; +import { EMPTY, merge, mergeMap, Observable, tap, throwError } from "rxjs"; import { Extractor } from "./interfaces/Extractor"; import { GeneralTransformer } from "./interfaces/GeneralTransformer"; @@ -97,7 +97,7 @@ export class Etl { ...this._extractors.map((extractor) => extractor.read(this._context)) ); - const pipe = this._generalTransformers + return this._generalTransformers .reduce( (observable, transformer) => transformer.process(observable, this._context), @@ -111,19 +111,18 @@ export class Etl { ) ) ) + ) + .pipe( + tap({ + error: (err) => { + this._state = EtlState.Error; + return throwError(() => err); + }, + complete: () => { + this._state = EtlState.Stopped; + }, + }) ); - - pipe.subscribe({ - error: (err) => { - this._state = EtlState.Error; - return throwError(() => err); - }, - complete: () => { - this._state = EtlState.Stopped; - }, - }); - - return pipe; } /** diff --git a/src/transformers/MatchMergeTransformer.ts b/src/transformers/MatchMergeTransformer.ts index 849d7c2..b99804f 100644 --- a/src/transformers/MatchMergeTransformer.ts +++ b/src/transformers/MatchMergeTransformer.ts @@ -1,4 +1,4 @@ -import { mergeMap, Observable, reduce } from "rxjs"; +import { from, mergeMap, Observable, reduce } from "rxjs"; import { GeneralTransformer } from "../interfaces/GeneralTransformer"; @@ -7,7 +7,9 @@ export abstract class MatchMergeTransformer implements GeneralTransformer { const matchMerge = (merged: any[], o2: any) => { return this.matchMerge(merged, o2, context); }; - return observable.pipe(reduce(matchMerge, [])).pipe(mergeMap((v) => v)); + return observable + .pipe(reduce(matchMerge, [])) + .pipe(mergeMap((v) => from(v))); } protected abstract match(o1: any, o2: any, context?: any): boolean; diff --git a/test/Etl.spec.ts b/test/Etl.spec.ts index eff1293..4f81bf7 100644 --- a/test/Etl.spec.ts +++ b/test/Etl.spec.ts @@ -113,6 +113,7 @@ describe("Etl", () => { .start() .subscribe({ complete: () => { + expect((dummyLoader.write as any).mock.calls).toHaveLength(1); expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ foo: "bar", hello: "world", @@ -129,6 +130,7 @@ describe("Etl", () => { .start() .subscribe({ complete: () => { + expect((dummyLoader.write as any).mock.calls).toHaveLength(3); expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ objId: 1, name: "foobar", From 7a196ec46b9709c6f5adbfacecce89bfd8bfeb9c Mon Sep 17 00:00:00 2001 From: Thilo Haas Date: Tue, 20 Dec 2022 10:26:19 +0100 Subject: [PATCH 12/12] feat: Fix subscribe ETL --- .eslintrc.json | 2 +- .prettierrc.json | 1 + package.json | 13 ++- src/Etl.ts | 35 ++----- src/extractors/JsonExtractor.ts | 6 +- src/index.ts | 16 ++-- src/interfaces/Extractor.ts | 2 +- src/interfaces/GeneralTransformer.ts | 2 +- src/interfaces/Loader.ts | 2 +- src/interfaces/Transformer.ts | 2 +- src/loaders/ConsoleLoader.ts | 4 +- src/transformers/MapTransformer.ts | 10 +- src/transformers/MatchMergeTransformer.ts | 8 +- test/Etl.spec.ts | 110 +++++++++------------- test/JsonExtractor.spec.ts | 38 ++++---- test/MapTransformer.spec.ts | 8 +- test/MatchMergeTransformer.spec.ts | 8 +- 17 files changed, 116 insertions(+), 151 deletions(-) create mode 100644 .prettierrc.json diff --git a/.eslintrc.json b/.eslintrc.json index fc2bc49..cbd4d25 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,3 +1,3 @@ { - "extends": ["@smartive/eslint-config/react"] + "extends": ["@smartive/eslint-config"] } diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..65c08dc --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1 @@ +"@smartive/prettier-config" diff --git a/package.json b/package.json index 8d7db85..26704a1 100644 --- a/package.json +++ b/package.json @@ -8,7 +8,12 @@ "clean": "del-cli ./build ./coverage", "build": "npm run clean && tsc -p ./config/tsconfig.build.json", "develop": "npm run clean && tsc -p .", - "lint": "eslint --max-warnings=-1", + "lint": "npm run lint:ts && npm run prettier", + "lint:fix": "npm run lint:ts:fix && npm run prettier:fix", + "lint:ts": "eslint --max-warnings=-1", + "lint:ts:fix": "eslint --max-warnings=-1 --fix", + "prettier": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\"", + "prettier:fix": "prettier --config .prettierrc.json --list-different \"./**/*.{ts,tsx}\" --write", "test": "npm run lint && npm run clean && jest -c ./jest.json", "test:watch": "npm run clean && jest -c ./jest.json --watch", "typedoc": "del-cli ./docs && typedoc --ignoreCompilerErrors --out ./docs --mode file --tsconfig ./config/tsconfig.build.json ./src/", @@ -30,11 +35,13 @@ "author": "Christoph Bühler ", "license": "MIT", "devDependencies": { - "@smartive/eslint-config": "3.1.1", + "@smartive/eslint-config": "^3.1.1", + "@smartive/prettier-config": "^3.0.0", "@types/jest": "^29.2.4", "del-cli": "^5.0.0", - "eslint": "8.30.0", + "eslint": "^8.30.0", "jest": "^29.3.1", + "prettier": "^2.8.1", "semantic-release": "^19.0.5", "ts-jest": "^29.0.3", "tsutils": "^3.21.0", diff --git a/src/Etl.ts b/src/Etl.ts index b4d5e87..72b1b12 100644 --- a/src/Etl.ts +++ b/src/Etl.ts @@ -1,10 +1,10 @@ -import { EMPTY, merge, mergeMap, Observable, tap, throwError } from "rxjs"; +import { EMPTY, merge, mergeMap, Observable, tap, throwError } from 'rxjs'; -import { Extractor } from "./interfaces/Extractor"; -import { GeneralTransformer } from "./interfaces/GeneralTransformer"; -import { Loader } from "./interfaces/Loader"; -import { Transformer } from "./interfaces/Transformer"; -import { MapTransformer } from "./transformers/MapTransformer"; +import { Extractor } from './interfaces/Extractor'; +import { GeneralTransformer } from './interfaces/GeneralTransformer'; +import { Loader } from './interfaces/Loader'; +import { Transformer } from './interfaces/Transformer'; +import { MapTransformer } from './transformers/MapTransformer'; export enum EtlState { Running, @@ -53,7 +53,7 @@ export class Etl { public setContext(context: any): this { if (this._state !== EtlState.Stopped) { this._state = EtlState.Error; - throw new Error("Tried to set context on invalid state."); + throw new Error('Tried to set context on invalid state.'); } this._context = context; return this; @@ -92,26 +92,11 @@ export class Etl { public start(observable: Observable = EMPTY): Observable { this._state = EtlState.Running; - const o: Observable = merge( - observable, - ...this._extractors.map((extractor) => extractor.read(this._context)) - ); + const o: Observable = merge(observable, ...this._extractors.map((extractor) => extractor.read(this._context))); return this._generalTransformers - .reduce( - (observable, transformer) => - transformer.process(observable, this._context), - o - ) - .pipe( - mergeMap((object) => - merge( - ...this._loaders.map((loader) => - loader.write(object, this._context) - ) - ) - ) - ) + .reduce((observable, transformer) => transformer.process(observable, this._context), o) + .pipe(mergeMap((object) => merge(...this._loaders.map((loader) => loader.write(object, this._context))))) .pipe( tap({ error: (err) => { diff --git a/src/extractors/JsonExtractor.ts b/src/extractors/JsonExtractor.ts index 5a3e956..7203896 100644 --- a/src/extractors/JsonExtractor.ts +++ b/src/extractors/JsonExtractor.ts @@ -1,7 +1,7 @@ -import { resolve } from "path"; -import { from, Observable, throwError } from "rxjs"; +import { resolve } from 'path'; +import { from, Observable, throwError } from 'rxjs'; -import { Extractor } from "../interfaces/Extractor"; +import { Extractor } from '../interfaces/Extractor'; /** * Extractor that reads a JSON file at a given filepath. The path is resolved relatively to the running tasks root dir. diff --git a/src/index.ts b/src/index.ts index 7b835ed..6091173 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,8 +1,8 @@ -export * from "./Etl"; -export * from "./extractors/JsonExtractor"; -export * from "./loaders/ConsoleLoader"; -export * from "./transformers/MapTransformer"; -export * from "./transformers/MatchMergeTransformer"; -export * from "./interfaces/Extractor"; -export * from "./interfaces/Transformer"; -export * from "./interfaces/Loader"; +export * from './Etl'; +export * from './extractors/JsonExtractor'; +export * from './loaders/ConsoleLoader'; +export * from './transformers/MapTransformer'; +export * from './transformers/MatchMergeTransformer'; +export * from './interfaces/Extractor'; +export * from './interfaces/Transformer'; +export * from './interfaces/Loader'; diff --git a/src/interfaces/Extractor.ts b/src/interfaces/Extractor.ts index 321cd74..ed4137d 100644 --- a/src/interfaces/Extractor.ts +++ b/src/interfaces/Extractor.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs'; /** * Extractor interface. Only provides "read()" method that returns an observable with the result. diff --git a/src/interfaces/GeneralTransformer.ts b/src/interfaces/GeneralTransformer.ts index e50776c..d2c88e2 100644 --- a/src/interfaces/GeneralTransformer.ts +++ b/src/interfaces/GeneralTransformer.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs'; /** * GeneralTransformer interface. Provides a "process(observable)" method that processes an observable. diff --git a/src/interfaces/Loader.ts b/src/interfaces/Loader.ts index b31cd8e..ffffcb4 100644 --- a/src/interfaces/Loader.ts +++ b/src/interfaces/Loader.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs'; /** * Loader interface. Provides ".write(obj)" method that returns an observable with the loaded value. diff --git a/src/interfaces/Transformer.ts b/src/interfaces/Transformer.ts index 9a96482..9d70b8e 100644 --- a/src/interfaces/Transformer.ts +++ b/src/interfaces/Transformer.ts @@ -1,4 +1,4 @@ -import { Observable } from "rxjs"; +import { Observable } from 'rxjs'; /** * Transformer interface. Only provides ".process(obj)" that returns an Observable with diff --git a/src/loaders/ConsoleLoader.ts b/src/loaders/ConsoleLoader.ts index f08d2e6..dbc25f7 100644 --- a/src/loaders/ConsoleLoader.ts +++ b/src/loaders/ConsoleLoader.ts @@ -1,6 +1,6 @@ -import { Observable, of } from "rxjs"; +import { Observable, of } from 'rxjs'; -import { Loader } from "../interfaces/Loader"; +import { Loader } from '../interfaces/Loader'; /** * Loader that outputs everything to the console. diff --git a/src/transformers/MapTransformer.ts b/src/transformers/MapTransformer.ts index aa90edc..00ee2ac 100644 --- a/src/transformers/MapTransformer.ts +++ b/src/transformers/MapTransformer.ts @@ -1,14 +1,12 @@ -import { mergeMap, Observable } from "rxjs"; +import { mergeMap, Observable } from 'rxjs'; -import { GeneralTransformer } from "../interfaces/GeneralTransformer"; -import { Transformer } from "../interfaces/Transformer"; +import { GeneralTransformer } from '../interfaces/GeneralTransformer'; +import { Transformer } from '../interfaces/Transformer'; export class MapTransformer implements GeneralTransformer { constructor(private transformer: Transformer) {} public process(observable: Observable, context?: any): Observable { - return observable.pipe( - mergeMap((o) => this.transformer.process(o, context)) - ); + return observable.pipe(mergeMap((o) => this.transformer.process(o, context))); } } diff --git a/src/transformers/MatchMergeTransformer.ts b/src/transformers/MatchMergeTransformer.ts index b99804f..41b8b89 100644 --- a/src/transformers/MatchMergeTransformer.ts +++ b/src/transformers/MatchMergeTransformer.ts @@ -1,15 +1,13 @@ -import { from, mergeMap, Observable, reduce } from "rxjs"; +import { from, mergeMap, Observable, reduce } from 'rxjs'; -import { GeneralTransformer } from "../interfaces/GeneralTransformer"; +import { GeneralTransformer } from '../interfaces/GeneralTransformer'; export abstract class MatchMergeTransformer implements GeneralTransformer { public process(observable: Observable, context?: any): Observable { const matchMerge = (merged: any[], o2: any) => { return this.matchMerge(merged, o2, context); }; - return observable - .pipe(reduce(matchMerge, [])) - .pipe(mergeMap((v) => from(v))); + return observable.pipe(reduce(matchMerge, [])).pipe(mergeMap((v) => from(v))); } protected abstract match(o1: any, o2: any, context?: any): boolean; diff --git a/test/Etl.spec.ts b/test/Etl.spec.ts index 4f81bf7..f9708bd 100644 --- a/test/Etl.spec.ts +++ b/test/Etl.spec.ts @@ -1,26 +1,12 @@ -import { from, of, reduce, throwError } from "rxjs"; +import { from, of, reduce, throwError } from 'rxjs'; -import { - Etl, - EtlState, - Extractor, - JsonExtractor, - Loader, - MatchMergeTransformer, - Transformer, -} from "../src"; +import { Etl, EtlState, Extractor, JsonExtractor, Loader, MatchMergeTransformer, Transformer } from '../src'; -describe("Etl", () => { +describe('Etl', () => { let etl: Etl; - const extractor: Extractor = new JsonExtractor( - "./test/.testdata/json-extractor.object.json" - ); - const arrayExtractor: Extractor = new JsonExtractor( - "./test/.testdata/json-extractor.array.json" - ); - const matchMergeExtractor: Extractor = new JsonExtractor( - "./test/.testdata/match-merge.json" - ); + const extractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.object.json'); + const arrayExtractor: Extractor = new JsonExtractor('./test/.testdata/json-extractor.array.json'); + const matchMergeExtractor: Extractor = new JsonExtractor('./test/.testdata/match-merge.json'); let o; let dummyExtractor: Extractor; let dummyTransformer: Transformer; @@ -29,7 +15,7 @@ describe("Etl", () => { beforeEach(() => { etl = new Etl(); - o = { _id: "001" }; + o = { _id: '001' }; dummyExtractor = { read: () => of(o), @@ -47,14 +33,14 @@ describe("Etl", () => { dummyLoader.write = jest.fn(dummyLoader.write); }); - it("should initialize with correct default params", () => { + it('should initialize with correct default params', () => { expect(etl.state).toBe(EtlState.Stopped); expect(etl.extractors.length).toBe(0); expect(etl.transformers.length).toBe(0); expect(etl.loaders.length).toBe(0); }); - it("should reset correctly", () => { + it('should reset correctly', () => { etl.addExtractor({ read: function () { return of(null); @@ -66,7 +52,7 @@ describe("Etl", () => { expect(etl.extractors.length).toBe(0); }); - it("should pass context down the pipeline", (done) => { + it('should pass context down the pipeline', (done) => { const context = 1; etl = new Etl(context); etl @@ -77,16 +63,14 @@ describe("Etl", () => { .subscribe({ complete: () => { expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); - expect((dummyTransformer.process as any).mock.calls[0]).toContain( - context - ); + expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); expect((dummyLoader.write as any).mock.calls[0]).toContain(context); done(); }, }); }); - it("should pass newly set context down the pipeline", (done) => { + it('should pass newly set context down the pipeline', (done) => { const context = 1; etl .addExtractor(dummyExtractor) @@ -97,16 +81,14 @@ describe("Etl", () => { .subscribe({ complete: () => { expect((dummyExtractor.read as any).mock.calls[0]).toContain(context); - expect((dummyTransformer.process as any).mock.calls[0]).toContain( - context - ); + expect((dummyTransformer.process as any).mock.calls[0]).toContain(context); expect((dummyLoader.write as any).mock.calls[0]).toContain(context); done(); }, }); }); - it("should process simple object", (done) => { + it('should process simple object', (done) => { etl .addExtractor(extractor) .addLoader(dummyLoader) @@ -115,15 +97,15 @@ describe("Etl", () => { complete: () => { expect((dummyLoader.write as any).mock.calls).toHaveLength(1); expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ - foo: "bar", - hello: "world", + foo: 'bar', + hello: 'world', }); done(); }, }); }); - it("should process simple array", (done) => { + it('should process simple array', (done) => { etl .addExtractor(arrayExtractor) .addLoader(dummyLoader) @@ -133,25 +115,25 @@ describe("Etl", () => { expect((dummyLoader.write as any).mock.calls).toHaveLength(3); expect((dummyLoader.write as any).mock.calls[0][0]).toMatchObject({ objId: 1, - name: "foobar", + name: 'foobar', }); expect((dummyLoader.write as any).mock.calls[1][0]).toMatchObject({ objId: 2, - name: "hello world", + name: 'hello world', }); expect((dummyLoader.write as any).mock.calls[2][0]).toMatchObject({ objId: 3, - name: "third test", + name: 'third test', }); done(); }, }); }); - it("should call error on extractor error", (done) => { + it('should call error on extractor error', (done) => { etl .addExtractor({ - read: () => throwError(() => new Error("test")), + read: () => throwError(() => new Error('test')), }) .addLoader(dummyLoader) .start() @@ -160,16 +142,16 @@ describe("Etl", () => { done(); }, complete: () => { - done(new Error("did not throw")); + done(new Error('did not throw')); }, }); }); - it("should call error on loader error", (done) => { + it('should call error on loader error', (done) => { etl .addExtractor(extractor) .addLoader({ - write: () => throwError(() => new Error("test")), + write: () => throwError(() => new Error('test')), }) .start() .subscribe({ @@ -177,17 +159,17 @@ describe("Etl", () => { done(); }, complete: () => { - done(new Error("did not throw")); + done(new Error('did not throw')); }, }); }); - it("should call error on transformer error", (done) => { + it('should call error on transformer error', (done) => { etl .addExtractor(extractor) .addLoader(dummyLoader) .addTransformer({ - process: () => throwError(() => new Error("test")), + process: () => throwError(() => new Error('test')), }) .start() .subscribe({ @@ -195,12 +177,12 @@ describe("Etl", () => { done(); }, complete: () => { - done(new Error("did not throw")); + done(new Error('did not throw')); }, }); }); - it("should process simple object with transformer", (done) => { + it('should process simple object with transformer', (done) => { const spy = jest.fn(); etl .addExtractor(extractor) @@ -212,7 +194,7 @@ describe("Etl", () => { .subscribe({ next: spy, error: () => { - done(new Error("did throw")); + done(new Error('did throw')); }, complete: () => { expect(spy.mock.calls.length).toBe(1); @@ -221,7 +203,7 @@ describe("Etl", () => { }); }); - it("should process simple array with transformer (flat)", (done) => { + it('should process simple array with transformer (flat)', (done) => { const spy = jest.fn(); etl .addExtractor(arrayExtractor) @@ -233,7 +215,7 @@ describe("Etl", () => { .subscribe({ next: spy, error: () => { - done(new Error("did throw")); + done(new Error('did throw')); }, complete: () => { expect(spy.mock.calls.length).toBe(6); @@ -242,7 +224,7 @@ describe("Etl", () => { }); }); - it("should process a general transformer", (done) => { + it('should process a general transformer', (done) => { const spy = jest.fn(); etl .addExtractor(arrayExtractor) @@ -254,7 +236,7 @@ describe("Etl", () => { .subscribe({ next: spy, error: () => { - done(new Error("did throw")); + done(new Error('did throw')); }, complete: () => { expect(spy.mock.calls.length).toBe(1); @@ -264,7 +246,7 @@ describe("Etl", () => { }); }); - it("should process a match-merge transformer", (done) => { + it('should process a match-merge transformer', (done) => { const spy = jest.fn(); class TestMatchTransformer extends MatchMergeTransformer { @@ -288,36 +270,34 @@ describe("Etl", () => { .subscribe({ next: spy, error: () => { - done(new Error("did throw")); + done(new Error('did throw')); }, complete: () => { expect(spy.mock.calls.length).toBe(2); expect(spy.mock.calls[0][0]).toMatchObject({ - location: "A", - things: ["a", "c"], + location: 'A', + things: ['a', 'c'], }); expect(spy.mock.calls[1][0]).toMatchObject({ - location: "B", - things: ["b", "d"], + location: 'B', + things: ['b', 'd'], }); done(); }, }); }); - it("should pipe inital observable", (done) => { + it('should pipe inital observable', (done) => { const context = 1; etl = new Etl(context); etl .addTransformer(dummyTransformer) .addLoader(dummyLoader) - .start(of("hi")) + .start(of('hi')) .subscribe({ complete: () => { - expect((dummyTransformer.process as any).mock.calls[0]).toContain( - "hi" - ); - expect((dummyLoader.write as any).mock.calls[0]).toContain("hi"); + expect((dummyTransformer.process as any).mock.calls[0]).toContain('hi'); + expect((dummyLoader.write as any).mock.calls[0]).toContain('hi'); done(); }, }); diff --git a/test/JsonExtractor.spec.ts b/test/JsonExtractor.spec.ts index f51f2ba..7ae23d5 100644 --- a/test/JsonExtractor.spec.ts +++ b/test/JsonExtractor.spec.ts @@ -1,39 +1,35 @@ -import { join } from "path"; +import { join } from 'path'; -import { JsonExtractor } from "../src"; +import { JsonExtractor } from '../src'; -describe("JsonExtractor", () => { - it("should return an observable", () => { - const ext = new JsonExtractor( - "./test/.testdata/json-extractor.object.json" - ); +describe('JsonExtractor', () => { + it('should return an observable', () => { + const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); expect(ext.read()).toBeInstanceOf(Object); }); - it("should get correct path", () => { - const ext = new JsonExtractor("hello"); + it('should get correct path', () => { + const ext = new JsonExtractor('hello'); const anyExt: any = ext; - const result = join(process.cwd(), "hello"); + const result = join(process.cwd(), 'hello'); expect(anyExt.filePath).toBe(result); }); - it("should receive a json object", (done) => { - const ext = new JsonExtractor( - "./test/.testdata/json-extractor.object.json" - ); + it('should receive a json object', (done) => { + const ext = new JsonExtractor('./test/.testdata/json-extractor.object.json'); ext.read().subscribe({ next: (obj) => { expect(obj).toMatchObject({ - foo: "bar", - hello: "world", + foo: 'bar', + hello: 'world', }); done(); }, }); }); - it("should receive a json array", (done) => { - const ext = new JsonExtractor("./test/.testdata/json-extractor.array.json"); + it('should receive a json array', (done) => { + const ext = new JsonExtractor('./test/.testdata/json-extractor.array.json'); const spy = jest.fn(); ext.read().subscribe({ next: spy, @@ -44,11 +40,11 @@ describe("JsonExtractor", () => { }); }); - it("should throw on not found file", (done) => { - const ext = new JsonExtractor("404.json"); + it('should throw on not found file', (done) => { + const ext = new JsonExtractor('404.json'); ext.read().subscribe({ next: () => { - done(new Error("did not throw")); + done(new Error('did not throw')); }, error: () => { done(); diff --git a/test/MapTransformer.spec.ts b/test/MapTransformer.spec.ts index 5e1aa0f..5f72208 100644 --- a/test/MapTransformer.spec.ts +++ b/test/MapTransformer.spec.ts @@ -1,9 +1,9 @@ -import { from, of } from "rxjs"; +import { from, of } from 'rxjs'; -import { MapTransformer } from "../src"; +import { MapTransformer } from '../src'; -describe("MapTransformer", () => { - it("should return an observable", () => { +describe('MapTransformer', () => { + it('should return an observable', () => { const spy = jest.fn(); const subt = { diff --git a/test/MatchMergeTransformer.spec.ts b/test/MatchMergeTransformer.spec.ts index dbbf935..e88e21e 100644 --- a/test/MatchMergeTransformer.spec.ts +++ b/test/MatchMergeTransformer.spec.ts @@ -1,9 +1,9 @@ -import { from } from "rxjs"; +import { from } from 'rxjs'; -import { MatchMergeTransformer } from "../src/transformers/MatchMergeTransformer"; +import { MatchMergeTransformer } from '../src/transformers/MatchMergeTransformer'; -describe("MatchMergeTransformer", () => { - it("should return an observable", () => { +describe('MatchMergeTransformer', () => { + it('should return an observable', () => { const spy = jest.fn(); class TestMatchMergeTransformer extends MatchMergeTransformer {