From 9c854b7667b538ca2f7fe43f75baf401b6465143 Mon Sep 17 00:00:00 2001 From: Divinewill Zero <51236443+zerothebahdman@users.noreply.github.com> Date: Wed, 17 Jul 2024 16:21:34 +0100 Subject: [PATCH] ref: log serializer (#20) * ref: refactored serializer for http requests * feat: add a `raw` method in http wrapper --------- Co-authored-by: Divinewill --- package.json | 2 + src/http/wrapper.ts | 103 ++++++++++++++++--------------------- src/log/serializers.ts | 114 +++++++++++++++++++++++------------------ yarn.lock | 6 +++ 4 files changed, 117 insertions(+), 108 deletions(-) diff --git a/package.json b/package.json index 2330a8b..275fb5a 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "helmet": "^7.1.0", "jose": "^5.3.0", "jsonwebtoken": "^9.0.2", + "lodash": "^4.17.21", "moment": "^2.30.1", "mongoose": "^8.1.1", "nanoid": "3.3.4", @@ -65,6 +66,7 @@ "@nestjs/testing": "^10.0.0", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/lodash": "^4.17.7", "@types/node": "^20.3.1", "@types/request-ip": "^0.0.41", "@types/supertest": "^6.0.0", diff --git a/src/http/wrapper.ts b/src/http/wrapper.ts index 76d0ac7..6b49abb 100644 --- a/src/http/wrapper.ts +++ b/src/http/wrapper.ts @@ -1,11 +1,5 @@ -import { - APIError, - HttpError, - NoAuthorizationTokenError, - NoRequestIDError, - TimeoutError, -} from './errors'; -import { AxiosError, AxiosInstance, AxiosRequestConfig } from 'axios'; +import { APIError, HttpError, NoAuthorizationTokenError, NoRequestIDError, TimeoutError } from './errors'; +import { AxiosError, AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios'; import FormData from 'form-data'; import { Request } from 'express'; @@ -14,8 +8,6 @@ import { encode } from './jwt'; import qs from 'qs'; import { v4 } from 'uuid'; -// import { encode } from './jwt'; - export interface AuthConfig { scheme: string; secret: string; @@ -31,10 +23,7 @@ export type Action = () => Promise; * A function that can configure an axios request. Use the `defer` function * to push async work till the end of configuration */ -export type Plugin = ( - req: Partial>, - defer: (action: Action) => void -) => void; +export type Plugin = (req: Partial>, defer: (action: Action) => void) => void; export type RequestData = T | FormData | string; @@ -80,15 +69,11 @@ export class RequestWrapper { type(t: 'json' | 'form' | 'urlencoded' = 'json') { switch (t) { case 'json': - Object.assign(this.request.headers as object, { - 'Content-Type': 'application/json', - }); + Object.assign(this.request.headers as object, { 'Content-Type': 'application/json' }); break; case 'urlencoded': this.request.data = qs.stringify(this.request.data); - Object.assign(this.request.headers as object, { - 'Content-Type': 'application/x-www-form-urlencoded', - }); + Object.assign(this.request.headers as object, { 'Content-Type': 'application/x-www-form-urlencoded' }); break; case 'form': const form = new FormData(); @@ -117,16 +102,9 @@ export class RequestWrapper { set(key: string, value: string): this; set(key: string | object, value?: string) { let headers = {}; - if (typeof key === 'string') { - headers[key] = value; - } else { - headers = key; - } + typeof key === 'string' ? (headers[key] = value) : (headers = key); - Object.assign( - this.request.headers as object, - typeof key === 'string' ? { [key]: value } : key - ); + Object.assign(this.request.headers as object, typeof key === 'string' ? { [key]: value } : key); return this; } @@ -136,12 +114,14 @@ export class RequestWrapper { * @param req source request if there's any */ track(req?: Request) { + // make sure request ID exists for non-base requests + if (req && !req.headers['x-request-id']) { + throw new NoRequestIDError(this.request.url); + } + Object.assign(this.request.headers as object, { - 'X-Request-ID': - !!req && req.headers['x-request-id'] - ? req.headers['x-request-id'] - : v4(), - 'X-Origin-Service': this.service, + 'X-Request-ID': !!req && req.headers['x-request-id'] ? req.headers['x-request-id'] : v4(), + 'X-Origin-Service': this.service }); return this; @@ -159,9 +139,7 @@ export class RequestWrapper { throw new NoAuthorizationTokenError(this.request.url); } - Object.assign(this.request.headers, { - Authorization: reqSession.headers.authorization, - }); + Object.assign(this.request.headers, { Authorization: reqSession.headers.authorization }); return this; } else { @@ -169,19 +147,15 @@ export class RequestWrapper { reqSession = { service: this.service, request_time: new Date(), - ...payload, + ...payload } as any; } // push till when the request is being made return this.defer(async () => { - const token = await encode( - this.authConfig.secret, - this.authConfig.timeout, - reqSession - ); + const token = await encode(this.authConfig.secret, this.authConfig.timeout, reqSession); Object.assign(this.request.headers, { - Authorization: `${this.authConfig.scheme} ${token}`, + Authorization: `${this.authConfig.scheme} ${token}` }); }); } @@ -198,23 +172,13 @@ export class RequestWrapper { } return this.instance({ timeout: timeout * 1000, ...this.request }).then( - (res) => res.data, + res => res.data, (err: AxiosError) => { if (err.response) { - throw new APIError( - err.config!.url as string, - err.response.status, - err.response.data - ); + throw new APIError(err.config!.url as string, err.response.status, err.response.data); } else if (err.request) { - if ( - err.code === AxiosError.ETIMEDOUT || - err.code === AxiosError.ECONNABORTED - ) { - throw new TimeoutError( - err.config!.url as string, - err.config!.timeout as number - ); + if (err.code === AxiosError.ETIMEDOUT || err.code === AxiosError.ECONNABORTED) { + throw new TimeoutError(err.config!.url as string, err.config!.timeout as number); } throw new HttpError(err.config!.url as string, err); } else { @@ -223,4 +187,27 @@ export class RequestWrapper { } ); } + + /** + * Same as do but gives direct access to the response. Also does not handle + * any error but timeouts + * @param timeout timeout for request in seconds + */ + async raw(timeout = 10): Promise> { + // call deferred actions + for (const iterator of this.asyncActions) { + await iterator(); + } + + return this.instance({ timeout: timeout * 1000, ...this.request }).then( + res => res, + (err: AxiosError) => { + if (err.code === AxiosError.ETIMEDOUT || err.code === AxiosError.ECONNABORTED) { + throw new TimeoutError(err.config.url, err.config.timeout); + } else { + throw new HttpError(err.config.url, err); + } + } + ); + } } diff --git a/src/log/serializers.ts b/src/log/serializers.ts index 1678332..a469a5f 100644 --- a/src/log/serializers.ts +++ b/src/log/serializers.ts @@ -1,16 +1,8 @@ import { AxiosRequestConfig, AxiosResponse } from 'axios'; import { Request, Response } from 'express'; -import unset from 'lodash/unset'; - -const axiosDefaultHeaders = [ - 'common', - 'delete', - 'get', - 'head', - 'post', - 'put', - 'patch', -]; +import { cloneDeep, isPlainObject, unset } from 'lodash'; + +const axiosDefaultHeaders = ['common', 'delete', 'get', 'head', 'post', 'put', 'patch']; /** * Create serializers for common log entries. This entries would @@ -25,9 +17,9 @@ export function defaultSerializers(...paths: string[]) { axios_req: axiosRequest(...paths), axios_res: axiosResponse(...paths), req: expressRequest(...paths), - res: expressResponse, + res: expressResponse(...paths), event: sanitized(...paths), - err: serializeErr, + err: serializeErr }; } @@ -63,17 +55,16 @@ export function serializeErr(err: any) { stack: getFullErrorStack(err), message: err.message, name: err.name, - ...err, + ...err }; } export function sanitized(...paths: string[]) { return (data: T) => { - if (!data || typeof data !== 'object' || Object.keys(data).length === 0) - return data; + if (!data || typeof data !== 'object' || Object.keys(data).length === 0) return data; const dataCopy = { ...data }; - paths.forEach((p) => unset(dataCopy, p)); + paths.forEach(p => unset(dataCopy, p)); return dataCopy; }; @@ -83,20 +74,13 @@ export function sanitized(...paths: string[]) { * Create serializer for axios requests * @param paths sensitive data pasths */ -export function axiosRequest( - ...paths: string[] -): (conf: AxiosRequestConfig) => object { +export function axiosRequest(...paths: string[]): (conf: AxiosRequestConfig) => object { return (conf: AxiosRequestConfig) => { - const log = { - method: conf.method, - url: conf.url, - headers: conf.headers, - params: conf.params, - }; + const log = { method: conf.method, url: conf.url, headers: conf.headers, params: conf.params }; // remove default header config const headers = Object.assign({}, conf.headers); - axiosDefaultHeaders.forEach((k) => { + axiosDefaultHeaders.forEach(k => { delete headers[k]; }); @@ -108,8 +92,7 @@ export function axiosRequest( } if (conf.data && Object.keys(conf.data).length !== 0) { - const logBody = { ...conf.data }; - paths.forEach((p) => unset(logBody, p)); + const logBody = deepSanitizeObj(conf.data, ...paths); log['data'] = logBody; } @@ -122,17 +105,10 @@ export function axiosRequest( * Serializer for axios responses * @param res axios response object */ -export function axiosResponse( - ...paths: string[] -): (res: AxiosResponse) => object { +export function axiosResponse(...paths: string[]): (res: AxiosResponse) => object { return (res: AxiosResponse) => { - const data = { ...res.data }; - paths.forEach((p) => unset(data, p)); - return { - statusCode: res.status, - headers: res.headers, - body: data, - }; + const data = deepSanitizeObj(res.data, ...paths); + return { statusCode: res.status, headers: res.headers, body: data }; }; } @@ -150,13 +126,11 @@ export function expressRequest(...paths: string[]): (req: Request) => object { headers: req.headers, params: req.params, remoteAddress: req.socket.remoteAddress, - remotePort: req.socket.remotePort, + remotePort: req.socket.remotePort }; if (req.body && Object.keys(req.body).length !== 0) { - const logBody = { ...req.body }; - paths.forEach((p) => unset(logBody, p)); - + const logBody = deepSanitizeObj(req.body, ...paths); log['body'] = logBody; } @@ -166,14 +140,54 @@ export function expressRequest(...paths: string[]): (req: Request) => object { /** * Serializer for express responses - * @param res express response object + * @param paths sensitive data paths */ -export function expressResponse(res: Response) { - if (!res || !res.statusCode) return res; +export function expressResponse(...paths: string[]): (res: Response) => object { + return (res: Response) => { + if (!res || !res.statusCode) return res; - return { - statusCode: res.statusCode, - headers: res.getHeaders(), - body: res.locals.body, + const log = { + statusCode: res.statusCode, + headers: res.getHeaders() + }; + + const body = + typeof res.locals.body === 'string' && isStringifiedObject(res.locals.body) + ? JSON.parse(res.locals.body) + : res.locals.body; + if (body && Object.keys(body).length !== 0) { + const logBody = deepSanitizeObj(body, ...paths); + + log['body'] = logBody; + } + + return log; }; } + +function isStringifiedObject(str: string): boolean { + try { + JSON.parse(str); + return true; + } catch (e) { + return false; + } +} + +function deepSanitizeObj(data: object, ...paths: string[]) { + const clone = cloneDeep(data); // Deep clone to avoid any reference issues + + function sanitizeNode(node: any) { + if (isPlainObject(node)) { + paths.forEach(path => unset(node, path)); + + Object.keys(node).forEach(key => sanitizeNode(node[key])); + } else if (Array.isArray(node)) { + node.forEach((item: any) => sanitizeNode(item)); + } + } + + sanitizeNode(clone); + + return clone; +} diff --git a/yarn.lock b/yarn.lock index d20a3dc..91e689d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1231,6 +1231,11 @@ resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841" integrity sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA== +"@types/lodash@^4.17.7": + version "4.17.7" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.17.7.tgz#2f776bcb53adc9e13b2c0dfd493dfcbd7de43612" + integrity sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA== + "@types/methods@^1.1.4": version "1.1.4" resolved "https://registry.yarnpkg.com/@types/methods/-/methods-1.1.4.tgz#d3b7ac30ac47c91054ea951ce9eed07b1051e547" @@ -5453,6 +5458,7 @@ word-wrap@^1.2.5: integrity sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA== "wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: + name wrap-ansi-cjs version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==