ref: log serializer (#20)

* ref: refactored serializer for http requests * feat: add a `raw` method in http wrapper --------- Co-authored-by: Divinewill <zerothebahdman@users.noreply.github.com>
blinkafrica · Jul 17, 2024 · 9c854b7 · 9c854b7
1 parent cb9f9a9
commit 9c854b7
Show file tree

Hide file tree

Showing 4 changed files with 117 additions and 108 deletions.
diff --git a/package.json b/package.json
@@ -47,6 +47,7 @@
     "helmet": "^7.1.0",
     "jose": "^5.3.0",
     "jsonwebtoken": "^9.0.2",
+    "lodash": "^4.17.21",
     "moment": "^2.30.1",
     "mongoose": "^8.1.1",
     "nanoid": "3.3.4",
@@ -65,6 +66,7 @@
     "@nestjs/testing": "^10.0.0",
     "@types/express": "^4.17.17",
     "@types/jest": "^29.5.2",
+    "@types/lodash": "^4.17.7",
     "@types/node": "^20.3.1",
     "@types/request-ip": "^0.0.41",
     "@types/supertest": "^6.0.0",

diff --git a/src/http/wrapper.ts b/src/http/wrapper.ts
@@ -1,11 +1,5 @@
-import {
-  APIError,
-  HttpError,
-  NoAuthorizationTokenError,
-  NoRequestIDError,
-  TimeoutError,
-} from './errors';
-import { AxiosError, AxiosInstance, AxiosRequestConfig } from 'axios';
+import { APIError, HttpError, NoAuthorizationTokenError, NoRequestIDError, TimeoutError } from './errors';
+import { AxiosError, AxiosInstance, AxiosRequestConfig, AxiosResponse } from 'axios';
 
 import FormData from 'form-data';
 import { Request } from 'express';
@@ -14,8 +8,6 @@ import { encode } from './jwt';
 import qs from 'qs';
 import { v4 } from 'uuid';
 
-// import { encode } from './jwt';
-
 export interface AuthConfig {
   scheme: string;
   secret: string;
@@ -31,10 +23,7 @@ export type Action = () => Promise<void>;
  * A function that can configure an axios request. Use the `defer` function
  * to push async work till the end of configuration
  */
-export type Plugin<T = any> = (
-  req: Partial<AxiosRequestConfig<T>>,
-  defer: (action: Action) => void
-) => void;
+export type Plugin<T = any> = (req: Partial<AxiosRequestConfig<T>>, defer: (action: Action) => void) => void;
 
 export type RequestData<T extends object> = T | FormData | string;
 
@@ -80,15 +69,11 @@ export class RequestWrapper<T extends object> {
   type(t: 'json' | 'form' | 'urlencoded' = 'json') {
     switch (t) {
       case 'json':
-        Object.assign(this.request.headers as object, {
-          'Content-Type': 'application/json',
-        });
+        Object.assign(this.request.headers as object, { 'Content-Type': 'application/json' });
         break;
       case 'urlencoded':
         this.request.data = qs.stringify(this.request.data);
-        Object.assign(this.request.headers as object, {
-          'Content-Type': 'application/x-www-form-urlencoded',
-        });
+        Object.assign(this.request.headers as object, { 'Content-Type': 'application/x-www-form-urlencoded' });
         break;
       case 'form':
         const form = new FormData();
@@ -117,16 +102,9 @@ export class RequestWrapper<T extends object> {
   set(key: string, value: string): this;
   set(key: string | object, value?: string) {
     let headers = {};
-    if (typeof key === 'string') {
-      headers[key] = value;
-    } else {
-      headers = key;
-    }
+    typeof key === 'string' ? (headers[key] = value) : (headers = key);
 
-    Object.assign(
-      this.request.headers as object,
-      typeof key === 'string' ? { [key]: value } : key
-    );
+    Object.assign(this.request.headers as object, typeof key === 'string' ? { [key]: value } : key);
 
     return this;
   }
@@ -136,12 +114,14 @@ export class RequestWrapper<T extends object> {
    * @param req source request if there's any
    */
   track(req?: Request) {
+    // make sure request ID exists for non-base requests
+    if (req && !req.headers['x-request-id']) {
+      throw new NoRequestIDError(this.request.url);
+    }
+
     Object.assign(this.request.headers as object, {
-      'X-Request-ID':
-        !!req && req.headers['x-request-id']
-          ? req.headers['x-request-id']
-          : v4(),
-      'X-Origin-Service': this.service,
+      'X-Request-ID': !!req && req.headers['x-request-id'] ? req.headers['x-request-id'] : v4(),
+      'X-Origin-Service': this.service
     });
 
     return this;
@@ -159,29 +139,23 @@ export class RequestWrapper<T extends object> {
         throw new NoAuthorizationTokenError(this.request.url);
       }
 
-      Object.assign(this.request.headers, {
-        Authorization: reqSession.headers.authorization,
-      });
+      Object.assign(this.request.headers, { Authorization: reqSession.headers.authorization });
 
       return this;
     } else {
       if (!reqSession) {
         reqSession = {
           service: this.service,
           request_time: new Date(),
-          ...payload,
+          ...payload
         } as any;
       }
 
       // push till when the request is being made
       return this.defer(async () => {
-        const token = await encode(
-          this.authConfig.secret,
-          this.authConfig.timeout,
-          reqSession
-        );
+        const token = await encode(this.authConfig.secret, this.authConfig.timeout, reqSession);
         Object.assign(this.request.headers, {
-          Authorization: `${this.authConfig.scheme} ${token}`,
+          Authorization: `${this.authConfig.scheme} ${token}`
         });
       });
     }
@@ -198,23 +172,13 @@ export class RequestWrapper<T extends object> {
     }
 
     return this.instance({ timeout: timeout * 1000, ...this.request }).then(
-      (res) => res.data,
+      res => res.data,
       (err: AxiosError) => {
         if (err.response) {
-          throw new APIError(
-            err.config!.url as string,
-            err.response.status,
-            err.response.data
-          );
+          throw new APIError(err.config!.url as string, err.response.status, err.response.data);
         } else if (err.request) {
-          if (
-            err.code === AxiosError.ETIMEDOUT ||
-            err.code === AxiosError.ECONNABORTED
-          ) {
-            throw new TimeoutError(
-              err.config!.url as string,
-              err.config!.timeout as number
-            );
+          if (err.code === AxiosError.ETIMEDOUT || err.code === AxiosError.ECONNABORTED) {
+            throw new TimeoutError(err.config!.url as string, err.config!.timeout as number);
           }
           throw new HttpError(err.config!.url as string, err);
         } else {
@@ -223,4 +187,27 @@ export class RequestWrapper<T extends object> {
       }
     );
   }
+
+  /**
+   * Same as do but gives direct access to the response. Also does not handle
+   * any error but timeouts
+   * @param timeout timeout for request in seconds
+   */
+  async raw<T = any>(timeout = 10): Promise<AxiosResponse<T>> {
+    // call deferred actions
+    for (const iterator of this.asyncActions) {
+      await iterator();
+    }
+
+    return this.instance({ timeout: timeout * 1000, ...this.request }).then(
+      res => res,
+      (err: AxiosError) => {
+        if (err.code === AxiosError.ETIMEDOUT || err.code === AxiosError.ECONNABORTED) {
+          throw new TimeoutError(err.config.url, err.config.timeout);
+        } else {
+          throw new HttpError(err.config.url, err);
+        }
+      }
+    );
+  }
 }
diff --git a/src/log/serializers.ts b/src/log/serializers.ts
@@ -1,16 +1,8 @@
 import { AxiosRequestConfig, AxiosResponse } from 'axios';
 import { Request, Response } from 'express';
-import unset from 'lodash/unset';
-
-const axiosDefaultHeaders = [
-  'common',
-  'delete',
-  'get',
-  'head',
-  'post',
-  'put',
-  'patch',
-];
+import { cloneDeep, isPlainObject, unset } from 'lodash';
+
+const axiosDefaultHeaders = ['common', 'delete', 'get', 'head', 'post', 'put', 'patch'];
 
 /**
  * Create serializers for common log entries. This entries would
@@ -25,9 +17,9 @@ export function defaultSerializers(...paths: string[]) {
     axios_req: axiosRequest(...paths),
     axios_res: axiosResponse(...paths),
     req: expressRequest(...paths),
-    res: expressResponse,
+    res: expressResponse(...paths),
     event: sanitized(...paths),
-    err: serializeErr,
+    err: serializeErr
   };
 }
 
@@ -63,17 +55,16 @@ export function serializeErr(err: any) {
     stack: getFullErrorStack(err),
     message: err.message,
     name: err.name,
-    ...err,
+    ...err
   };
 }
 
 export function sanitized<T = any>(...paths: string[]) {
   return (data: T) => {
-    if (!data || typeof data !== 'object' || Object.keys(data).length === 0)
-      return data;
+    if (!data || typeof data !== 'object' || Object.keys(data).length === 0) return data;
 
     const dataCopy = { ...data };
-    paths.forEach((p) => unset(dataCopy, p));
+    paths.forEach(p => unset(dataCopy, p));
 
     return dataCopy;
   };
@@ -83,20 +74,13 @@ export function sanitized<T = any>(...paths: string[]) {
  * Create serializer for axios requests
  * @param paths sensitive data pasths
  */
-export function axiosRequest(
-  ...paths: string[]
-): (conf: AxiosRequestConfig) => object {
+export function axiosRequest(...paths: string[]): (conf: AxiosRequestConfig) => object {
   return (conf: AxiosRequestConfig) => {
-    const log = {
-      method: conf.method,
-      url: conf.url,
-      headers: conf.headers,
-      params: conf.params,
-    };
+    const log = { method: conf.method, url: conf.url, headers: conf.headers, params: conf.params };
 
     // remove default header config
     const headers = Object.assign({}, conf.headers);
-    axiosDefaultHeaders.forEach((k) => {
+    axiosDefaultHeaders.forEach(k => {
       delete headers[k];
     });
 
@@ -108,8 +92,7 @@ export function axiosRequest(
     }
 
     if (conf.data && Object.keys(conf.data).length !== 0) {
-      const logBody = { ...conf.data };
-      paths.forEach((p) => unset(logBody, p));
+      const logBody = deepSanitizeObj(conf.data, ...paths);
 
       log['data'] = logBody;
     }
@@ -122,17 +105,10 @@ export function axiosRequest(
  * Serializer for axios responses
  * @param res axios response object
  */
-export function axiosResponse(
-  ...paths: string[]
-): (res: AxiosResponse<any>) => object {
+export function axiosResponse(...paths: string[]): (res: AxiosResponse<any>) => object {
   return (res: AxiosResponse<any>) => {
-    const data = { ...res.data };
-    paths.forEach((p) => unset(data, p));
-    return {
-      statusCode: res.status,
-      headers: res.headers,
-      body: data,
-    };
+    const data = deepSanitizeObj(res.data, ...paths);
+    return { statusCode: res.status, headers: res.headers, body: data };
   };
 }
 
@@ -150,13 +126,11 @@ export function expressRequest(...paths: string[]): (req: Request) => object {
       headers: req.headers,
       params: req.params,
       remoteAddress: req.socket.remoteAddress,
-      remotePort: req.socket.remotePort,
+      remotePort: req.socket.remotePort
     };
 
     if (req.body && Object.keys(req.body).length !== 0) {
-      const logBody = { ...req.body };
-      paths.forEach((p) => unset(logBody, p));
-
+      const logBody = deepSanitizeObj(req.body, ...paths);
       log['body'] = logBody;
     }
 
@@ -166,14 +140,54 @@ export function expressRequest(...paths: string[]): (req: Request) => object {
 
 /**
  * Serializer for express responses
- * @param res express response object
+ * @param paths sensitive data paths
  */
-export function expressResponse(res: Response) {
-  if (!res || !res.statusCode) return res;
+export function expressResponse(...paths: string[]): (res: Response) => object {
+  return (res: Response) => {
+    if (!res || !res.statusCode) return res;
 
-  return {
-    statusCode: res.statusCode,
-    headers: res.getHeaders(),
-    body: res.locals.body,
+    const log = {
+      statusCode: res.statusCode,
+      headers: res.getHeaders()
+    };
+
+    const body =
+      typeof res.locals.body === 'string' && isStringifiedObject(res.locals.body)
+        ? JSON.parse(res.locals.body)
+        : res.locals.body;
+    if (body && Object.keys(body).length !== 0) {
+      const logBody = deepSanitizeObj(body, ...paths);
+
+      log['body'] = logBody;
+    }
+
+    return log;
   };
 }
+
+function isStringifiedObject(str: string): boolean {
+  try {
+    JSON.parse(str);
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+function deepSanitizeObj(data: object, ...paths: string[]) {
+  const clone = cloneDeep(data); // Deep clone to avoid any reference issues
+
+  function sanitizeNode(node: any) {
+    if (isPlainObject(node)) {
+      paths.forEach(path => unset(node, path));
+
+      Object.keys(node).forEach(key => sanitizeNode(node[key]));
+    } else if (Array.isArray(node)) {
+      node.forEach((item: any) => sanitizeNode(item));
+    }
+  }
+
+  sanitizeNode(clone);
+
+  return clone;
+}