diff --git a/auth/utils/read-external-sources.ts b/auth/utils/read-external-sources.ts index b9c78bcd9..a290123e8 100644 --- a/auth/utils/read-external-sources.ts +++ b/auth/utils/read-external-sources.ts @@ -81,7 +81,7 @@ function filterPropertiesByServiceName(envObj: any, serviceName: string): any { } }); - // all env variables are parsed as strings, convert disable ssl vars to boolean + // all env variables are parsed as strings, convert boolean vars as needed if (typeof credentials.disableSsl === 'string') { credentials.disableSsl = credentials.disableSsl === 'true'; } @@ -90,6 +90,10 @@ function filterPropertiesByServiceName(envObj: any, serviceName: string): any { credentials.authDisableSsl = credentials.authDisableSsl === 'true'; } + if (typeof credentials.enableGzip === 'string') { + credentials.enableGzip = credentials.enableGzip === 'true'; + } + return credentials; } diff --git a/lib/base-service.ts b/lib/base-service.ts index 3a5508ca7..017b5e07d 100644 --- a/lib/base-service.ts +++ b/lib/base-service.ts @@ -145,6 +145,15 @@ export class BaseService { } } + /** + * Turn request body compression on or off. + * + * @param {boolean} setting Will turn it on if 'true', off if 'false'. + */ + public setEnableGzipCompression(setting: boolean): void { + this.requestWrapperInstance.compressRequestData = setting; + } + /** * Configure the service using external configuration * @@ -205,8 +214,12 @@ export class BaseService { const properties = readExternalSources(serviceName); if (properties !== null) { - // the user can define two client-level variables in the credentials file: url and disableSsl - const { url, disableSsl } = properties; + // the user can define the following client-level variables in the credentials file: + // - url + // - disableSsl + // - enableGzip + + const { url, disableSsl, enableGzip } = properties; if (url) { results.serviceUrl = stripTrailingSlash(url); @@ -214,6 +227,9 @@ export class BaseService { if (disableSsl === true) { results.disableSslVerification = disableSsl; } + if (enableGzip === true) { + results.enableGzipCompression = enableGzip; + } } return results; diff --git a/lib/request-wrapper.ts b/lib/request-wrapper.ts index 7ffc8bc81..6e5566a0e 100644 --- a/lib/request-wrapper.ts +++ b/lib/request-wrapper.ts @@ -19,20 +19,25 @@ import { AxiosRequestConfig } from 'axios'; import axiosCookieJarSupport from 'axios-cookiejar-support'; import extend = require('extend'); import FormData = require('form-data'); +import { OutgoingHttpHeaders } from 'http'; import https = require('https'); +import isStream = require('isstream'); import querystring = require('querystring'); -import { PassThrough as readableStream } from 'stream'; +import zlib = require('zlib'); import { buildRequestFileObject, getMissingParams, isEmptyObject, isFileData, isFileWithMetadata, stripTrailingSlash } from './helper'; import logger from './logger'; +import { streamToPromise } from './stream-to-promise'; const isBrowser = typeof window === 'object'; const globalTransactionId = 'x-global-transaction-id'; export class RequestWrapper { private axiosInstance; + private compressRequestData: boolean; constructor(axiosOptions?) { axiosOptions = axiosOptions || {}; + this.compressRequestData = Boolean(axiosOptions.enableGzipCompression); // override several axios defaults // axios sets the default Content-Type for `post`, `put`, and `patch` operations @@ -137,7 +142,7 @@ export class RequestWrapper { * @returns {ReadableStream|undefined} * @throws {Error} */ - public sendRequest(parameters): Promise { + public async sendRequest(parameters): Promise { const options = extend(true, {}, parameters.defaultOptions, parameters.options); const { path, body, form, formData, qs, method, serviceUrl } = options; let { headers, url } = options; @@ -205,6 +210,11 @@ export class RequestWrapper { // accept gzip encoded responses if Accept-Encoding is not already set headers['Accept-Encoding'] = headers['Accept-Encoding'] || 'gzip'; + // compress request body data if enabled + if (this.compressRequestData) { + data = await this.gzipRequestBody(data, headers); + } + const requestParams = { url, method, @@ -311,6 +321,48 @@ export class RequestWrapper { return error; } + + private async gzipRequestBody(data: any, headers: OutgoingHttpHeaders): Promise { + // skip compression if user has set the encoding header to gzip + const contentSetToGzip = + headers['Content-Encoding'] && + headers['Content-Encoding'].toString().includes('gzip'); + + if (!data || contentSetToGzip) { + return data; + } + + let reqBuffer: Buffer; + + try { + if (isStream(data)) { + reqBuffer = Buffer.from(await streamToPromise(data)); + } else if (data.toString && data.toString() !== '[object Object]' && !Array.isArray(data)) { + // this handles pretty much any primitive that isnt a JSON object or array + reqBuffer = Buffer.from(data.toString()); + } else { + reqBuffer = Buffer.from(JSON.stringify(data)); + } + } catch (err) { + logger.error('Error converting request body to a buffer - data will not be compressed.'); + logger.debug(err); + return data; + } + + try { + data = zlib.gzipSync(reqBuffer); + + // update the headers by reference - only if the data was actually compressed + headers['Content-Encoding'] = 'gzip'; + } catch (err) { + // if an exception is caught, `data` will still be in its original form + // we can just proceed with the request uncompressed + logger.error('Error compressing request body - data will not be compressed.'); + logger.debug(err); + } + + return data; + } } /** diff --git a/test/unit/base-service.test.js b/test/unit/base-service.test.js index d4d7a7b54..96d598a2f 100644 --- a/test/unit/base-service.test.js +++ b/test/unit/base-service.test.js @@ -103,6 +103,22 @@ describe('Base Service', () => { expect(testService.baseOptions.serviceUrl).toBe(newUrl); }); + it('should support enabling gzip compression after instantiation', () => { + const testService = new TestService({ + authenticator: AUTHENTICATOR, + }); + + expect(testService.baseOptions.enableGzipCompression).toBeFalsy(); + + const on = true; + testService.setEnableGzipCompression(on); + expect(testService.requestWrapperInstance.compressRequestData).toBe(on); + + const off = false; + testService.setEnableGzipCompression(off); + expect(testService.requestWrapperInstance.compressRequestData).toBe(off); + }); + it('should throw an error if an authenticator is not passed in', () => { expect(() => new TestService()).toThrow(); }); @@ -374,6 +390,7 @@ describe('Base Service', () => { readExternalSourcesMock.mockImplementation(() => ({ url: 'abc123.com', disableSsl: true, + enableGzip: true, })); testService.configureService(DEFAULT_NAME); @@ -381,6 +398,7 @@ describe('Base Service', () => { expect(readExternalSourcesMock).toHaveBeenCalled(); expect(testService.baseOptions.serviceUrl).toEqual('abc123.com'); expect(testService.baseOptions.disableSslVerification).toEqual(true); + expect(testService.baseOptions.enableGzipCompression).toEqual(true); }); it('configureService method should throw error if service name is not provided', () => { diff --git a/test/unit/read-external-sources.test.js b/test/unit/read-external-sources.test.js index 2ef57e8da..089a6d945 100644 --- a/test/unit/read-external-sources.test.js +++ b/test/unit/read-external-sources.test.js @@ -132,12 +132,14 @@ describe('Read External Sources Module', () => { expect(properties.scope).toBe(SCOPE); }); - it('should convert disableSsl values from string to boolean', () => { + it('should convert certain values from string to boolean', () => { process.env.TEST_SERVICE_DISABLE_SSL = 'true'; process.env.TEST_SERVICE_AUTH_DISABLE_SSL = 'true'; + process.env.TEST_SERVICE_ENABLE_GZIP = 'true'; const properties = readExternalSources(SERVICE_NAME); expect(typeof properties.disableSsl).toBe('boolean'); expect(typeof properties.authDisableSsl).toBe('boolean'); + expect(typeof properties.enableGzip).toBe('boolean'); }); }); diff --git a/test/unit/request-wrapper.test.js b/test/unit/request-wrapper.test.js index 3c8651fcf..476a91a53 100644 --- a/test/unit/request-wrapper.test.js +++ b/test/unit/request-wrapper.test.js @@ -1,6 +1,9 @@ 'use strict'; const fs = require('fs'); const https = require('https'); +const { Readable } = require('stream'); +const zlib = require('zlib'); +const logger = require('../../dist/lib/logger').default; process.env.NODE_DEBUG = 'axios'; jest.mock('axios'); const axios = require('axios'); @@ -48,6 +51,9 @@ describe('RequestWrapper constructor', () => { it('should handle scenario that no arguments are provided', () => { const rw = new RequestWrapper(); expect(rw).toBeInstanceOf(RequestWrapper); + + // without any arguments, the constructor should set its member variables accordingly + expect(rw.compressRequestData).toBe(false); }); it('should set defaults for certain axios configurations', () => { @@ -98,6 +104,11 @@ describe('RequestWrapper constructor', () => { expect(createdAxiosConfig.httpsAgent.options).toBeDefined(); expect(createdAxiosConfig.httpsAgent.options.rejectUnauthorized).toBe(false); }); + + it('should set `compressRequestData` based on user options', () => { + const rw = new RequestWrapper({ enableGzipCompression: true }); + expect(rw.compressRequestData).toBe(true); + }); }); describe('sendRequest', () => { @@ -481,6 +492,33 @@ describe('sendRequest', () => { done(); }); + it('should call `gzipRequestBody` if configured to do so', async () => { + const parameters = { + defaultOptions: { + body: 'post=body', + formData: '', + qs: {}, + method: 'POST', + url: '/trailing/slash/', + serviceUrl: 'https://example.ibm.com/', + headers: { + 'test-header': 'test-header-value', + }, + responseType: 'buffer', + }, + }; + + requestWrapperInstance.compressRequestData = true; + + mockAxiosInstance.mockResolvedValue(axiosResolveValue); + const gzipSpy = jest.spyOn(requestWrapperInstance, 'gzipRequestBody'); + await requestWrapperInstance.sendRequest(parameters); + expect(gzipSpy).toHaveBeenCalled(); + + // reset the alteration of the requestWrapperInstance + requestWrapperInstance.compressRequestData = false; + }); + // Need to rewrite this to test instantiation with userOptions // it('should keep parameters in options that are not explicitly set in requestwrapper', async done => { @@ -680,3 +718,152 @@ describe('formatError', () => { expect(error.message).toBe('error in building the request'); }); }); + +describe('gzipRequestBody', () => { + const gzipSpy = jest.spyOn(zlib, 'gzipSync'); + const errorLogSpy = jest.spyOn(logger, 'error').mockImplementation(() => {}); + const debugLogSpy = jest.spyOn(logger, 'debug').mockImplementation(() => {}); + + afterEach(() => { + gzipSpy.mockClear(); + errorLogSpy.mockClear(); + debugLogSpy.mockClear(); + }); + + it('should return unaltered data if encoding header is already set to gzip', async () => { + let data = 42; + const headers = { 'Content-Encoding': 'gzip' }; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBe(42); + }); + + it('should return unaltered data if encoding header values include gzip', async () => { + let data = 42; + const headers = { 'Content-Encoding': ['gzip', 'other-value'] }; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBe(42); + }); + + it('should return unaltered data if data is null', async () => { + let data = null; + const headers = {}; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(gzipSpy).not.toHaveBeenCalled(); + expect(data).toBe(null); + }); + + it('should compress json data into a buffer', async () => { + let data = { key: 'value' }; + const headers = {}; + + const jsonSpy = jest.spyOn(JSON, 'stringify'); + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBeInstanceOf(Buffer); + expect(gzipSpy).toHaveBeenCalled(); + expect(jsonSpy).toHaveBeenCalled(); + expect(headers['Content-Encoding']).toBe('gzip'); + + jsonSpy.mockRestore(); + }); + + it('should compress array data into a buffer', async () => { + let data = ['request', 'body']; + const headers = {}; + + const jsonSpy = jest.spyOn(JSON, 'stringify'); + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBeInstanceOf(Buffer); + expect(gzipSpy).toHaveBeenCalled(); + expect(jsonSpy).toHaveBeenCalled(); + expect(headers['Content-Encoding']).toBe('gzip'); + + jsonSpy.mockRestore(); + }); + + it('should compress string data into a buffer', async () => { + let data = 'body'; + const headers = {}; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBeInstanceOf(Buffer); + expect(gzipSpy).toHaveBeenCalled(); + expect(headers['Content-Encoding']).toBe('gzip'); + }); + + it('should compress number data into a buffer', async () => { + let data = 42; + const headers = {}; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBeInstanceOf(Buffer); + expect(gzipSpy).toHaveBeenCalled(); + expect(headers['Content-Encoding']).toBe('gzip'); + }); + + it('should compress stream data into a buffer', async () => { + let data = Readable.from(['request body']); + const headers = {}; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBeInstanceOf(Buffer); + expect(gzipSpy).toHaveBeenCalled(); + expect(headers['Content-Encoding']).toBe('gzip'); + }); + + it('should log an error and return data unaltered if data cant be stringified', async () => { + let data = { key: 'value' }; + data.circle = data; + const headers = {}; + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data.key).toBe('value'); + expect(errorLogSpy).toHaveBeenCalled(); + expect(errorLogSpy.mock.calls[0][0]).toBe( + 'Error converting request body to a buffer - data will not be compressed.' + ); + expect(debugLogSpy).toHaveBeenCalled(); + expect(debugLogSpy.mock.calls[0][0].message).toMatch('Converting circular structure to JSON'); + }); + + it('should log an error and return data unaltered if data cant be gzipped', async () => { + let data = 42; + const headers = {}; + + gzipSpy.mockImplementationOnce(() => { + throw new Error('bad zip'); + }); + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBe(42); + expect(errorLogSpy).toHaveBeenCalled(); + expect(errorLogSpy.mock.calls[0][0]).toBe( + 'Error compressing request body - data will not be compressed.' + ); + expect(debugLogSpy).toHaveBeenCalled(); + expect(debugLogSpy.mock.calls[0][0].message).toMatch('bad zip'); + }); + + it('should not update header if data cant be gzipped', async () => { + let data = 42; + const headers = {}; + + gzipSpy.mockImplementationOnce(() => { + throw new Error('bad zip'); + }); + + data = await requestWrapperInstance.gzipRequestBody(data, headers); + expect(data).toBe(42); + expect(headers['Content-Encoding']).toBeUndefined(); + expect(errorLogSpy).toHaveBeenCalled(); + expect(errorLogSpy.mock.calls[0][0]).toBe( + 'Error compressing request body - data will not be compressed.' + ); + expect(debugLogSpy).toHaveBeenCalled(); + expect(debugLogSpy.mock.calls[0][0].message).toMatch('bad zip'); + }); +});