From db23db3221f39972b3073f50a4b952f9696dff95 Mon Sep 17 00:00:00 2001 From: Joshua Date: Thu, 4 Nov 2021 13:19:10 -0700 Subject: [PATCH] Use advanced settings for csv separator and visual report timezone (#209) --- .../components/context_menu/context_menu.js | 6 +-- public/components/main/main_utils.tsx | 10 +---- public/components/utils/settings_service.ts | 14 +++++++ server/routes/lib/createReport.ts | 3 ++ server/routes/report.ts | 7 +++- .../__tests__/savedSearchReportHelper.test.ts | 41 ++++++++++++++++++- server/routes/utils/dataReportHelpers.ts | 4 +- .../routes/utils/savedSearchReportHelper.ts | 5 ++- 8 files changed, 73 insertions(+), 17 deletions(-) diff --git a/public/components/context_menu/context_menu.js b/public/components/context_menu/context_menu.js index 6cf87e5d..30602ec4 100644 --- a/public/components/context_menu/context_menu.js +++ b/public/components/context_menu/context_menu.js @@ -105,9 +105,9 @@ const generateInContextReport = async ( }; fetch( - `../api/reporting/generateReport?timezone=${ - Intl.DateTimeFormat().resolvedOptions().timeZone - }&dateFormat=${uiSettingsService.get('dateFormat')}`, + `../api/reporting/generateReport?${new URLSearchParams( + uiSettingsService.getSearchParams() + )}`, { headers: { 'Content-Type': 'application/json', diff --git a/public/components/main/main_utils.tsx b/public/components/main/main_utils.tsx index c32a8791..f784573b 100644 --- a/public/components/main/main_utils.tsx +++ b/public/components/main/main_utils.tsx @@ -181,10 +181,7 @@ export const generateReportFromDefinitionId = async ( headers: { 'Content-Type': 'application/json', }, - query: { - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, - dateFormat: uiSettingsService.get('dateFormat'), - }, + query: uiSettingsService.getSearchParams(), }) .then(async (response: any) => { // for emailing a report, this API response doesn't have response body @@ -217,10 +214,7 @@ export const generateReportById = async ( ) => { await httpClient .get(`../api/reporting/generateReport/${reportId}`, { - query: { - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, - dateFormat: uiSettingsService.get('dateFormat'), - }, + query: uiSettingsService.getSearchParams(), }) .then(async (response) => { //TODO: duplicate code, extract to be a function that can reuse. e.g. handleResponse(response) diff --git a/public/components/utils/settings_service.ts b/public/components/utils/settings_service.ts index f1baf906..5df595f5 100644 --- a/public/components/utils/settings_service.ts +++ b/public/components/utils/settings_service.ts @@ -20,4 +20,18 @@ export const uiSettingsService = { get: (key: string, defaultOverride?: any) => { return uiSettings?.get(key, defaultOverride) || ''; }, + getSearchParams: function () { + const rawTimeZone = this.get('dateFormat:tz'); + const timezone = + !rawTimeZone || rawTimeZone === 'Browser' + ? Intl.DateTimeFormat().resolvedOptions().timeZone + : rawTimeZone; + const dateFormat = this.get('dateFormat'); + const csvSeparator = this.get('csv:separator'); + return { + timezone, + dateFormat, + csvSeparator, + }; + }, }; diff --git a/server/routes/lib/createReport.ts b/server/routes/lib/createReport.ts index 70058a87..56bf0d39 100644 --- a/server/routes/lib/createReport.ts +++ b/server/routes/lib/createReport.ts @@ -71,6 +71,8 @@ export const createReport = async ( // @ts-ignore const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; + // @ts-ignore + const csvSeparator = request.query.csvSeparator || ','; const { basePath, serverInfo: { protocol, port, hostname }, @@ -98,6 +100,7 @@ export const createReport = async ( report, opensearchClient, dateFormat, + csvSeparator, isScheduledTask, logger ); diff --git a/server/routes/report.ts b/server/routes/report.ts index be13d7ee..268a0234 100644 --- a/server/routes/report.ts +++ b/server/routes/report.ts @@ -58,6 +58,7 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { query: schema.object({ timezone: schema.maybe(schema.string()), dateFormat: schema.maybe(schema.string()), + csvSeparator: schema.maybe(schema.string()), }), }, }, @@ -122,7 +123,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { }), query: schema.object({ timezone: schema.string(), - dateFormat: schema.maybe(schema.string()), + dateFormat: schema.string(), + csvSeparator: schema.string(), }), }, }, @@ -188,7 +190,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { }), query: schema.object({ timezone: schema.string(), - dateFormat: schema.maybe(schema.string()), + dateFormat: schema.string(), + csvSeparator: schema.string(), }), }, }, diff --git a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 536bdbaf..58f877bd 100644 --- a/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -64,7 +64,7 @@ const input = { }, }; -const mockDateFormat = 'date_hour_minute_second_fraction'; +const mockDateFormat = 'MM/DD/YYYY h:mm:ss.SSS a'; /** * Max result window size in OpenSearch index settings. @@ -84,6 +84,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -95,6 +96,7 @@ describe('test create saved search report', () => { input, mockOpenSearchClient([]), mockDateFormat, + ',', undefined, mockLogger ); @@ -105,6 +107,7 @@ describe('test create saved search report', () => { input, mockOpenSearchClient([]), mockDateFormat, + ',', undefined, mockLogger ); @@ -118,6 +121,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -137,6 +141,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -170,6 +175,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -206,6 +212,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -234,6 +241,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -266,6 +274,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -292,6 +301,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -304,6 +314,30 @@ describe('test create saved search report', () => { ); }, 20000); + test('create report for data set with comma and custom separator', async () => { + const hits = [ + hit({ category: ',c1', customer_gender: 'Ma,le' }), + hit({ category: 'c2,', customer_gender: 'M,ale' }), + hit({ category: ',,c3', customer_gender: 'Male,,,' }), + ]; + const client = mockOpenSearchClient(hits); + const { dataUrl } = await createSavedSearchReport( + input, + client, + mockDateFormat, + '|', + undefined, + mockLogger + ); + + expect(dataUrl).toEqual( + 'category|customer_gender\n' + + ',c1|Ma,le\n' + + 'c2,|M,ale\n' + + ',,c3|Male,,,' + ); + }, 20000); + test('create report for data set with nested fields', async () => { const hits = [ hit({ @@ -324,6 +358,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -348,6 +383,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -378,6 +414,7 @@ describe('test create saved search report', () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -404,6 +441,7 @@ test('create report for data set contains null field value', async () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); @@ -434,6 +472,7 @@ test('create report for data set with metadata fields', async () => { input, client, mockDateFormat, + ',', undefined, mockLogger ); diff --git a/server/routes/utils/dataReportHelpers.ts b/server/routes/utils/dataReportHelpers.ts index f313e2a8..3df73377 100644 --- a/server/routes/utils/dataReportHelpers.ts +++ b/server/routes/utils/dataReportHelpers.ts @@ -163,10 +163,10 @@ export const getOpenSearchData = ( }; //Convert the data to Csv format -export const convertToCSV = async (dataset) => { +export const convertToCSV = async (dataset, csvSeparator) => { let convertedData: any = []; const options = { - delimiter: { field: ',', eol: '\n' }, + delimiter: { field: csvSeparator, eol: '\n' }, emptyFieldValue: ' ', }; await converter.json2csvAsync(dataset[0], options).then((csv) => { diff --git a/server/routes/utils/savedSearchReportHelper.ts b/server/routes/utils/savedSearchReportHelper.ts index 9f25ea74..3138844d 100644 --- a/server/routes/utils/savedSearchReportHelper.ts +++ b/server/routes/utils/savedSearchReportHelper.ts @@ -50,6 +50,7 @@ export async function createSavedSearchReport( report: any, client: ILegacyClusterClient | ILegacyScopedClusterClient, dateFormat: string, + csvSeparator: string, isScheduledTask: boolean = true, logger: Logger ): Promise { @@ -62,6 +63,7 @@ export async function createSavedSearchReport( client, params.core_params, dateFormat, + csvSeparator, isScheduledTask, logger ); @@ -148,6 +150,7 @@ async function generateReportData( client: ILegacyClusterClient | ILegacyScopedClusterClient, params: any, dateFormat: string, + csvSeparator: string, isScheduledTask: boolean, logger: Logger ) { @@ -277,6 +280,6 @@ async function generateReportData( async function convertOpenSearchDataToCsv() { const dataset: any = []; dataset.push(getOpenSearchData(arrayHits, report, params, dateFormat)); - return await convertToCSV(dataset); + return await convertToCSV(dataset, csvSeparator); } }