From 65274644c0898489245752d61305a856b738cf93 Mon Sep 17 00:00:00 2001 From: Dai Date: Mon, 14 Sep 2020 11:53:29 -0700 Subject: [PATCH 01/21] Integrate with saved search report --- .../server/routes/savedSearchReportHelper.ts | 374 ++++++++++++++++++ .../server/routes/utils/reportHelper.ts | 4 +- 2 files changed, 376 insertions(+), 2 deletions(-) create mode 100644 kibana-reports/server/routes/savedSearchReportHelper.ts diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts new file mode 100644 index 00000000..64bfe9df --- /dev/null +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -0,0 +1,374 @@ +import { metaData, getSelectedFields, buildQuery, getEsData, convertToCSV } from './utils/dataReportHelpers'; +import { + IClusterClient, + IScopedClusterClient, +} from '../../../../src/core/server'; + +export async function createSavedSearchReport( + report: any, + client: IClusterClient | IScopedClusterClient +) { + await populateMetaData(client, report.report_params); + return await generateReport(client); +} + +async function populateMetaData( + client: IClusterClient | IScopedClusterClient, + reportParams: any +) { + metaData.saved_search_id = reportParams.saved_search_id; + metaData.report_format = reportParams.report_format; + metaData.start = reportParams.start; + metaData.end = reportParams.end; + let resIndexPattern: any = {}; + //get the saved search infos + const ssParams = { + index: '.kibana', + id: 'search:' + reportParams.saved_search_id, + }; + + const ssInfos = await client.callAsInternalUser( + 'get', + ssParams + ); + + // get the sorting + metaData.sorting = ssInfos._source.search.sort; + + // get the saved search type + metaData.type = ssInfos._source.type; + + // get the filters + metaData.filters = + ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; + + //get the list of selected columns in the saved search.Otherwise select all the fields under the _source + await getSelectedFields(ssInfos._source.search.columns); + + //Get index name + for (let item of ssInfos._source.references) { + if (item.name === JSON.parse(metaData.filters).indexRefName) { + //Get index-pattern informations + const indexPattern = await client.callAsInternalUser( + 'get', + { + index: '.kibana', + id: 'index-pattern:' + item.id, + } + ); + resIndexPattern = indexPattern._source['index-pattern']; + metaData.paternName = resIndexPattern.title; + (metaData.timeFieldName = resIndexPattern.timeFieldName), + (metaData.fields = resIndexPattern.fields); //Get all fields + //Getting fields of type Date + for (let item of JSON.parse(metaData.fields)) { + if (item.type === 'date') { + metaData.dateFields.push(item.name); + } + } + } + } +} + +async function generateReport( + client: IClusterClient | IScopedClusterClient +) { + let report = { _source: metaData }; + let nbRows: number = 0; + let scroll_size: number = 0; + + let dataset: any = []; + let arrayHits: any = []; + let esData: any = {}; + let message: string = 'success'; + let fetch_size: number = 0; + let nbScroll: number = 0; + //try { + + //fetch ES query max size windows + const indexPattern: string = report._source.paternName; + let settings = await client.callAsInternalUser( + 'indices.getSettings', + { + index: indexPattern, + includeDefaults: true, + } + ); + const default_max_size: number = + settings[indexPattern].defaults.index.max_result_window; + + //build the ES Count query + const countReq = buildQuery(report, 1); + //Count the Data in ES + const esCount = await client.callAsInternalUser( + 'count', + { + index: indexPattern, + body: countReq.toJSON(), + } + ); + + //If No data in elasticsearch + if (esCount.count === 0) { + /*return response.custom({ + statusCode: 200, + body: 'No data in Elasticsearch.', + });*/ + return { + data: {}, + filename: "", + }; + } + + //build the ES query + const reqBody = buildQuery(report, 0); + + //first case: No args passed. No need to scroll + if (!nbRows && !scroll_size) { + if (esCount.count > default_max_size) { + message = `Truncated Data! The requested data has reached the limit of Elasticsearch query size of ${default_max_size}. Please increase the limit and try again !`; + } + esData = await fetchData(report, reqBody, default_max_size); + arrayHits.push(esData.hits); + } + + //Second case: 1 arg passed + + //Only Number of Rows is passed + + if (nbRows && !scroll_size) { + let rows = 0; + if (nbRows > default_max_size) { + //fetch the data + fetch_size = default_max_size; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + if (nbRows > esCount.count) { + rows = esCount.count; + nbScroll = Math.floor(esCount.count / default_max_size); + } else { + rows = nbRows; + nbScroll = Math.floor(nbRows / default_max_size); + } + + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await client.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = rows % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData(report, reqBody, extra_fetch); + arrayHits.push(extra_esData.hits); + } + } else { + fetch_size = nbRows; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + } + } else if (scroll_size && !nbRows) { + //Only scroll_size is passed + if (esCount.count > default_max_size) { + fetch_size = scroll_size; + nbScroll = Math.floor(esCount.count / scroll_size); + if (scroll_size > default_max_size) { + fetch_size = default_max_size; + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + nbScroll = Math.floor(esCount.count / default_max_size); + } + //fetch the data + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await client.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = esCount.count % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData(report, reqBody, extra_fetch); + arrayHits.push(extra_esData.hits); + } + } else { + //no need to scroll + esData = await fetchData(report, reqBody, esCount.count); + arrayHits.push(esData.hits); + } + } + //Third case: 2 args passed + if (scroll_size && nbRows) { + if (nbRows > esCount.count) { + if (esCount.count > default_max_size) { + //perform the scroll + if (scroll_size > default_max_size) { + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + fetch_size = default_max_size; + nbScroll = Math.floor(esCount.count / default_max_size); + } else { + fetch_size = scroll_size; + nbScroll = Math.floor(esCount.count / scroll_size); + } + + //fetch the data then perform the scroll + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + + //perform the scroll + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await client.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = esCount.count % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData( + report, + reqBody, + extra_fetch + ); + arrayHits.push(extra_esData.hits); + } + } else { + //no need to perform the scroll just fetch the data + fetch_size = esCount.count; + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + } + } else { + if (nbRows > default_max_size) { + if (scroll_size > default_max_size) { + message = + 'cannot perform a scroll with a scroll size bigger than the max fetch size'; + fetch_size = default_max_size; + nbScroll = Math.floor(nbRows / default_max_size); + } else { + fetch_size = scroll_size; + nbScroll = Math.floor(nbRows / scroll_size); + } + //fetch the data then perform the scroll + esData = await fetchData(report, reqBody, fetch_size); + arrayHits.push(esData.hits); + //perform the scroll + + for (let i = 0; i < nbScroll - 1; i++) { + let resScroll = await client.callAsInternalUser( + 'scroll', + { + scrollId: esData._scroll_id, + scroll: '1m', + } + ); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); + } + } + let extra_fetch = nbRows % fetch_size; + if (extra_fetch > 0) { + let extra_esData = await fetchData( + report, + reqBody, + extra_fetch + ); + arrayHits.push(extra_esData.hits); + } + } else { + //just fetch the data no need of scroll + esData = await fetchData(report, reqBody, nbRows); + arrayHits.push(esData.hits); + } + } + } + + //Get data + dataset.push(getEsData(arrayHits, report)); + //Convert To csv + const csv = await convertToCSV(dataset); + + const data = { + default_max_size, + message, + nbScroll, + total: esCount.count, + datasetCount: dataset[0].length, + dataset, + csv, + }; + + + const timeCreated = new Date().toISOString(); + return { + timeCreated, + dataUrl: csv, + fileName: "", + }; + + // To do: return the data + /* return response.ok({ + body: data, + headers: { + 'content-type': 'application/json', + }, + }); + } catch (error) { + //@ts-ignore + context.reporting_plugin.logger.error( + `Fail to generate the report: ${error}` + ); + return response.custom({ + statusCode: error.statusCode || 500, + body: parseEsErrorResponse(error), + }); + }*/ + + //Fecth the data from ES + async function fetchData(report, reqBody, fetch_size) { + const docvalues = []; + for (let dateType of report._source.dateFields) { + docvalues.push({ + field: dateType, + format: 'date_hour_minute', + }); + } + + const newBody = { + query: reqBody.toJSON().query, + docvalue_fields: docvalues, + }; + + const esData = await client.callAsInternalUser( + 'search', + { + index: report._source.paternName, + scroll: '1m', + body: newBody, + size: fetch_size, + } + ); + return esData; + } +} diff --git a/kibana-reports/server/routes/utils/reportHelper.ts b/kibana-reports/server/routes/utils/reportHelper.ts index 58569960..6e2b8b73 100644 --- a/kibana-reports/server/routes/utils/reportHelper.ts +++ b/kibana-reports/server/routes/utils/reportHelper.ts @@ -21,6 +21,7 @@ import { IClusterClient, IScopedClusterClient, } from '../../../../../src/core/server'; +import { createSavedSearchReport } from '../savedSearchReportHelper'; export const createVisualReport = async ( report: any @@ -119,8 +120,7 @@ export const createReport = async ( const reportSource = report.report_source; if (reportSource === REPORT_TYPE.savedSearch) { - // TODO: Add createDataReport(report) - console.log('placeholder for createDataReport'); + createReportResult = await createSavedSearchReport(report, client); } else if ( reportSource === REPORT_TYPE.dashboard || reportSource === REPORT_TYPE.visualization From 1d690a1f737d5a76f6f6cf328331a0d8f9325e11 Mon Sep 17 00:00:00 2001 From: Dai Date: Mon, 14 Sep 2020 13:43:38 -0700 Subject: [PATCH 02/21] Integrate with saved search report --- .../server/routes/savedSearchReportHelper.ts | 43 +++++++++++++------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 64bfe9df..c80d63d7 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -1,3 +1,19 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import { v1 as uuidv1 } from 'uuid'; import { metaData, getSelectedFields, buildQuery, getEsData, convertToCSV } from './utils/dataReportHelpers'; import { IClusterClient, @@ -9,7 +25,19 @@ export async function createSavedSearchReport( client: IClusterClient | IScopedClusterClient ) { await populateMetaData(client, report.report_params); - return await generateReport(client); + const data = await generateReport(client); + + const timeCreated = new Date().toISOString(); + const fileName = getFileName() + '.csv'; + return { + timeCreated, + dataUrl: data, + fileName, + }; + + function getFileName(): string { + return `${report.report_name}_${timeCreated}_${uuidv1()}`; + } } async function populateMetaData( @@ -114,10 +142,7 @@ async function generateReport( statusCode: 200, body: 'No data in Elasticsearch.', });*/ - return { - data: {}, - filename: "", - }; + return {}; } //build the ES query @@ -319,13 +344,7 @@ async function generateReport( csv, }; - - const timeCreated = new Date().toISOString(); - return { - timeCreated, - dataUrl: csv, - fileName: "", - }; + return csv; // To do: return the data /* return response.ok({ From fd8ab9c5c86eb64da86be293bf410e7d03882565 Mon Sep 17 00:00:00 2001 From: Dai Date: Mon, 14 Sep 2020 15:07:46 -0700 Subject: [PATCH 03/21] Fix index setting read issue --- .../server/routes/savedSearchReportHelper.ts | 369 ++++-------------- 1 file changed, 78 insertions(+), 291 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index c80d63d7..fde88087 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -14,11 +14,8 @@ */ import { v1 as uuidv1 } from 'uuid'; -import { metaData, getSelectedFields, buildQuery, getEsData, convertToCSV } from './utils/dataReportHelpers'; -import { - IClusterClient, - IScopedClusterClient, -} from '../../../../src/core/server'; +import { buildQuery, convertToCSV, getEsData, getSelectedFields, metaData } from './utils/dataReportHelpers'; +import { IClusterClient, IScopedClusterClient } from '../../../../src/core/server'; export async function createSavedSearchReport( report: any, @@ -49,16 +46,13 @@ async function populateMetaData( metaData.start = reportParams.start; metaData.end = reportParams.end; let resIndexPattern: any = {}; - //get the saved search infos + // get the saved search infos const ssParams = { index: '.kibana', id: 'search:' + reportParams.saved_search_id, }; - const ssInfos = await client.callAsInternalUser( - 'get', - ssParams - ); + const ssInfos = await client.callAsInternalUser('get', ssParams); // get the sorting metaData.sorting = ssInfos._source.search.sort; @@ -70,26 +64,23 @@ async function populateMetaData( metaData.filters = ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; - //get the list of selected columns in the saved search.Otherwise select all the fields under the _source + // get the list of selected columns in the saved search.Otherwise select all the fields under the _source await getSelectedFields(ssInfos._source.search.columns); - //Get index name - for (let item of ssInfos._source.references) { + // Get index name + for (const item of ssInfos._source.references) { if (item.name === JSON.parse(metaData.filters).indexRefName) { - //Get index-pattern informations - const indexPattern = await client.callAsInternalUser( - 'get', - { - index: '.kibana', - id: 'index-pattern:' + item.id, - } - ); + // Get index-pattern information + const indexPattern = await client.callAsInternalUser('get', { + index: '.kibana', + id: 'index-pattern:' + item.id, + }); resIndexPattern = indexPattern._source['index-pattern']; metaData.paternName = resIndexPattern.title; (metaData.timeFieldName = resIndexPattern.timeFieldName), - (metaData.fields = resIndexPattern.fields); //Get all fields - //Getting fields of type Date - for (let item of JSON.parse(metaData.fields)) { + (metaData.fields = resIndexPattern.fields); // Get all fields + // Getting fields of type Date + for (const item of JSON.parse(metaData.fields)) { if (item.type === 'date') { metaData.dateFields.push(item.name); } @@ -98,276 +89,76 @@ async function populateMetaData( } } -async function generateReport( - client: IClusterClient | IScopedClusterClient -) { - let report = { _source: metaData }; - let nbRows: number = 0; - let scroll_size: number = 0; +async function generateReport(client: IClusterClient | IScopedClusterClient) { + const report = { _source: metaData }; - let dataset: any = []; - let arrayHits: any = []; + const dataset: any = []; + const arrayHits: any = []; let esData: any = {}; - let message: string = 'success'; - let fetch_size: number = 0; - let nbScroll: number = 0; - //try { - - //fetch ES query max size windows - const indexPattern: string = report._source.paternName; - let settings = await client.callAsInternalUser( - 'indices.getSettings', - { - index: indexPattern, - includeDefaults: true, - } - ); - const default_max_size: number = - settings[indexPattern].defaults.index.max_result_window; - - //build the ES Count query - const countReq = buildQuery(report, 1); - //Count the Data in ES - const esCount = await client.callAsInternalUser( - 'count', - { - index: indexPattern, - body: countReq.toJSON(), - } - ); - - //If No data in elasticsearch - if (esCount.count === 0) { - /*return response.custom({ - statusCode: 200, - body: 'No data in Elasticsearch.', - });*/ - return {}; - } - //build the ES query - const reqBody = buildQuery(report, 0); - - //first case: No args passed. No need to scroll - if (!nbRows && !scroll_size) { - if (esCount.count > default_max_size) { - message = `Truncated Data! The requested data has reached the limit of Elasticsearch query size of ${default_max_size}. Please increase the limit and try again !`; - } - esData = await fetchData(report, reqBody, default_max_size); - arrayHits.push(esData.hits); - } + // fetch ES query max size windows + const indexPattern: string = report._source.paternName; + const settings = await client.callAsInternalUser('indices.getSettings', { + index: indexPattern, + includeDefaults: true, + }); + const defaultMaxSize: number = + settings[indexPattern].settings.index.max_result_window != null + ? settings[indexPattern].settings.index.max_result_window + : settings[indexPattern].defaults.index.max_result_window; + + // build the ES Count query + const countReq = buildQuery(report, 1); + // Count the Data in ES + const esCount = await client.callAsInternalUser('count', { + index: indexPattern, + body: countReq.toJSON(), + }); + + // If No data in elasticsearch + const total = esCount.count; + if (total === 0) { + return {}; + } - //Second case: 1 arg passed + // build the ES query + const reqBody = buildQuery(report, 0); - //Only Number of Rows is passed + if (total > defaultMaxSize) { + // fetch the data + esData = await fetchData(report, reqBody, defaultMaxSize); + arrayHits.push(esData.hits); - if (nbRows && !scroll_size) { - let rows = 0; - if (nbRows > default_max_size) { - //fetch the data - fetch_size = default_max_size; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - if (nbRows > esCount.count) { - rows = esCount.count; - nbScroll = Math.floor(esCount.count / default_max_size); - } else { - rows = nbRows; - nbScroll = Math.floor(nbRows / default_max_size); - } + const nbScroll = Math.floor(total / defaultMaxSize); - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await client.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = rows % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData(report, reqBody, extra_fetch); - arrayHits.push(extra_esData.hits); - } - } else { - fetch_size = nbRows; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - } - } else if (scroll_size && !nbRows) { - //Only scroll_size is passed - if (esCount.count > default_max_size) { - fetch_size = scroll_size; - nbScroll = Math.floor(esCount.count / scroll_size); - if (scroll_size > default_max_size) { - fetch_size = default_max_size; - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - nbScroll = Math.floor(esCount.count / default_max_size); - } - //fetch the data - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await client.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = esCount.count % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData(report, reqBody, extra_fetch); - arrayHits.push(extra_esData.hits); - } - } else { - //no need to scroll - esData = await fetchData(report, reqBody, esCount.count); - arrayHits.push(esData.hits); + for (let i = 0; i < nbScroll - 1; i++) { + const resScroll = await client.callAsInternalUser('scroll', { + scrollId: esData._scroll_id, + scroll: '1m', + }); + if (Object.keys(resScroll.hits.hits).length > 0) { + arrayHits.push(resScroll.hits); } } - //Third case: 2 args passed - if (scroll_size && nbRows) { - if (nbRows > esCount.count) { - if (esCount.count > default_max_size) { - //perform the scroll - if (scroll_size > default_max_size) { - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - fetch_size = default_max_size; - nbScroll = Math.floor(esCount.count / default_max_size); - } else { - fetch_size = scroll_size; - nbScroll = Math.floor(esCount.count / scroll_size); - } - - //fetch the data then perform the scroll - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - - //perform the scroll - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await client.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = esCount.count % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData( - report, - reqBody, - extra_fetch - ); - arrayHits.push(extra_esData.hits); - } - } else { - //no need to perform the scroll just fetch the data - fetch_size = esCount.count; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - } - } else { - if (nbRows > default_max_size) { - if (scroll_size > default_max_size) { - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - fetch_size = default_max_size; - nbScroll = Math.floor(nbRows / default_max_size); - } else { - fetch_size = scroll_size; - nbScroll = Math.floor(nbRows / scroll_size); - } - //fetch the data then perform the scroll - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await client.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = nbRows % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData( - report, - reqBody, - extra_fetch - ); - arrayHits.push(extra_esData.hits); - } - } else { - //just fetch the data no need of scroll - esData = await fetchData(report, reqBody, nbRows); - arrayHits.push(esData.hits); - } - } + const extraFetch = total % defaultMaxSize; + if (extraFetch > 0) { + const extraEsData = await fetchData(report, reqBody, extraFetch); + arrayHits.push(extraEsData.hits); } + } else { + esData = await fetchData(report, reqBody, total); + arrayHits.push(esData.hits); + } - //Get data - dataset.push(getEsData(arrayHits, report)); - //Convert To csv - const csv = await convertToCSV(dataset); - - const data = { - default_max_size, - message, - nbScroll, - total: esCount.count, - datasetCount: dataset[0].length, - dataset, - csv, - }; - - return csv; + // Get data + dataset.push(getEsData(arrayHits, report)); + // Convert To csv + return await convertToCSV(dataset); - // To do: return the data - /* return response.ok({ - body: data, - headers: { - 'content-type': 'application/json', - }, - }); - } catch (error) { - //@ts-ignore - context.reporting_plugin.logger.error( - `Fail to generate the report: ${error}` - ); - return response.custom({ - statusCode: error.statusCode || 500, - body: parseEsErrorResponse(error), - }); - }*/ - - //Fecth the data from ES - async function fetchData(report, reqBody, fetch_size) { + // Fetch the data from ES + async function fetchData(report, reqBody, fetchSize) { const docvalues = []; - for (let dateType of report._source.dateFields) { + for (const dateType of report._source.dateFields) { docvalues.push({ field: dateType, format: 'date_hour_minute', @@ -379,15 +170,11 @@ async function generateReport( docvalue_fields: docvalues, }; - const esData = await client.callAsInternalUser( - 'search', - { - index: report._source.paternName, - scroll: '1m', - body: newBody, - size: fetch_size, - } - ); - return esData; + return await client.callAsInternalUser('search', { + index: report._source.paternName, + scroll: '1m', + body: newBody, + size: fetchSize, + }); } } From e4b9a595a92dc47634f855370adc5309fb70bb99 Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 14:08:13 -0700 Subject: [PATCH 04/21] Add UT --- .../server/routes/savedSearchReportHelper.ts | 10 +- .../__tests__/savedSearchReportHelper.test.ts | 166 ++++++++++++++++++ 2 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index fde88087..2726004d 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -102,7 +102,7 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { index: indexPattern, includeDefaults: true, }); - const defaultMaxSize: number = + const maxResultSize: number = settings[indexPattern].settings.index.max_result_window != null ? settings[indexPattern].settings.index.max_result_window : settings[indexPattern].defaults.index.max_result_window; @@ -124,12 +124,12 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { // build the ES query const reqBody = buildQuery(report, 0); - if (total > defaultMaxSize) { + if (total > maxResultSize) { // fetch the data - esData = await fetchData(report, reqBody, defaultMaxSize); + esData = await fetchData(report, reqBody, maxResultSize); arrayHits.push(esData.hits); - const nbScroll = Math.floor(total / defaultMaxSize); + const nbScroll = Math.floor(total / maxResultSize); for (let i = 0; i < nbScroll - 1; i++) { const resScroll = await client.callAsInternalUser('scroll', { @@ -140,7 +140,7 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { arrayHits.push(resScroll.hits); } } - const extraFetch = total % defaultMaxSize; + const extraFetch = total % maxResultSize; if (extraFetch > 0) { const extraEsData = await fetchData(report, reqBody, extraFetch); arrayHits.push(extraEsData.hits); diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts new file mode 100644 index 00000000..02cb7657 --- /dev/null +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -0,0 +1,166 @@ +/* + * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +import 'regenerator-runtime/runtime'; +import { createSavedSearchReport } from '../../savedSearchReportHelper'; + +function mockSavedSearch() { + return JSON.parse(` + { + "type": "search", + "id": "ddd8f430-f2ef-11ea-8c86-81a0b21b4b67", + "search": { + "title": "Show category and gender", + "description": "", + "hits": 0, + "columns": [ + "category", + "customer_gender" + ], + "sort": [], + "version": 1, + "kibanaSavedObjectMeta": { + "searchSourceJSON": "{\\"highlightAll\\":true,\\"version\\":true,\\"query\\":{\\"query\\":\\"\\",\\"language\\":\\"kuery\\"},\\"indexRefName\\":\\"kibanaSavedObjectMeta.searchSourceJSON.index\\",\\"filter\\":[]}" + } + }, + "references": [ + { + "name": "kibanaSavedObjectMeta.searchSourceJSON.index", + "type": "index-pattern", + "id": "ff959d40-b880-11e8-a6d9-e546fe2bba5f" + } + ] + } + `); +} + +function mockIndexPattern() { + return JSON.parse(` + { + "index-pattern": { + "title": "kibana_sample_data_ecommerce", + "timeFieldName": "order_date", + "fields": "[{\\"name\\":\\"_id\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"_id\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":false},{\\"name\\":\\"_index\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"_index\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":false},{\\"name\\":\\"_score\\",\\"type\\":\\"number\\",\\"count\\":0,\\"scripted\\":false,\\"searchable\\":false,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"_source\\",\\"type\\":\\"_source\\",\\"esTypes\\":[\\"_source\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":false,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"_type\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"_type\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":false},{\\"name\\":\\"category\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":2,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"category.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"category\\"}}},{\\"name\\":\\"currency\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"customer_birth_date\\",\\"type\\":\\"date\\",\\"esTypes\\":[\\"date\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"customer_first_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"customer_first_name.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"customer_first_name\\"}}},{\\"name\\":\\"customer_full_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"customer_full_name.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"customer_full_name\\"}}},{\\"name\\":\\"customer_gender\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":2,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"customer_id\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"customer_last_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"customer_last_name.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"customer_last_name\\"}}},{\\"name\\":\\"customer_phone\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"day_of_week\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"day_of_week_i\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"integer\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"email\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"geoip.city_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"geoip.continent_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"geoip.country_iso_code\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"geoip.location\\",\\"type\\":\\"geo_point\\",\\"esTypes\\":[\\"geo_point\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"geoip.region_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"manufacturer\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"manufacturer.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"manufacturer\\"}}},{\\"name\\":\\"order_date\\",\\"type\\":\\"date\\",\\"esTypes\\":[\\"date\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"order_id\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products._id\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"products._id.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"products._id\\"}}},{\\"name\\":\\"products.base_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.base_unit_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.category\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"products.category.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"products.category\\"}}},{\\"name\\":\\"products.created_on\\",\\"type\\":\\"date\\",\\"esTypes\\":[\\"date\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.discount_amount\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.discount_percentage\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.manufacturer\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"products.manufacturer.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"products.manufacturer\\"}}},{\\"name\\":\\"products.min_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.product_id\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"long\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.product_name\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"text\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":false,\\"readFromDocValues\\":false},{\\"name\\":\\"products.product_name.keyword\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true,\\"subType\\":{\\"multi\\":{\\"parent\\":\\"products.product_name\\"}}},{\\"name\\":\\"products.quantity\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"integer\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.sku\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.tax_amount\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.taxful_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.taxless_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"products.unit_discount_amount\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"sku\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"taxful_total_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"taxless_total_price\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"half_float\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"total_quantity\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"integer\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"total_unique_products\\",\\"type\\":\\"number\\",\\"esTypes\\":[\\"integer\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"type\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true},{\\"name\\":\\"user\\",\\"type\\":\\"string\\",\\"esTypes\\":[\\"keyword\\"],\\"count\\":0,\\"scripted\\":false,\\"searchable\\":true,\\"aggregatable\\":true,\\"readFromDocValues\\":true}]", + "fieldFormatMap": "{\\"taxful_total_price\\":{\\"id\\":\\"number\\",\\"params\\":{\\"parsedUrl\\":{\\"origin\\":\\"http://localhost:5601\\",\\"pathname\\":\\"/app/kibana\\",\\"basePath\\":\\"\\"},\\"pattern\\":\\"$0,0.[00]\\"}}}" + } + } + `); +} + +function mockIndexSettings() { + return JSON.parse(` + { + "kibana_sample_data_ecommerce": { + "settings": { + "index": { + "number_of_shards": "1", + "auto_expand_replicas": "0-1", + "provided_name": "kibana_sample_data_ecommerce", + "max_result_window": "10", + "creation_date": "1594417718898", + "number_of_replicas": "0", + "uuid": "0KnfmEsaTYKg39ONcrA5Eg", + "version": { + "created": "7080099" + } + } + } + } + } + `); +} + +function hit(kv: any) { + return { + _source: kv, + }; +} + +describe('test create saved search report', () => { + test('create report by single search', async () => { + const client = jest.fn(); + client.callAsInternalUser = jest + .fn() + .mockImplementation((endpoint: string, params: any) => { + switch (endpoint) { + case 'get': + return { + _source: params.id.startsWith('index-pattern:') + ? mockIndexPattern() + : mockSavedSearch(), + }; + case 'indices.getSettings': + return mockIndexSettings(); + case 'count': + return { + count: 10, + }; + case 'search': + return { + hits: { + hits: [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + hit({ category: 'c6', customer_gender: 'Male' }), + hit({ category: 'c7', customer_gender: 'Male' }), + hit({ category: 'c8', customer_gender: 'Male' }), + hit({ category: 'c9', customer_gender: 'Male' }), + hit({ category: 'c10', customer_gender: 'Male' }), + ], + }, + }; + default: + fail('Fail due to unexpected function call on client', endpoint); + } + }); + + const input = { + report_name: 'Test', + report_source: 'Saved search', + report_type: 'Download', + description: 'Hi this is your saved search', + report_params: { + saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', + start: '1343576635300', + end: '1596037435301', + report_format: 'csv', + }, + }; + + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(fileName).toContain(`${input.report_name}_${timeCreated}`); + expect(fileName).toContain(`.${input.report_params.report_format}`); + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender,' + + '5.category,5.customer_gender,' + + '6.category,6.customer_gender,' + + '7.category,7.customer_gender,' + + '8.category,8.customer_gender,' + + '9.category,9.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,c6,Male,c7,Male,c8,Male,c9,Male,c10,Male' + ); + }, 20000); +}); From bc7673efc6d754eb9343ae66bfc8edec3be44335 Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 14:17:30 -0700 Subject: [PATCH 05/21] Add UT --- .../__tests__/savedSearchReportHelper.test.ts | 172 ++++++++++-------- 1 file changed, 95 insertions(+), 77 deletions(-) diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 02cb7657..a38ce737 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -16,6 +16,95 @@ import 'regenerator-runtime/runtime'; import { createSavedSearchReport } from '../../savedSearchReportHelper'; +describe('test create saved search report', () => { + test('create report by single search', async () => { + const hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + hit({ category: 'c6', customer_gender: 'Male' }), + hit({ category: 'c7', customer_gender: 'Male' }), + hit({ category: 'c8', customer_gender: 'Male' }), + hit({ category: 'c9', customer_gender: 'Male' }), + hit({ category: 'c10', customer_gender: 'Male' }), + ]; + const client = mockEsClient(hits); + const input = { + report_name: 'Test', + report_source: 'Saved search', + report_type: 'Download', + description: 'Hi this is your saved search', + report_params: { + saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', + start: '1343576635300', + end: '1596037435301', + report_format: 'csv', + }, + }; + + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(fileName).toContain(`${input.report_name}_${timeCreated}`); + expect(fileName).toContain(`.${input.report_params.report_format}`); + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender,' + + '5.category,5.customer_gender,' + + '6.category,6.customer_gender,' + + '7.category,7.customer_gender,' + + '8.category,8.customer_gender,' + + '9.category,9.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,c6,Male,c7,Male,c8,Male,c9,Male,c10,Male' + ); + }, 20000); +}); + +/** + * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. + */ +function mockEsClient(mockHits: Array<{ _source: any }>) { + const client = jest.fn(); + client.callAsInternalUser = jest + .fn() + .mockImplementation((endpoint: string, params: any) => { + switch (endpoint) { + case 'get': + return { + _source: params.id.startsWith('index-pattern:') + ? mockIndexPattern() + : mockSavedSearch(), + }; + case 'indices.getSettings': + return mockIndexSettings(); + case 'count': + return { + count: 10, + }; + case 'search': + return { + hits: { + hits: mockHits, + }, + }; + default: + fail('Fail due to unexpected function call on client', endpoint); + } + }); + return client; +} + + +/** + * Mock a saved search for kibana_sample_data_ecommerce with 2 selected fields: category and customer_gender + */ function mockSavedSearch() { return JSON.parse(` { @@ -46,6 +135,9 @@ function mockSavedSearch() { `); } +/** + * Mock index pattern for kibana_sample_data_ecommerce. + */ function mockIndexPattern() { return JSON.parse(` { @@ -59,6 +151,9 @@ function mockIndexPattern() { `); } +/** + * Mock index settings for kibana_sample_data_ecommerce. + */ function mockIndexSettings() { return JSON.parse(` { @@ -87,80 +182,3 @@ function hit(kv: any) { _source: kv, }; } - -describe('test create saved search report', () => { - test('create report by single search', async () => { - const client = jest.fn(); - client.callAsInternalUser = jest - .fn() - .mockImplementation((endpoint: string, params: any) => { - switch (endpoint) { - case 'get': - return { - _source: params.id.startsWith('index-pattern:') - ? mockIndexPattern() - : mockSavedSearch(), - }; - case 'indices.getSettings': - return mockIndexSettings(); - case 'count': - return { - count: 10, - }; - case 'search': - return { - hits: { - hits: [ - hit({ category: 'c1', customer_gender: 'Male' }), - hit({ category: 'c2', customer_gender: 'Male' }), - hit({ category: 'c3', customer_gender: 'Male' }), - hit({ category: 'c4', customer_gender: 'Male' }), - hit({ category: 'c5', customer_gender: 'Male' }), - hit({ category: 'c6', customer_gender: 'Male' }), - hit({ category: 'c7', customer_gender: 'Male' }), - hit({ category: 'c8', customer_gender: 'Male' }), - hit({ category: 'c9', customer_gender: 'Male' }), - hit({ category: 'c10', customer_gender: 'Male' }), - ], - }, - }; - default: - fail('Fail due to unexpected function call on client', endpoint); - } - }); - - const input = { - report_name: 'Test', - report_source: 'Saved search', - report_type: 'Download', - description: 'Hi this is your saved search', - report_params: { - saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', - start: '1343576635300', - end: '1596037435301', - report_format: 'csv', - }, - }; - - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); - - expect(fileName).toContain(`${input.report_name}_${timeCreated}`); - expect(fileName).toContain(`.${input.report_params.report_format}`); - expect(dataUrl).toEqual( - '0.category,0.customer_gender,' + - '1.category,1.customer_gender,' + - '2.category,2.customer_gender,' + - '3.category,3.customer_gender,' + - '4.category,4.customer_gender,' + - '5.category,5.customer_gender,' + - '6.category,6.customer_gender,' + - '7.category,7.customer_gender,' + - '8.category,8.customer_gender,' + - '9.category,9.customer_gender\n' + - 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,c6,Male,c7,Male,c8,Male,c9,Male,c10,Male' - ); - }, 20000); -}); From ad6d55f5f586a7b6e30440d9c13d3e772e9d936f Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 15:49:45 -0700 Subject: [PATCH 06/21] Add UT for scroll --- .../server/routes/savedSearchReportHelper.ts | 4 +- .../__tests__/savedSearchReportHelper.test.ts | 88 ++++++++++++++----- 2 files changed, 67 insertions(+), 25 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 2726004d..4f717881 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -131,7 +131,7 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { const nbScroll = Math.floor(total / maxResultSize); - for (let i = 0; i < nbScroll - 1; i++) { + for (let i = 0; i < nbScroll; i++) { const resScroll = await client.callAsInternalUser('scroll', { scrollId: esData._scroll_id, scroll: '1m', @@ -140,11 +140,13 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { arrayHits.push(resScroll.hits); } } + /* const extraFetch = total % maxResultSize; if (extraFetch > 0) { const extraEsData = await fetchData(report, reqBody, extraFetch); arrayHits.push(extraEsData.hits); } + */ } else { esData = await fetchData(report, reqBody, total); arrayHits.push(esData.hits); diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index a38ce737..bcebee93 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -16,6 +16,22 @@ import 'regenerator-runtime/runtime'; import { createSavedSearchReport } from '../../savedSearchReportHelper'; +/** + * The mock and sample input for saved search export function. + */ +const input = { + report_name: 'Test', + report_source: 'Saved search', + report_type: 'Download', + description: 'Hi this is your saved search', + report_params: { + saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', + start: '1343576635300', + end: '1596037435301', + report_format: 'csv', + }, +}; + describe('test create saved search report', () => { test('create report by single search', async () => { const hits = [ @@ -24,26 +40,40 @@ describe('test create saved search report', () => { hit({ category: 'c3', customer_gender: 'Male' }), hit({ category: 'c4', customer_gender: 'Male' }), hit({ category: 'c5', customer_gender: 'Male' }), - hit({ category: 'c6', customer_gender: 'Male' }), - hit({ category: 'c7', customer_gender: 'Male' }), - hit({ category: 'c8', customer_gender: 'Male' }), - hit({ category: 'c9', customer_gender: 'Male' }), - hit({ category: 'c10', customer_gender: 'Male' }), ]; const client = mockEsClient(hits); - const input = { - report_name: 'Test', - report_source: 'Saved search', - report_type: 'Download', - description: 'Hi this is your saved search', - report_params: { - saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', - start: '1343576635300', - end: '1596037435301', - report_format: 'csv', - }, - }; + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(fileName).toContain(`${input.report_name}_${timeCreated}`); + expect(fileName).toContain(`.${input.report_params.report_format}`); + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male' + ); + }, 20000); + test('create report by scroll', async () => { + const hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + hit({ category: 'c6', customer_gender: 'Female' }), + hit({ category: 'c7', customer_gender: 'Female' }), + hit({ category: 'c8', customer_gender: 'Female' }), + hit({ category: 'c9', customer_gender: 'Female' }), + hit({ category: 'c10', customer_gender: 'Female' }), + hit({ category: 'c11', customer_gender: 'Male' }), + ]; + const client = mockEsClient(hits); const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( input, client @@ -61,8 +91,11 @@ describe('test create saved search report', () => { '6.category,6.customer_gender,' + '7.category,7.customer_gender,' + '8.category,8.customer_gender,' + - '9.category,9.customer_gender\n' + - 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,c6,Male,c7,Male,c8,Male,c9,Male,c10,Male' + '9.category,9.customer_gender,' + + '10.category,10.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,' + + 'c6,Female,c7,Female,c8,Female,c9,Female,c10,Female,' + + 'c11,Male' ); }, 20000); }); @@ -71,6 +104,7 @@ describe('test create saved search report', () => { * Mock Elasticsearch client and return different mock objects based on endpoint and parameters. */ function mockEsClient(mockHits: Array<{ _source: any }>) { + let call = 0; const client = jest.fn(); client.callAsInternalUser = jest .fn() @@ -86,12 +120,19 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { return mockIndexSettings(); case 'count': return { - count: 10, + count: mockHits.length, }; case 'search': return { hits: { - hits: mockHits, + hits: mockHits.slice(0, 5), + }, + }; + case 'scroll': + call++; + return { + hits: { + hits: mockHits.slice(5 * call, 5 * (call + 1)), }, }; default: @@ -101,9 +142,8 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { return client; } - /** - * Mock a saved search for kibana_sample_data_ecommerce with 2 selected fields: category and customer_gender + * Mock a saved search for kibana_sample_data_ecommerce with 2 selected fields: category and customer_gender. */ function mockSavedSearch() { return JSON.parse(` @@ -163,7 +203,7 @@ function mockIndexSettings() { "number_of_shards": "1", "auto_expand_replicas": "0-1", "provided_name": "kibana_sample_data_ecommerce", - "max_result_window": "10", + "max_result_window": "5", "creation_date": "1594417718898", "number_of_replicas": "0", "uuid": "0KnfmEsaTYKg39ONcrA5Eg", From 244084e1244e5c6aea01b72282f04eff8906aabc Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 16:14:02 -0700 Subject: [PATCH 07/21] Fix scroll issue --- .../server/routes/savedSearchReportHelper.ts | 69 ++++++++----------- .../__tests__/savedSearchReportHelper.test.ts | 16 ++++- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 4f717881..4f5dc837 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -45,26 +45,21 @@ async function populateMetaData( metaData.report_format = reportParams.report_format; metaData.start = reportParams.start; metaData.end = reportParams.end; + + // Get saved search info let resIndexPattern: any = {}; - // get the saved search infos const ssParams = { index: '.kibana', id: 'search:' + reportParams.saved_search_id, }; - const ssInfos = await client.callAsInternalUser('get', ssParams); - // get the sorting metaData.sorting = ssInfos._source.search.sort; - - // get the saved search type metaData.type = ssInfos._source.type; - - // get the filters metaData.filters = ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; - // get the list of selected columns in the saved search.Otherwise select all the fields under the _source + // Get the list of selected columns in the saved search.Otherwise select all the fields under the _source await getSelectedFields(ssInfos._source.search.columns); // Get index name @@ -91,12 +86,11 @@ async function populateMetaData( async function generateReport(client: IClusterClient | IScopedClusterClient) { const report = { _source: metaData }; - const dataset: any = []; const arrayHits: any = []; let esData: any = {}; - // fetch ES query max size windows + // Fetch ES query max size windows to decide search or scroll const indexPattern: string = report._source.paternName; const settings = await client.callAsInternalUser('indices.getSettings', { index: indexPattern, @@ -107,30 +101,32 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { ? settings[indexPattern].settings.index.max_result_window : settings[indexPattern].defaults.index.max_result_window; - // build the ES Count query + // Build the ES Count query to count the size of result const countReq = buildQuery(report, 1); - // Count the Data in ES const esCount = await client.callAsInternalUser('count', { index: indexPattern, body: countReq.toJSON(), }); - // If No data in elasticsearch + // Return nothing if No data in elasticsearch const total = esCount.count; if (total === 0) { return {}; } - // build the ES query - const reqBody = buildQuery(report, 0); - + const reqBody = buildRequestBody(buildQuery(report, 0)); if (total > maxResultSize) { - // fetch the data - esData = await fetchData(report, reqBody, maxResultSize); + // Open scroll context by fetching first batch + esData = await client.callAsInternalUser('search', { + index: report._source.paternName, + scroll: '1m', + body: reqBody, + size: maxResultSize, + }); arrayHits.push(esData.hits); + // Start scrolling till the end const nbScroll = Math.floor(total / maxResultSize); - for (let i = 0; i < nbScroll; i++) { const resScroll = await client.callAsInternalUser('scroll', { scrollId: esData._scroll_id, @@ -140,25 +136,25 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { arrayHits.push(resScroll.hits); } } - /* - const extraFetch = total % maxResultSize; - if (extraFetch > 0) { - const extraEsData = await fetchData(report, reqBody, extraFetch); - arrayHits.push(extraEsData.hits); - } - */ + + // Clear scroll context + await client.callAsInternalUser('clearScroll', { + scrollId: esData._scroll_id, + }); } else { - esData = await fetchData(report, reqBody, total); + esData = await client.callAsInternalUser('search', { + index: report._source.paternName, + body: reqBody, + size: total, + }); arrayHits.push(esData.hits); } - // Get data + // Parse ES data and convert to CSV dataset.push(getEsData(arrayHits, report)); - // Convert To csv return await convertToCSV(dataset); - // Fetch the data from ES - async function fetchData(report, reqBody, fetchSize) { + function buildRequestBody(query: any) { const docvalues = []; for (const dateType of report._source.dateFields) { docvalues.push({ @@ -167,16 +163,9 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { }); } - const newBody = { - query: reqBody.toJSON().query, + return { + query: query.toJSON().query, docvalue_fields: docvalues, }; - - return await client.callAsInternalUser('search', { - index: report._source.paternName, - scroll: '1m', - body: newBody, - size: fetchSize, - }); } } diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index bcebee93..509110b8 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -32,6 +32,11 @@ const input = { }, }; +/** + * Max result window size in ES index settings. + */ +const maxResultSize = 5; + describe('test create saved search report', () => { test('create report by single search', async () => { const hits = [ @@ -125,16 +130,21 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { case 'search': return { hits: { - hits: mockHits.slice(0, 5), + hits: mockHits.slice(0, maxResultSize), }, }; case 'scroll': call++; return { hits: { - hits: mockHits.slice(5 * call, 5 * (call + 1)), + hits: mockHits.slice( + maxResultSize * call, + maxResultSize * (call + 1) + ), }, }; + case 'clearScroll': + return null; default: fail('Fail due to unexpected function call on client', endpoint); } @@ -203,7 +213,7 @@ function mockIndexSettings() { "number_of_shards": "1", "auto_expand_replicas": "0-1", "provided_name": "kibana_sample_data_ecommerce", - "max_result_window": "5", + "max_result_window": "${maxResultSize}", "creation_date": "1594417718898", "number_of_replicas": "0", "uuid": "0KnfmEsaTYKg39ONcrA5Eg", From 0d832303653ef63466e2e6362e8b5cce3d1b22b5 Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 16:39:16 -0700 Subject: [PATCH 08/21] Refactor generate csv data function --- .../server/routes/savedSearchReportHelper.ts | 68 +++++++++++-------- 1 file changed, 41 insertions(+), 27 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 4f5dc837..fe1ab27e 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -22,7 +22,7 @@ export async function createSavedSearchReport( client: IClusterClient | IScopedClusterClient ) { await populateMetaData(client, report.report_params); - const data = await generateReport(client); + const data = await generateCsvData(client); const timeCreated = new Date().toISOString(); const fileName = getFileName() + '.csv'; @@ -84,29 +84,14 @@ async function populateMetaData( } } -async function generateReport(client: IClusterClient | IScopedClusterClient) { - const report = { _source: metaData }; - const dataset: any = []; - const arrayHits: any = []; +async function generateCsvData(client: IClusterClient | IScopedClusterClient) { let esData: any = {}; - - // Fetch ES query max size windows to decide search or scroll + const arrayHits: any = []; + const dataset: any = []; + const report = { _source: metaData }; const indexPattern: string = report._source.paternName; - const settings = await client.callAsInternalUser('indices.getSettings', { - index: indexPattern, - includeDefaults: true, - }); - const maxResultSize: number = - settings[indexPattern].settings.index.max_result_window != null - ? settings[indexPattern].settings.index.max_result_window - : settings[indexPattern].defaults.index.max_result_window; - - // Build the ES Count query to count the size of result - const countReq = buildQuery(report, 1); - const esCount = await client.callAsInternalUser('count', { - index: indexPattern, - body: countReq.toJSON(), - }); + const maxResultSize: number = await getMaxResultSize(); + const esCount = await getEsDataSize(); // Return nothing if No data in elasticsearch const total = esCount.count; @@ -116,6 +101,37 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { const reqBody = buildRequestBody(buildQuery(report, 0)); if (total > maxResultSize) { + await getEsDataByScroll(); + } else { + await getEsDataBySearch(); + } + + // Parse ES data and convert to CSV + dataset.push(getEsData(arrayHits, report)); + return await convertToCSV(dataset); + + // Fetch ES query max size windows to decide search or scroll + async function getMaxResultSize() { + const settings = await client.callAsInternalUser('indices.getSettings', { + index: indexPattern, + includeDefaults: true, + }); + // The location of max result window differs if set by user. + return settings[indexPattern].settings.index.max_result_window != null + ? settings[indexPattern].settings.index.max_result_window + : settings[indexPattern].defaults.index.max_result_window; + } + + // Build the ES Count query to count the size of result + async function getEsDataSize() { + const countReq = buildQuery(report, 1); + return await client.callAsInternalUser('count', { + index: indexPattern, + body: countReq.toJSON(), + }); + } + + async function getEsDataByScroll() { // Open scroll context by fetching first batch esData = await client.callAsInternalUser('search', { index: report._source.paternName, @@ -141,7 +157,9 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { await client.callAsInternalUser('clearScroll', { scrollId: esData._scroll_id, }); - } else { + } + + async function getEsDataBySearch() { esData = await client.callAsInternalUser('search', { index: report._source.paternName, body: reqBody, @@ -150,10 +168,6 @@ async function generateReport(client: IClusterClient | IScopedClusterClient) { arrayHits.push(esData.hits); } - // Parse ES data and convert to CSV - dataset.push(getEsData(arrayHits, report)); - return await convertToCSV(dataset); - function buildRequestBody(query: any) { const docvalues = []; for (const dateType of report._source.dateFields) { From fe6d21727fa4107ed466cdbf9fb7406ac7ccbffb Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 15 Sep 2020 16:50:45 -0700 Subject: [PATCH 09/21] Refactor generate csv data function --- .../server/routes/savedSearchReportHelper.ts | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index fe1ab27e..8e18a3f8 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -17,6 +17,11 @@ import { v1 as uuidv1 } from 'uuid'; import { buildQuery, convertToCSV, getEsData, getSelectedFields, metaData } from './utils/dataReportHelpers'; import { IClusterClient, IScopedClusterClient } from '../../../../src/core/server'; +/** + * Specify how long scroll context should be maintained for scrolled search + */ +const scrollTimeout = '1m'; + export async function createSavedSearchReport( report: any, client: IClusterClient | IScopedClusterClient @@ -37,6 +42,11 @@ export async function createSavedSearchReport( } } +/** + * Populate parameters and saved search info related to meta data object. + * @param client ES client + * @param reportParams CSV export specific parameters + */ async function populateMetaData( client: IClusterClient | IScopedClusterClient, reportParams: any @@ -84,16 +94,18 @@ async function populateMetaData( } } +/** + * Generate CSV data by query and convert ES data set. + * @param client ES client + */ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { let esData: any = {}; const arrayHits: any = []; - const dataset: any = []; const report = { _source: metaData }; const indexPattern: string = report._source.paternName; const maxResultSize: number = await getMaxResultSize(); const esCount = await getEsDataSize(); - // Return nothing if No data in elasticsearch const total = esCount.count; if (total === 0) { return {}; @@ -105,10 +117,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { } else { await getEsDataBySearch(); } - - // Parse ES data and convert to CSV - dataset.push(getEsData(arrayHits, report)); - return await convertToCSV(dataset); + return convertEsDataToCsv(); // Fetch ES query max size windows to decide search or scroll async function getMaxResultSize() { @@ -116,7 +125,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { index: indexPattern, includeDefaults: true, }); - // The location of max result window differs if set by user. + // The location of max result window differs if default overridden. return settings[indexPattern].settings.index.max_result_window != null ? settings[indexPattern].settings.index.max_result_window : settings[indexPattern].defaults.index.max_result_window; @@ -135,7 +144,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { // Open scroll context by fetching first batch esData = await client.callAsInternalUser('search', { index: report._source.paternName, - scroll: '1m', + scroll: scrollTimeout, body: reqBody, size: maxResultSize, }); @@ -146,7 +155,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { for (let i = 0; i < nbScroll; i++) { const resScroll = await client.callAsInternalUser('scroll', { scrollId: esData._scroll_id, - scroll: '1m', + scroll: scrollTimeout, }); if (Object.keys(resScroll.hits.hits).length > 0) { arrayHits.push(resScroll.hits); @@ -182,4 +191,11 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { docvalue_fields: docvalues, }; } + + // Parse ES data and convert to CSV + async function convertEsDataToCsv() { + const dataset: any = []; + dataset.push(getEsData(arrayHits, report)); + return await convertToCSV(dataset); + } } From 4f2275aab541e40a3bce7baedc93276152205167 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 11:11:37 -0700 Subject: [PATCH 10/21] Refactor generate csv data function --- .../server/routes/savedSearchReportHelper.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 8e18a3f8..73201a38 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -14,8 +14,17 @@ */ import { v1 as uuidv1 } from 'uuid'; -import { buildQuery, convertToCSV, getEsData, getSelectedFields, metaData } from './utils/dataReportHelpers'; -import { IClusterClient, IScopedClusterClient } from '../../../../src/core/server'; +import { + buildQuery, + convertToCSV, + getEsData, + getSelectedFields, + metaData, +} from './utils/dataReportHelpers'; +import { + IClusterClient, + IScopedClusterClient, +} from '../../../../src/core/server'; /** * Specify how long scroll context should be maintained for scrolled search From 828cd1a27e7064752210d89291e01a8a40d3d883 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 11:54:15 -0700 Subject: [PATCH 11/21] Change according to latest schema --- .../server/routes/savedSearchReportHelper.ts | 26 ++++---- .../__tests__/savedSearchReportHelper.test.ts | 63 +++++++++++++++---- 2 files changed, 65 insertions(+), 24 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 73201a38..98bf2082 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -35,7 +35,7 @@ export async function createSavedSearchReport( report: any, client: IClusterClient | IScopedClusterClient ) { - await populateMetaData(client, report.report_params); + await populateMetaData(client, report); const data = await generateCsvData(client); const timeCreated = new Date().toISOString(); @@ -47,29 +47,33 @@ export async function createSavedSearchReport( }; function getFileName(): string { - return `${report.report_name}_${timeCreated}_${uuidv1()}`; + return `${ + report.report_definition.report_params.report_name + }_${timeCreated}_${uuidv1()}`; } } /** * Populate parameters and saved search info related to meta data object. - * @param client ES client - * @param reportParams CSV export specific parameters + * @param client ES client + * @param report Report input */ async function populateMetaData( client: IClusterClient | IScopedClusterClient, - reportParams: any + report: any ) { - metaData.saved_search_id = reportParams.saved_search_id; - metaData.report_format = reportParams.report_format; - metaData.start = reportParams.start; - metaData.end = reportParams.end; + metaData.saved_search_id = + report.report_definition.report_params.core_params.saved_search_id; + metaData.report_format = + report.report_definition.report_params.core_params.report_format; + metaData.start = report.time_from; + metaData.end = report.time_to; // Get saved search info let resIndexPattern: any = {}; const ssParams = { index: '.kibana', - id: 'search:' + reportParams.saved_search_id, + id: 'search:' + metaData.saved_search_id, }; const ssInfos = await client.callAsInternalUser('get', ssParams); @@ -117,7 +121,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { const total = esCount.count; if (total === 0) { - return {}; + return ''; } const reqBody = buildRequestBody(buildQuery(report, 0)); diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 509110b8..f9e710cd 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -20,15 +20,33 @@ import { createSavedSearchReport } from '../../savedSearchReportHelper'; * The mock and sample input for saved search export function. */ const input = { - report_name: 'Test', - report_source: 'Saved search', - report_type: 'Download', - description: 'Hi this is your saved search', - report_params: { - saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', - start: '1343576635300', - end: '1596037435301', - report_format: 'csv', + query_url: + 'http://localhost:5601/app/kibana#/search/7adfa750-4c81-11e8-b3d7-01146121b73d', + time_from: 1343576635300, + time_to: 1596037435301, + report_definition: { + report_params: { + report_name: 'test report table order', + report_source: 'Saved search', + description: 'Hi this is your saved search on demand', + core_params: { + base_url: + 'http://localhost:5601/app/kibana#/search/7adfa750-4c81-11e8-b3d7-01146121b73d', + saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', + report_format: 'csv', + time_duration: 'PT5M', + }, + }, + delivery: { + recipients: ['kibanaUser:dfajfopdasf'], + title: 'fake title', + description: { + text: 'dasdasdasd', + }, + }, + trigger: { + trigger_type: 'On demand', + }, }, }; @@ -38,6 +56,29 @@ const input = { const maxResultSize = 5; describe('test create saved search report', () => { + + test('create report with expected file name', async () => { + const hits: Array<{ _source: any }> = []; + const client = mockEsClient(hits); + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(fileName).toContain(`test report table order_${timeCreated}`); + expect(fileName).toContain('.csv'); + }, 20000); + + test('create report for empty result', async () => { + const hits: Array<{ _source: any }> = []; + const client = mockEsClient(hits); + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + expect(dataUrl).toEqual(''); + }, 20000); + test('create report by single search', async () => { const hits = [ hit({ category: 'c1', customer_gender: 'Male' }), @@ -52,8 +93,6 @@ describe('test create saved search report', () => { client ); - expect(fileName).toContain(`${input.report_name}_${timeCreated}`); - expect(fileName).toContain(`.${input.report_params.report_format}`); expect(dataUrl).toEqual( '0.category,0.customer_gender,' + '1.category,1.customer_gender,' + @@ -84,8 +123,6 @@ describe('test create saved search report', () => { client ); - expect(fileName).toContain(`${input.report_name}_${timeCreated}`); - expect(fileName).toContain(`.${input.report_params.report_format}`); expect(dataUrl).toEqual( '0.category,0.customer_gender,' + '1.category,1.customer_gender,' + From c3f17cf2ec4ae7a3a2e7a1d7b0fc2a4f59f5f48a Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 13:51:56 -0700 Subject: [PATCH 12/21] Add limit param to schema --- kibana-reports/server/model/index.ts | 1 + .../server/routes/savedSearchReportHelper.ts | 12 +++++-- .../__tests__/savedSearchReportHelper.test.ts | 31 ++++++++++++++++++- 3 files changed, 40 insertions(+), 4 deletions(-) diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index 77f3b151..00416a5d 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -30,6 +30,7 @@ export const dataReportSchema = schema.object({ time_duration: schema.string(), //TODO: future support schema.literal('xlsx') report_format: schema.oneOf([schema.literal(FORMAT.csv)]), + limit: schema.number({ defaultValue: 10000 }), }); const visualReportSchema = schema.object({ diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 98bf2082..742390ef 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -36,7 +36,10 @@ export async function createSavedSearchReport( client: IClusterClient | IScopedClusterClient ) { await populateMetaData(client, report); - const data = await generateCsvData(client); + const data = await generateCsvData( + client, + report.report_definition.report_params.core_params.limit + ); const timeCreated = new Date().toISOString(); const fileName = getFileName() + '.csv'; @@ -111,7 +114,10 @@ async function populateMetaData( * Generate CSV data by query and convert ES data set. * @param client ES client */ -async function generateCsvData(client: IClusterClient | IScopedClusterClient) { +async function generateCsvData( + client: IClusterClient | IScopedClusterClient, + limit: number +) { let esData: any = {}; const arrayHits: any = []; const report = { _source: metaData }; @@ -119,7 +125,7 @@ async function generateCsvData(client: IClusterClient | IScopedClusterClient) { const maxResultSize: number = await getMaxResultSize(); const esCount = await getEsDataSize(); - const total = esCount.count; + const total = Math.min(esCount.count, limit); if (total === 0) { return ''; } diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index f9e710cd..a970e42b 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -15,6 +15,7 @@ import 'regenerator-runtime/runtime'; import { createSavedSearchReport } from '../../savedSearchReportHelper'; +import { reportSchema } from '../../../model'; /** * The mock and sample input for saved search export function. @@ -35,6 +36,7 @@ const input = { saved_search_id: 'ddd8f430-f2ef-11ea-8c86-81a0b21b4b67', report_format: 'csv', time_duration: 'PT5M', + limit: 10000, }, }, delivery: { @@ -57,6 +59,11 @@ const maxResultSize = 5; describe('test create saved search report', () => { + test('create report with valid input', async () => { + // Check if the assumption of input is up-to-date + reportSchema.validate(input); + }, 20000); + test('create report with expected file name', async () => { const hits: Array<{ _source: any }> = []; const client = mockEsClient(hits); @@ -140,6 +147,28 @@ describe('test create saved search report', () => { 'c11,Male' ); }, 20000); + + test('create report with limit', async () => { + const hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + hit({ category: 'c6', customer_gender: 'Female' }), + ]; + + // Assign a smaller limit than default to test + input.report_definition.report_params.core_params.limit = 1; + + const client = mockEsClient(hits); + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(dataUrl).toEqual('0.category,0.customer_gender\nc1,Male'); + }, 20000); }); /** @@ -167,7 +196,7 @@ function mockEsClient(mockHits: Array<{ _source: any }>) { case 'search': return { hits: { - hits: mockHits.slice(0, maxResultSize), + hits: mockHits.slice(0, params.size), }, }; case 'scroll': From 96019c6511b8dce48027b4b124a890666441fec1 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 16:22:03 -0700 Subject: [PATCH 13/21] Truncate to limit size --- .../server/routes/savedSearchReportHelper.ts | 2 +- .../__tests__/savedSearchReportHelper.test.ts | 73 +++++++++++++++++-- .../server/routes/utils/dataReportHelpers.ts | 7 +- 3 files changed, 73 insertions(+), 9 deletions(-) diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/savedSearchReportHelper.ts index 742390ef..1d517aec 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/savedSearchReportHelper.ts @@ -214,7 +214,7 @@ async function generateCsvData( // Parse ES data and convert to CSV async function convertEsDataToCsv() { const dataset: any = []; - dataset.push(getEsData(arrayHits, report)); + dataset.push(getEsData(arrayHits, report, limit)); return await convertToCSV(dataset); } } diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index a970e42b..ef46fa46 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -76,7 +76,7 @@ describe('test create saved search report', () => { expect(fileName).toContain('.csv'); }, 20000); - test('create report for empty result', async () => { + test('create report for empty data set', async () => { const hits: Array<{ _source: any }> = []; const client = mockEsClient(hits); const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( @@ -86,7 +86,7 @@ describe('test create saved search report', () => { expect(dataUrl).toEqual(''); }, 20000); - test('create report by single search', async () => { + test('create report for small data set by single search', async () => { const hits = [ hit({ category: 'c1', customer_gender: 'Male' }), hit({ category: 'c2', customer_gender: 'Male' }), @@ -110,7 +110,7 @@ describe('test create saved search report', () => { ); }, 20000); - test('create report by scroll', async () => { + test('create report for large data set by scroll', async () => { const hits = [ hit({ category: 'c1', customer_gender: 'Male' }), hit({ category: 'c2', customer_gender: 'Male' }), @@ -148,7 +148,30 @@ describe('test create saved search report', () => { ); }, 20000); - test('create report with limit', async () => { + test('create report with limit smaller than max result size', async () => { + // Assign a smaller limit than default to test + input.report_definition.report_params.core_params.limit = 1; + + const hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + ]; + const client = mockEsClient(hits); + const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + input, + client + ); + + expect(dataUrl).toEqual('0.category,0.customer_gender\nc1,Male'); + }, 20000); + + test('create report with limit greater than max result size', async () => { + // Assign a limit just a little greater than max result size (5) + input.report_definition.report_params.core_params.limit = 6; + const hits = [ hit({ category: 'c1', customer_gender: 'Male' }), hit({ category: 'c2', customer_gender: 'Male' }), @@ -156,18 +179,54 @@ describe('test create saved search report', () => { hit({ category: 'c4', customer_gender: 'Male' }), hit({ category: 'c5', customer_gender: 'Male' }), hit({ category: 'c6', customer_gender: 'Female' }), + hit({ category: 'c7', customer_gender: 'Female' }), + hit({ category: 'c8', customer_gender: 'Female' }), + hit({ category: 'c9', customer_gender: 'Female' }), + hit({ category: 'c10', customer_gender: 'Female' }), ]; + const client = mockEsClient(hits); + const { dataUrl } = await createSavedSearchReport(input, client); - // Assign a smaller limit than default to test - input.report_definition.report_params.core_params.limit = 1; + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender,' + + '5.category,5.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,' + + 'c6,Female' + ); + }, 20000); + test('create report with limit greater than total result size', async () => { + // Assign a limit even greater than the result size + input.report_definition.report_params.core_params.limit = 10; + + const hits = [ + hit({ category: 'c1', customer_gender: 'Male' }), + hit({ category: 'c2', customer_gender: 'Male' }), + hit({ category: 'c3', customer_gender: 'Male' }), + hit({ category: 'c4', customer_gender: 'Male' }), + hit({ category: 'c5', customer_gender: 'Male' }), + hit({ category: 'c6', customer_gender: 'Female' }), + ]; const client = mockEsClient(hits); const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( input, client ); - expect(dataUrl).toEqual('0.category,0.customer_gender\nc1,Male'); + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender,' + + '5.category,5.customer_gender\n' + + 'c1,Male,c2,Male,c3,Male,c4,Male,c5,Male,' + + 'c6,Female' + ); }, 20000); }); diff --git a/kibana-reports/server/routes/utils/dataReportHelpers.ts b/kibana-reports/server/routes/utils/dataReportHelpers.ts index d43ff7b0..8b233a55 100644 --- a/kibana-reports/server/routes/utils/dataReportHelpers.ts +++ b/kibana-reports/server/routes/utils/dataReportHelpers.ts @@ -152,7 +152,7 @@ export const buildQuery = (report, is_count) => { }; // Fetch the data from ES -export const getEsData = (arrayHits, report) => { +export const getEsData = (arrayHits, report, limit: number) => { let hits: any = []; for (let valueRes of arrayHits) { for (let data of valueRes.hits) { @@ -172,6 +172,11 @@ export const getEsData = (arrayHits, report) => { } else { hits.push(data); } + + // Truncate to expected limit size + if (hits.length >= limit) { + return hits; + } } } return hits; From bd4698a2ae400232d0c31fa56e0cfd719412cc82 Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 16:34:20 -0700 Subject: [PATCH 14/21] Delete unused old report files --- kibana-reports/server/routes/dataReport.ts | 345 --------- .../server/routes/dataReportMetadata.ts | 131 ---- kibana-reports/server/routes/index.ts | 4 - .../routes/utils/__tests__/dataReport.test.ts | 677 ------------------ 4 files changed, 1157 deletions(-) delete mode 100644 kibana-reports/server/routes/dataReport.ts delete mode 100644 kibana-reports/server/routes/dataReportMetadata.ts delete mode 100644 kibana-reports/server/routes/utils/__tests__/dataReport.test.ts diff --git a/kibana-reports/server/routes/dataReport.ts b/kibana-reports/server/routes/dataReport.ts deleted file mode 100644 index 60fbd273..00000000 --- a/kibana-reports/server/routes/dataReport.ts +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import { schema } from '@kbn/config-schema'; -import { - IRouter, - IKibanaResponse, - ResponseError, -} from '../../../../src/core/server'; -import { API_PREFIX } from '../../common'; -import { parseEsErrorResponse } from './utils/helpers'; -import { buildQuery, getEsData, convertToCSV } from './utils/dataReportHelpers'; - -export default function (router: IRouter) { - //download the data-report from meta data - router.get( - { - path: `${API_PREFIX}/data-report/generate/{reportId}`, - validate: { - params: schema.object({ - reportId: schema.string(), - }), - query: schema.object({ - nbRows: schema.maybe(schema.number({ min: 1 })), - scroll_size: schema.maybe(schema.number({ min: 1 })), - }), - }, - }, - async ( - context, - request, - response - ): Promise> => { - try { - let { nbRows, scroll_size } = request.query as { - nbRows?: number; - scroll_size?: number; - }; - - let dataset: any = []; - let arrayHits: any = []; - let esData: any = {}; - let message: string = 'success'; - let fetch_size: number = 0; - let nbScroll: number = 0; - - //get the metadata of the report from ES using reportId - const report = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'get', - { - index: 'datareport', - id: request.params.reportId, - } - ); - - //fetch ES query max size windows - const indexPattern: string = report._source.paternName; - let settings = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'indices.getSettings', - { - index: indexPattern, - includeDefaults: true, - } - ); - const default_max_size: number = - settings[indexPattern].defaults.index.max_result_window; - - //build the ES Count query - const countReq = buildQuery(report, 1); - //Count the Data in ES - const esCount = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'count', - { - index: indexPattern, - body: countReq.toJSON(), - } - ); - - //If No data in elasticsearch - if (esCount.count === 0) { - return response.custom({ - statusCode: 200, - body: 'No data in Elasticsearch.', - }); - } - - //build the ES query - const reqBody = buildQuery(report, 0); - - //first case: No args passed. No need to scroll - if (!nbRows && !scroll_size) { - if (esCount.count > default_max_size) { - message = `Truncated Data! The requested data has reached the limit of Elasticsearch query size of ${default_max_size}. Please increase the limit and try again !`; - } - esData = await fetchData(report, reqBody, default_max_size); - arrayHits.push(esData.hits); - } - - //Second case: 1 arg passed - - //Only Number of Rows is passed - - if (nbRows && !scroll_size) { - let rows = 0; - if (nbRows > default_max_size) { - //fetch the data - fetch_size = default_max_size; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - if (nbRows > esCount.count) { - rows = esCount.count; - nbScroll = Math.floor(esCount.count / default_max_size); - } else { - rows = nbRows; - nbScroll = Math.floor(nbRows / default_max_size); - } - - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = rows % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData(report, reqBody, extra_fetch); - arrayHits.push(extra_esData.hits); - } - } else { - fetch_size = nbRows; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - } - } else if (scroll_size && !nbRows) { - //Only scroll_size is passed - if (esCount.count > default_max_size) { - fetch_size = scroll_size; - nbScroll = Math.floor(esCount.count / scroll_size); - if (scroll_size > default_max_size) { - fetch_size = default_max_size; - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - nbScroll = Math.floor(esCount.count / default_max_size); - } - //fetch the data - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = esCount.count % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData(report, reqBody, extra_fetch); - arrayHits.push(extra_esData.hits); - } - } else { - //no need to scroll - esData = await fetchData(report, reqBody, esCount.count); - arrayHits.push(esData.hits); - } - } - //Third case: 2 args passed - if (scroll_size && nbRows) { - if (nbRows > esCount.count) { - if (esCount.count > default_max_size) { - //perform the scroll - if (scroll_size > default_max_size) { - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - fetch_size = default_max_size; - nbScroll = Math.floor(esCount.count / default_max_size); - } else { - fetch_size = scroll_size; - nbScroll = Math.floor(esCount.count / scroll_size); - } - - //fetch the data then perform the scroll - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - - //perform the scroll - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = esCount.count % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData( - report, - reqBody, - extra_fetch - ); - arrayHits.push(extra_esData.hits); - } - } else { - //no need to perform the scroll just fetch the data - fetch_size = esCount.count; - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - } - } else { - if (nbRows > default_max_size) { - if (scroll_size > default_max_size) { - message = - 'cannot perform a scroll with a scroll size bigger than the max fetch size'; - fetch_size = default_max_size; - nbScroll = Math.floor(nbRows / default_max_size); - } else { - fetch_size = scroll_size; - nbScroll = Math.floor(nbRows / scroll_size); - } - //fetch the data then perform the scroll - esData = await fetchData(report, reqBody, fetch_size); - arrayHits.push(esData.hits); - //perform the scroll - - for (let i = 0; i < nbScroll - 1; i++) { - let resScroll = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'scroll', - { - scrollId: esData._scroll_id, - scroll: '1m', - } - ); - if (Object.keys(resScroll.hits.hits).length > 0) { - arrayHits.push(resScroll.hits); - } - } - let extra_fetch = nbRows % fetch_size; - if (extra_fetch > 0) { - let extra_esData = await fetchData( - report, - reqBody, - extra_fetch - ); - arrayHits.push(extra_esData.hits); - } - } else { - //just fetch the data no need of scroll - esData = await fetchData(report, reqBody, nbRows); - arrayHits.push(esData.hits); - } - } - } - - //Get data - dataset.push(getEsData(arrayHits, report)); - - //Convert To csv - const csv = await convertToCSV(dataset); - - const data = { - default_max_size, - message, - nbScroll, - total: esCount.count, - datasetCount: dataset[0].length, - dataset, - csv, - }; - - // To do: return the data - return response.ok({ - body: data, - headers: { - 'content-type': 'application/json', - }, - }); - } catch (error) { - //@ts-ignore - context.reporting_plugin.logger.error( - `Fail to generate the report: ${error}` - ); - return response.custom({ - statusCode: error.statusCode || 500, - body: parseEsErrorResponse(error), - }); - } - - //Fecth the data from ES - async function fetchData(report, reqBody, fetch_size) { - const docvalues = []; - for (let dateType of report._source.dateFields) { - docvalues.push({ - field: dateType, - format: 'date_hour_minute', - }); - } - - const newBody = { - query: reqBody.toJSON().query, - docvalue_fields: docvalues, - }; - - const esData = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'search', - { - index: report._source.paternName, - scroll: '1m', - body: newBody, - size: fetch_size, - } - ); - return esData; - } - } - ); -} diff --git a/kibana-reports/server/routes/dataReportMetadata.ts b/kibana-reports/server/routes/dataReportMetadata.ts deleted file mode 100644 index e3f694fc..00000000 --- a/kibana-reports/server/routes/dataReportMetadata.ts +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import { schema } from '@kbn/config-schema'; -import { - IRouter, - IKibanaResponse, - ResponseError, -} from '../../../../src/core/server'; -import { API_PREFIX } from '../../common'; -import { dataReportSchema } from '../model'; -import { parseEsErrorResponse } from './utils/helpers'; -import { metaData, getSelectedFields } from './utils/dataReportHelpers'; -const axios = require('axios'); - -export default function (router: IRouter) { - //generate report csv meta data - router.post( - { - path: `${API_PREFIX}/data-report/metadata`, - validate: { - body: schema.any(), - }, - }, - async ( - context, - request, - response - ): Promise> => { - // input validation - try { - dataReportSchema.validate(request.body); - } catch (error) { - return response.badRequest({ body: error }); - } - try { - let dataReport = request.body; - metaData.saved_search_id = dataReport.saved_search_id; - metaData.report_format = dataReport.report_format; - metaData.start = dataReport.start; - metaData.end = dataReport.end; - let resIndexPattern: any = {}; - - //get the saved search infos - const ssParams = { - index: '.kibana', - id: 'search:' + dataReport.saved_search_id, - }; - - const ssInfos = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'get', - ssParams - ); - - // get the sorting - metaData.sorting = ssInfos._source.search.sort; - - // get the saved search type - metaData.type = ssInfos._source.type; - - // get the filters - metaData.filters = - ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; - - //get the list of selected columns in the saved search.Otherwise select all the fields under the _source - await getSelectedFields(ssInfos._source.search.columns); - - //Get index name - for (let item of ssInfos._source.references) { - if (item.name === JSON.parse(metaData.filters).indexRefName) { - //Get index-pattern informations - const indexPattern = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'get', - { - index: '.kibana', - id: 'index-pattern:' + item.id, - } - ); - resIndexPattern = indexPattern._source['index-pattern']; - metaData.paternName = resIndexPattern.title; - (metaData.timeFieldName = resIndexPattern.timeFieldName), - (metaData.fields = resIndexPattern.fields); //Get all fields - //Getting fields of type Date - for (let item of JSON.parse(metaData.fields)) { - if (item.type === 'date') { - metaData.dateFields.push(item.name); - } - } - } - } - - //save the meta data to the dataReport index to be updated with the right mapping - const report = await context.core.elasticsearch.adminClient.callAsInternalUser( - 'index', - { - index: 'datareport', - body: metaData, - } - ); - - return response.ok({ - body: { report, metaData }, - headers: { - 'content-type': 'application/json', - }, - }); - } catch (error) { - //@ts-ignore - context.reporting_plugin.logger.error( - `Failed to generate the report meta data: ${error}` - ); - return response.custom({ - statusCode: error.statusCode || 500, - body: parseEsErrorResponse(error), - }); - } - } - ); -} diff --git a/kibana-reports/server/routes/index.ts b/kibana-reports/server/routes/index.ts index e7ec68e3..f75b15cf 100644 --- a/kibana-reports/server/routes/index.ts +++ b/kibana-reports/server/routes/index.ts @@ -15,15 +15,11 @@ import registerReportRoute from './report'; import registerReportDefinitionRoute from './reportDefinition'; -import registerDataReport from './dataReport'; -import registerDataReportMetadata from './dataReportMetadata'; import registerReportSourceRoute from './getReportSource'; import { IRouter } from '../../../../src/core/server'; export default function (router: IRouter) { registerReportRoute(router); registerReportDefinitionRoute(router); - registerDataReportMetadata(router); - registerDataReport(router); registerReportSourceRoute(router); } diff --git a/kibana-reports/server/routes/utils/__tests__/dataReport.test.ts b/kibana-reports/server/routes/utils/__tests__/dataReport.test.ts deleted file mode 100644 index be38a587..00000000 --- a/kibana-reports/server/routes/utils/__tests__/dataReport.test.ts +++ /dev/null @@ -1,677 +0,0 @@ -/* - * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -import 'regenerator-runtime/runtime'; -import DATA_REPORT_TEST_DATA_LARGE from './test_data/dataReportTestDataLarge.json'; -import DATA_REPORT_TEST_DATA_SMALL from './test_data/dataReportTestDataSmall.json'; -const axios = require('axios'); - -axios.defaults.adapter = require('axios/lib/adapters/http'); - -beforeAll(async () => { - // create the datareport index - await axios - .put('http://localhost:9200/datareport?pretty') - .then((res) => {}) - .catch((error) => { - console.log('error in creating datareport index:', error.response); - }); - - // add first document - await axios - .post( - 'http://localhost:9200/datareport/_doc/RwLFIXQBdaQgV0jgh80O?pretty', - DATA_REPORT_TEST_DATA_LARGE - ) - .then(() => {}) - .catch((error) => { - console.log('error in adding large sample data:', error); - }); - - // add second document - await axios - .post( - 'http://localhost:9200/datareport/_doc/FgLXJnQBdaQgV0jgTtEp?pretty', - DATA_REPORT_TEST_DATA_SMALL - ) - .then(() => {}) - .catch((error) => { - console.log('error in adding small sample data:', error); - }); -}); - -afterAll(async () => { - // delete datareport index after test? -}); - -describe('data report metadata tests suites', () => { - test('test to generate data report meta data successfully', async () => { - expect.assertions(5); - const url = '/api/reporting/data-report/metadata'; - const input = { - saved_search_id: '571aaf70-4c88-11e8-b3d7-01146121b73d', - start: '1343576635300', - end: '1596037435301', - report_format: 'csv', - }; - let response: any = {}; - - const report = await axios({ - method: 'POST', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - data: input, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'Error in test to generate data report meta data successfully:', - error - ); - }); - - const { - saved_search_id, - report_format, - start, - end, - paternName, - } = response.metaData; - expect(saved_search_id).toEqual(input.saved_search_id); - expect(paternName).toEqual('kibana_sample_data_flights'); - expect(start).toEqual(input.start); - expect(end).toEqual(input.end); - expect(report_format).toEqual('csv'); - }, 20000); - - test("test to generate data report meta data Report doesn't exist", async () => { - expect.assertions(1); - const url = '/api/reporting/data-report/metadata'; - const input = { - saved_search_id: '571aaf70-4c88-11e8-b3d7-01146121b73d0', - start: '1343576635300', - end: '1596037435301', - report_format: 'csv', - }; - let response: any = {}; - let message: string = ''; - const report = await axios({ - method: 'POST', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - data: input, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - message = error.response.data.message; - }); - expect(message).toEqual("Saved Search doesn't exist !"); - }, 20000); -}); - -describe('data report data generation tests suites', () => { - describe('test for Case 1 No args ', () => { - test('esCount > default_fetch_size => default_fetch_size', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'error in esCount > default_fetch_size => default_fetch_size:', - error - ); - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.default_max_size); - }, 20000); - - test('esCount < default_fetch_size => esCount ', async () => { - expect.assertions(1); - const input = { - reportId: 'FgLXJnQBdaQgV0jgTtEp', - esCount: 445, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'error in esCount < default_fetch_size => esCount:', - error - ); - }); - const { datasetCount } = response; - expect(datasetCount).toEqual(input.esCount); - }, 20000); - }); - - describe('test for Case 2: arg => nbRows ', () => { - test('nbRows == esCount => esCount', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 13059, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log('error in nbRows == esCount => esCount:', error); - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.esCount); - }, 20000); - - test('nbRows > esCount < default_max_size => esCount', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 200000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'error in nbRows > esCount < default_max_size => esCount:', - error - ); - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.esCount); - }, 20000); - - test('nbRows > esCount > default_max_size => default_max_size', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 200000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'error in nbRows > esCount > default_max_size => default_max_size:', - error - ); - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.esCount); - }, 20000); - - test('nbRows > default_max_size && nbRows < esCount => nbRows ', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 12000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - }, - }) - .then((res) => { - response = res.data; - }) - .catch((error) => { - console.log( - 'error in nbRows > default_max_size && nbRows < esCount => nbRows:', - error - ); - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.nbRows); - }, 20000); - - test('nbRows < default_max_size => nbRows', async () => { - expect.assertions(1); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 5000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - }, - }).then((res) => { - response = res.data; - }); - const { datasetCount } = response; - - expect(datasetCount).toEqual(input.nbRows); - }, 20000); - }); - - describe('test for Case 2: arg => scroll_size ', () => { - test('esCount > default_max_size && scroll_size > default_max_size => esCount', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - scroll_size: 200000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - let scrolls = Math.floor(input.esCount / input.default_max_size); - const { datasetCount, nbScroll } = response; - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount > default_max_size && scroll_size < default_max_size ', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - scroll_size: 8000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - let scrolls = Math.floor(input.esCount / input.scroll_size); - const { datasetCount, nbScroll } = response; - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount < default_max_size ', async () => { - expect.assertions(2); - const input = { - reportId: 'FgLXJnQBdaQgV0jgTtEp', - scroll_size: 200000, - esCount: 445, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - let scrolls = 0; - const { datasetCount, nbScroll } = response; - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - }); - - describe('test for Case 3: args => nbRows && scroll_size ', () => { - test('nbRows > esCount > default_max_size && scroll_size < default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 200000, - scroll_size: 200, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - let scrolls = Math.floor(input.esCount / input.scroll_size); - const { datasetCount, nbScroll } = response; - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('nbRows > esCount > default_max_size && scroll_size > default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 200000, - scroll_size: 12000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = Math.floor(input.esCount / input.default_max_size); - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('nbRows > esCount < default_max_size && scroll_size > default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'FgLXJnQBdaQgV0jgTtEp', - nbRows: 200000, - scroll_size: 18000, - esCount: 445, - default_max_size: 10000, - }; - - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = 0; - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('nbRows > esCount < default_max_size && scroll_size < default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'FgLXJnQBdaQgV0jgTtEp', - nbRows: 200000, - scroll_size: 8000, - esCount: 445, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = 0; - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.esCount); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount > nbRows < default_max_size && scroll_size > default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 5000, - scroll_size: 28000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = 0; - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.nbRows); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount > nbRows < default_max_size && scroll_size < default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 5000, - scroll_size: 8000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = 0; - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.nbRows); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount > nbRows > default_max_size && scroll_size > default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 12000, - scroll_size: 18000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = Math.floor(input.nbRows / input.default_max_size); - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.nbRows); - expect(nbScroll).toEqual(scrolls); - }, 20000); - - test('esCount > nbRows > default_max_size && scroll_size < default_max_size', async () => { - expect.assertions(2); - const input = { - reportId: 'RwLFIXQBdaQgV0jgh80O', - nbRows: 12000, - scroll_size: 7000, - esCount: 13059, - default_max_size: 10000, - }; - let url = '/api/reporting/data-report/generate/' + input.reportId; - let response: any = {}; - const data = await axios({ - method: 'GET', - proxy: { host: '127.0.0.1', port: 5601 }, - url, - headers: { 'kbn-xsrf': 'reporting' }, - params: { - nbRows: input.nbRows, - scroll_size: input.scroll_size, - }, - }).then((res) => { - response = res.data; - }); - - let scrolls = Math.floor(input.nbRows / input.scroll_size); - const { datasetCount, nbScroll } = response; - - expect(datasetCount).toEqual(input.nbRows); - expect(nbScroll).toEqual(scrolls); - }, 20000); - }); -}); From 967ce361ebc8af1a576fdab3d1fff8ad7a59668c Mon Sep 17 00:00:00 2001 From: Dai Date: Wed, 16 Sep 2020 16:46:10 -0700 Subject: [PATCH 15/21] Move saved search file to util folder --- .../routes/utils/__tests__/savedSearchReportHelper.test.ts | 2 +- kibana-reports/server/routes/utils/reportHelper.ts | 2 +- .../server/routes/{ => utils}/savedSearchReportHelper.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename kibana-reports/server/routes/{ => utils}/savedSearchReportHelper.ts (98%) diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index ef46fa46..33367642 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -14,7 +14,7 @@ */ import 'regenerator-runtime/runtime'; -import { createSavedSearchReport } from '../../savedSearchReportHelper'; +import { createSavedSearchReport } from '../savedSearchReportHelper'; import { reportSchema } from '../../../model'; /** diff --git a/kibana-reports/server/routes/utils/reportHelper.ts b/kibana-reports/server/routes/utils/reportHelper.ts index 8dfbc01f..574e2380 100644 --- a/kibana-reports/server/routes/utils/reportHelper.ts +++ b/kibana-reports/server/routes/utils/reportHelper.ts @@ -26,7 +26,7 @@ import { IClusterClient, IScopedClusterClient, } from '../../../../../src/core/server'; -import { createSavedSearchReport } from '../savedSearchReportHelper'; +import { createSavedSearchReport } from './savedSearchReportHelper'; import { ReportSchemaType } from '../../model'; export const createVisualReport = async ( diff --git a/kibana-reports/server/routes/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts similarity index 98% rename from kibana-reports/server/routes/savedSearchReportHelper.ts rename to kibana-reports/server/routes/utils/savedSearchReportHelper.ts index 1d517aec..2504c772 100644 --- a/kibana-reports/server/routes/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -20,11 +20,11 @@ import { getEsData, getSelectedFields, metaData, -} from './utils/dataReportHelpers'; +} from './dataReportHelpers'; import { IClusterClient, IScopedClusterClient, -} from '../../../../src/core/server'; +} from '../../../../../src/core/server'; /** * Specify how long scroll context should be maintained for scrolled search From c425b6dae3d03bda4bfc7582a527ba57ff04fbe1 Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 17 Sep 2020 10:15:11 -0700 Subject: [PATCH 16/21] Address PR comments --- kibana-reports/server/model/index.ts | 2 +- .../__tests__/savedSearchReportHelper.test.ts | 11 +++++++++-- .../routes/utils/savedSearchReportHelper.ts | 16 ++++++++-------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index 00416a5d..ed6e3fde 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -30,7 +30,7 @@ export const dataReportSchema = schema.object({ time_duration: schema.string(), //TODO: future support schema.literal('xlsx') report_format: schema.oneOf([schema.literal(FORMAT.csv)]), - limit: schema.number({ defaultValue: 10000 }), + limit: schema.maybe(schema.number({ defaultValue: 10000 })), }); const visualReportSchema = schema.object({ diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 33367642..b95939ab 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -71,9 +71,16 @@ describe('test create saved search report', () => { input, client ); - expect(fileName).toContain(`test report table order_${timeCreated}`); - expect(fileName).toContain('.csv'); + }, 20000); + + test('create report with expected file name extension', async () => { + const csvReport = await createSavedSearchReport(input, mockEsClient([])); + expect(csvReport.fileName).toContain('.csv'); + + input.report_definition.report_params.core_params.report_format = 'xlsx'; + const xlsxReport = await createSavedSearchReport(input, mockEsClient([])); + expect(xlsxReport.fileName).toContain('.xlsx'); }, 20000); test('create report for empty data set', async () => { diff --git a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts index 2504c772..35553fcb 100644 --- a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -35,14 +35,16 @@ export async function createSavedSearchReport( report: any, client: IClusterClient | IScopedClusterClient ) { + const params = report.report_definition.report_params; + const limit = params.core_params.limit; + const format = params.core_params.report_format; + const name = params.report_name; + await populateMetaData(client, report); - const data = await generateCsvData( - client, - report.report_definition.report_params.core_params.limit - ); + const data = await generateCsvData(client, limit); const timeCreated = new Date().toISOString(); - const fileName = getFileName() + '.csv'; + const fileName = getFileName() + '.' + format; return { timeCreated, dataUrl: data, @@ -50,9 +52,7 @@ export async function createSavedSearchReport( }; function getFileName(): string { - return `${ - report.report_definition.report_params.report_name - }_${timeCreated}_${uuidv1()}`; + return `${name}_${timeCreated}_${uuidv1()}`; } } From 31d855518e048f8c4a09144c5731c35e1c27d0fb Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 17 Sep 2020 10:19:18 -0700 Subject: [PATCH 17/21] Remove unused var --- .../__tests__/savedSearchReportHelper.test.ts | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index b95939ab..0dca0c1b 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -67,7 +67,7 @@ describe('test create saved search report', () => { test('create report with expected file name', async () => { const hits: Array<{ _source: any }> = []; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( + const { timeCreated, fileName } = await createSavedSearchReport( input, client ); @@ -86,10 +86,7 @@ describe('test create saved search report', () => { test('create report for empty data set', async () => { const hits: Array<{ _source: any }> = []; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); + const { dataUrl } = await createSavedSearchReport(input, client); expect(dataUrl).toEqual(''); }, 20000); @@ -102,10 +99,7 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); + const { dataUrl } = await createSavedSearchReport(input, client); expect(dataUrl).toEqual( '0.category,0.customer_gender,' + @@ -132,10 +126,7 @@ describe('test create saved search report', () => { hit({ category: 'c11', customer_gender: 'Male' }), ]; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); + const { dataUrl } = await createSavedSearchReport(input, client); expect(dataUrl).toEqual( '0.category,0.customer_gender,' + @@ -167,10 +158,7 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); + const { dataUrl } = await createSavedSearchReport(input, client); expect(dataUrl).toEqual('0.category,0.customer_gender\nc1,Male'); }, 20000); @@ -219,10 +207,7 @@ describe('test create saved search report', () => { hit({ category: 'c6', customer_gender: 'Female' }), ]; const client = mockEsClient(hits); - const { timeCreated, dataUrl, fileName } = await createSavedSearchReport( - input, - client - ); + const { dataUrl } = await createSavedSearchReport(input, client); expect(dataUrl).toEqual( '0.category,0.customer_gender,' + From c7db8bf29f9cc1db641fe7548f7c4af1c6473ebd Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 17 Sep 2020 11:50:57 -0700 Subject: [PATCH 18/21] Move getFileName and callCluster to util file --- .../__tests__/savedSearchReportHelper.test.ts | 2 +- kibana-reports/server/routes/utils/helpers.ts | 28 +++++++++++++++++++ .../server/routes/utils/reportHelper.ts | 20 +------------ .../routes/utils/savedSearchReportHelper.ts | 18 ++++++------ 4 files changed, 38 insertions(+), 30 deletions(-) diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 0dca0c1b..1c59d172 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -71,7 +71,7 @@ describe('test create saved search report', () => { input, client ); - expect(fileName).toContain(`test report table order_${timeCreated}`); + expect(fileName).toContain(`test report table order_`); }, 20000); test('create report with expected file name extension', async () => { diff --git a/kibana-reports/server/routes/utils/helpers.ts b/kibana-reports/server/routes/utils/helpers.ts index 79585650..9cce8903 100644 --- a/kibana-reports/server/routes/utils/helpers.ts +++ b/kibana-reports/server/routes/utils/helpers.ts @@ -14,6 +14,11 @@ */ import { KibanaResponseFactory } from '../../../../../src/core/server'; +import { v1 as uuidv1 } from 'uuid'; +import { + IClusterClient, + IScopedClusterClient, +} from '../../../../../src/core/server'; export function parseEsErrorResponse(error: any) { if (error.response) { @@ -33,3 +38,26 @@ export function errorResponse(response: KibanaResponseFactory, error: any) { body: parseEsErrorResponse(error), }); } + +/** + * Generate report file name based on name and timestamp. + * @param itemName report item name + * @param timeCreated timestamp when this is being created + */ +export function getFileName(itemName: string, timeCreated: Date): string { + return `${itemName}_${timeCreated.toISOString()}_${uuidv1()}`; +} + +export const callCluster = async ( + client: IClusterClient | IScopedClusterClient, + endpoint: string, + params: any +) => { + let esResp; + if ('callAsCurrentUser' in client) { + esResp = await client.callAsCurrentUser(endpoint, params); + } else { + esResp = await client.callAsInternalUser(endpoint, params); + } + return esResp; +}; diff --git a/kibana-reports/server/routes/utils/reportHelper.ts b/kibana-reports/server/routes/utils/reportHelper.ts index 574e2380..b223e905 100644 --- a/kibana-reports/server/routes/utils/reportHelper.ts +++ b/kibana-reports/server/routes/utils/reportHelper.ts @@ -14,7 +14,6 @@ */ import puppeteer from 'puppeteer'; -import { v1 as uuidv1 } from 'uuid'; import { FORMAT, REPORT_TYPE, @@ -22,6 +21,7 @@ import { CONFIG_INDEX_NAME, } from './constants'; import { RequestParams } from '@elastic/elasticsearch'; +import { getFileName, callCluster } from './helpers'; import { IClusterClient, IScopedClusterClient, @@ -185,21 +185,3 @@ export const createReport = async ( return createReportResult; }; - -function getFileName(itemName: string, timeCreated: Date): string { - return `${itemName}_${timeCreated.toISOString()}_${uuidv1()}`; -} - -const callCluster = async ( - client: IClusterClient | IScopedClusterClient, - endpoint: string, - params: any -) => { - let esResp; - if ('callAsCurrentUser' in client) { - esResp = await client.callAsCurrentUser(endpoint, params); - } else { - esResp = await client.callAsInternalUser(endpoint, params); - } - return esResp; -}; diff --git a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts index 35553fcb..37255b27 100644 --- a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -13,7 +13,6 @@ * permissions and limitations under the License. */ -import { v1 as uuidv1 } from 'uuid'; import { buildQuery, convertToCSV, @@ -25,6 +24,7 @@ import { IClusterClient, IScopedClusterClient, } from '../../../../../src/core/server'; +import { getFileName, callCluster } from './helpers'; /** * Specify how long scroll context should be maintained for scrolled search @@ -34,26 +34,23 @@ const scrollTimeout = '1m'; export async function createSavedSearchReport( report: any, client: IClusterClient | IScopedClusterClient -) { +): Promise<{ timeCreated: number; dataUrl: string; fileName: string }> { const params = report.report_definition.report_params; const limit = params.core_params.limit; - const format = params.core_params.report_format; - const name = params.report_name; + const reportFormat = params.core_params.report_format; + const reportName = params.report_name; await populateMetaData(client, report); const data = await generateCsvData(client, limit); - const timeCreated = new Date().toISOString(); - const fileName = getFileName() + '.' + format; + const curTime = new Date(); + const timeCreated = curTime.valueOf(); + const fileName = getFileName(reportName, curTime) + '.' + reportFormat; return { timeCreated, dataUrl: data, fileName, }; - - function getFileName(): string { - return `${name}_${timeCreated}_${uuidv1()}`; - } } /** @@ -113,6 +110,7 @@ async function populateMetaData( /** * Generate CSV data by query and convert ES data set. * @param client ES client + * @param limit limit size of result data set */ async function generateCsvData( client: IClusterClient | IScopedClusterClient, From 3ced3fe08bfccff66ee9e4dba49b4166ce7d72b6 Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 17 Sep 2020 15:05:59 -0700 Subject: [PATCH 19/21] Use callCluster --- kibana-reports/server/routes/utils/helpers.ts | 6 ++++++ .../routes/utils/savedSearchReportHelper.ts | 16 ++++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/kibana-reports/server/routes/utils/helpers.ts b/kibana-reports/server/routes/utils/helpers.ts index 9cce8903..816b8655 100644 --- a/kibana-reports/server/routes/utils/helpers.ts +++ b/kibana-reports/server/routes/utils/helpers.ts @@ -48,6 +48,12 @@ export function getFileName(itemName: string, timeCreated: Date): string { return `${itemName}_${timeCreated.toISOString()}_${uuidv1()}`; } +/** + * Call ES cluster function. + * @param client ES client + * @param endpoint ES API method + * @param params ES API parameters + */ export const callCluster = async ( client: IClusterClient | IScopedClusterClient, endpoint: string, diff --git a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts index 37255b27..bddf7013 100644 --- a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -75,7 +75,7 @@ async function populateMetaData( index: '.kibana', id: 'search:' + metaData.saved_search_id, }; - const ssInfos = await client.callAsInternalUser('get', ssParams); + const ssInfos = await callCluster(client, 'get', ssParams); metaData.sorting = ssInfos._source.search.sort; metaData.type = ssInfos._source.type; @@ -89,7 +89,7 @@ async function populateMetaData( for (const item of ssInfos._source.references) { if (item.name === JSON.parse(metaData.filters).indexRefName) { // Get index-pattern information - const indexPattern = await client.callAsInternalUser('get', { + const indexPattern = await callCluster(client, 'get', { index: '.kibana', id: 'index-pattern:' + item.id, }); @@ -138,7 +138,7 @@ async function generateCsvData( // Fetch ES query max size windows to decide search or scroll async function getMaxResultSize() { - const settings = await client.callAsInternalUser('indices.getSettings', { + const settings = await callCluster(client, 'indices.getSettings', { index: indexPattern, includeDefaults: true, }); @@ -151,7 +151,7 @@ async function generateCsvData( // Build the ES Count query to count the size of result async function getEsDataSize() { const countReq = buildQuery(report, 1); - return await client.callAsInternalUser('count', { + return await callCluster(client, 'count', { index: indexPattern, body: countReq.toJSON(), }); @@ -159,7 +159,7 @@ async function generateCsvData( async function getEsDataByScroll() { // Open scroll context by fetching first batch - esData = await client.callAsInternalUser('search', { + esData = await callCluster(client, 'search', { index: report._source.paternName, scroll: scrollTimeout, body: reqBody, @@ -170,7 +170,7 @@ async function generateCsvData( // Start scrolling till the end const nbScroll = Math.floor(total / maxResultSize); for (let i = 0; i < nbScroll; i++) { - const resScroll = await client.callAsInternalUser('scroll', { + const resScroll = await callCluster(client, 'scroll', { scrollId: esData._scroll_id, scroll: scrollTimeout, }); @@ -180,13 +180,13 @@ async function generateCsvData( } // Clear scroll context - await client.callAsInternalUser('clearScroll', { + await callCluster(client, 'clearScroll', { scrollId: esData._scroll_id, }); } async function getEsDataBySearch() { - esData = await client.callAsInternalUser('search', { + esData = await callCluster(client, 'search', { index: report._source.paternName, body: reqBody, size: total, From 2bd508c3ea987bfee380ee32c45e88b02f26718e Mon Sep 17 00:00:00 2001 From: Dai Date: Thu, 17 Sep 2020 17:27:40 -0700 Subject: [PATCH 20/21] Add excel option to sanitize --- kibana-reports/server/model/index.ts | 1 + .../__tests__/savedSearchReportHelper.test.ts | 63 +++++++++++++++++++ .../server/routes/utils/dataReportHelpers.ts | 27 ++++++-- .../routes/utils/savedSearchReportHelper.ts | 9 ++- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/kibana-reports/server/model/index.ts b/kibana-reports/server/model/index.ts index ed6e3fde..17b523c8 100644 --- a/kibana-reports/server/model/index.ts +++ b/kibana-reports/server/model/index.ts @@ -31,6 +31,7 @@ export const dataReportSchema = schema.object({ //TODO: future support schema.literal('xlsx') report_format: schema.oneOf([schema.literal(FORMAT.csv)]), limit: schema.maybe(schema.number({ defaultValue: 10000 })), + excel: schema.maybe(schema.boolean({ defaultValue: true })), }); const visualReportSchema = schema.object({ diff --git a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 1c59d172..2ba58b3f 100644 --- a/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/kibana-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -37,6 +37,7 @@ const input = { report_format: 'csv', time_duration: 'PT5M', limit: 10000, + excel: true, }, }, delivery: { @@ -220,6 +221,68 @@ describe('test create saved search report', () => { 'c6,Female' ); }, 20000); + + test('create report for data set with comma', async () => { + const hits = [ + hit({ category: ',c1', customer_gender: 'Ma,le' }), + hit({ category: 'c2,', customer_gender: 'M,ale' }), + hit({ category: ',,c3', customer_gender: 'Male,,,' }), + ]; + const client = mockEsClient(hits); + const { dataUrl } = await createSavedSearchReport(input, client); + + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender\n' + + '",c1","Ma,le","c2,","M,ale",",,c3","Male,,,"' + ); + }, 20000); + + test('create report by sanitizing data set for Excel', async () => { + const hits = [ + hit({ category: 'c1', customer_gender: '=Male' }), + hit({ category: 'c2', customer_gender: 'Male=' }), + hit({ category: 'c3', customer_gender: '+Ma,le' }), + hit({ category: ',-c4', customer_gender: 'Male' }), + hit({ category: ',,,@c5', customer_gender: 'Male' }), + ]; + const client = mockEsClient(hits); + const { dataUrl } = await createSavedSearchReport(input, client); + + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender\n' + + `c1,'=Male,c2,Male=,c3,"'+Ma,le",",-c4",Male,",,,@c5",Male` + ); + }, 20000); + + test('create report by not sanitizing data set for Excel', async () => { + // Enable Excel escape option + input.report_definition.report_params.core_params.excel = false; + + const hits = [ + hit({ category: 'c1', customer_gender: '=Male' }), + hit({ category: 'c2', customer_gender: 'Male=' }), + hit({ category: 'c3', customer_gender: '+Ma,le' }), + hit({ category: ',-c4', customer_gender: 'Male' }), + hit({ category: ',,,@c5', customer_gender: 'Male' }), + ]; + const client = mockEsClient(hits); + const { dataUrl } = await createSavedSearchReport(input, client); + + expect(dataUrl).toEqual( + '0.category,0.customer_gender,' + + '1.category,1.customer_gender,' + + '2.category,2.customer_gender,' + + '3.category,3.customer_gender,' + + '4.category,4.customer_gender\n' + + `c1,=Male,c2,Male=,c3,"+Ma,le",",-c4",Male,",,,@c5",Male` + ); + }, 20000); }); /** diff --git a/kibana-reports/server/routes/utils/dataReportHelpers.ts b/kibana-reports/server/routes/utils/dataReportHelpers.ts index 8b233a55..ccee8820 100644 --- a/kibana-reports/server/routes/utils/dataReportHelpers.ts +++ b/kibana-reports/server/routes/utils/dataReportHelpers.ts @@ -152,7 +152,7 @@ export const buildQuery = (report, is_count) => { }; // Fetch the data from ES -export const getEsData = (arrayHits, report, limit: number) => { +export const getEsData = (arrayHits, report, params) => { let hits: any = []; for (let valueRes of arrayHits) { for (let data of valueRes.hits) { @@ -168,13 +168,13 @@ export const getEsData = (arrayHits, report, limit: number) => { delete data['fields']; if (report._source.fields_exist === true) { let result = traverse(data, report._source.selectedFields); - hits.push(result); + hits.push(params.excel ? sanitize(result) : result); } else { - hits.push(data); + hits.push(params.excel ? sanitize(data) : data); } // Truncate to expected limit size - if (hits.length >= limit) { + if (hits.length >= params.limit) { return hits; } } @@ -214,3 +214,22 @@ function traverse(data, keys, result = {}) { } return result; } + +/** + * Escape special characters if field value prefixed with. + * This is intend to avoid CSV injection in Microsoft Excel. + * @param doc document + */ +function sanitize(doc: any) { + for (const field in doc) { + if ( + doc[field].startsWith('+') || + doc[field].startsWith('-') || + doc[field].startsWith('=') || + doc[field].startsWith('@') + ) { + doc[field] = "'" + doc[field]; + } + } + return doc; +} diff --git a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts index bddf7013..d31d86c4 100644 --- a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -36,12 +36,11 @@ export async function createSavedSearchReport( client: IClusterClient | IScopedClusterClient ): Promise<{ timeCreated: number; dataUrl: string; fileName: string }> { const params = report.report_definition.report_params; - const limit = params.core_params.limit; const reportFormat = params.core_params.report_format; const reportName = params.report_name; await populateMetaData(client, report); - const data = await generateCsvData(client, limit); + const data = await generateCsvData(client, params.core_params); const curTime = new Date(); const timeCreated = curTime.valueOf(); @@ -114,7 +113,7 @@ async function populateMetaData( */ async function generateCsvData( client: IClusterClient | IScopedClusterClient, - limit: number + params: any ) { let esData: any = {}; const arrayHits: any = []; @@ -123,7 +122,7 @@ async function generateCsvData( const maxResultSize: number = await getMaxResultSize(); const esCount = await getEsDataSize(); - const total = Math.min(esCount.count, limit); + const total = Math.min(esCount.count, params.limit); if (total === 0) { return ''; } @@ -212,7 +211,7 @@ async function generateCsvData( // Parse ES data and convert to CSV async function convertEsDataToCsv() { const dataset: any = []; - dataset.push(getEsData(arrayHits, report, limit)); + dataset.push(getEsData(arrayHits, report, params)); return await convertToCSV(dataset); } } From 8691b6a9da219b344441e1d3ae65d209044049ab Mon Sep 17 00:00:00 2001 From: Dai Date: Tue, 22 Sep 2020 10:55:26 -0700 Subject: [PATCH 21/21] Address PR comments --- kibana-reports/server/routes/utils/savedSearchReportHelper.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts index d31d86c4..13780c08 100644 --- a/kibana-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/kibana-reports/server/routes/utils/savedSearchReportHelper.ts @@ -40,7 +40,7 @@ export async function createSavedSearchReport( const reportName = params.report_name; await populateMetaData(client, report); - const data = await generateCsvData(client, params.core_params); + const data = await generateReportData(client, params.core_params); const curTime = new Date(); const timeCreated = curTime.valueOf(); @@ -111,7 +111,7 @@ async function populateMetaData( * @param client ES client * @param limit limit size of result data set */ -async function generateCsvData( +async function generateReportData( client: IClusterClient | IScopedClusterClient, params: any ) {