diff --git a/src/legacy/core_plugins/kibana/public/dashboard/store/panel_actions_store.ts b/src/legacy/core_plugins/kibana/public/dashboard/store/panel_actions_store.ts index 449125d0ecfa4..69a9a93b1828a 100644 --- a/src/legacy/core_plugins/kibana/public/dashboard/store/panel_actions_store.ts +++ b/src/legacy/core_plugins/kibana/public/dashboard/store/panel_actions_store.ts @@ -28,7 +28,9 @@ class PanelActionsStore { */ public initializeFromRegistry(panelActionsRegistry: ContextMenuAction[]) { panelActionsRegistry.forEach(panelAction => { - this.actions.push(panelAction); + if (!this.actions.includes(panelAction)) { + this.actions.push(panelAction); + } }); } } diff --git a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts index c41092682283d..fe5b9994c574e 100644 --- a/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts +++ b/src/legacy/core_plugins/kibana/public/discover/embeddable/search_embeddable.ts @@ -110,6 +110,10 @@ export class SearchEmbeddable extends Embeddable { return this.inspectorAdaptors; } + public getPanelTitle() { + return this.panelTitle; + } + public onContainerStateChanged(containerState: ContainerState) { this.customization = containerState.embeddableCustomization || {}; this.filters = containerState.filters; diff --git a/x-pack/plugins/reporting/__snapshots__/index.test.js.snap b/x-pack/plugins/reporting/__snapshots__/index.test.js.snap index f19785d5df0e8..ed5d0d236e603 100644 --- a/x-pack/plugins/reporting/__snapshots__/index.test.js.snap +++ b/x-pack/plugins/reporting/__snapshots__/index.test.js.snap @@ -25,6 +25,7 @@ Object { "zoom": 2, }, "csv": Object { + "enablePanelActionDownload": false, "maxSizeBytes": 10485760, "scroll": Object { "duration": "30s", @@ -85,6 +86,7 @@ Object { "zoom": 2, }, "csv": Object { + "enablePanelActionDownload": false, "maxSizeBytes": 10485760, "scroll": Object { "duration": "30s", @@ -144,6 +146,7 @@ Object { "zoom": 2, }, "csv": Object { + "enablePanelActionDownload": false, "maxSizeBytes": 10485760, "scroll": Object { "duration": "30s", @@ -204,6 +207,7 @@ Object { "zoom": 2, }, "csv": Object { + "enablePanelActionDownload": false, "maxSizeBytes": 10485760, "scroll": Object { "duration": "30s", diff --git a/x-pack/plugins/reporting/server/lib/esqueue/helpers/cancellation_token.js b/x-pack/plugins/reporting/common/cancellation_token.ts similarity index 86% rename from x-pack/plugins/reporting/server/lib/esqueue/helpers/cancellation_token.js rename to x-pack/plugins/reporting/common/cancellation_token.ts index 8d5633cd1c1d8..293431d9f1338 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/helpers/cancellation_token.js +++ b/x-pack/plugins/reporting/common/cancellation_token.ts @@ -7,12 +7,15 @@ import { isFunction } from 'lodash'; export class CancellationToken { + private isCancelled: boolean; + private _callbacks: any[]; + constructor() { this.isCancelled = false; this._callbacks = []; } - on = (callback) => { + on(callback: Function) { if (!isFunction(callback)) { throw new Error('Expected callback to be a function'); } @@ -23,10 +26,10 @@ export class CancellationToken { } this._callbacks.push(callback); - }; + } - cancel = () => { + cancel() { this.isCancelled = true; this._callbacks.forEach(callback => callback()); - }; + } } diff --git a/x-pack/plugins/reporting/common/constants.ts b/x-pack/plugins/reporting/common/constants.ts index cf2f6555a34a1..baca113331268 100644 --- a/x-pack/plugins/reporting/common/constants.ts +++ b/x-pack/plugins/reporting/common/constants.ts @@ -9,12 +9,16 @@ export const PLUGIN_ID = 'reporting'; export const JOB_COMPLETION_NOTIFICATIONS_SESSION_KEY = 'xpack.reporting.jobCompletionNotifications'; -export const API_BASE_URL = '/api/reporting'; +export const API_BASE_URL = '/api/reporting'; // "Generation URL" from share menu +export const API_BASE_URL_V1 = '/api/reporting/v1'; // +export const API_BASE_GENERATE_V1 = `${API_BASE_URL_V1}/generate`; + +export const CONTENT_TYPE_CSV = 'text/csv'; export const WHITELISTED_JOB_CONTENT_TYPES = [ 'application/json', 'application/pdf', - 'text/csv', + CONTENT_TYPE_CSV, 'image/png', ]; @@ -41,4 +45,5 @@ export const KIBANA_REPORTING_TYPE = 'reporting'; export const PDF_JOB_TYPE = 'printable_pdf'; export const PNG_JOB_TYPE = 'PNG'; export const CSV_JOB_TYPE = 'csv'; +export const CSV_FROM_SAVEDOBJECT_JOB_TYPE = 'csv_from_savedobject'; export const USES_HEADLESS_JOB_TYPES = [PDF_JOB_TYPE, PNG_JOB_TYPE]; diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.test.ts b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.test.ts index 815d0169be2af..117b72e59bf23 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.test.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.test.ts @@ -15,7 +15,15 @@ beforeEach(() => { test(`fails if no URL is passed`, async () => { await expect( addForceNowQuerystring({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, server: mockServer, }) ).rejects.toBeDefined(); @@ -24,7 +32,17 @@ test(`fails if no URL is passed`, async () => { test(`adds forceNow to hash's query, if it exists`, async () => { const forceNow = '2000-01-01T00:00:00.000Z'; const { urls } = await addForceNowQuerystring({ - job: { relativeUrl: '/app/kibana#/something', forceNow }, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + relativeUrl: '/app/kibana#/something', + forceNow, + }, server: mockServer, }); @@ -38,6 +56,13 @@ test(`appends forceNow to hash's query, if it exists`, async () => { const { urls } = await addForceNowQuerystring({ job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, relativeUrl: '/app/kibana#/something?_g=something', forceNow, }, @@ -52,6 +77,13 @@ test(`appends forceNow to hash's query, if it exists`, async () => { test(`doesn't append forceNow query to url, if it doesn't exists`, async () => { const { urls } = await addForceNowQuerystring({ job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, relativeUrl: '/app/kibana#/something', }, server: mockServer, diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts index f9bf39e495b92..fda91c51ef0c2 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/add_force_now_query_string.ts @@ -3,12 +3,12 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -// @ts-ignore + import url from 'url'; import { getAbsoluteUrlFactory } from '../../../common/get_absolute_url'; -import { ConditionalHeaders, KbnServer, ReportingJob } from '../../../types'; +import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types'; -function getSavedObjectAbsoluteUrl(job: ReportingJob, relativeUrl: string, server: KbnServer) { +function getSavedObjectAbsoluteUrl(job: JobDocPayload, relativeUrl: string, server: KbnServer) { const getAbsoluteUrl: any = getAbsoluteUrlFactory(server); const { pathname: path, hash, search } = url.parse(relativeUrl); @@ -21,7 +21,7 @@ export const addForceNowQuerystring = async ({ logo, server, }: { - job: ReportingJob; + job: JobDocPayload; conditionalHeaders?: ConditionalHeaders; logo?: any; server: KbnServer; @@ -34,7 +34,7 @@ export const addForceNowQuerystring = async ({ job.urls = [getSavedObjectAbsoluteUrl(job, job.relativeUrl, server)]; } - const urls = job.urls.map(jobUrl => { + const urls = job.urls.map((jobUrl: string) => { if (!job.forceNow) { return jobUrl; } diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.test.ts b/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.test.ts index a289093d2ebaa..7318beadd6a69 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.test.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.test.ts @@ -23,7 +23,17 @@ describe('headers', () => { test(`fails if it can't decrypt headers`, async () => { await expect( decryptJobHeaders({ - job: { relativeUrl: '/app/kibana#/something', timeRange: {} }, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + relativeUrl: '/app/kibana#/something', + timeRange: {}, + }, server: mockServer, }) ).rejects.toBeDefined(); @@ -37,7 +47,17 @@ describe('headers', () => { const encryptedHeaders = await encryptHeaders(headers); const { decryptedHeaders } = await decryptJobHeaders({ - job: { relativeUrl: '/app/kibana#/something', headers: encryptedHeaders }, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + relativeUrl: '/app/kibana#/something', + headers: encryptedHeaders, + }, server: mockServer, }); expect(decryptedHeaders).toEqual(headers); diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.ts b/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.ts index d10d8b51f84ec..e933056c79441 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/decrypt_job_headers.ts @@ -5,13 +5,13 @@ */ // @ts-ignore import { cryptoFactory } from '../../../server/lib/crypto'; -import { CryptoFactory, KbnServer, ReportingJob } from '../../../types'; +import { CryptoFactory, JobDocPayload, KbnServer } from '../../../types'; export const decryptJobHeaders = async ({ job, server, }: { - job: ReportingJob; + job: JobDocPayload; server: KbnServer; }) => { const crypto: CryptoFactory = cryptoFactory(server); diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.test.ts b/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.test.ts index 8106286de423a..768c25e548717 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.test.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.test.ts @@ -26,7 +26,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -43,7 +51,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -64,7 +80,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -81,7 +105,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -96,7 +128,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -119,7 +159,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -136,7 +184,15 @@ describe('conditions', () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -152,7 +208,15 @@ test('uses basePath from job when creating saved object service', async () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -162,7 +226,16 @@ test('uses basePath from job when creating saved object service', async () => { const jobBasePath = '/sbp/s/marketing'; await getCustomLogo({ - job: { basePath: jobBasePath }, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + basePath: jobBasePath, + }, conditionalHeaders, server: mockServer, }); @@ -179,7 +252,15 @@ test(`uses basePath from server if job doesn't have a basePath when creating sav }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); @@ -188,7 +269,15 @@ test(`uses basePath from server if job doesn't have a basePath when creating sav mockServer.uiSettingsServiceFactory().get.mockReturnValue(logo); await getCustomLogo({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, conditionalHeaders, server: mockServer, }); @@ -202,7 +291,15 @@ describe('config formatting', () => { test(`lowercases server.host`, async () => { mockServer = createMockServer({ settings: { 'server.host': 'COOL-HOSTNAME' } }); const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: {}, server: mockServer, }); @@ -214,7 +311,15 @@ describe('config formatting', () => { settings: { 'xpack.reporting.kibanaServer.hostname': 'GREAT-HOSTNAME' }, }); const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: {}, server: mockServer, }); diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.ts b/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.ts index 0da4b9892b7cd..04959712a117a 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/get_conditional_headers.ts @@ -3,14 +3,14 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { ConditionalHeaders, ConfigObject, KbnServer, ReportingJob } from '../../../types'; +import { ConditionalHeaders, ConfigObject, JobDocPayload, KbnServer } from '../../../types'; export const getConditionalHeaders = ({ job, filteredHeaders, server, }: { - job: ReportingJob; + job: JobDocPayload; filteredHeaders: Record; server: KbnServer; }) => { diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.test.ts b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.test.ts index 8ab96af979cd8..029b641bd73e5 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.test.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.test.ts @@ -19,13 +19,29 @@ test(`gets logo from uiSettings`, async () => { }; const { conditionalHeaders } = await getConditionalHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, filteredHeaders: permittedHeaders, server: mockServer, }); const { logo } = await getCustomLogo({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, conditionalHeaders, server: mockServer, }); diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts index 0b25187a938ae..b41e4c8218614 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/get_custom_logo.ts @@ -3,15 +3,16 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ + import { UI_SETTINGS_CUSTOM_PDF_LOGO } from '../../../common/constants'; -import { ConditionalHeaders, KbnServer, ReportingJob } from '../../../types'; +import { ConditionalHeaders, JobDocPayload, KbnServer } from '../../../types'; export const getCustomLogo = async ({ job, conditionalHeaders, server, }: { - job: ReportingJob; + job: JobDocPayload; conditionalHeaders: ConditionalHeaders; server: KbnServer; }) => { diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.test.ts b/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.test.ts index 5ee21d89aed56..c030fc4c662cb 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.test.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.test.ts @@ -27,7 +27,15 @@ test(`omits blacklisted headers`, async () => { }; const { filteredHeaders } = await omitBlacklistedHeaders({ - job: {}, + job: { + title: 'cool-job-bro', + type: 'csv', + jobParams: { + savedObjectId: 'abc-123', + isImmediate: false, + savedObjectType: 'search', + }, + }, decryptedHeaders: { ...permittedHeaders, ...blacklistedHeaders, diff --git a/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.ts b/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.ts index a334d57a73f68..44a9ed5a8ee51 100644 --- a/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.ts +++ b/x-pack/plugins/reporting/export_types/common/execute_job/omit_blacklisted_headers.ts @@ -5,14 +5,14 @@ */ import { omit } from 'lodash'; import { KBN_SCREENSHOT_HEADER_BLACKLIST } from '../../../common/constants'; -import { KbnServer, ReportingJob } from '../../../types'; +import { JobDocPayload, KbnServer } from '../../../types'; export const omitBlacklistedHeaders = ({ job, decryptedHeaders, server, }: { - job: ReportingJob; + job: JobDocPayload; decryptedHeaders: Record; server: KbnServer; }) => { diff --git a/x-pack/plugins/reporting/export_types/csv/server/__tests__/execute_job.js b/x-pack/plugins/reporting/export_types/csv/server/__tests__/execute_job.js index b8799f7a516d9..5ef2a897d10fd 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/__tests__/execute_job.js +++ b/x-pack/plugins/reporting/export_types/csv/server/__tests__/execute_job.js @@ -9,7 +9,7 @@ import Puid from 'puid'; import sinon from 'sinon'; import nodeCrypto from '@elastic/node-crypto'; -import { CancellationToken } from '../../../../server/lib/esqueue/helpers/cancellation_token'; +import { CancellationToken } from '../../../../common/cancellation_token'; import { FieldFormat } from '../../../../../../../src/legacy/ui/field_formats/field_format.js'; import { FieldFormatsService } from '../../../../../../../src/legacy/ui/field_formats/field_formats_service.js'; import { createStringFormat } from '../../../../../../../src/legacy/core_plugins/kibana/common/field_formats/types/string.js'; diff --git a/x-pack/plugins/reporting/export_types/csv/server/lib/flatten_hit.js b/x-pack/plugins/reporting/export_types/csv/server/lib/flatten_hit.js index 38bcd98aa1de4..375b096e32651 100644 --- a/x-pack/plugins/reporting/export_types/csv/server/lib/flatten_hit.js +++ b/x-pack/plugins/reporting/export_types/csv/server/lib/flatten_hit.js @@ -6,6 +6,7 @@ import _ from 'lodash'; +// TODO this logic should be re-used with Discover export function createFlattenHit(fields, metaFields, conflictedTypesFields) { const flattenSource = (flat, obj, keyPrefix) => { keyPrefix = keyPrefix ? keyPrefix + '.' : ''; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.d.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.d.ts new file mode 100644 index 0000000000000..3f35f7e237f5b --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.d.ts @@ -0,0 +1,150 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface SavedObjectServiceError { + statusCode: number; + error?: string; + message?: string; +} + +export interface SavedObjectMetaJSON { + searchSourceJSON: string; +} + +export interface SavedObjectMeta { + searchSource: SearchSource; +} + +export interface SavedSearchObjectAttributesJSON { + title: string; + sort: any[]; + columns: string[]; + kibanaSavedObjectMeta: SavedObjectMetaJSON; + uiState: any; +} + +export interface SavedSearchObjectAttributes { + title: string; + sort: any[]; + columns?: string[]; + kibanaSavedObjectMeta: SavedObjectMeta; + uiState: any; +} + +export interface VisObjectAttributesJSON { + title: string; + visState: string; // JSON string + type: string; + params: any; + uiStateJSON: string; // also JSON string + aggs: any[]; + sort: any[]; + kibanaSavedObjectMeta: SavedObjectMeta; +} + +export interface VisObjectAttributes { + title: string; + visState: string; // JSON string + type: string; + params: any; + uiState: { + vis: { + params: { + sort: { + columnIndex: string; + direction: string; + }; + }; + }; + }; + aggs: any[]; + sort: any[]; + kibanaSavedObjectMeta: SavedObjectMeta; +} + +export interface SavedObjectReference { + name: string; // should be kibanaSavedObjectMeta.searchSourceJSON.index + type: string; // should be index-pattern + id: string; +} + +export interface SavedObject { + attributes: any; + references: SavedObjectReference[]; +} + +/* This object is passed to different helpers in different parts of the code + - packages/kbn-es-query/src/es_query/build_es_query + - x-pack/plugins/reporting/export_types/csv/server/lib/field_format_map + The structure has redundant parts and json-parsed / json-unparsed versions of the same data + */ +export interface IndexPatternSavedObject { + title: string; + timeFieldName: string; + fields: any[]; + attributes: { + fieldFormatMap: string; + fields: string; + }; +} + +export interface TimeRangeParams { + timezone: string; + min: Date | string | number; + max: Date | string | number; +} + +export interface VisPanel { + indexPatternSavedObjectId?: string; + savedSearchObjectId?: string; + attributes: VisObjectAttributes; + timerange: TimeRangeParams; +} + +export interface SearchPanel { + indexPatternSavedObjectId: string; + attributes: SavedSearchObjectAttributes; + timerange: TimeRangeParams; +} + +export interface SearchSourceQuery { + isSearchSourceQuery: boolean; +} + +export interface SearchSource { + query: SearchSourceQuery; + filter: any[]; +} + +export interface SearchRequest { + index: string; + body: + | { + _source: { + excludes: string[]; + includes: string[]; + }; + docvalue_fields: string[]; + query: + | { + bool: { + filter: any[]; + must_not: any[]; + should: any[]; + must: any[]; + }; + } + | any; + script_fields: any; + sort: Array<{ + [key: string]: { + order: string; + }; + }>; + stored_fields: string[]; + } + | any; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts new file mode 100644 index 0000000000000..6cc56cb5fa4f1 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './index.d'; + +/* + * These functions are exported to share with the API route handler that + * generates csv from saved object immediately on request. + */ +export { executeJobFactory } from './server/execute_job'; +export { createJobFactory } from './server/create_job'; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/metadata.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/metadata.ts new file mode 100644 index 0000000000000..fcef889e52fe4 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/metadata.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../common/constants'; + +export const metadata = { + id: CSV_FROM_SAVEDOBJECT_JOB_TYPE, + name: CSV_FROM_SAVEDOBJECT_JOB_TYPE, +}; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts new file mode 100644 index 0000000000000..3ba54473474ee --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { notFound, notImplemented } from 'boom'; +import { Request } from 'hapi'; +import { get } from 'lodash'; + +import { cryptoFactory, LevelLogger, oncePerServer } from '../../../../server/lib'; +import { JobDocPayload, JobParams, KbnServer } from '../../../../types'; +import { + SavedObject, + SavedObjectServiceError, + SavedSearchObjectAttributesJSON, + SearchPanel, + TimeRangeParams, + VisObjectAttributesJSON, +} from '../../'; +import { createJobSearch } from './create_job_search'; + +interface VisData { + title: string; + visType: string; + panel: SearchPanel; +} + +type CreateJobFn = (jobParams: JobParams, headers: any, req: Request) => Promise; + +function createJobFn(server: KbnServer): CreateJobFn { + const crypto = cryptoFactory(server); + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); + + return async function createJob( + jobParams: JobParams, + headers: any, + req: Request + ): Promise { + const { savedObjectType, savedObjectId } = jobParams; + const serializedEncryptedHeaders = await crypto.encrypt(headers); + const client = req.getSavedObjectsClient(); + + const { panel, title, visType }: VisData = await Promise.resolve() + .then(() => client.get(savedObjectType, savedObjectId)) + .then(async (savedObject: SavedObject) => { + const { attributes, references } = savedObject; + const { + kibanaSavedObjectMeta: kibanaSavedObjectMetaJSON, + } = attributes as SavedSearchObjectAttributesJSON; + const { timerange } = req.payload as { timerange: TimeRangeParams }; + + if (!kibanaSavedObjectMetaJSON) { + throw new Error('Could not parse saved object data!'); + } + + const kibanaSavedObjectMeta = { + ...kibanaSavedObjectMetaJSON, + searchSource: JSON.parse(kibanaSavedObjectMetaJSON.searchSourceJSON), + }; + + const { visState: visStateJSON } = attributes as VisObjectAttributesJSON; + if (visStateJSON) { + throw notImplemented('Visualization types are not yet implemented'); + } + + // saved search type + return await createJobSearch(timerange, attributes, references, kibanaSavedObjectMeta); + }) + .catch((err: Error) => { + const boomErr = (err as unknown) as { isBoom: boolean }; + if (boomErr.isBoom) { + throw err; + } + const errPayload: SavedObjectServiceError = get(err, 'output.payload', { statusCode: 0 }); + if (errPayload.statusCode === 404) { + throw notFound(errPayload.message); + } + if (err.stack) { + logger.error(err.stack); + } + throw new Error(`Unable to create a job from saved object data! Error: ${err}`); + }); + + return { + basePath: req.getBasePath(), + headers: serializedEncryptedHeaders, + jobParams: { ...jobParams, panel, visType }, + type: null, // resolved in executeJob + objects: null, // resolved in executeJob + title, + }; + }; +} + +export const createJobFactory = oncePerServer(createJobFn); diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job_search.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job_search.ts new file mode 100644 index 0000000000000..c98a0f965aa99 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/create_job_search.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + SavedObjectMeta, + SavedObjectReference, + SavedSearchObjectAttributes, + SearchPanel, + TimeRangeParams, +} from '../../'; + +interface SearchPanelData { + title: string; + visType: string; + panel: SearchPanel; +} + +export async function createJobSearch( + timerange: TimeRangeParams, + attributes: SavedSearchObjectAttributes, + references: SavedObjectReference[], + kibanaSavedObjectMeta: SavedObjectMeta +): Promise { + const { searchSource } = kibanaSavedObjectMeta; + if (!searchSource || !references) { + throw new Error('The saved search object is missing configuration fields!'); + } + + const indexPatternMeta = references.find( + (ref: SavedObjectReference) => ref.type === 'index-pattern' + ); + if (!indexPatternMeta) { + throw new Error('Could not find index pattern for the saved search!'); + } + + const sPanel = { + attributes: { + ...attributes, + kibanaSavedObjectMeta: { searchSource }, + }, + indexPatternSavedObjectId: indexPatternMeta.id, + timerange, + }; + + return { panel: sPanel, title: attributes.title, visType: 'search' }; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/index.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/index.ts new file mode 100644 index 0000000000000..e129e3e5f47ec --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/create_job/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { createJobFactory } from './create_job'; diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts new file mode 100644 index 0000000000000..67bcea92785e1 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/execute_job.ts @@ -0,0 +1,102 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request } from 'hapi'; +import { i18n } from '@kbn/i18n'; + +import { cryptoFactory, LevelLogger, oncePerServer } from '../../../server/lib'; +import { JobDocOutputExecuted, JobDocPayload, KbnServer } from '../../../types'; +import { CONTENT_TYPE_CSV } from '../../../common/constants'; +import { CsvResultFromSearch, createGenerateCsv } from './lib'; + +interface FakeRequest { + headers: any; + getBasePath: (opts: any) => string; + server: KbnServer; +} + +type ExecuteJobFn = (job: JobDocPayload, realRequest?: Request) => Promise; + +function executeJobFn(server: KbnServer): ExecuteJobFn { + const crypto = cryptoFactory(server); + const config = server.config(); + const serverBasePath = config.get('server.basePath'); + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); + const generateCsv = createGenerateCsv(logger); + + return async function executeJob( + job: JobDocPayload, + realRequest?: Request + ): Promise { + const { basePath, jobParams } = job; + const { isImmediate, panel, visType } = jobParams; + + logger.debug(`Execute job generating [${visType}] csv`); + + let requestObject: Request | FakeRequest; + if (isImmediate && realRequest) { + logger.debug(`executing job from immediate API`); + requestObject = realRequest; + } else { + logger.debug(`executing job async using encrypted headers`); + let decryptedHeaders; + const serializedEncryptedHeaders = job.headers; + try { + decryptedHeaders = await crypto.decrypt(serializedEncryptedHeaders); + } catch (err) { + throw new Error( + i18n.translate( + 'xpack.reporting.exportTypes.csv_from_savedobject.executeJob.failedToDecryptReportJobDataErrorMessage', + { + defaultMessage: + 'Failed to decrypt report job data. Please ensure that {encryptionKey} is set and re-generate this report. {err}', + values: { encryptionKey: 'xpack.reporting.encryptionKey', err }, + } + ) + ); + } + + requestObject = { + headers: decryptedHeaders, + getBasePath: () => basePath || serverBasePath, + server, + }; + } + + let content: string; + let maxSizeReached = false; + let size = 0; + try { + const generateResults: CsvResultFromSearch = await generateCsv( + requestObject, + server, + visType as string, + panel, + jobParams + ); + + ({ + result: { content, maxSizeReached, size }, + } = generateResults); + } catch (err) { + logger.error(`Generate CSV Error! ${err}`); + throw err; + } + + if (maxSizeReached) { + logger.warn(`Max size reached: CSV output truncated to ${size} bytes`); + } + + return { + content_type: CONTENT_TYPE_CSV, + content, + max_size_reached: maxSizeReached, + size, + }; + }; +} + +export const executeJobFactory = oncePerServer(executeJobFn); diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/index.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/index.ts new file mode 100644 index 0000000000000..b614fd3c681b3 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/index.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants'; +import { ExportTypesRegistry } from '../../../types'; +import { metadata } from '../metadata'; +import { createJobFactory } from './create_job'; +import { executeJobFactory } from './execute_job'; + +export function register(registry: ExportTypesRegistry) { + registry.register({ + ...metadata, + jobType: CSV_FROM_SAVEDOBJECT_JOB_TYPE, + jobContentExtension: 'csv', + createJobFactory, + executeJobFactory, + validLicenses: ['trial', 'basic', 'standard', 'gold', 'platinum'], + }); +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts new file mode 100644 index 0000000000000..cf2d621b7d201 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { badRequest } from 'boom'; +import { Request } from 'hapi'; +import { KbnServer, Logger, JobParams } from '../../../../types'; +import { SearchPanel, VisPanel } from '../../'; +import { generateCsvSearch } from './generate_csv_search'; + +interface FakeRequest { + headers: any; + getBasePath: (opts: any) => string; + server: KbnServer; +} + +export function createGenerateCsv(logger: Logger) { + return async function generateCsv( + request: Request | FakeRequest, + server: KbnServer, + visType: string, + panel: VisPanel | SearchPanel, + jobParams: JobParams + ) { + // This should support any vis type that is able to fetch + // and model data on the server-side + + // This structure will not be needed when the vis data just consists of an + // expression that we could run through the interpreter to get csv + switch (visType) { + case 'search': + return await generateCsvSearch( + request as Request, + server, + logger, + panel as SearchPanel, + jobParams + ); + default: + throw badRequest(`Unsupported or unrecognized saved object type: ${visType}`); + } + }; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts new file mode 100644 index 0000000000000..9d0b588b02da7 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/generate_csv_search.ts @@ -0,0 +1,166 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request } from 'hapi'; + +// @ts-ignore no module definition +import { buildEsQuery } from '@kbn/es-query'; +// @ts-ignore no module definition +import { createGenerateCsv } from '../../../csv/server/lib/generate_csv'; + +import { CancellationToken } from '../../../../common/cancellation_token'; + +import { KbnServer, Logger, JobParams } from '../../../../types'; +import { + IndexPatternSavedObject, + SavedSearchObjectAttributes, + SearchPanel, + SearchRequest, + SearchSource, + SearchSourceQuery, +} from '../../'; +import { + CsvResultFromSearch, + ESQueryConfig, + GenerateCsvParams, + Filter, + IndexPatternField, + QueryFilter, +} from './'; +import { getDataSource } from './get_data_source'; +import { getFilters } from './get_filters'; + +const getEsQueryConfig = async (config: any) => { + const configs = await Promise.all([ + config.get('query:allowLeadingWildcards'), + config.get('query:queryString:options'), + config.get('courier:ignoreFilterIfFieldNotInIndex'), + ]); + const [allowLeadingWildcards, queryStringOptions, ignoreFilterIfFieldNotInIndex] = configs; + return { allowLeadingWildcards, queryStringOptions, ignoreFilterIfFieldNotInIndex }; +}; + +const getUiSettings = async (config: any) => { + const configs = await Promise.all([config.get('csv:separator'), config.get('csv:quoteValues')]); + const [separator, quoteValues] = configs; + return { separator, quoteValues }; +}; + +export async function generateCsvSearch( + req: Request, + server: KbnServer, + logger: Logger, + searchPanel: SearchPanel, + jobParams: JobParams +): Promise { + const { savedObjects, uiSettingsServiceFactory } = server; + const savedObjectsClient = savedObjects.getScopedSavedObjectsClient(req); + const { indexPatternSavedObjectId, timerange } = searchPanel; + const savedSearchObjectAttr = searchPanel.attributes as SavedSearchObjectAttributes; + const { indexPatternSavedObject } = await getDataSource( + savedObjectsClient, + indexPatternSavedObjectId + ); + const uiConfig = uiSettingsServiceFactory({ savedObjectsClient }); + const esQueryConfig = await getEsQueryConfig(uiConfig); + + const { + kibanaSavedObjectMeta: { + searchSource: { + filter: [searchSourceFilter], + query: searchSourceQuery, + }, + }, + } = savedSearchObjectAttr as { kibanaSavedObjectMeta: { searchSource: SearchSource } }; + + const { + timeFieldName: indexPatternTimeField, + title: esIndex, + fields: indexPatternFields, + } = indexPatternSavedObject; + + let payloadQuery: QueryFilter | undefined; + let payloadSort: any[] = []; + if (jobParams.post && jobParams.post.state) { + ({ + post: { state: { query: payloadQuery, sort: payloadSort = [] } }, + } = jobParams); + } + + const { includes, timezone, combinedFilter } = getFilters( + indexPatternSavedObjectId, + indexPatternTimeField, + timerange, + savedSearchObjectAttr, + searchSourceFilter, + payloadQuery + ); + + const [savedSortField, savedSortOrder] = savedSearchObjectAttr.sort; + const sortConfig = [...payloadSort, { [savedSortField]: { order: savedSortOrder } }]; + + const scriptFieldsConfig = indexPatternFields + .filter((f: IndexPatternField) => f.scripted) + .reduce((accum: any, curr: IndexPatternField) => { + return { + ...accum, + [curr.name]: { + script: { + source: curr.script, + lang: curr.lang, + }, + }, + }; + }, {}); + const docValueFields = indexPatternTimeField ? [indexPatternTimeField] : undefined; + + // this array helps ensure the params are passed to buildEsQuery (non-Typescript) in the right order + const buildCsvParams: [IndexPatternSavedObject, SearchSourceQuery, Filter[], ESQueryConfig] = [ + indexPatternSavedObject, + searchSourceQuery, + combinedFilter, + esQueryConfig, + ]; + + const searchRequest: SearchRequest = { + index: esIndex, + body: { + _source: { includes }, + docvalue_fields: docValueFields, + query: buildEsQuery(...buildCsvParams), + script_fields: scriptFieldsConfig, + sort: sortConfig, + }, + }; + + const { callWithRequest } = server.plugins.elasticsearch.getCluster('data'); + const callCluster = (...params: any[]) => callWithRequest(req, ...params); + const config = server.config(); + const uiSettings = await getUiSettings(uiConfig); + + const generateCsvParams: GenerateCsvParams = { + searchRequest, + callEndpoint: callCluster, + fields: includes, + formatsMap: new Map(), // there is no field formatting in this API; this is required for generateCsv + metaFields: [], + conflictedTypesFields: [], + cancellationToken: new CancellationToken(), + settings: { + ...uiSettings, + maxSizeBytes: config.get('xpack.reporting.csv.maxSizeBytes'), + scroll: config.get('xpack.reporting.csv.scroll'), + timezone, + }, + }; + + const generateCsv = createGenerateCsv(logger); + + return { + type: 'CSV from Saved Search', + result: await generateCsv(generateCsvParams), + }; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_data_source.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_data_source.ts new file mode 100644 index 0000000000000..920c55da4f83d --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_data_source.ts @@ -0,0 +1,59 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { + IndexPatternSavedObject, + SavedObjectReference, + SavedSearchObjectAttributesJSON, + SearchSource, +} from '../../'; + +export async function getDataSource( + savedObjectsClient: any, + indexPatternId?: string, + savedSearchObjectId?: string +): Promise<{ + indexPatternSavedObject: IndexPatternSavedObject; + searchSource: SearchSource | null; +}> { + let indexPatternSavedObject: IndexPatternSavedObject; + let searchSource: SearchSource | null = null; + + if (savedSearchObjectId) { + try { + const { attributes, references } = (await savedObjectsClient.get( + 'search', + savedSearchObjectId + )) as { attributes: SavedSearchObjectAttributesJSON; references: SavedObjectReference[] }; + searchSource = JSON.parse(attributes.kibanaSavedObjectMeta.searchSourceJSON); + const { id: indexPatternFromSearchId } = references.find( + ({ type }) => type === 'index-pattern' + ) as { id: string }; + ({ indexPatternSavedObject } = await getDataSource( + savedObjectsClient, + indexPatternFromSearchId + )); + return { searchSource, indexPatternSavedObject }; + } catch (err) { + throw new Error(`Could not get saved search info! ${err}`); + } + } + try { + const { attributes } = await savedObjectsClient.get('index-pattern', indexPatternId); + const { fields, title, timeFieldName } = attributes; + const parsedFields = fields ? JSON.parse(fields) : []; + + indexPatternSavedObject = { + fields: parsedFields, + title, + timeFieldName, + attributes, + }; + } catch (err) { + throw new Error(`Could not get index pattern saved object! ${err}`); + } + return { indexPatternSavedObject, searchSource }; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.test.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.test.ts new file mode 100644 index 0000000000000..e87cfc212d404 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.test.ts @@ -0,0 +1,207 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { SavedSearchObjectAttributes, TimeRangeParams } from '../../'; +import { QueryFilter, SearchSourceFilter } from './'; +import { getFilters } from './get_filters'; + +interface Args { + indexPatternId: string; + indexPatternTimeField: string | null; + timerange: TimeRangeParams | null; + savedSearchObjectAttr: SavedSearchObjectAttributes; + searchSourceFilter: SearchSourceFilter; + queryFilter: QueryFilter; +} + +describe('CSV from Saved Object: get_filters', () => { + let args: Args; + beforeEach(() => { + args = { + indexPatternId: 'logs-test-*', + indexPatternTimeField: 'testtimestamp', + timerange: { + timezone: 'UTC', + min: '1901-01-01T00:00:00.000Z', + max: '1902-01-01T00:00:00.000Z', + }, + savedSearchObjectAttr: { + title: 'test', + sort: [{ sortField: { order: 'asc' } }], + kibanaSavedObjectMeta: { + searchSource: { + query: { isSearchSourceQuery: true }, + filter: ['hello searchSource filter 1'], + }, + }, + columns: ['larry'], + uiState: null, + }, + searchSourceFilter: { isSearchSourceFilter: true, isFilter: true }, + queryFilter: { isQueryFilter: true, isFilter: true }, + }; + }); + + describe('search', () => { + it('for timebased search', () => { + const filters = getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr, + args.searchSourceFilter, + args.queryFilter + ); + + expect(filters).toEqual({ + combinedFilter: [ + { + range: { + testtimestamp: { + format: 'strict_date_time', + gte: '1901-01-01T00:00:00Z', + lte: '1902-01-01T00:00:00Z', + }, + }, + }, + { isFilter: true, isSearchSourceFilter: true }, + { isFilter: true, isQueryFilter: true }, + ], + includes: ['testtimestamp', 'larry'], + timezone: 'UTC', + }); + }); + + it('for non-timebased search', () => { + args.indexPatternTimeField = null; + args.timerange = null; + + const filters = getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr, + args.searchSourceFilter, + args.queryFilter + ); + + expect(filters).toEqual({ + combinedFilter: [ + { isFilter: true, isSearchSourceFilter: true }, + { isFilter: true, isQueryFilter: true }, + ], + includes: ['larry'], + timezone: null, + }); + }); + }); + + describe('errors', () => { + it('throw if timebased and timerange is missing', () => { + args.timerange = null; + + const throwFn = () => + getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr, + args.searchSourceFilter, + args.queryFilter + ); + + expect(throwFn).toThrow( + 'Time range params are required for index pattern [logs-test-*], using time field [testtimestamp]' + ); + }); + }); + + it('composes the defined filters', () => { + expect( + getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr, + undefined, + undefined + ) + ).toEqual({ + combinedFilter: [ + { + range: { + testtimestamp: { + format: 'strict_date_time', + gte: '1901-01-01T00:00:00Z', + lte: '1902-01-01T00:00:00Z', + }, + }, + }, + ], + includes: ['testtimestamp', 'larry'], + timezone: 'UTC', + }); + + expect( + getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr, + undefined, + args.queryFilter + ) + ).toEqual({ + combinedFilter: [ + { + range: { + testtimestamp: { + format: 'strict_date_time', + gte: '1901-01-01T00:00:00Z', + lte: '1902-01-01T00:00:00Z', + }, + }, + }, + { isFilter: true, isQueryFilter: true }, + ], + includes: ['testtimestamp', 'larry'], + timezone: 'UTC', + }); + }); + + describe('timefilter', () => { + it('formats the datetime to the provided timezone', () => { + args.timerange = { + timezone: 'MST', + min: '1901-01-01T00:00:00Z', + max: '1902-01-01T00:00:00Z', + }; + + expect( + getFilters( + args.indexPatternId, + args.indexPatternTimeField, + args.timerange, + args.savedSearchObjectAttr + ) + ).toEqual({ + combinedFilter: [ + { + range: { + testtimestamp: { + format: 'strict_date_time', + gte: '1900-12-31T17:00:00-07:00', + lte: '1901-12-31T17:00:00-07:00', + }, + }, + }, + ], + includes: ['testtimestamp', 'larry'], + timezone: 'MST', + }); + }); + }); +}); diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.ts new file mode 100644 index 0000000000000..bb3c6b8283f33 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/get_filters.ts @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { badRequest } from 'boom'; +import moment from 'moment-timezone'; + +import { SavedSearchObjectAttributes, TimeRangeParams } from '../../'; +import { QueryFilter, Filter, SearchSourceFilter } from './'; + +export function getFilters( + indexPatternId: string, + indexPatternTimeField: string | null, + timerange: TimeRangeParams | null, + savedSearchObjectAttr: SavedSearchObjectAttributes, + searchSourceFilter?: SearchSourceFilter, + queryFilter?: QueryFilter +) { + let includes: string[]; + let timeFilter: any | null; + let timezone: string | null; + + if (indexPatternTimeField) { + if (!timerange) { + throw badRequest( + `Time range params are required for index pattern [${indexPatternId}], using time field [${indexPatternTimeField}]` + ); + } + + timezone = timerange.timezone; + const { min: gte, max: lte } = timerange; + timeFilter = { + range: { + [indexPatternTimeField]: { + format: 'strict_date_time', + gte: moment.tz(moment(gte), timezone).format(), + lte: moment.tz(moment(lte), timezone).format(), + }, + }, + }; + + const savedSearchCols = savedSearchObjectAttr.columns || []; + includes = [indexPatternTimeField, ...savedSearchCols]; + } else { + includes = savedSearchObjectAttr.columns || []; + timeFilter = null; + timezone = null; + } + + const combinedFilter: Filter[] = [timeFilter, searchSourceFilter, queryFilter].filter(Boolean); // builds an array of defined filters + + return { timezone, combinedFilter, includes }; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts new file mode 100644 index 0000000000000..183a9c29da0c3 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.d.ts @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CancellationToken } from '../../../../common/cancellation_token'; +import { SavedSearchObjectAttributes, SearchPanel, SearchRequest, SearchSource } from '../../'; + +export interface SavedSearchGeneratorResult { + content: string; + maxSizeReached: boolean; + size: number; +} + +export interface CsvResultFromSearch { + type: string; + result: SavedSearchGeneratorResult; +} + +type EndpointCaller = (method: string, params: any) => Promise; +type FormatsMap = Map< + string, + { + id: string; + params: { + pattern: string; + }; + } +>; + +export interface GenerateCsvParams { + searchRequest: SearchRequest; + callEndpoint: EndpointCaller; + fields: string[]; + formatsMap: FormatsMap; + metaFields: string[]; // FIXME not sure what this is for + conflictedTypesFields: string[]; // FIXME not sure what this is for + cancellationToken: CancellationToken; + settings: { + separator: string; + quoteValues: boolean; + timezone: string | null; + maxSizeBytes: number; + scroll: { duration: string; size: number }; + }; +} + +/* + * These filter types are stub types to help ensure things get passed to + * non-Typescript functions in the right order. An actual structure is not + * needed because the code doesn't look into the properties; just combines them + * and passes them through to other non-TS modules. + */ +export interface Filter { + isFilter: boolean; +} +export interface TimeFilter extends Filter { + isTimeFilter: boolean; +} +export interface QueryFilter extends Filter { + isQueryFilter: boolean; +} +export interface SearchSourceFilter extends Filter { + isSearchSourceFilter: boolean; +} + +export interface ESQueryConfig { + allowLeadingWildcards: boolean; + queryStringOptions: boolean; + ignoreFilterIfFieldNotInIndex: boolean; +} + +export interface IndexPatternField { + scripted: boolean; + lang?: string; + script?: string; + name: string; +} diff --git a/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.ts b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.ts new file mode 100644 index 0000000000000..048cf5b457794 --- /dev/null +++ b/x-pack/plugins/reporting/export_types/csv_from_savedobject/server/lib/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export * from './index.d'; +export { createGenerateCsv } from './generate_csv'; diff --git a/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js b/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js index f128ae7c97f34..95100836d134d 100644 --- a/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js +++ b/x-pack/plugins/reporting/export_types/printable_pdf/server/execute_job/index.js @@ -58,7 +58,8 @@ function executeJobFn(server) { })) ); - const stop$ = Rx.fromEventPattern(cancellationToken.on); + const boundCancelOn = cancellationToken.on.bind(cancellationToken); + const stop$ = Rx.fromEventPattern(boundCancelOn); return process$.pipe(takeUntil(stop$)).toPromise(); }); diff --git a/x-pack/plugins/reporting/index.js b/x-pack/plugins/reporting/index.js index d7837528034e0..a73d674017744 100644 --- a/x-pack/plugins/reporting/index.js +++ b/x-pack/plugins/reporting/index.js @@ -36,12 +36,17 @@ export const reporting = (kibana) => { 'plugins/reporting/share_context_menu/register_csv_reporting', 'plugins/reporting/share_context_menu/register_reporting', ], + contextMenuActions: [ + 'plugins/reporting/panel_actions/get_csv_panel_action', + ], hacks: ['plugins/reporting/hacks/job_completion_notifier'], home: ['plugins/reporting/register_feature'], managementSections: ['plugins/reporting/views/management'], injectDefaultVars(server, options) { + const config = server.config(); return { - reportingPollConfig: options.poll + reportingPollConfig: options.poll, + enablePanelActionDownload: config.get('xpack.reporting.csv.enablePanelActionDownload'), }; }, uiSettingDefaults: { @@ -117,6 +122,7 @@ export const reporting = (kibana) => { }).default() }).default(), csv: Joi.object({ + enablePanelActionDownload: Joi.boolean().default(false), maxSizeBytes: Joi.number().integer().default(1024 * 1024 * 10), // bytes in a kB * kB in a mB * 10 scroll: Joi.object({ duration: Joi.string().regex(/^[0-9]+(d|h|m|s|ms|micros|nanos)$/, { name: 'DurationString' }).default('30s'), diff --git a/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx new file mode 100644 index 0000000000000..939aef9586cc1 --- /dev/null +++ b/x-pack/plugins/reporting/public/panel_actions/get_csv_panel_action.tsx @@ -0,0 +1,154 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import dateMath from '@elastic/datemath'; +import { i18n } from '@kbn/i18n'; +import moment from 'moment-timezone'; + +import { ContextMenuAction, ContextMenuActionsRegistryProvider } from 'ui/embeddable'; +import { PanelActionAPI } from 'ui/embeddable/context_menu_actions/types'; +import { kfetch } from 'ui/kfetch'; +import { toastNotifications } from 'ui/notify'; +import chrome from 'ui/chrome'; +import { API_BASE_URL_V1 } from '../../common/constants'; + +const API_BASE_URL = `${API_BASE_URL_V1}/generate/immediate/csv/saved-object`; + +class GetCsvReportPanelAction extends ContextMenuAction { + private isDownloading: boolean; + + constructor() { + super( + { + id: 'downloadCsvReport', + parentPanelId: 'mainMenu', + }, + { + icon: 'document', + getDisplayName: () => + i18n.translate('xpack.reporting.dashboard.downloadCsvPanelTitle', { + defaultMessage: 'Download CSV', + }), + } + ); + + this.isDownloading = false; + } + + public async getSearchRequestBody({ searchEmbeddable }: { searchEmbeddable: any }) { + const adapters = searchEmbeddable.getInspectorAdapters(); + if (!adapters) { + return {}; + } + + if (adapters.requests.requests.length === 0) { + return {}; + } + + return searchEmbeddable.searchScope.searchSource.getSearchRequestBody(); + } + + public isVisible = (panelActionAPI: PanelActionAPI): boolean => { + const enablePanelActionDownload = chrome.getInjected('enablePanelActionDownload'); + + if (!enablePanelActionDownload) { + return false; + } + + const { embeddable, containerState } = panelActionAPI; + + return ( + containerState.viewMode !== 'edit' && !!embeddable && embeddable.hasOwnProperty('savedSearch') + ); + }; + + public onClick = async (panelActionAPI: PanelActionAPI) => { + const { embeddable } = panelActionAPI as any; + const { + timeRange: { from, to }, + } = embeddable; + + if (!embeddable || this.isDownloading) { + return; + } + + const searchEmbeddable = embeddable; + const searchRequestBody = await this.getSearchRequestBody({ searchEmbeddable }); + const state = _.pick(searchRequestBody, ['sort', 'docvalue_fields', 'query']); + const kibanaTimezone = chrome.getUiSettingsClient().get('dateFormat:tz'); + + const id = `search:${embeddable.savedSearch.id}`; + const filename = embeddable.getPanelTitle(); + const timezone = kibanaTimezone === 'Browser' ? moment.tz.guess() : kibanaTimezone; + const fromTime = dateMath.parse(from); + const toTime = dateMath.parse(to); + + if (!fromTime || !toTime) { + return this.onGenerationFail( + new Error(`Invalid time range: From: ${fromTime}, To: ${toTime}`) + ); + } + + const body = JSON.stringify({ + timerange: { + min: fromTime.format(), + max: toTime.format(), + timezone, + }, + state, + }); + + this.isDownloading = true; + + toastNotifications.addSuccess({ + title: i18n.translate('xpack.reporting.dashboard.csvDownloadStartedTitle', { + defaultMessage: `CSV Download Started`, + }), + text: i18n.translate('xpack.reporting.dashboard.csvDownloadStartedMessage', { + defaultMessage: `Your CSV will download momentarily.`, + }), + 'data-test-subj': 'csvDownloadStarted', + }); + + await kfetch({ method: 'POST', pathname: `${API_BASE_URL}/${id}`, body }) + .then((rawResponse: string) => { + this.isDownloading = false; + + const download = `${filename}.csv`; + const blob = new Blob([rawResponse], { type: 'text/csv;charset=utf-8;' }); + + // Hack for IE11 Support + if (window.navigator.msSaveOrOpenBlob) { + return window.navigator.msSaveOrOpenBlob(blob, download); + } + + const a = window.document.createElement('a'); + const downloadObject = window.URL.createObjectURL(blob); + + a.href = downloadObject; + a.download = download; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(downloadObject); + document.body.removeChild(a); + }) + .catch(this.onGenerationFail.bind(this)); + }; + + private onGenerationFail(error: Error) { + this.isDownloading = false; + toastNotifications.addDanger({ + title: i18n.translate('xpack.reporting.dashboard.failedCsvDownloadTitle', { + defaultMessage: `CSV download failed`, + }), + text: i18n.translate('xpack.reporting.dashboard.failedCsvDownloadMessage', { + defaultMessage: `We couldn't generate your CSV at this time.`, + }), + 'data-test-subj': 'downloadCsvFail', + }); + } +} + +ContextMenuActionsRegistryProvider.register(() => new GetCsvReportPanelAction()); diff --git a/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index 4bdd35ca381f2..f1b8936f412c5 100644 --- a/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -33,6 +33,7 @@ export class HeadlessChromiumDriver { constructor(page: Chrome.Page, { logger }: ChromiumDriverOptions) { this.page = page; + // @ts-ignore https://github.com/elastic/kibana/issues/32140 this.logger = logger.clone(['headless-chromium-driver']); } diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/cancellation_token.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/cancellation_token.js index b40765810fb63..9a7c263a76f08 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/cancellation_token.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/helpers/cancellation_token.js @@ -6,7 +6,7 @@ import expect from '@kbn/expect'; import sinon from 'sinon'; -import { CancellationToken } from '../../helpers/cancellation_token'; +import { CancellationToken } from '../../../../../common/cancellation_token'; describe('CancellationToken', function () { let cancellationToken; @@ -17,12 +17,14 @@ describe('CancellationToken', function () { describe('on', function () { [true, null, undefined, 1, 'string', {}, []].forEach(function (value) { it(`should throw an Error if value is ${value}`, function () { - expect(cancellationToken.on).withArgs(value).to.throwError(); + const boundOn = cancellationToken.on.bind(cancellationToken); + expect(boundOn).withArgs(value).to.throwError(); }); }); it('accepts a function', function () { - expect(cancellationToken.on).withArgs(function () {}).to.not.throwError(); + const boundOn = cancellationToken.on.bind(cancellationToken); + expect(boundOn).withArgs(function () {}).not.to.throwError(); }); it(`calls function if cancel has previously been called`, function () { @@ -35,7 +37,8 @@ describe('CancellationToken', function () { describe('cancel', function () { it('should be a function accepting no parameters', function () { - expect(cancellationToken.cancel).withArgs().to.not.throwError(); + const boundCancel = cancellationToken.cancel.bind(cancellationToken); + expect(boundCancel).withArgs().to.not.throwError(); }); it('should call a single callback', function () { diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index 0670b72ad99cb..c71ec211804cd 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -9,7 +9,7 @@ import Puid from 'puid'; import moment from 'moment'; import { constants } from './constants'; import { WorkerTimeoutError, UnspecifiedWorkerError } from './helpers/errors'; -import { CancellationToken } from './helpers/cancellation_token'; +import { CancellationToken } from '../../../common/cancellation_token'; import { Poller } from '../../../../../common/poller'; const puid = new Puid(); diff --git a/x-pack/plugins/reporting/server/lib/export_types_registry.js b/x-pack/plugins/reporting/server/lib/export_types_registry.js index 7631161c4f899..da9fb79d1efaa 100644 --- a/x-pack/plugins/reporting/server/lib/export_types_registry.js +++ b/x-pack/plugins/reporting/server/lib/export_types_registry.js @@ -21,7 +21,7 @@ function scan(pattern) { }); } -const pattern = resolve(__dirname, '../../export_types/*/server/index.js'); +const pattern = resolve(__dirname, '../../export_types/*/server/index.[jt]s'); async function exportTypesRegistryFn(server) { const exportTypesRegistry = new ExportTypesRegistry(); const files = await scan(pattern); diff --git a/x-pack/plugins/reporting/server/lib/index.ts b/x-pack/plugins/reporting/server/lib/index.ts new file mode 100644 index 0000000000000..026c444ca35d7 --- /dev/null +++ b/x-pack/plugins/reporting/server/lib/index.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// @ts-ignore untyped module +export { createTaggedLogger } from './create_tagged_logger'; +// @ts-ignore untyped module +export { cryptoFactory } from './crypto'; +// @ts-ignore untyped module +export { oncePerServer } from './once_per_server'; + +export { LevelLogger } from './level_logger'; diff --git a/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts b/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts new file mode 100644 index 0000000000000..f2a254f6077f6 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/generate_from_savedobject.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request, ResponseToolkit } from 'hapi'; +import { get } from 'lodash'; + +import { API_BASE_GENERATE_V1, CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../common/constants'; +import { KbnServer } from '../../types'; +import { HandlerErrorFunction, HandlerFunction, QueuedJobPayload } from './types'; +import { getRouteOptions } from './lib/route_config_factories'; +import { getJobParamsFromRequest } from './lib/get_job_params_from_request'; + +/* + * 1. Build `jobParams` object: job data that execution will need to reference in various parts of the lifecycle + * 2. Pass the jobParams and other common params to `handleRoute`, a shared function to enqueue the job with the params + * 3. Ensure that details for a queued job were returned + */ +const getJobFromRouteHandler = async ( + handleRoute: HandlerFunction, + handleRouteError: HandlerErrorFunction, + request: Request, + h: ResponseToolkit +): Promise => { + let result: QueuedJobPayload; + try { + const jobParams = getJobParamsFromRequest(request, { isImmediate: false }); + result = await handleRoute(CSV_FROM_SAVEDOBJECT_JOB_TYPE, jobParams, request, h); + } catch (err) { + throw handleRouteError(CSV_FROM_SAVEDOBJECT_JOB_TYPE, err); + } + + if (get(result, 'source.job') == null) { + throw new Error(`The Export handler is expected to return a result with job info! ${result}`); + } + + return result; +}; + +/* + * This function registers API Endpoints for queuing Reporting jobs. The API inputs are: + * - saved object type and ID + * - time range and time zone + * - application state: + * - filters + * - query bar + * - local (transient) changes the user made to the saved object + */ +export function registerGenerateCsvFromSavedObject( + server: KbnServer, + handleRoute: HandlerFunction, + handleRouteError: HandlerErrorFunction +) { + const routeOptions = getRouteOptions(server); + + server.route({ + path: `${API_BASE_GENERATE_V1}/csv/saved-object/{savedObjectType}:{savedObjectId}`, + method: 'POST', + options: routeOptions, + handler: async (request: Request, h: ResponseToolkit) => { + return getJobFromRouteHandler(handleRoute, handleRouteError, request, h); + }, + }); +} diff --git a/x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts b/x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts new file mode 100644 index 0000000000000..10f634aedcb4c --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/generate_from_savedobject_immediate.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request, ResponseObject, ResponseToolkit } from 'hapi'; + +import { API_BASE_GENERATE_V1 } from '../../common/constants'; +import { createJobFactory, executeJobFactory } from '../../export_types/csv_from_savedobject'; +import { JobDocPayload, JobDocOutputExecuted, KbnServer } from '../../types'; +import { LevelLogger } from '../lib/level_logger'; +import { getRouteOptions } from './lib/route_config_factories'; +import { getJobParamsFromRequest } from './lib/get_job_params_from_request'; + +interface KibanaResponse extends ResponseObject { + isBoom: boolean; +} + +/* + * This function registers API Endpoints for immediate Reporting jobs. The API inputs are: + * - saved object type and ID + * - time range and time zone + * - application state: + * - filters + * - query bar + * - local (transient) changes the user made to the saved object + */ +export function registerGenerateCsvFromSavedObjectImmediate(server: KbnServer) { + const routeOptions = getRouteOptions(server); + + /* + * CSV export with the `immediate` option does not queue a job with Reporting's ESQueue to run the job async. Instead, this does: + * - re-use the createJob function to build up es query config + * - re-use the executeJob function to run the scan and scroll queries and capture the entire CSV in a result object. + */ + server.route({ + path: `${API_BASE_GENERATE_V1}/immediate/csv/saved-object/{savedObjectType}:{savedObjectId}`, + method: 'POST', + options: routeOptions, + handler: async (request: Request, h: ResponseToolkit) => { + const logger = LevelLogger.createForServer(server, ['reporting', 'savedobject-csv']); + const jobParams = getJobParamsFromRequest(request, { isImmediate: true }); + const createJobFn = createJobFactory(server); + const executeJobFn = executeJobFactory(server, request); + const jobDocPayload: JobDocPayload = await createJobFn(jobParams, request.headers, request); + const { + content_type: jobOutputContentType, + content: jobOutputContent, + size: jobOutputSize, + }: JobDocOutputExecuted = await executeJobFn(jobDocPayload, request); + + logger.info(`job output size: ${jobOutputSize} bytes`); + + /* + * ESQueue worker function defaults `content` to null, even if the + * executeJob returned undefined. + * + * This converts null to undefined so the value can be sent to h.response() + */ + if (jobOutputContent === null) { + logger.warn('CSV Job Execution created empty content result'); + } + const response = h + .response(jobOutputContent ? jobOutputContent : undefined) + .type(jobOutputContentType); + + // Set header for buffer download, not streaming + const { isBoom } = response as KibanaResponse; + if (isBoom == null) { + response.header('accept-ranges', 'none'); + } + + return response; + }, + }); +} diff --git a/x-pack/plugins/reporting/server/routes/index.ts b/x-pack/plugins/reporting/server/routes/index.ts index f51634056b1b3..9fbdfae7206f2 100644 --- a/x-pack/plugins/reporting/server/routes/index.ts +++ b/x-pack/plugins/reporting/server/routes/index.ts @@ -11,6 +11,8 @@ import { KbnServer } from '../../types'; // @ts-ignore import { enqueueJobFactory } from '../lib/enqueue_job'; import { registerGenerate } from './generate'; +import { registerGenerateCsvFromSavedObject } from './generate_from_savedobject'; +import { registerGenerateCsvFromSavedObjectImmediate } from './generate_from_savedobject_immediate'; import { registerJobs } from './jobs'; import { registerLegacy } from './legacy'; @@ -20,7 +22,15 @@ export function registerRoutes(server: KbnServer) { const { errors: esErrors } = server.plugins.elasticsearch.getCluster('admin'); const enqueueJob = enqueueJobFactory(server); - async function handler(exportTypeId: any, jobParams: any, request: Request, h: ResponseToolkit) { + /* + * Generates enqueued job details to use in responses + */ + async function handler( + exportTypeId: string, + jobParams: any, + request: Request, + h: ResponseToolkit + ) { // @ts-ignore const user = request.pre.user; const headers = request.headers; @@ -38,12 +48,12 @@ export function registerRoutes(server: KbnServer) { .type('application/json'); } - function handleError(exportType: any, err: Error) { + function handleError(exportTypeId: string, err: Error) { if (err instanceof esErrors['401']) { return boom.unauthorized(`Sorry, you aren't authenticated`); } if (err instanceof esErrors['403']) { - return boom.forbidden(`Sorry, you are not authorized to create ${exportType} reports`); + return boom.forbidden(`Sorry, you are not authorized to create ${exportTypeId} reports`); } if (err instanceof esErrors['404']) { return boom.boomify(err, { statusCode: 404 }); @@ -53,5 +63,12 @@ export function registerRoutes(server: KbnServer) { registerGenerate(server, handler, handleError); registerLegacy(server, handler, handleError); + + // Register beta panel-action download-related API's + if (config.get('xpack.reporting.csv.enablePanelActionDownload')) { + registerGenerateCsvFromSavedObject(server, handler, handleError); + registerGenerateCsvFromSavedObjectImmediate(server); + } + registerJobs(server); } diff --git a/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts b/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts new file mode 100644 index 0000000000000..097b41c4fb697 --- /dev/null +++ b/x-pack/plugins/reporting/server/routes/lib/get_job_params_from_request.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { Request } from 'hapi'; +import { JobParamPostPayload, JobParams } from '../../../types'; + +export function getJobParamsFromRequest( + request: Request, + opts: { isImmediate: boolean } +): JobParams { + const { savedObjectType, savedObjectId } = request.params; + const { timerange, state } = request.payload as JobParamPostPayload; + const post = timerange || state ? { timerange, state } : undefined; + + return { + isImmediate: opts.isImmediate, + savedObjectType, + savedObjectId, + post, + }; +} diff --git a/x-pack/plugins/reporting/server/routes/lib/route_config_factories.ts b/x-pack/plugins/reporting/server/routes/lib/route_config_factories.ts index 2396694d5c56f..e09db6e536fee 100644 --- a/x-pack/plugins/reporting/server/routes/lib/route_config_factories.ts +++ b/x-pack/plugins/reporting/server/routes/lib/route_config_factories.ts @@ -5,11 +5,13 @@ */ import { Request } from 'hapi'; +import Joi from 'joi'; import { KbnServer } from '../../../types'; // @ts-ignore import { authorizedUserPreRoutingFactory } from './authorized_user_pre_routing'; // @ts-ignore import { reportingFeaturePreRoutingFactory } from './reporting_feature_pre_routing'; +import { CSV_FROM_SAVEDOBJECT_JOB_TYPE } from '../../../common/constants'; const API_TAG = 'api'; @@ -41,6 +43,27 @@ export function getRouteConfigFactoryReportingPre(server: KbnServer) { }; } +export function getRouteOptions(server: KbnServer) { + const getRouteConfig = getRouteConfigFactoryReportingPre(server); + return { + ...getRouteConfig(() => CSV_FROM_SAVEDOBJECT_JOB_TYPE), + validate: { + params: Joi.object({ + savedObjectType: Joi.string().required(), + savedObjectId: Joi.string().required(), + }).required(), + payload: Joi.object({ + state: Joi.object().default({}), + timerange: Joi.object({ + timezone: Joi.string().default('UTC'), + min: Joi.date().required(), + max: Joi.date().required(), + }).optional(), + }), + }, + }; +} + export function getRouteConfigFactoryManagementPre(server: KbnServer) { const authorizedUserPreRouting = authorizedUserPreRoutingFactory(server); const reportingFeaturePreRouting = reportingFeaturePreRoutingFactory(server); diff --git a/x-pack/plugins/reporting/server/routes/types.d.ts b/x-pack/plugins/reporting/server/routes/types.d.ts index 90040870778bd..6004811d8a2b4 100644 --- a/x-pack/plugins/reporting/server/routes/types.d.ts +++ b/x-pack/plugins/reporting/server/routes/types.d.ts @@ -5,6 +5,7 @@ */ import { Request, ResponseToolkit } from 'hapi'; +import { JobDocPayload } from '../../types'; export type HandlerFunction = ( exportType: any, @@ -14,3 +15,12 @@ export type HandlerFunction = ( ) => any; export type HandlerErrorFunction = (exportType: any, err: Error) => any; + +export interface QueuedJobPayload { + error?: boolean; + source: { + job: { + payload: JobDocPayload; + }; + }; +} diff --git a/x-pack/plugins/reporting/types.d.ts b/x-pack/plugins/reporting/types.d.ts index c0284dc3e8007..235bf04e58437 100644 --- a/x-pack/plugins/reporting/types.d.ts +++ b/x-pack/plugins/reporting/types.d.ts @@ -18,6 +18,7 @@ export interface KbnServer { plugins: Record; route: any; log: any; + fieldFormatServiceFactory: (uiConfig: any) => any; savedObjects: { getScopedSavedObjectsClient: ( fakeRequest: { headers: object; getBasePath: () => string } @@ -28,6 +29,20 @@ export interface KbnServer { ) => UiSettings; } +export interface ExportTypeDefinition { + id: string; + name: string; + jobType: string; + jobContentExtension: string; + createJobFactory: () => any; + executeJobFactory: () => any; + validLicenses: string[]; +} + +export interface ExportTypesRegistry { + register: (exportTypeDefinition: ExportTypeDefinition) => void; +} + export interface ConfigObject { get: (path?: string) => any; } @@ -41,7 +56,7 @@ export interface Logger { debug: (message: string) => void; error: (message: string) => void; warning: (message: string) => void; - clone: (tags: string[]) => Logger; + clone?: (tags: string[]) => Logger; } export interface ViewZoomWidthHeight { @@ -92,20 +107,58 @@ export interface CryptoFactory { decrypt: (headers?: Record) => string; } -export interface ReportingJob { - headers?: Record; +export interface TimeRangeParams { + timezone: string; + min: Date | string | number; + max: Date | string | number; +} + +type PostPayloadState = Partial<{ + state: { + query: any; + sort: any[]; + columns: string[]; // TODO + }; +}>; + +// retain POST payload data, needed for async +interface JobParamPostPayload extends PostPayloadState { + timerange: TimeRangeParams; +} + +// params that come into a request +export interface JobParams { + savedObjectType: string; + savedObjectId: string; + isImmediate: boolean; + post?: JobParamPostPayload; + panel?: any; // has to be resolved by the request handler + visType?: string; // has to be resolved by the request handler +} + +export interface JobDocPayload { basePath?: string; - urls?: string[]; - relativeUrl?: string; forceNow?: string; + headers?: Record; + jobParams: JobParams; + relativeUrl?: string; timeRange?: any; - objects?: [any]; + title: string; + urls?: string[]; + type?: string | null; // string if completed job; null if incomplete job; + objects?: string | null; // string if completed job; null if incomplete job; +} + +export interface JobDocOutput { + content: string; // encoded content + contentType: string; } export interface JobDoc { - output: any; jobtype: string; - payload: ReportingJob; + output: JobDocOutput; + payload: JobDocPayload; + status: string; // completed, failed, etc } export interface JobSource { @@ -113,6 +166,25 @@ export interface JobSource { _source: JobDoc; } +/* + * A snake_cased field is the only significant difference in structure of + * JobDocOutputExecuted vs JobDocOutput. + * + * JobDocOutput is the structure of the object returned by getDocumentPayload + * + * data in the _source fields of the + * Reporting index. + * + * The ESQueueWorker internals have executed job objects returned with this + * structure. See `_formatOutput` in reporting/server/lib/esqueue/worker.js + */ +export interface JobDocOutputExecuted { + content_type: string; // vs `contentType` above + content: string | null; // defaultOutput is null + max_size_reached: boolean; + size: number; +} + export interface ESQueueWorker { on: (event: string, handler: any) => void; } diff --git a/x-pack/scripts/functional_tests.js b/x-pack/scripts/functional_tests.js index fe46b447c9090..385678852e9a1 100644 --- a/x-pack/scripts/functional_tests.js +++ b/x-pack/scripts/functional_tests.js @@ -8,6 +8,7 @@ require('@kbn/plugin-helpers').babelRegister(); require('@kbn/test').runTestsCli([ require.resolve('../test/reporting/configs/chromium_api.js'), require.resolve('../test/reporting/configs/chromium_functional.js'), + require.resolve('../test/reporting/configs/generate_api'), require.resolve('../test/functional/config.js'), require.resolve('../test/api_integration/config_security_basic.js'), require.resolve('../test/api_integration/config.js'), diff --git a/x-pack/test/api_integration/config.js b/x-pack/test/api_integration/config.js index eafad98a0e655..bce6cbc61260f 100644 --- a/x-pack/test/api_integration/config.js +++ b/x-pack/test/api_integration/config.js @@ -21,16 +21,11 @@ import { SpacesServiceProvider, } from '../common/services'; -export default async function ({ readConfigFile }) { - const kibanaAPITestsConfig = await readConfigFile( - require.resolve('../../../test/api_integration/config.js') - ); - const xPackFunctionalTestsConfig = await readConfigFile( - require.resolve('../functional/config.js') - ); - const kibanaCommonConfig = await readConfigFile( - require.resolve('../../../test/common/config.js') - ); +export async function getApiIntegrationConfig({ readConfigFile }) { + + const kibanaAPITestsConfig = await readConfigFile(require.resolve('../../../test/api_integration/config.js')); + const xPackFunctionalTestsConfig = await readConfigFile(require.resolve('../functional/config.js')); + const kibanaCommonConfig = await readConfigFile(require.resolve('../../../test/common/config.js')); return { testFiles: [require.resolve('./apis')], @@ -73,3 +68,5 @@ export default async function ({ readConfigFile }) { }, }; } + +export default getApiIntegrationConfig; diff --git a/x-pack/test/functional/config.js b/x-pack/test/functional/config.js index 6b8dc8a2fc106..43fade16af105 100644 --- a/x-pack/test/functional/config.js +++ b/x-pack/test/functional/config.js @@ -192,6 +192,8 @@ export default async function ({ readConfigFile }) { '--server.uuid=5b2de169-2785-441b-ae8c-186a1936b17d', '--xpack.xpack_main.telemetry.enabled=false', '--xpack.maps.showMapsInspectorAdapter=true', + '--xpack.reporting.queue.pollInterval=3000', // make it explicitly the default + '--xpack.reporting.csv.maxSizeBytes=2850', // small-ish limit for cutting off a 1999 byte report '--stats.maximumWaitTimeForAllCollectorsInS=0', '--xpack.security.encryptionKey="wuGNaIhoMpk5sO4UBxgr3NyW1sFcLgIf"', // server restarts should not invalidate active sessions '--xpack.code.security.enableGitCertCheck=false', // Disable git certificate check diff --git a/x-pack/test/functional/es_archives/reporting/logs/data.json.gz b/x-pack/test/functional/es_archives/reporting/logs/data.json.gz new file mode 100644 index 0000000000000..ff16bc0c594bd Binary files /dev/null and b/x-pack/test/functional/es_archives/reporting/logs/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/logs/mappings.json b/x-pack/test/functional/es_archives/reporting/logs/mappings.json new file mode 100644 index 0000000000000..adf4050bb88c4 --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/logs/mappings.json @@ -0,0 +1,302 @@ +{ + "type": "index", + "value": { + "index": ".kibana", + "mappings": { + "properties": { + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "dashboard": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "graph-workspace": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "dynamic": "strict", + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "search": { + "dynamic": "strict", + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "dynamic": "strict", + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "spaceId": { + "type": "keyword" + }, + "timelion-sheet": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "url": { + "dynamic": "strict", + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} \ No newline at end of file diff --git a/x-pack/test/functional/es_archives/reporting/sales/data.json.gz b/x-pack/test/functional/es_archives/reporting/sales/data.json.gz new file mode 100644 index 0000000000000..3552044c378d0 Binary files /dev/null and b/x-pack/test/functional/es_archives/reporting/sales/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/sales/mappings.json b/x-pack/test/functional/es_archives/reporting/sales/mappings.json new file mode 100644 index 0000000000000..3249708537b73 --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/sales/mappings.json @@ -0,0 +1,334 @@ +{ + "type": "index", + "value": { + "index": ".kibana", + "mappings": { + "properties": { + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + } + } + }, + "dashboard": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "graph-workspace": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "dynamic": "strict", + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + } + } + }, + "search": { + "dynamic": "strict", + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "dynamic": "strict", + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "spaceId": { + "type": "keyword" + }, + "timelion-sheet": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "url": { + "dynamic": "strict", + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "visualization": { + "dynamic": "strict", + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchId": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "index": "sales", + "mappings": { + "properties": { + "@date": { + "type": "date" + }, + "metric": { + "type": "integer" + }, + "name": { + "type": "keyword" + }, + "power": { + "type": "long" + }, + "success": { + "type": "boolean" + } + } + }, + "settings": { + "index": { + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/reporting/scripted/data.json.gz b/x-pack/test/functional/es_archives/reporting/scripted/data.json.gz new file mode 100644 index 0000000000000..093a6461c6e8f Binary files /dev/null and b/x-pack/test/functional/es_archives/reporting/scripted/data.json.gz differ diff --git a/x-pack/test/functional/es_archives/reporting/scripted/mappings.json b/x-pack/test/functional/es_archives/reporting/scripted/mappings.json new file mode 100644 index 0000000000000..d36bbc72f4ffa --- /dev/null +++ b/x-pack/test/functional/es_archives/reporting/scripted/mappings.json @@ -0,0 +1,742 @@ +{ + "type": "index", + "value": { + "aliases": { + ".kibana": { + } + }, + "index": ".kibana_1", + "mappings": { + "_meta": { + "migrationMappingPropertyHashes": { + "apm-telemetry": "0383a570af33654a51c8a1352417bc6b", + "canvas-workpad": "b0a1706d356228dbdcb4a17e6b9eb231", + "config": "87aca8fdb053154f11383fce3dbf3edf", + "dashboard": "eb3789e1af878e73f85304333240f65f", + "graph-workspace": "cd7ba1330e6682e9cc00b78850874be1", + "index-pattern": "66eccb05066c5a89924f48a9e9736499", + "infrastructure-ui-source": "10acdf67d9a06d462e198282fd6d4b81", + "kql-telemetry": "d12a98a6f19a2d273696597547e064ee", + "map": "23d7aa4a720d4938ccde3983f87bd58d", + "maps-telemetry": "a4229f8b16a6820c6d724b7e0c1f729d", + "migrationVersion": "4a1746014a75ade3a714e1db5763276f", + "ml-telemetry": "257fd1d4b4fdbb9cb4b8a3b27da201e9", + "namespace": "2f4316de49999235636386fe51dc06c1", + "references": "7997cf5a56cc02bdc9c93361bde732b0", + "sample-data-telemetry": "7d3cfeb915303c9641c59681967ffeb4", + "search": "181661168bbadd1eff5902361e2a0d5c", + "server": "ec97f1c5da1a19609a60874e5af1100c", + "space": "0d5011d73a0ef2f0f615bb42f26f187e", + "telemetry": "e1c8bc94e443aefd9458932cc0697a4d", + "timelion-sheet": "9a2a2748877c7a7b582fef201ab1d4cf", + "type": "2f4316de49999235636386fe51dc06c1", + "updated_at": "00da57df13e94e9d98437d13ace4bfe0", + "upgrade-assistant-reindex-operation": "a53a20fe086b72c9a86da3cc12dad8a6", + "upgrade-assistant-telemetry": "56702cec857e0a9dacfb696655b4ff7b", + "url": "c7f66a0df8b1b52f17c28c4adb111105", + "user-action": "0d409297dc5ebe1e3a1da691c6ee32e3", + "visualization": "52d7a13ad68a150c4525b292d23e12cc" + } + }, + "dynamic": "strict", + "properties": { + "apm-telemetry": { + "properties": { + "has_any_services": { + "type": "boolean" + }, + "services_per_agent": { + "properties": { + "go": { + "null_value": 0, + "type": "long" + }, + "java": { + "null_value": 0, + "type": "long" + }, + "js-base": { + "null_value": 0, + "type": "long" + }, + "nodejs": { + "null_value": 0, + "type": "long" + }, + "python": { + "null_value": 0, + "type": "long" + }, + "ruby": { + "null_value": 0, + "type": "long" + }, + "rum-js": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "canvas-workpad": { + "dynamic": "false", + "properties": { + "@created": { + "type": "date" + }, + "@timestamp": { + "type": "date" + }, + "name": { + "fields": { + "keyword": { + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "config": { + "dynamic": "true", + "properties": { + "buildNum": { + "type": "keyword" + }, + "dateFormat:tz": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "defaultIndex": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "search:queryLanguage": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "dashboard": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "optionsJSON": { + "type": "text" + }, + "panelsJSON": { + "type": "text" + }, + "refreshInterval": { + "properties": { + "display": { + "type": "keyword" + }, + "pause": { + "type": "boolean" + }, + "section": { + "type": "integer" + }, + "value": { + "type": "integer" + } + } + }, + "timeFrom": { + "type": "keyword" + }, + "timeRestore": { + "type": "boolean" + }, + "timeTo": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "graph-workspace": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "numLinks": { + "type": "integer" + }, + "numVertices": { + "type": "integer" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "wsState": { + "type": "text" + } + } + }, + "index-pattern": { + "properties": { + "fieldFormatMap": { + "type": "text" + }, + "fields": { + "type": "text" + }, + "intervalName": { + "type": "keyword" + }, + "notExpandable": { + "type": "boolean" + }, + "sourceFilters": { + "type": "text" + }, + "timeFieldName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "type": { + "type": "keyword" + }, + "typeMeta": { + "type": "keyword" + } + } + }, + "infrastructure-ui-source": { + "properties": { + "description": { + "type": "text" + }, + "fields": { + "properties": { + "container": { + "type": "keyword" + }, + "host": { + "type": "keyword" + }, + "pod": { + "type": "keyword" + }, + "tiebreaker": { + "type": "keyword" + }, + "timestamp": { + "type": "keyword" + } + } + }, + "logAlias": { + "type": "keyword" + }, + "metricAlias": { + "type": "keyword" + }, + "name": { + "type": "text" + } + } + }, + "kql-telemetry": { + "properties": { + "optInCount": { + "type": "long" + }, + "optOutCount": { + "type": "long" + } + } + }, + "map": { + "properties": { + "bounds": { + "type": "geo_shape" + }, + "description": { + "type": "text" + }, + "layerListJSON": { + "type": "text" + }, + "mapStateJSON": { + "type": "text" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "maps-telemetry": { + "properties": { + "attributesPerMap": { + "properties": { + "dataSourcesCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + }, + "emsVectorLayersCount": { + "dynamic": "true", + "type": "object" + }, + "layerTypesCount": { + "dynamic": "true", + "type": "object" + }, + "layersCount": { + "properties": { + "avg": { + "type": "long" + }, + "max": { + "type": "long" + }, + "min": { + "type": "long" + } + } + } + } + }, + "mapsTotalCount": { + "type": "long" + }, + "timeCaptured": { + "type": "date" + } + } + }, + "migrationVersion": { + "dynamic": "true", + "properties": { + "dashboard": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "index-pattern": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "search": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + }, + "visualization": { + "fields": { + "keyword": { + "ignore_above": 256, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "ml-telemetry": { + "properties": { + "file_data_visualizer": { + "properties": { + "index_creation_count": { + "type": "long" + } + } + } + } + }, + "namespace": { + "type": "keyword" + }, + "references": { + "properties": { + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + }, + "type": "nested" + }, + "sample-data-telemetry": { + "properties": { + "installCount": { + "type": "long" + }, + "unInstallCount": { + "type": "long" + } + } + }, + "search": { + "properties": { + "columns": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "sort": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "server": { + "properties": { + "uuid": { + "type": "keyword" + } + } + }, + "space": { + "properties": { + "_reserved": { + "type": "boolean" + }, + "disabledFeatures": { + "type": "keyword" + }, + "color": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "disabledFeatures": { + "type": "keyword" + }, + "initials": { + "type": "keyword" + }, + "name": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "telemetry": { + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "timelion-sheet": { + "properties": { + "description": { + "type": "text" + }, + "hits": { + "type": "integer" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "timelion_chart_height": { + "type": "integer" + }, + "timelion_columns": { + "type": "integer" + }, + "timelion_interval": { + "type": "keyword" + }, + "timelion_other_interval": { + "type": "keyword" + }, + "timelion_rows": { + "type": "integer" + }, + "timelion_sheet": { + "type": "text" + }, + "title": { + "type": "text" + }, + "version": { + "type": "integer" + } + } + }, + "type": { + "type": "keyword" + }, + "updated_at": { + "type": "date" + }, + "upgrade-assistant-reindex-operation": { + "dynamic": "true", + "properties": { + "indexName": { + "type": "keyword" + }, + "status": { + "type": "integer" + } + } + }, + "upgrade-assistant-telemetry": { + "properties": { + "features": { + "properties": { + "deprecation_logging": { + "properties": { + "enabled": { + "null_value": true, + "type": "boolean" + } + } + } + } + }, + "ui_open": { + "properties": { + "cluster": { + "null_value": 0, + "type": "long" + }, + "indices": { + "null_value": 0, + "type": "long" + }, + "overview": { + "null_value": 0, + "type": "long" + } + } + }, + "ui_reindex": { + "properties": { + "close": { + "null_value": 0, + "type": "long" + }, + "open": { + "null_value": 0, + "type": "long" + }, + "start": { + "null_value": 0, + "type": "long" + }, + "stop": { + "null_value": 0, + "type": "long" + } + } + } + } + }, + "url": { + "properties": { + "accessCount": { + "type": "long" + }, + "accessDate": { + "type": "date" + }, + "createDate": { + "type": "date" + }, + "url": { + "fields": { + "keyword": { + "ignore_above": 2048, + "type": "keyword" + } + }, + "type": "text" + } + } + }, + "user-action": { + "properties": { + "count": { + "type": "integer" + } + } + }, + "visualization": { + "properties": { + "description": { + "type": "text" + }, + "kibanaSavedObjectMeta": { + "properties": { + "searchSourceJSON": { + "type": "text" + } + } + }, + "savedSearchRefName": { + "type": "keyword" + }, + "title": { + "type": "text" + }, + "uiStateJSON": { + "type": "text" + }, + "version": { + "type": "integer" + }, + "visState": { + "type": "text" + } + } + } + } + }, + "settings": { + "index": { + "auto_expand_replicas": "0-1", + "number_of_replicas": "0", + "number_of_shards": "1" + } + } + } +} + +{ + "type": "index", + "value": { + "aliases": { + }, + "index": "babynames", + "mappings": { + "properties": { + "date": { + "type": "date" + }, + "gender": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "percent": { + "type": "float" + }, + "value": { + "type": "integer" + }, + "year": { + "type": "integer" + } + } + }, + "settings": { + "index": { + "mapping": { + "coerce": "false" + }, + "number_of_replicas": "0", + "number_of_shards": "2" + } + } + } +} diff --git a/x-pack/test/reporting/api/generate/csv_saved_search.ts b/x-pack/test/reporting/api/generate/csv_saved_search.ts new file mode 100644 index 0000000000000..9e234fbe26ec1 --- /dev/null +++ b/x-pack/test/reporting/api/generate/csv_saved_search.ts @@ -0,0 +1,346 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; +import supertest from 'supertest'; +import { + CSV_RESULT_HUGE, + CSV_RESULT_SCRIPTED, + CSV_RESULT_SCRIPTED_REQUERY, + CSV_RESULT_SCRIPTED_RESORTED, + CSV_RESULT_TIMEBASED, + CSV_RESULT_TIMELESS, +} from './fixtures'; + +interface GenerateOpts { + timerange?: { + timezone: string; + min: number | string | Date; + max: number | string | Date; + }; + state: any; +} + +// eslint-disable-next-line import/no-default-export +export default function({ getService }: { getService: any }) { + const esArchiver = getService('esArchiver'); + const supertestSvc = getService('supertest'); + const generateAPI = { + getCsvFromSavedSearch: async ( + id: string, + { timerange, state }: GenerateOpts, + isImmediate = true + ) => { + return await supertestSvc + .post(`/api/reporting/v1/generate/${isImmediate ? 'immediate/' : ''}csv/saved-object/${id}`) + .set('kbn-xsrf', 'xxx') + .send({ timerange, state }); + }, + }; + + describe('Generation from Saved Search ID', () => { + describe('Saved Search Features', () => { + it('With filters and timebased data', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/logs'); + await esArchiver.load('logstash_functional'); + + // TODO: check headers for inline filename + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7', + { + timerange: { + timezone: 'UTC', + min: '2015-09-19T10:00:00.000Z', + max: '2015-09-21T10:00:00.000Z', + }, + state: {}, + } + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_TIMEBASED); + + await esArchiver.unload('reporting/logs'); + await esArchiver.unload('logstash_functional'); + }); + + it('With filters and non-timebased data', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/sales'); + + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:71e3ee20-3f99-11e9-b8ee-6b9604f2f877', + { + state: {}, + } + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_TIMELESS); + + await esArchiver.unload('reporting/sales'); + }); + + it('With scripted fields and field formatters', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + { + timerange: { + timezone: 'UTC', + min: '1979-01-01T10:00:00Z', + max: '1981-01-01T10:00:00Z', + }, + state: {}, + } + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_SCRIPTED); + + await esArchiver.unload('reporting/scripted'); + }); + }); + + describe('API Features', () => { + it('Return a 404', async () => { + const { body } = (await generateAPI.getCsvFromSavedSearch('search:gobbledygook', { + timerange: { timezone: 'UTC', min: 63097200000, max: 126255599999 }, + state: {}, + })) as supertest.Response; + const expectedBody = { + error: 'Not Found', + message: 'Saved object [search/gobbledygook] not found', + statusCode: 404, + }; + expect(body).to.eql(expectedBody); + }); + + it('Return 400 if time range param is needed but missing', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/logs'); + await esArchiver.load('logstash_functional'); + + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:d7a79750-3edd-11e9-99cc-4d80163ee9e7', + { state: {} } + )) as supertest.Response; + + expect(resStatus).to.eql(400); + expect(resType).to.eql('application/json'); + const { message: errorMessage } = JSON.parse(resText); + expect(errorMessage).to.eql( + 'Time range params are required for index pattern [logstash-*], using time field [@timestamp]' + ); + + await esArchiver.unload('reporting/logs'); + await esArchiver.unload('logstash_functional'); + }); + + it('Stops at Max Size Reached', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + { + timerange: { + timezone: 'UTC', + min: '1960-01-01T10:00:00Z', + max: '1999-01-01T10:00:00Z', + }, + state: {}, + } + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_HUGE); + + await esArchiver.unload('reporting/scripted'); + }); + }); + + describe('Merge user state into the query', () => { + it('for query', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const params = { + searchId: 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + postPayload: { + timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore + state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } } // prettier-ignore + }, + isImmediate: true, + }; + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + params.searchId, + params.postPayload, + params.isImmediate + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_SCRIPTED_REQUERY); + + await esArchiver.unload('reporting/scripted'); + }); + + it('for sort', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + { + timerange: { + timezone: 'UTC', + min: '1979-01-01T10:00:00Z', + max: '1981-01-01T10:00:00Z', + }, + state: { sort: [{ name: { order: 'asc', unmapped_type: 'boolean' } }] }, + } + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('text/csv'); + expect(resText).to.eql(CSV_RESULT_SCRIPTED_RESORTED); + + await esArchiver.unload('reporting/scripted'); + }); + }); + + describe('Non-Immediate', () => { + it('using queries in job params', async () => { + // load test data that contains a saved search and documents + await esArchiver.load('reporting/scripted'); + + const params = { + searchId: 'search:f34bf440-5014-11e9-bce7-4dabcb8bef24', + postPayload: { + timerange: { timezone: 'UTC', min: '1979-01-01T10:00:00Z', max: '1981-01-01T10:00:00Z' }, // prettier-ignore + state: { query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } } // prettier-ignore + }, + isImmediate: false, + }; + const { + status: resStatus, + text: resText, + type: resType, + } = (await generateAPI.getCsvFromSavedSearch( + params.searchId, + params.postPayload, + params.isImmediate + )) as supertest.Response; + + expect(resStatus).to.eql(200); + expect(resType).to.eql('application/json'); + const { + path: jobDownloadPath, + job: { index: jobIndex, jobtype: jobType, created_by: jobCreatedBy, payload: jobPayload }, + } = JSON.parse(resText); + + expect(jobDownloadPath.slice(0, 29)).to.equal('/api/reporting/jobs/download/'); + expect(jobIndex.slice(0, 11)).to.equal('.reporting-'); + expect(jobType).to.be('csv_from_savedobject'); + expect(jobCreatedBy).to.be('elastic'); + + const { + title: payloadTitle, + objects: payloadObjects, + jobParams: payloadParams, + } = jobPayload; + expect(payloadTitle).to.be('EVERYBABY2'); + expect(payloadObjects).to.be(null); // value for non-immediate + expect(payloadParams.savedObjectType).to.be('search'); + expect(payloadParams.savedObjectId).to.be('f34bf440-5014-11e9-bce7-4dabcb8bef24'); + expect(payloadParams.isImmediate).to.be(false); + + const { state: postParamState, timerange: postParamTimerange } = payloadParams.post; + expect(postParamState).to.eql({ + query: { bool: { filter: [ { bool: { filter: [ { bool: { minimum_should_match: 1, should: [{ query_string: { fields: ['name'], query: 'Fe*' } }] } } ] } } ] } } // prettier-ignore + }); + expect(postParamTimerange).to.eql({ + max: '1981-01-01T10:00:00.000Z', + min: '1979-01-01T10:00:00.000Z', + timezone: 'UTC', + }); + + const { + indexPatternSavedObjectId: payloadPanelIndexPatternSavedObjectId, + timerange: payloadPanelTimerange, + } = payloadParams.panel; + expect(payloadPanelIndexPatternSavedObjectId).to.be('89655130-5013-11e9-bce7-4dabcb8bef24'); + expect(payloadPanelTimerange).to.eql({ + timezone: 'UTC', + min: '1979-01-01T10:00:00.000Z', + max: '1981-01-01T10:00:00.000Z', + }); + + expect(payloadParams.visType).to.be('search'); + + // check the resource at jobDownloadPath + const downloadFromPath = async (downloadPath: string) => { + const { status, text, type } = await supertestSvc + .get(downloadPath) + .set('kbn-xsrf', 'xxx'); + return { + status, + text, + type, + }; + }; + + await new Promise(resolve => { + setTimeout(async () => { + const { status, text, type } = await downloadFromPath(jobDownloadPath); + expect(status).to.eql(200); + expect(type).to.eql('text/csv'); + expect(text).to.eql(CSV_RESULT_SCRIPTED_REQUERY); + resolve(); + }, 5000); // x-pack/test/functional/config settings are inherited, uses 3 seconds for polling interval. + }); + + await esArchiver.unload('reporting/scripted'); + }); + }); + }); +} diff --git a/x-pack/test/reporting/api/generate/fixtures.ts b/x-pack/test/reporting/api/generate/fixtures.ts new file mode 100644 index 0000000000000..2cac49f650842 --- /dev/null +++ b/x-pack/test/reporting/api/generate/fixtures.ts @@ -0,0 +1,175 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export const CSV_RESULT_TIMEBASED = `"@timestamp",clientip,extension +"2015-09-20T10:26:48.725Z","74.214.76.90",jpg +"2015-09-20T10:26:48.540Z","146.86.123.109",jpg +"2015-09-20T10:26:48.353Z","233.126.159.144",jpg +"2015-09-20T10:26:45.468Z","153.139.156.196",png +"2015-09-20T10:26:34.063Z","25.140.171.133",css +"2015-09-20T10:26:11.181Z","239.249.202.59",jpg +"2015-09-20T10:26:00.639Z","95.59.225.31",css +"2015-09-20T10:26:00.094Z","247.174.57.245",jpg +"2015-09-20T10:25:55.744Z","116.126.47.226",css +"2015-09-20T10:25:54.701Z","169.228.188.120",jpg +"2015-09-20T10:25:52.360Z","74.224.77.232",css +"2015-09-20T10:25:49.913Z","97.83.96.39",css +"2015-09-20T10:25:44.979Z","175.188.44.145",css +"2015-09-20T10:25:40.968Z","89.143.125.181",jpg +"2015-09-20T10:25:36.331Z","231.169.195.137",css +"2015-09-20T10:25:34.064Z","137.205.146.206",jpg +"2015-09-20T10:25:32.312Z","53.0.188.251",jpg +"2015-09-20T10:25:27.254Z","111.214.104.239",jpg +"2015-09-20T10:25:22.561Z","111.46.85.146",jpg +"2015-09-20T10:25:06.674Z","55.100.60.111",jpg +"2015-09-20T10:25:05.114Z","34.197.178.155",jpg +"2015-09-20T10:24:55.114Z","163.123.136.118",jpg +"2015-09-20T10:24:54.818Z","11.195.163.57",jpg +"2015-09-20T10:24:53.742Z","96.222.137.213",png +"2015-09-20T10:24:48.798Z","227.228.214.218",jpg +"2015-09-20T10:24:20.223Z","228.53.110.116",jpg +"2015-09-20T10:24:01.794Z","196.131.253.111",png +"2015-09-20T10:23:49.521Z","125.163.133.47",jpg +"2015-09-20T10:23:45.816Z","148.47.216.255",jpg +"2015-09-20T10:23:36.052Z","51.105.100.214",jpg +"2015-09-20T10:23:34.323Z","41.210.252.157",gif +"2015-09-20T10:23:27.213Z","248.163.75.193",png +"2015-09-20T10:23:14.866Z","48.43.210.167",png +"2015-09-20T10:23:10.578Z","33.95.78.209",css +"2015-09-20T10:23:07.001Z","96.40.73.208",css +"2015-09-20T10:23:02.876Z","174.32.230.63",jpg +"2015-09-20T10:23:00.019Z","140.233.207.177",jpg +"2015-09-20T10:22:47.447Z","37.127.124.65",jpg +"2015-09-20T10:22:45.803Z","130.171.208.139",png +"2015-09-20T10:22:45.590Z","39.250.210.253",jpg +"2015-09-20T10:22:43.997Z","248.239.221.43",css +"2015-09-20T10:22:36.107Z","232.64.207.109",gif +"2015-09-20T10:22:30.527Z","24.186.122.118",jpg +"2015-09-20T10:22:25.697Z","23.3.174.206",jpg +"2015-09-20T10:22:08.272Z","185.170.80.142",php +"2015-09-20T10:21:40.822Z","202.22.74.232",png +"2015-09-20T10:21:36.210Z","39.227.27.167",jpg +"2015-09-20T10:21:19.154Z","140.233.207.177",jpg +"2015-09-20T10:21:09.852Z","22.151.97.227",jpg +"2015-09-20T10:21:06.079Z","157.39.25.197",css +"2015-09-20T10:21:01.357Z","37.127.124.65",jpg +"2015-09-20T10:20:56.519Z","23.184.94.58",jpg +"2015-09-20T10:20:40.189Z","80.83.92.252",jpg +"2015-09-20T10:20:27.012Z","66.194.157.171",png +"2015-09-20T10:20:24.450Z","15.191.218.38",jpg +"2015-09-20T10:19:45.764Z","199.113.69.162",jpg +"2015-09-20T10:19:43.754Z","171.243.18.67",gif +"2015-09-20T10:19:41.208Z","126.87.234.213",jpg +"2015-09-20T10:19:40.307Z","78.216.173.242",css +`; + +export const CSV_RESULT_TIMELESS = `name,power +"Jonelle-Jane Marth","1.1768" +"Suzie-May Rishel","1.824" +"Suzie-May Rishel","2.077" +"Rosana Casto","2.8084" +"Stephen Cortez","4.9856" +"Jonelle-Jane Marth","6.156" +"Jonelle-Jane Marth","7.0966" +"Florinda Alejandro","10.3734" +"Jonelle-Jane Marth","14.8074" +"Suzie-May Rishel","19.7377" +"Suzie-May Rishel","20.9198" +"Florinda Alejandro","22.2092" +`; + +export const CSV_RESULT_SCRIPTED = `date,year,name,value,"years_ago" +"1981-01-01T00:00:00.000Z",1981,Fetty,1763,38 +"1981-01-01T00:00:00.000Z",1981,Fonnie,2330,38 +"1981-01-01T00:00:00.000Z",1981,Farbara,6456,38 +"1981-01-01T00:00:00.000Z",1981,Felinda,1886,38 +"1981-01-01T00:00:00.000Z",1981,Frenda,7162,38 +"1981-01-01T00:00:00.000Z",1981,Feth,3685,38 +"1981-01-01T00:00:00.000Z",1981,Feverly,1987,38 +"1981-01-01T00:00:00.000Z",1981,Fecky,1930,38 +"1980-01-01T00:00:00.000Z",1980,Fonnie,2748,39 +"1980-01-01T00:00:00.000Z",1980,Frenda,8335,39 +"1980-01-01T00:00:00.000Z",1980,Fetty,1967,39 +"1980-01-01T00:00:00.000Z",1980,Farbara,8026,39 +"1980-01-01T00:00:00.000Z",1980,Feth,4246,39 +"1980-01-01T00:00:00.000Z",1980,Feverly,2249,39 +"1980-01-01T00:00:00.000Z",1980,Fecky,2071,39 +`; + +export const CSV_RESULT_SCRIPTED_REQUERY = `date,year,name,value,"years_ago" +"1981-01-01T00:00:00.000Z",1981,Fetty,1763,38 +"1981-01-01T00:00:00.000Z",1981,Felinda,1886,38 +"1981-01-01T00:00:00.000Z",1981,Feth,3685,38 +"1981-01-01T00:00:00.000Z",1981,Feverly,1987,38 +"1981-01-01T00:00:00.000Z",1981,Fecky,1930,38 +"1980-01-01T00:00:00.000Z",1980,Fetty,1967,39 +"1980-01-01T00:00:00.000Z",1980,Feth,4246,39 +"1980-01-01T00:00:00.000Z",1980,Feverly,2249,39 +"1980-01-01T00:00:00.000Z",1980,Fecky,2071,39 +`; + +export const CSV_RESULT_SCRIPTED_RESORTED = `date,year,name,value,"years_ago" +"1981-01-01T00:00:00.000Z",1981,Farbara,6456,38 +"1980-01-01T00:00:00.000Z",1980,Farbara,8026,39 +"1981-01-01T00:00:00.000Z",1981,Fecky,1930,38 +"1980-01-01T00:00:00.000Z",1980,Fecky,2071,39 +"1981-01-01T00:00:00.000Z",1981,Felinda,1886,38 +"1981-01-01T00:00:00.000Z",1981,Feth,3685,38 +"1980-01-01T00:00:00.000Z",1980,Feth,4246,39 +"1981-01-01T00:00:00.000Z",1981,Fetty,1763,38 +"1980-01-01T00:00:00.000Z",1980,Fetty,1967,39 +"1981-01-01T00:00:00.000Z",1981,Feverly,1987,38 +"1980-01-01T00:00:00.000Z",1980,Feverly,2249,39 +"1981-01-01T00:00:00.000Z",1981,Fonnie,2330,38 +"1980-01-01T00:00:00.000Z",1980,Fonnie,2748,39 +"1981-01-01T00:00:00.000Z",1981,Frenda,7162,38 +"1980-01-01T00:00:00.000Z",1980,Frenda,8335,39 +`; + +export const CSV_RESULT_HUGE = `date,year,name,value,"years_ago" +"1984-01-01T00:00:00.000Z",1984,Fobby,2791,35 +"1984-01-01T00:00:00.000Z",1984,Frent,3416,35 +"1984-01-01T00:00:00.000Z",1984,Frett,2679,35 +"1984-01-01T00:00:00.000Z",1984,Filly,3366,35 +"1984-01-01T00:00:00.000Z",1984,Frian,34468,35 +"1984-01-01T00:00:00.000Z",1984,Fenjamin,7191,35 +"1984-01-01T00:00:00.000Z",1984,Frandon,5863,35 +"1984-01-01T00:00:00.000Z",1984,Fruce,1855,35 +"1984-01-01T00:00:00.000Z",1984,Fryan,7236,35 +"1984-01-01T00:00:00.000Z",1984,Frad,2482,35 +"1984-01-01T00:00:00.000Z",1984,Fradley,5175,35 +"1983-01-01T00:00:00.000Z",1983,Fryan,7114,36 +"1983-01-01T00:00:00.000Z",1983,Fradley,4752,36 +"1983-01-01T00:00:00.000Z",1983,Frian,35717,36 +"1983-01-01T00:00:00.000Z",1983,Farbara,4434,36 +"1983-01-01T00:00:00.000Z",1983,Fenjamin,5235,36 +"1983-01-01T00:00:00.000Z",1983,Fruce,1914,36 +"1983-01-01T00:00:00.000Z",1983,Fobby,2888,36 +"1983-01-01T00:00:00.000Z",1983,Frett,3031,36 +"1982-01-01T00:00:00.000Z",1982,Fonnie,1853,37 +"1982-01-01T00:00:00.000Z",1982,Frandy,2082,37 +"1982-01-01T00:00:00.000Z",1982,Fecky,1786,37 +"1982-01-01T00:00:00.000Z",1982,Frandi,2056,37 +"1982-01-01T00:00:00.000Z",1982,Fridget,1864,37 +"1982-01-01T00:00:00.000Z",1982,Farbara,5081,37 +"1982-01-01T00:00:00.000Z",1982,Feth,2818,37 +"1982-01-01T00:00:00.000Z",1982,Frenda,6270,37 +"1981-01-01T00:00:00.000Z",1981,Fetty,1763,38 +"1981-01-01T00:00:00.000Z",1981,Fonnie,2330,38 +"1981-01-01T00:00:00.000Z",1981,Farbara,6456,38 +"1981-01-01T00:00:00.000Z",1981,Felinda,1886,38 +"1981-01-01T00:00:00.000Z",1981,Frenda,7162,38 +"1981-01-01T00:00:00.000Z",1981,Feth,3685,38 +"1981-01-01T00:00:00.000Z",1981,Feverly,1987,38 +"1981-01-01T00:00:00.000Z",1981,Fecky,1930,38 +"1980-01-01T00:00:00.000Z",1980,Fonnie,2748,39 +"1980-01-01T00:00:00.000Z",1980,Frenda,8335,39 +"1980-01-01T00:00:00.000Z",1980,Fetty,1967,39 +"1980-01-01T00:00:00.000Z",1980,Farbara,8026,39 +"1980-01-01T00:00:00.000Z",1980,Feth,4246,39 +"1980-01-01T00:00:00.000Z",1980,Feverly,2249,39 +"1980-01-01T00:00:00.000Z",1980,Fecky,2071,39 +`; diff --git a/x-pack/test/reporting/api/generate/index.js b/x-pack/test/reporting/api/generate/index.js new file mode 100644 index 0000000000000..99286762f44f1 --- /dev/null +++ b/x-pack/test/reporting/api/generate/index.js @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export default function ({ loadTestFile }) { + describe('CSV', function () { + this.tags('ciGroup2'); + loadTestFile(require.resolve('./csv_saved_search')); + }); +} diff --git a/x-pack/test/reporting/configs/api.js b/x-pack/test/reporting/configs/api.js index 7c30f299f3f46..b93e89b568648 100644 --- a/x-pack/test/reporting/configs/api.js +++ b/x-pack/test/reporting/configs/api.js @@ -24,14 +24,18 @@ export async function getReportingApiConfig({ readConfigFile }) { junit: { reportName: 'X-Pack Reporting API Tests', }, + testFiles: [require.resolve('../api/generate')], esTestCluster: apiConfig.get('esTestCluster'), kbnTestServer: { ...apiConfig.get('kbnTestServer'), serverArgs: [ ...apiConfig.get('kbnTestServer.serverArgs'), + '--xpack.reporting.csv.enablePanelActionDownload=true', `--optimize.enabled=true`, '--logging.events.log', JSON.stringify(['info', 'warning', 'error', 'fatal', 'optimize', 'reporting']) ], }, }; } + +export default getReportingApiConfig; diff --git a/x-pack/test/reporting/configs/chromium_api.js b/x-pack/test/reporting/configs/chromium_api.js index 461c6c0df5271..1377a4af9de9f 100644 --- a/x-pack/test/reporting/configs/chromium_api.js +++ b/x-pack/test/reporting/configs/chromium_api.js @@ -20,6 +20,7 @@ export default async function ({ readConfigFile }) { ...reportingApiConfig.kbnTestServer, serverArgs: [ ...reportingApiConfig.kbnTestServer.serverArgs, + '--xpack.reporting.csv.enablePanelActionDownload=true', `--xpack.reporting.capture.browser.type=chromium`, `--xpack.spaces.enabled=false`, ], diff --git a/x-pack/test/reporting/configs/chromium_functional.js b/x-pack/test/reporting/configs/chromium_functional.js index 80a9024c06ab8..69d8ae9d55b57 100644 --- a/x-pack/test/reporting/configs/chromium_functional.js +++ b/x-pack/test/reporting/configs/chromium_functional.js @@ -20,6 +20,7 @@ export default async function ({ readConfigFile }) { ...functionalConfig.kbnTestServer, serverArgs: [ ...functionalConfig.kbnTestServer.serverArgs, + '--xpack.reporting.csv.enablePanelActionDownload=true', `--xpack.reporting.capture.browser.type=chromium`, ], }, diff --git a/x-pack/test/reporting/configs/functional.js b/x-pack/test/reporting/configs/functional.js index 4eafa10369e93..24d1a0228a70c 100644 --- a/x-pack/test/reporting/configs/functional.js +++ b/x-pack/test/reporting/configs/functional.js @@ -25,6 +25,7 @@ export async function getFunctionalConfig({ readConfigFile }) { ...xPackFunctionalTestsConfig.get('kbnTestServer'), serverArgs: [ ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--xpack.reporting.csv.enablePanelActionDownload=true', '--logging.events.log', JSON.stringify(['info', 'warning', 'error', 'fatal', 'optimize', 'reporting']) ], }, diff --git a/x-pack/test/reporting/configs/generate_api.js b/x-pack/test/reporting/configs/generate_api.js new file mode 100644 index 0000000000000..cd81dea583fff --- /dev/null +++ b/x-pack/test/reporting/configs/generate_api.js @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getApiIntegrationConfig } from '../../api_integration/config'; +import { getReportingApiConfig } from './api'; + +export default async function ({ readConfigFile }) { + const apiTestConfig = await getApiIntegrationConfig({ readConfigFile }); + const reportingApiConfig = await getReportingApiConfig({ readConfigFile }); + const xPackFunctionalTestsConfig = await readConfigFile(require.resolve('../../functional/config.js')); + + return { + ...reportingApiConfig, + junit: { reportName: 'X-Pack Reporting Generate API Integration Tests' }, + testFiles: [require.resolve('../api/generate')], + services: { + ...apiTestConfig.services, + ...reportingApiConfig.services, + }, + kbnTestServer: { + ...xPackFunctionalTestsConfig.get('kbnTestServer'), + serverArgs: [ + ...xPackFunctionalTestsConfig.get('kbnTestServer.serverArgs'), + '--xpack.reporting.csv.enablePanelActionDownload=true', + ], + }, + esArchiver: apiTestConfig.esArchiver, + }; +}