From 3a0eb46eede8461d4159383bd51ded76e77a3903 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 2 Apr 2020 14:41:28 +0100 Subject: [PATCH 1/7] [ML] Show better file structure finder explanations --- x-pack/plugins/ml/common/types/errors.ts | 1 + .../ml/common/types/file_datavisualizer.ts | 6 ++ .../{about_panel.js => about_panel.tsx} | 14 ++-- .../about_panel/{index.js => index.ts} | 0 ...welcome_content.js => welcome_content.tsx} | 25 +++--- ...alysis_summary.js => analysis_summary.tsx} | 11 +-- .../analysis_summary/{index.js => index.ts} | 0 .../experimental_badge/experimental_badge.js | 27 ------ .../experimental_badge/experimental_badge.tsx | 25 ++++++ .../experimental_badge/{index.js => index.ts} | 0 .../explanation_flyout/explanation_flyout.tsx | 83 +++++++++++++++++++ .../index.js => explanation_flyout/index.ts} | 2 +- .../{file_contents.js => file_contents.tsx} | 16 ++-- .../file_contents/{index.js => index.ts} | 0 .../file_datavisualizer_view.js | 43 +++++++--- ...or_callouts.js => file_error_callouts.tsx} | 45 ++++++++-- .../import_errors/{errors.js => errors.tsx} | 36 ++++---- .../import_errors/{index.js => index.ts} | 0 ...import_progress.js => import_progress.tsx} | 34 +++++--- .../components/import_progress/index.ts | 7 ++ .../results_view/{index.js => index.ts} | 0 .../{results_view.js => results_view.tsx} | 50 +++++++++-- .../ml/server/client/elasticsearch_ml.ts | 4 +- .../plugins/ml/server/client/error_wrapper.ts | 8 +- .../file_data_visualizer.ts | 16 +--- 25 files changed, 327 insertions(+), 126 deletions(-) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/{about_panel.js => about_panel.tsx} (91%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/{welcome_content.js => welcome_content.tsx} (90%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/{analysis_summary.js => analysis_summary.tsx} (89%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/{index.js => index.ts} (100%) delete mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.js create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/{index.js => index.ts} (100%) create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/{import_progress/index.js => explanation_flyout/index.ts} (78%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/{file_contents.js => file_contents.tsx} (83%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/{file_error_callouts.js => file_error_callouts.tsx} (73%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/{errors.js => errors.tsx} (85%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/{import_progress.js => import_progress.tsx} (92%) create mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/{results_view.js => results_view.tsx} (58%) diff --git a/x-pack/plugins/ml/common/types/errors.ts b/x-pack/plugins/ml/common/types/errors.ts index 63e222490082bb..284250bd5ce55b 100644 --- a/x-pack/plugins/ml/common/types/errors.ts +++ b/x-pack/plugins/ml/common/types/errors.ts @@ -9,6 +9,7 @@ export interface ErrorResponse { statusCode: number; error: string; message: string; + attributes?: any; }; name: string; } diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index bc03f82673a1fe..60108f1d06c2f5 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -28,4 +28,10 @@ export interface FindFileStructureResponse { need_client_timezone: boolean; num_lines_analyzed: number; column_names: string[]; + explanation?: string[]; + grok_pattern?: string; + multiline_start_pattern?: string; + java_timestamp_formats?: string[]; + joda_timestamp_formats?: string[]; + timestamp_field?: string; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx similarity index 91% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx index edecc925591d37..2bddf0de0499da 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/about_panel.tsx @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFlexGroup, @@ -23,7 +23,11 @@ import { import { WelcomeContent } from './welcome_content'; -export function AboutPanel({ onFilePickerChange }) { +interface Props { + onFilePickerChange(files: FileList | null): void; +} + +export const AboutPanel: FC = ({ onFilePickerChange }) => { return ( @@ -54,9 +58,9 @@ export function AboutPanel({ onFilePickerChange }) { ); -} +}; -export function LoadingPanel() { +export const LoadingPanel: FC = () => { return ( @@ -79,4 +83,4 @@ export function LoadingPanel() { ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx similarity index 90% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx index 73d12122879f8c..c56ab021e2f2f6 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/about_panel/welcome_content.tsx @@ -5,7 +5,8 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiFlexGroup, @@ -19,7 +20,14 @@ import { import { ExperimentalBadge } from '../experimental_badge'; -export function WelcomeContent() { +export const WelcomeContent: FC = () => { + const toolTipContent = i18n.translate( + 'xpack.ml.fileDatavisualizer.welcomeContent.experimentalFeatureTooltip', + { + defaultMessage: "Experimental feature. We'd love to hear your feedback.", + } + ); + return ( @@ -32,16 +40,7 @@ export function WelcomeContent() { id="xpack.ml.fileDatavisualizer.welcomeContent.visualizeDataFromLogFileTitle" defaultMessage="Visualize data from a log file {experimentalBadge}" values={{ - experimentalBadge: ( - - } - /> - ), + experimentalBadge: , }} /> @@ -144,4 +143,4 @@ export function WelcomeContent() { ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx similarity index 89% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx index b3bf4ea4daf83f..b80db3b27fa7e5 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/analysis_summary/analysis_summary.tsx @@ -5,11 +5,12 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTitle, EuiSpacer, EuiDescriptionList } from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; -export function AnalysisSummary({ results }) { +export const AnalysisSummary: FC<{ results: FindFileStructureResponse }> = ({ results }) => { const items = createDisplayItems(results); return ( @@ -28,10 +29,10 @@ export function AnalysisSummary({ results }) { ); -} +}; -function createDisplayItems(results) { - const items = [ +function createDisplayItems(results: FindFileStructureResponse) { + const items: Array<{ title: any; description: string | number }> = [ { title: ( - - } - tooltipContent={tooltipContent} - /> - - ); -} diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx new file mode 100644 index 00000000000000..cd82416b344c33 --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FormattedMessage } from '@kbn/i18n/react'; +import React, { FC } from 'react'; + +import { EuiBetaBadge } from '@elastic/eui'; + +export const ExperimentalBadge: FC<{ tooltipContent: string }> = ({ tooltipContent }) => ( + + + } + tooltipContent={tooltipContent} + /> + +); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx new file mode 100644 index 00000000000000..324dde729fbd9c --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyout, + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiFlyoutHeader, + EuiButtonEmpty, + EuiTitle, + EuiFlyoutBody, + EuiSpacer, + EuiText, + EuiSubSteps, +} from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; + +interface Props { + results: FindFileStructureResponse; + closeFlyout(): void; +} +export const ExplanationFlyout: FC = ({ results, closeFlyout }) => { + const explanation = results.explanation!; + return ( + + + +

+ +

+ + + + + + + + + + + + + + + + ); +}; + +const Content: FC<{ explanation: string[] }> = ({ explanation }) => ( + <> + + The logical steps that have produced the analysis results. + + +
    + {explanation.map((e, i) => ( +
  • + {e} + +
    • + ))} +
+ + + +); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts similarity index 78% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts index f75d869f4667c2..e2880504038878 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/index.ts @@ -4,4 +4,4 @@ * you may not use this file except in compliance with the Elastic License. */ -export { ImportProgress, IMPORT_STATUS } from './import_progress'; +export { ExplanationFlyout } from './explanation_flyout'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx similarity index 83% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx index ed5ab57a2588d0..6564b9a1f4d838 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/file_contents.tsx @@ -5,13 +5,19 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTitle, EuiSpacer } from '@elastic/eui'; import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; -export function FileContents({ data, format, numberOfLines }) { +interface Props { + data: string; + format: string; + numberOfLines: number; +} + +export const FileContents: FC = ({ data, format, numberOfLines }) => { let mode = ML_EDITOR_MODE.TEXT; if (format === ML_EDITOR_MODE.JSON) { mode = ML_EDITOR_MODE.JSON; @@ -35,7 +41,7 @@ export function FileContents({ data, format, numberOfLines }) { id="xpack.ml.fileDatavisualizer.fileContents.firstLinesDescription" defaultMessage="First {numberOfLines, plural, zero {# line} one {# line} other {# lines}}" values={{ - numberOfLines: numberOfLines, + numberOfLines, }} /> @@ -51,9 +57,9 @@ export function FileContents({ data, format, numberOfLines }) { /> ); -} +}; -function limitByNumberOfLines(data, numberOfLines) { +function limitByNumberOfLines(data: string, numberOfLines: number) { return data .split('\n') .slice(0, numberOfLines) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_contents/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js index 12e5a14b518713..ce221d5eddb7d2 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js @@ -17,9 +17,9 @@ import { BottomBar } from '../bottom_bar'; import { ResultsView } from '../results_view'; import { FileCouldNotBeRead, FileTooLarge } from './file_error_callouts'; import { EditFlyout } from '../edit_flyout'; +import { ExplanationFlyout } from '../explanation_flyout'; import { ImportView } from '../import_view'; import { MAX_BYTES } from '../../../../../../common/constants/file_datavisualizer'; -import { isErrorResponse } from '../../../../../../common/types/errors'; import { readFile, createUrlOverrides, @@ -42,12 +42,14 @@ export class FileDataVisualizerView extends Component { fileSize: 0, fileTooLarge: false, fileCouldNotBeRead: false, - serverErrorMessage: '', + serverError: null, loading: false, loaded: false, results: undefined, + explanation: undefined, mode: MODE.READ, isEditFlyoutVisible: false, + isExplanationFlyoutVisible: false, bottomBarVisible: false, hasPermissionToImport: false, }; @@ -78,8 +80,9 @@ export class FileDataVisualizerView extends Component { fileSize: 0, fileTooLarge: false, fileCouldNotBeRead: false, - serverErrorMessage: '', + serverError: null, results: undefined, + explanation: undefined, }, () => { if (files.length) { @@ -172,6 +175,7 @@ export class FileDataVisualizerView extends Component { this.setState({ results: resp.results, + explanation: resp.explanation, loaded: true, loading: false, fileCouldNotBeRead: isRetry, @@ -179,19 +183,13 @@ export class FileDataVisualizerView extends Component { } catch (error) { console.error(error); - let serverErrorMsg; - if (isErrorResponse(error) === true) { - serverErrorMsg = `${error.body.error}: ${error.body.message}`; - } else { - serverErrorMsg = JSON.stringify(error, null, 2); - } - this.setState({ results: undefined, + explanation: undefined, loaded: false, loading: false, fileCouldNotBeRead: true, - serverErrorMessage: serverErrorMsg, + serverError: error, }); // as long as the previous overrides are different to the current overrides, @@ -216,6 +214,16 @@ export class FileDataVisualizerView extends Component { this.hideBottomBar(); }; + closeExplanationFlyout = () => { + this.setState({ isExplanationFlyoutVisible: false }); + this.showBottomBar(); + }; + + showExplanationFlyout = () => { + this.setState({ isExplanationFlyoutVisible: true }); + this.hideBottomBar(); + }; + showBottomBar = () => { this.setState({ bottomBarVisible: true }); }; @@ -252,14 +260,16 @@ export class FileDataVisualizerView extends Component { loading, loaded, results, + explanation, fileContents, fileName, fileSize, fileTooLarge, fileCouldNotBeRead, - serverErrorMessage, + serverError, mode, isEditFlyoutVisible, + isExplanationFlyoutVisible, bottomBarVisible, hasPermissionToImport, } = this.state; @@ -281,7 +291,7 @@ export class FileDataVisualizerView extends Component { {fileCouldNotBeRead && loading === false && ( - + )} @@ -289,9 +299,12 @@ export class FileDataVisualizerView extends Component { {loaded && ( this.showEditFlyout()} + showExplanationFlyout={() => this.showExplanationFlyout()} + disableButtons={isEditFlyoutVisible || isExplanationFlyoutVisible} /> )} + {isExplanationFlyoutVisible && ( + + )} + {bottomBarVisible && loaded && ( = ({ fileSize, maxFileSize }) => { const fileSizeFormatted = numeral(fileSize).format(FILE_SIZE_DISPLAY_FORMAT); const maxFileSizeFormatted = numeral(maxFileSize).format(FILE_SIZE_DISPLAY_FORMAT); @@ -67,9 +73,15 @@ export function FileTooLarge({ fileSize, maxFileSize }) { {errorText} ); +}; + +interface FileCouldNotBeReadProps { + error: ErrorResponse; + loaded: boolean; } -export function FileCouldNotBeRead({ error, loaded }) { +export const FileCouldNotBeRead: FC = ({ error, loaded }) => { + const message = error.body.message; return ( - {error !== undefined &&

{error}

} + {message} + {loaded && ( -

+ <> + -

+ )} ); -} +}; + +export const Explanation: FC<{ error: ErrorResponse }> = ({ error }) => { + if (!error.body.attributes?.body?.error?.suppressed?.length) { + return null; + } + const reason: string = error.body.attributes.body.error.suppressed[0].reason; + return ( + <> + + {reason.split('\n').map((m, i) => ( +
{m}
+ ))} + + ); +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx similarity index 85% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx index 6629c0109feafd..d1e8d138f6c3ed 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx @@ -5,13 +5,19 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; +import { i18n } from '@kbn/i18n'; import { EuiCallOut, EuiAccordion } from '@elastic/eui'; -import { IMPORT_STATUS } from '../import_progress'; +import { IMPORT_STATUS, Statuses } from '../import_progress'; -export function ImportErrors({ errors, statuses }) { +interface Props { + errors: any[]; + statuses: Statuses; +} + +export const ImportErrors: FC = ({ errors, statuses }) => { return ( {errors.map((e, i) => ( @@ -19,9 +25,9 @@ export function ImportErrors({ errors, statuses }) { ))} ); -} +}; -function title(statuses) { +function title(statuses: Statuses) { switch (IMPORT_STATUS.FAILED) { case statuses.readStatus: return ( @@ -82,7 +88,7 @@ function title(statuses) { } } -function ImportError(error, key) { +function ImportError(error: any, key: number) { const errorObj = toString(error); return ( @@ -106,7 +112,12 @@ function ImportError(error, key) { ); } -function toString(error) { +interface ImportError { + msg: string; + more?: string; +} + +function toString(error: any): ImportError { if (typeof error === 'string') { return { msg: error }; } @@ -118,7 +129,7 @@ function toString(error) { if (typeof error.error === 'object') { if (error.error.msg !== undefined) { // this will catch a bulk ingest failure - const errorObj = { msg: error.error.msg }; + const errorObj: ImportError = { msg: error.error.msg }; if (error.error.body !== undefined) { errorObj.more = error.error.response; } @@ -139,11 +150,8 @@ function toString(error) { } return { - msg: ( - - ), + msg: i18n.translate('xpack.ml.fileDatavisualizer.importErrors.unknownErrorMessage', { + defaultMessage: 'Unknown error', + }), }; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx similarity index 92% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx index 272ec2979ad2f0..533fec4ac50c8c 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/import_progress.tsx @@ -6,17 +6,31 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiStepsHorizontal, EuiProgress, EuiSpacer } from '@elastic/eui'; -export const IMPORT_STATUS = { - INCOMPLETE: 'incomplete', - COMPLETE: 'complete', - FAILED: 'danger', -}; +export enum IMPORT_STATUS { + INCOMPLETE = 'incomplete', + COMPLETE = 'complete', + FAILED = 'danger', +} + +export interface Statuses { + reading: boolean; + readStatus: IMPORT_STATUS; + parseJSONStatus: IMPORT_STATUS; + indexCreatedStatus: IMPORT_STATUS; + ingestPipelineCreatedStatus: IMPORT_STATUS; + indexPatternCreatedStatus: IMPORT_STATUS; + uploadProgress: number; + uploadStatus: IMPORT_STATUS; + createIndexPattern: boolean; + createPipeline: boolean; + permissionCheckStatus: IMPORT_STATUS; +} -export function ImportProgress({ statuses }) { +export const ImportProgress: FC<{ statuses: Statuses }> = ({ statuses }) => { const { reading, readStatus, @@ -271,9 +285,9 @@ export function ImportProgress({ statuses }) { )} ); -} +}; -function UploadFunctionProgress({ progress }) { +const UploadFunctionProgress: FC<{ progress: number }> = ({ progress }) => { return (

@@ -290,4 +304,4 @@ function UploadFunctionProgress({ progress }) { )} ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts new file mode 100644 index 00000000000000..9b0c6ca2264cbd --- /dev/null +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_progress/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { ImportProgress, IMPORT_STATUS, Statuses } from './import_progress'; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx similarity index 58% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx index 96116e5cefa010..1358b42bd2d371 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx @@ -7,10 +7,10 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; - +import React, { FC } from 'react'; import { EuiButton, + EuiButtonEmpty, EuiPage, EuiPageBody, EuiPageContentHeader, @@ -18,13 +18,33 @@ import { EuiTabbedContent, EuiSpacer, EuiTitle, + EuiFlexGroup, + EuiFlexItem, } from '@elastic/eui'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; import { FileContents } from '../file_contents'; import { AnalysisSummary } from '../analysis_summary'; +// @ts-ignore import { FieldsStats } from '../fields_stats'; -export const ResultsView = ({ data, fileName, results, showEditFlyout }) => { +interface Props { + data: string; + fileName: string; + results: FindFileStructureResponse; + showEditFlyout(): void; + showExplanationFlyout(): void; + disableButtons: boolean; +} + +export const ResultsView: FC = ({ + data, + fileName, + results, + showEditFlyout, + showExplanationFlyout, + disableButtons, +}) => { const tabs = [ { id: 'file-stats', @@ -60,12 +80,24 @@ export const ResultsView = ({ data, fileName, results, showEditFlyout }) => { - showEditFlyout()}> - - + + + showEditFlyout()} disabled={disableButtons}> + + + + + showExplanationFlyout()} disabled={disableButtons}> + + + + diff --git a/x-pack/plugins/ml/server/client/elasticsearch_ml.ts b/x-pack/plugins/ml/server/client/elasticsearch_ml.ts index caedaed92e5b1b..d5c7882a30d206 100644 --- a/x-pack/plugins/ml/server/client/elasticsearch_ml.ts +++ b/x-pack/plugins/ml/server/client/elasticsearch_ml.ts @@ -740,7 +740,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) urls: [ { fmt: - '/_ml/find_file_structure?&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', + '/_ml/find_file_structure?&explain=true&charset=<%=charset%>&format=<%=format%>&has_header_row=<%=has_header_row%>&column_names=<%=column_names%>&delimiter=<%=delimiter%>"e=<%=quote%>&should_trim_fields=<%=should_trim_fields%>&grok_pattern=<%=grok_pattern%>×tamp_field=<%=timestamp_field%>×tamp_format=<%=timestamp_format%>&lines_to_sample=<%=lines_to_sample%>', req: { charset: { type: 'string', @@ -778,7 +778,7 @@ export const elasticsearchJsPlugin = (Client: any, config: any, components: any) }, }, { - fmt: '/_ml/find_file_structure', + fmt: '/_ml/find_file_structure?&explain=true', }, ], needBody: true, diff --git a/x-pack/plugins/ml/server/client/error_wrapper.ts b/x-pack/plugins/ml/server/client/error_wrapper.ts index 7f691732954821..afce0c1af686fb 100644 --- a/x-pack/plugins/ml/server/client/error_wrapper.ts +++ b/x-pack/plugins/ml/server/client/error_wrapper.ts @@ -9,9 +9,13 @@ import { ResponseError, CustomHttpResponseOptions } from 'kibana/server'; export function wrapError(error: any): CustomHttpResponseOptions { const boom = isBoom(error) ? error : boomify(error, { statusCode: error.status }); + const statusCode = boom.output.statusCode; return { - body: boom, + body: { + ...boom, + ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}), + }, headers: boom.output.headers, - statusCode: boom.output.statusCode, + statusCode, }; } diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts index 9af755c6918fbe..c6c0c373646b3f 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -4,7 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -import Boom from 'boom'; import { APICaller } from 'kibana/server'; import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer'; @@ -27,17 +26,10 @@ export interface AnalysisResult { export function fileDataVisualizerProvider(callAsCurrentUser: APICaller) { async function analyzeFile(data: any, overrides: any): Promise { - let results = []; - - try { - results = await callAsCurrentUser('ml.fileStructure', { - body: data, - ...overrides, - }); - } catch (error) { - const err = error.message !== undefined ? error.message : error; - throw Boom.badRequest(err); - } + const results = await callAsCurrentUser('ml.fileStructure', { + body: data, + ...overrides, + }); const { hasOverrides, reducedOverrides } = formatOverrides(overrides); From 460460c4e867bd9337e5652b1f9fe3959dc55336 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Apr 2020 14:54:18 +0100 Subject: [PATCH 2/7] more typescript changes --- .../ml/common/types/file_datavisualizer.ts | 53 ++++++++++++ .../file_datavisualizer_view.js | 2 +- .../{advanced.js => advanced.tsx} | 47 ++++++++--- ...import_settings.js => import_settings.tsx} | 22 ++++- .../import_settings/{index.js => index.ts} | 0 .../import_settings/{simple.js => simple.tsx} | 13 ++- .../{import_summary.js => import_summary.tsx} | 46 +++++++--- .../import_summary/{index.js => index.ts} | 0 .../importer/{importer.js => importer.ts} | 83 +++++++++++++------ ...mporter_factory.js => importer_factory.ts} | 3 +- .../importer/{index.js => index.ts} | 0 ...essage_importer.js => message_importer.ts} | 29 ++++--- ...{ndjson_importer.js => ndjson_importer.ts} | 10 +-- .../components/utils/{index.js => index.ts} | 0 .../file_based/components/utils/overrides.js | 21 ----- .../components/utils/{utils.js => utils.ts} | 40 ++++++--- .../services/ml_api_service/datavisualizer.ts | 5 +- .../file_data_visualizer.ts | 21 ++--- .../file_data_visualizer/import_data.ts | 39 +++------ .../models/file_data_visualizer/index.ts | 9 +- .../ml/server/routes/file_data_visualizer.ts | 12 +-- 21 files changed, 291 insertions(+), 164 deletions(-) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/{advanced.js => advanced.tsx} (84%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/{import_settings.js => import_settings.tsx} (82%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/{simple.js => simple.tsx} (88%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/{import_summary.js => import_summary.tsx} (86%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/{importer.js => importer.ts} (74%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/{importer_factory.js => importer_factory.ts} (86%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/{index.js => index.ts} (100%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/{message_importer.js => message_importer.ts} (76%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/{ndjson_importer.js => ndjson_importer.ts} (80%) rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/{index.js => index.ts} (100%) delete mode 100644 x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js rename x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/{utils.js => utils.ts} (79%) diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index 60108f1d06c2f5..88ca915c94addc 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -4,6 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ +export interface InputOverrides { + [key: string]: string; +} + +export type FormattedOverrides = InputOverrides & { + column_names: string[]; + has_header_row: boolean; + should_trim_fields: boolean; +}; + +export interface AnalysisResult { + results: FindFileStructureResponse; + overrides?: FormattedOverrides; +} + export interface FindFileStructureResponse { charset: string; has_header_row: boolean; @@ -34,4 +49,42 @@ export interface FindFileStructureResponse { java_timestamp_formats?: string[]; joda_timestamp_formats?: string[]; timestamp_field?: string; + should_trim_fields?: boolean; +} + +export interface ImportResponse { + success: boolean; + id: string; + index?: string; + pipelineId?: string; + docCount: number; + failures: ImportFailure[]; + error?: any; + ingestError?: boolean; +} + +export interface ImportFailure { + item: number; + reason: string; + doc: Doc; +} + +export interface Doc { + message: string; +} + +export interface Settings { + pipeline?: string; + index: string; + body: any[]; + [key: string]: any; +} + +export interface Mappings { + [key: string]: any; +} + +export interface IngestPipeline { + id: string; + pipeline: any; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js index ce221d5eddb7d2..02f14a9e4553e5 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/file_datavisualizer_view/file_datavisualizer_view.js @@ -131,7 +131,7 @@ export class FileDataVisualizerView extends Component { console.log('overrides', overrides); const { analyzeFile } = ml.fileDatavisualizer; const resp = await analyzeFile(lessData, overrides); - const serverSettings = processResults(resp.results); + const serverSettings = processResults(resp); const serverOverrides = resp.overrides; this.previousOverrides = this.overrides; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx similarity index 84% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx index 14cbe67662ed6f..570c020916f9a2 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx @@ -6,7 +6,7 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFieldText, @@ -20,7 +20,25 @@ import { import { MLJobEditor, ML_EDITOR_MODE } from '../../../../jobs/jobs_list/components/ml_job_editor'; const EDITOR_HEIGHT = '300px'; -export function AdvancedSettings({ +interface Props { + index: string; + indexPattern: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + onIndexPatternChange(): void; + indexSettingsString: string; + mappingsString: string; + pipelineString: string; + onIndexSettingsStringChange(): void; + onMappingsStringChange(): void; + onPipelineStringChange(): void; + indexNameError: string; + indexPatternNameError: string; +} + +export const AdvancedSettings: FC = ({ index, indexPattern, initialized, @@ -36,7 +54,7 @@ export function AdvancedSettings({ onPipelineStringChange, indexNameError, indexPatternNameError, -}) { +}) => { return ( } - disabled={createIndexPattern === false || initialized === true} + // disabled={createIndexPattern === false || initialized === true} isInvalid={indexPatternNameError !== ''} error={[indexPatternNameError]} > @@ -133,9 +151,15 @@ export function AdvancedSettings({ ); +}; + +interface JsonEditorProps { + initialized: boolean; + data: string; + onChange(): void; } -function IndexSettings({ initialized, data, onChange }) { +const IndexSettings: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; -function Mappings({ initialized, data, onChange }) { +const Mappings: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; -function IngestPipeline({ initialized, data, onChange }) { +const IngestPipeline: FC = ({ initialized, data, onChange }) => { return ( } - disabled={initialized === true} fullWidth > ); -} +}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx similarity index 82% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx index ba637c472333d3..02cf647caaf16d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/import_settings.tsx @@ -5,14 +5,32 @@ */ import { i18n } from '@kbn/i18n'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiTabbedContent, EuiSpacer } from '@elastic/eui'; import { SimpleSettings } from './simple'; import { AdvancedSettings } from './advanced'; -export const ImportSettings = ({ +interface Props { + index: string; + indexPattern: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + onIndexPatternChange(): void; + indexSettingsString: string; + mappingsString: string; + pipelineString: string; + onIndexSettingsStringChange(): void; + onMappingsStringChange(): void; + onPipelineStringChange(): void; + indexNameError: string; + indexPatternNameError: string; +} + +export const ImportSettings: FC = ({ index, indexPattern, initialized, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx similarity index 88% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx index 271b9493aa1f35..1e716824729e33 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/simple.tsx @@ -6,11 +6,20 @@ import { i18n } from '@kbn/i18n'; import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiFieldText, EuiFormRow, EuiCheckbox, EuiSpacer } from '@elastic/eui'; -export const SimpleSettings = ({ +interface Props { + index: string; + initialized: boolean; + onIndexChange(): void; + createIndexPattern: boolean; + onCreateIndexPatternChange(): void; + indexNameError: string; +} + +export const SimpleSettings: FC = ({ index, initialized, onIndexChange, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx similarity index 86% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx index 0e67807a39fd92..6ee17d401bd707 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/import_summary.tsx @@ -5,11 +5,29 @@ */ import { FormattedMessage } from '@kbn/i18n/react'; -import React from 'react'; +import React, { FC } from 'react'; import { EuiSpacer, EuiDescriptionList, EuiCallOut, EuiAccordion } from '@elastic/eui'; -export function ImportSummary({ +interface Props { + index: string; + indexPattern: string; + ingestPipelineId: string; + docCount: number; + importFailures: DocFailure[]; + createIndexPattern: boolean; + createPipeline: boolean; +} + +interface DocFailure { + item: number; + reason: string; + doc: { + message: string; + }; +} + +export const ImportSummary: FC = ({ index, indexPattern, ingestPipelineId, @@ -17,7 +35,7 @@ export function ImportSummary({ importFailures, createIndexPattern, createPipeline, -}) { +}) => { const items = createDisplayItems( index, indexPattern, @@ -75,9 +93,13 @@ export function ImportSummary({ )} ); +}; + +interface FailuresProps { + failedDocs: DocFailure[]; } -function Failures({ failedDocs }) { +const Failures: FC = ({ failedDocs }) => { return ( ); -} +}; function createDisplayItems( - index, - indexPattern, - ingestPipelineId, - docCount, - importFailures, - createIndexPattern, - createPipeline + index: string, + indexPattern: string, + ingestPipelineId: string, + docCount: number, + importFailures: DocFailure[], + createIndexPattern: boolean, + createPipeline: boolean ) { const items = [ { diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_summary/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts similarity index 74% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts index 27899a58beed2f..71e89affc9d73d 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts @@ -4,30 +4,53 @@ * you may not use this file except in compliance with the Elastic License. */ -import { ml } from '../../../../../services/ml_api_service'; import { chunk } from 'lodash'; import moment from 'moment'; import { i18n } from '@kbn/i18n'; +import { ml } from '../../../../../services/ml_api_service'; +import { + Doc, + ImportFailure, + ImportResponse, + Mappings, + Settings, + IngestPipeline, +} from '../../../../../../../common/types/file_datavisualizer'; const CHUNK_SIZE = 5000; const MAX_CHUNK_CHAR_COUNT = 1000000; const IMPORT_RETRIES = 5; +export interface ImportConfig { + settings: Settings; + mappings: Mappings; + pipeline: IngestPipeline; +} + +export interface ImportResults { + success: boolean; + failures?: any[]; + docCount?: number; + error?: any; +} + export class Importer { - constructor({ settings, mappings, pipeline }) { - this.settings = settings; - this.mappings = mappings; - this.pipeline = pipeline; - - this.data = []; - this.docArray = []; - this.docSizeArray = []; + private _settings: Settings; + private _mappings: Mappings; + private _pipeline: IngestPipeline; + + protected _docArray: Doc[] = []; + + constructor({ settings, mappings, pipeline }: ImportConfig) { + this._settings = settings; + this._mappings = mappings; + this._pipeline = pipeline; } - async initializeImport(index) { - const settings = this.settings; - const mappings = this.mappings; - const pipeline = this.pipeline; + async initializeImport(index: string) { + const settings = this._settings; + const mappings = this._mappings; + const pipeline = this._pipeline; updatePipelineTimezone(pipeline); // if no pipeline has been supplied, @@ -52,7 +75,12 @@ export class Importer { return createIndexResp; } - async import(id, index, pipelineId, setImportProgress) { + async import( + id: string, + index: string, + pipelineId: string, + setImportProgress: (progress: number) => void + ): Promise { if (!id || !index) { return { success: false, @@ -65,14 +93,14 @@ export class Importer { }; } - const chunks = createDocumentChunks(this.docArray); + const chunks = createDocumentChunks(this._docArray); const ingestPipeline = { id: pipelineId, }; let success = true; - const failures = []; + const failures: ImportFailure[] = []; let error; for (let i = 0; i < chunks.length; i++) { @@ -86,10 +114,13 @@ export class Importer { }; let retries = IMPORT_RETRIES; - let resp = { + let resp: ImportResponse = { success: false, failures: [], docCount: 0, + id: '', + index: '', + pipelineId: '', }; while (resp.success === false && retries > 0) { @@ -97,12 +128,13 @@ export class Importer { resp = await ml.fileDatavisualizer.import(aggs); if (retries < IMPORT_RETRIES) { + // eslint-disable-next-line no-console console.log(`Retrying import ${IMPORT_RETRIES - retries}`); } retries--; } catch (err) { - resp = { success: false, error: err }; + resp.success = false; retries = 0; } } @@ -110,6 +142,7 @@ export class Importer { if (resp.success) { setImportProgress(((i + 1) / chunks.length) * 100); } else { + // eslint-disable-next-line no-console console.error(resp); success = false; error = resp.error; @@ -120,10 +153,10 @@ export class Importer { populateFailures(resp, failures, i); } - const result = { + const result: ImportResults = { success, failures, - docCount: this.docArray.length, + docCount: this._docArray.length, }; if (success) { @@ -136,7 +169,7 @@ export class Importer { } } -function populateFailures(error, failures, chunkCount) { +function populateFailures(error: ImportResponse, failures: ImportFailure[], chunkCount: number) { if (error.failures && error.failures.length) { // update the item value to include the chunk count // e.g. item 3 in chunk 2 is actually item 20003 @@ -155,10 +188,10 @@ function populateFailures(error, failures, chunkCount) { // But it's not sending every single field that Filebeat would add, so the ingest pipeline // cannot look for a event.timezone variable in each input record. // Therefore we need to replace {{ event.timezone }} with the actual browser timezone -function updatePipelineTimezone(ingestPipeline) { +function updatePipelineTimezone(ingestPipeline: any) { if (ingestPipeline !== undefined && ingestPipeline.processors && ingestPipeline.processors) { const dateProcessor = ingestPipeline.processors.find( - p => p.date !== undefined && p.date.timezone === '{{ event.timezone }}' + (p: any) => p.date !== undefined && p.date.timezone === '{{ event.timezone }}' ); if (dateProcessor) { @@ -167,8 +200,8 @@ function updatePipelineTimezone(ingestPipeline) { } } -function createDocumentChunks(docArray) { - const chunks = []; +function createDocumentChunks(docArray: Doc[]) { + const chunks: Doc[][] = []; // chop docArray into 5000 doc chunks const tempChunks = chunk(docArray, CHUNK_SIZE); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts similarity index 86% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts index 381e8ef6044528..8dde513aa36a58 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts @@ -6,8 +6,9 @@ import { MessageImporter } from './message_importer'; import { NdjsonImporter } from './ndjson_importer'; +import { ImportConfig } from './importer'; -export function importerFactory(format, results, settings) { +export function importerFactory(format: string, results: any, settings: ImportConfig) { switch (format) { // delimited and semi-structured text are both handled by splitting the // file into messages, then sending these to ES for further processing diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts similarity index 76% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index c2d3ac69f0963f..ff46f570c63082 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -4,17 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Importer } from './importer'; +import { Importer, ImportConfig } from './importer'; +import { Doc } from '../../../../../../../common/types/file_datavisualizer'; export class MessageImporter extends Importer { - constructor(results, settings) { + private _excludeLinesRegex: RegExp | null; + private _multilineStartRegex: RegExp | null; + + constructor(results: any, settings: ImportConfig) { super(settings); - this.excludeLinesRegex = + this._excludeLinesRegex = results.exclude_lines_pattern === undefined ? null : new RegExp(results.exclude_lines_pattern); - this.multilineStartRegex = + this._multilineStartRegex = results.multiline_start_pattern === undefined ? null : new RegExp(results.multiline_start_pattern); @@ -26,9 +30,9 @@ export class MessageImporter extends Importer { // multiline_start_pattern regex // if it does, it is a legitimate end of line and can be pushed into the list, // if not, it must be a newline char inside a field value, so keep looking. - read(text) { + read(text: string) { try { - const data = []; + const data: Doc[] = []; let message = ''; let line = ''; @@ -57,14 +61,13 @@ export class MessageImporter extends Importer { data.shift(); } - this.data = data; - this.docArray = this.data; + this._docArray = data; return { success: true, }; } catch (error) { - console.error(error); + // console.error(error); return { success: false, error, @@ -72,9 +75,9 @@ export class MessageImporter extends Importer { } } - processLine(data, message, line) { - if (this.excludeLinesRegex === null || line.match(this.excludeLinesRegex) === null) { - if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) { + processLine(data: Doc[], message: string, line: string) { + if (this._excludeLinesRegex === null || line.match(this._excludeLinesRegex) === null) { + if (this._multilineStartRegex === null || line.match(this._multilineStartRegex) !== null) { this.addMessage(data, message); message = ''; } else if (data.length === 0) { @@ -90,7 +93,7 @@ export class MessageImporter extends Importer { return message; } - addMessage(data, message) { + addMessage(data: Doc[], message: string) { // if the message ended \r\n (Windows line endings) // then omit the \r as well as the \n for consistency message = message.replace(/\r$/, ''); diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts similarity index 80% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts index 887bf1a41200ad..e93c70bbf1d84e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts @@ -4,18 +4,18 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Importer } from './importer'; +import { Importer, ImportConfig } from './importer'; export class NdjsonImporter extends Importer { - constructor(results, settings) { + constructor(results: any, settings: ImportConfig) { super(settings); } - read(json) { + read(json: string) { try { const splitJson = json.split(/}\s*\n/); - const ndjson = []; + const ndjson: any[] = []; for (let i = 0; i < splitJson.length; i++) { if (splitJson[i] !== '') { // note the extra } at the end of the line, adding back @@ -24,7 +24,7 @@ export class NdjsonImporter extends Importer { } } - this.docArray = ndjson; + this._docArray = ndjson; return { success: true, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts similarity index 100% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/index.ts diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js deleted file mode 100644 index f8a90c87b9dc8d..00000000000000 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/overrides.js +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -export const DEFAULT_LINES_TO_SAMPLE = 1000; - -export const overrideDefaults = { - timestampFormat: undefined, - timestampField: undefined, - format: undefined, - delimiter: undefined, - quote: undefined, - hasHeaderRow: undefined, - charset: undefined, - columnNames: undefined, - shouldTrimFields: undefined, - grokPattern: undefined, - linesToSample: undefined, -}; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts similarity index 79% rename from x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js rename to x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts index 39cd25ba87d8c1..5048065ae60fa8 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/utils/utils.ts @@ -4,11 +4,27 @@ * you may not use this file except in compliance with the Elastic License. */ -import { overrideDefaults, DEFAULT_LINES_TO_SAMPLE } from './overrides'; import { isEqual } from 'lodash'; import { ml } from '../../../../services/ml_api_service'; - -export function readFile(file) { +import { AnalysisResult, InputOverrides } from '../../../../../../common/types/file_datavisualizer'; + +const DEFAULT_LINES_TO_SAMPLE = 1000; + +const overrideDefaults = { + timestampFormat: undefined, + timestampField: undefined, + format: undefined, + delimiter: undefined, + quote: undefined, + hasHeaderRow: undefined, + charset: undefined, + columnNames: undefined, + shouldTrimFields: undefined, + grokPattern: undefined, + linesToSample: undefined, +}; + +export function readFile(file: File) { return new Promise((resolve, reject) => { if (file && file.size) { const reader = new FileReader(); @@ -23,14 +39,14 @@ export function readFile(file) { resolve({ data }); } }; - })(file); + })(); } else { reject(); } }); } -export function reduceData(data, mb) { +export function reduceData(data: string, mb: number) { // assuming ascii characters in the file where 1 char is 1 byte // TODO - change this when other non UTF-8 formats are // supported for the read data @@ -38,8 +54,8 @@ export function reduceData(data, mb) { return data.length >= size ? data.slice(0, size) : data; } -export function createUrlOverrides(overrides, originalSettings) { - const formattedOverrides = {}; +export function createUrlOverrides(overrides: InputOverrides, originalSettings: InputOverrides) { + const formattedOverrides: InputOverrides = {}; for (const o in overrideDefaults) { if (overrideDefaults.hasOwnProperty(o)) { let value = overrides[o]; @@ -93,15 +109,15 @@ export function createUrlOverrides(overrides, originalSettings) { return formattedOverrides; } -export function processResults(results) { +export function processResults({ results, overrides }: AnalysisResult) { const timestampFormat = results.java_timestamp_formats !== undefined && results.java_timestamp_formats.length ? results.java_timestamp_formats[0] : undefined; const linesToSample = - results.overrides !== undefined && results.overrides.lines_to_sample !== undefined - ? results.overrides.lines_to_sample + overrides !== undefined && overrides.lines_to_sample !== undefined + ? overrides.lines_to_sample : DEFAULT_LINES_TO_SAMPLE; return { @@ -125,8 +141,8 @@ export function processResults(results) { * @param {string} indexName * @returns {Promise} */ -export async function hasImportPermission(indexName) { - const priv = { +export async function hasImportPermission(indexName: string) { + const priv: { cluster: string[]; index?: any } = { cluster: ['cluster:monitor/nodes/info', 'cluster:admin/ingest/pipeline/put'], }; diff --git a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts index 9b492530d303d2..20332546d9cdec 100644 --- a/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts +++ b/x-pack/plugins/ml/public/application/services/ml_api_service/datavisualizer.ts @@ -7,6 +7,7 @@ import { http } from '../http_service'; import { basePath } from './index'; +import { ImportResponse } from '../../../../common/types/file_datavisualizer'; export const fileDatavisualizer = { analyzeFile(file: string, params: Record = {}) { @@ -27,7 +28,7 @@ export const fileDatavisualizer = { mappings, ingestPipeline, }: { - id: string; + id: string | undefined; index: string; data: any; settings: any; @@ -43,7 +44,7 @@ export const fileDatavisualizer = { ingestPipeline, }); - return http({ + return http({ path: `${basePath()}/file_data_visualizer/import`, method: 'POST', query, diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts index c6c0c373646b3f..d53378b886a995 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -5,25 +5,14 @@ */ import { APICaller } from 'kibana/server'; -import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer'; +import { + AnalysisResult, + FormattedOverrides, + InputOverrides, +} from '../../../common/types/file_datavisualizer'; export type InputData = any[]; -export interface InputOverrides { - [key: string]: string; -} - -export type FormattedOverrides = InputOverrides & { - column_names: string[]; - has_header_row: boolean; - should_trim_fields: boolean; -}; - -export interface AnalysisResult { - results: FindFileStructureResponse; - overrides?: FormattedOverrides; -} - export function fileDataVisualizerProvider(callAsCurrentUser: APICaller) { async function analyzeFile(data: any, overrides: any): Promise { const results = await callAsCurrentUser('ml.fileStructure', { diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts index ab8c702cbb12ad..bae51a335e28f7 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -6,39 +6,24 @@ import { APICaller } from 'kibana/server'; import { INDEX_META_DATA_CREATED_BY } from '../../../common/constants/file_datavisualizer'; +import { + ImportResponse, + ImportFailure, + Settings, + Mappings, + IngestPipeline, +} from '../../../common/types/file_datavisualizer'; import { InputData } from './file_data_visualizer'; -export interface Settings { - pipeline?: string; - index: string; - body: any[]; - [key: string]: any; -} - -export interface Mappings { - [key: string]: any; -} - -export interface InjectPipeline { - id: string; - pipeline: any; -} - -interface Failure { - item: number; - reason: string; - doc: any; -} - export function importDataProvider(callAsCurrentUser: APICaller) { async function importData( id: string, index: string, settings: Settings, mappings: Mappings, - ingestPipeline: InjectPipeline, + ingestPipeline: IngestPipeline, data: InputData - ) { + ): Promise { let createdIndex; let createdPipelineId; const docCount = data.length; @@ -66,7 +51,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { createdPipelineId = pipelineId; } - let failures: Failure[] = []; + let failures: ImportFailure[] = []; if (data.length) { const resp = await indexData(index, createdPipelineId, data); if (resp.success === false) { @@ -144,7 +129,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { }; } } catch (error) { - let failures: Failure[] = []; + let failures: ImportFailure[] = []; let ingestError = false; if (error.errors !== undefined && Array.isArray(error.items)) { // an expected error where some or all of the bulk request @@ -169,7 +154,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { return await callAsCurrentUser('ingest.putPipeline', { id, body: pipeline }); } - function getFailures(items: any[], data: InputData): Failure[] { + function getFailures(items: any[], data: InputData): ImportFailure[] { const failures = []; for (let i = 0; i < items.length; i++) { const item = items[i]; diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts index 94529dc111696e..f8a27fdcd7e1ae 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/index.ts @@ -4,11 +4,6 @@ * you may not use this file except in compliance with the Elastic License. */ -export { - fileDataVisualizerProvider, - InputOverrides, - InputData, - AnalysisResult, -} from './file_data_visualizer'; +export { fileDataVisualizerProvider, InputData } from './file_data_visualizer'; -export { importDataProvider, Settings, InjectPipeline, Mappings } from './import_data'; +export { importDataProvider } from './import_data'; diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts index fcfd6e121c9f16..138f94ac987458 100644 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts @@ -7,15 +7,17 @@ import { schema } from '@kbn/config-schema'; import { RequestHandlerContext } from 'kibana/server'; import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; -import { wrapError } from '../client/error_wrapper'; import { InputOverrides, + Settings, + IngestPipeline, + Mappings, +} from '../../common/types/file_datavisualizer'; +import { wrapError } from '../client/error_wrapper'; +import { InputData, fileDataVisualizerProvider, importDataProvider, - Settings, - InjectPipeline, - Mappings, } from '../models/file_data_visualizer'; import { RouteInitialization } from '../types'; @@ -32,7 +34,7 @@ function importData( index: string, settings: Settings, mappings: Mappings, - ingestPipeline: InjectPipeline, + ingestPipeline: IngestPipeline, data: InputData ) { const { importData: importDataFunc } = importDataProvider(context.ml!.mlClient.callAsCurrentUser); From 3538c914efb7ea6f703d58d97bb6a9887b3d199e Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Apr 2020 15:14:27 +0100 Subject: [PATCH 3/7] changing function format --- .../experimental_badge/experimental_badge.tsx | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx index cd82416b344c33..0d2cd2dcdb5d06 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/experimental_badge/experimental_badge.tsx @@ -9,17 +9,19 @@ import React, { FC } from 'react'; import { EuiBetaBadge } from '@elastic/eui'; -export const ExperimentalBadge: FC<{ tooltipContent: string }> = ({ tooltipContent }) => ( - - - } - tooltipContent={tooltipContent} - /> - -); +export const ExperimentalBadge: FC<{ tooltipContent: string }> = ({ tooltipContent }) => { + return ( + + + } + tooltipContent={tooltipContent} + /> + + ); +}; From 09c4d8eb1621bffaa91b71e5f714e2e0c308e002 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Fri, 3 Apr 2020 15:32:20 +0100 Subject: [PATCH 4/7] fixing some types --- x-pack/plugins/ml/common/types/file_datavisualizer.ts | 10 ++++++++-- .../file_based/components/import_errors/errors.tsx | 10 +++++----- .../file_based/components/import_settings/advanced.tsx | 1 - .../components/import_view/importer/importer.ts | 3 ++- .../import_view/importer/importer_factory.ts | 7 ++++++- .../import_view/importer/message_importer.ts | 7 +++++-- .../components/import_view/importer/ndjson_importer.ts | 3 ++- .../server/models/file_data_visualizer/import_data.ts | 4 ++-- .../plugins/ml/server/routes/file_data_visualizer.ts | 4 ++-- 9 files changed, 32 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/plugins/ml/common/types/file_datavisualizer.ts index 88ca915c94addc..f771547b978114 100644 --- a/x-pack/plugins/ml/common/types/file_datavisualizer.ts +++ b/x-pack/plugins/ml/common/types/file_datavisualizer.ts @@ -46,6 +46,7 @@ export interface FindFileStructureResponse { explanation?: string[]; grok_pattern?: string; multiline_start_pattern?: string; + exclude_lines_pattern?: string; java_timestamp_formats?: string[]; joda_timestamp_formats?: string[]; timestamp_field?: string; @@ -84,7 +85,12 @@ export interface Mappings { [key: string]: any; } -export interface IngestPipeline { +export interface IngestPipelineWrapper { id: string; - pipeline: any; + pipeline: IngestPipeline; +} + +export interface IngestPipeline { + description: string; + processors: any[]; } diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx index d1e8d138f6c3ed..f723ad1a752bf4 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_errors/errors.tsx @@ -12,6 +12,11 @@ import { EuiCallOut, EuiAccordion } from '@elastic/eui'; import { IMPORT_STATUS, Statuses } from '../import_progress'; +interface ImportError { + msg: string; + more?: string; +} + interface Props { errors: any[]; statuses: Statuses; @@ -112,11 +117,6 @@ function ImportError(error: any, key: number) { ); } -interface ImportError { - msg: string; - more?: string; -} - function toString(error: any): ImportError { if (typeof error === 'string') { return { msg: error }; diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx index 570c020916f9a2..a79a7d36f32945 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_settings/advanced.tsx @@ -111,7 +111,6 @@ export const AdvancedSettings: FC = ({ defaultMessage="Index pattern name" /> } - // disabled={createIndexPattern === false || initialized === true} isInvalid={indexPatternNameError !== ''} error={[indexPatternNameError]} > diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts index 71e89affc9d73d..c97f1c147c4543 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer.ts @@ -135,6 +135,7 @@ export class Importer { retries--; } catch (err) { resp.success = false; + resp.error = err; retries = 0; } } @@ -188,7 +189,7 @@ function populateFailures(error: ImportResponse, failures: ImportFailure[], chun // But it's not sending every single field that Filebeat would add, so the ingest pipeline // cannot look for a event.timezone variable in each input record. // Therefore we need to replace {{ event.timezone }} with the actual browser timezone -function updatePipelineTimezone(ingestPipeline: any) { +function updatePipelineTimezone(ingestPipeline: IngestPipeline) { if (ingestPipeline !== undefined && ingestPipeline.processors && ingestPipeline.processors) { const dateProcessor = ingestPipeline.processors.find( (p: any) => p.date !== undefined && p.date.timezone === '{{ event.timezone }}' diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts index 8dde513aa36a58..a656b192203682 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/importer_factory.ts @@ -7,8 +7,13 @@ import { MessageImporter } from './message_importer'; import { NdjsonImporter } from './ndjson_importer'; import { ImportConfig } from './importer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; -export function importerFactory(format: string, results: any, settings: ImportConfig) { +export function importerFactory( + format: string, + results: FindFileStructureResponse, + settings: ImportConfig +) { switch (format) { // delimited and semi-structured text are both handled by splitting the // file into messages, then sending these to ES for further processing diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index ff46f570c63082..71afc58fb22844 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -5,13 +5,16 @@ */ import { Importer, ImportConfig } from './importer'; -import { Doc } from '../../../../../../../common/types/file_datavisualizer'; +import { + Doc, + FindFileStructureResponse, +} from '../../../../../../../common/types/file_datavisualizer'; export class MessageImporter extends Importer { private _excludeLinesRegex: RegExp | null; private _multilineStartRegex: RegExp | null; - constructor(results: any, settings: ImportConfig) { + constructor(results: FindFileStructureResponse, settings: ImportConfig) { super(settings); this._excludeLinesRegex = diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts index e93c70bbf1d84e..7f5f37abc5246c 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/ndjson_importer.ts @@ -5,9 +5,10 @@ */ import { Importer, ImportConfig } from './importer'; +import { FindFileStructureResponse } from '../../../../../../../common/types/file_datavisualizer'; export class NdjsonImporter extends Importer { - constructor(results: any, settings: ImportConfig) { + constructor(results: FindFileStructureResponse, settings: ImportConfig) { super(settings); } diff --git a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts index bae51a335e28f7..9d7009955124f4 100644 --- a/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts +++ b/x-pack/plugins/ml/server/models/file_data_visualizer/import_data.ts @@ -11,7 +11,7 @@ import { ImportFailure, Settings, Mappings, - IngestPipeline, + IngestPipelineWrapper, } from '../../../common/types/file_datavisualizer'; import { InputData } from './file_data_visualizer'; @@ -21,7 +21,7 @@ export function importDataProvider(callAsCurrentUser: APICaller) { index: string, settings: Settings, mappings: Mappings, - ingestPipeline: IngestPipeline, + ingestPipeline: IngestPipelineWrapper, data: InputData ): Promise { let createdIndex; diff --git a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts index 138f94ac987458..b915d13aa97206 100644 --- a/x-pack/plugins/ml/server/routes/file_data_visualizer.ts +++ b/x-pack/plugins/ml/server/routes/file_data_visualizer.ts @@ -10,7 +10,7 @@ import { MAX_BYTES } from '../../common/constants/file_datavisualizer'; import { InputOverrides, Settings, - IngestPipeline, + IngestPipelineWrapper, Mappings, } from '../../common/types/file_datavisualizer'; import { wrapError } from '../client/error_wrapper'; @@ -34,7 +34,7 @@ function importData( index: string, settings: Settings, mappings: Mappings, - ingestPipeline: IngestPipeline, + ingestPipeline: IngestPipelineWrapper, data: InputData ) { const { importData: importDataFunc } = importDataProvider(context.ml!.mlClient.callAsCurrentUser); From 5585255027cc61d94bdf14e871f881b1e9e220e4 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Sat, 4 Apr 2020 08:27:27 +0100 Subject: [PATCH 5/7] fixing translation id --- .../file_based/components/results_view/results_view.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx index 1358b42bd2d371..f9de03c119d28f 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/results_view/results_view.tsx @@ -92,7 +92,7 @@ export const ResultsView: FC = ({ showExplanationFlyout()} disabled={disableButtons}> From 60e2a552f3f051f220461622960d6e650b6bdca6 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 6 Apr 2020 14:40:12 +0100 Subject: [PATCH 6/7] fix boom error reporting --- x-pack/plugins/ml/server/client/error_wrapper.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/ml/server/client/error_wrapper.ts b/x-pack/plugins/ml/server/client/error_wrapper.ts index afce0c1af686fb..de53e4d4345a9b 100644 --- a/x-pack/plugins/ml/server/client/error_wrapper.ts +++ b/x-pack/plugins/ml/server/client/error_wrapper.ts @@ -12,7 +12,7 @@ export function wrapError(error: any): CustomHttpResponseOptions const statusCode = boom.output.statusCode; return { body: { - ...boom, + message: boom, ...(statusCode !== 500 && error.body ? { attributes: { body: error.body } } : {}), }, headers: boom.output.headers, From 799a10bcc3825ec9c816dada7f46a4acc19faf13 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 6 Apr 2020 15:19:48 +0100 Subject: [PATCH 7/7] changes based on review --- .../explanation_flyout/explanation_flyout.tsx | 11 +++++------ .../file_based/components/import_view/import_view.js | 1 - .../import_view/importer/message_importer.ts | 1 - 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx index 324dde729fbd9c..8e44a296126b61 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/explanation_flyout/explanation_flyout.tsx @@ -3,11 +3,6 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ import React, { FC } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; @@ -66,7 +61,11 @@ export const ExplanationFlyout: FC = ({ results, closeFlyout }) => { const Content: FC<{ explanation: string[] }> = ({ explanation }) => ( <> - The logical steps that have produced the analysis results. + +

    diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js index 0a58153e374dfc..4c9579bfd4b46e 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js @@ -623,7 +623,6 @@ async function createKibanaIndexPattern( id, }; } catch (error) { - console.error(error); return { success: false, error, diff --git a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts index 71afc58fb22844..7ccc5a8d673f42 100644 --- a/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts +++ b/x-pack/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.ts @@ -70,7 +70,6 @@ export class MessageImporter extends Importer { success: true, }; } catch (error) { - // console.error(error); return { success: false, error,