From d61564d5c207427cd460dcb2ffb241590c628cbc Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Thu, 20 Feb 2020 17:17:20 +0000 Subject: [PATCH 01/10] [ML] Adding filebeat config to file dataviz --- .../plugins/ml/public/application/app.tsx | 5 + .../contexts/kibana/kibana_context.ts | 2 + .../filebeat_config_flyout/filebeat_config.ts | 68 ++++++ .../filebeat_config_flyout.tsx | 116 +++++++++++ .../filebeat_config_flyout/index.ts | 7 + .../components/import_view/import_view.js | 24 +++ .../import_view/importer/message_importer.js | 5 + .../results_links/{index.js => index.ts} | 0 .../components/results_links/results_links.js | 182 ---------------- .../results_links/results_links.tsx | 194 ++++++++++++++++++ .../application/util/dependency_cache.ts | 10 + x-pack/legacy/plugins/ml/public/legacy.ts | 4 +- x-pack/legacy/plugins/ml/public/plugin.ts | 3 +- 13 files changed, 436 insertions(+), 184 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts create mode 100644 x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx create mode 100644 x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts rename x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/{index.js => index.ts} (100%) delete mode 100644 x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.js create mode 100644 x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx diff --git a/x-pack/legacy/plugins/ml/public/application/app.tsx b/x-pack/legacy/plugins/ml/public/application/app.tsx index 24cbfbfb346dd..980f8c7e6f946 100644 --- a/x-pack/legacy/plugins/ml/public/application/app.tsx +++ b/x-pack/legacy/plugins/ml/public/application/app.tsx @@ -12,6 +12,7 @@ import 'ace'; import { AppMountParameters, CoreStart } from 'kibana/public'; import { DataPublicPluginStart } from 'src/plugins/data/public'; +import { SecurityPluginSetup } from '../../../../../plugins/security/public'; import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public'; import { setDependencyCache, clearCache } from './util/dependency_cache'; @@ -20,6 +21,7 @@ import { MlRouter } from './routing'; export interface MlDependencies extends AppMountParameters { data: DataPublicPluginStart; + security: SecurityPluginSetup; __LEGACY: { XSRF: string; APP_URL: string; @@ -49,6 +51,7 @@ const App: FC = ({ coreStart, deps }) => { APP_URL: deps.__LEGACY.APP_URL, application: coreStart.application, http: coreStart.http, + security: deps.security, }); deps.onAppLeave(actions => { clearCache(); @@ -64,6 +67,7 @@ const App: FC = ({ coreStart, deps }) => { const services = { appName: 'ML', data: deps.data, + security: deps.security, ...coreStart, }; @@ -79,6 +83,7 @@ const App: FC = ({ coreStart, deps }) => { export const renderApp = (coreStart: CoreStart, depsStart: object, deps: MlDependencies) => { ReactDOM.render(, deps.element); + // const user = deps.security.authc.getCurrentUser().then(console.log); return () => ReactDOM.unmountComponentAtNode(deps.element); }; diff --git a/x-pack/legacy/plugins/ml/public/application/contexts/kibana/kibana_context.ts b/x-pack/legacy/plugins/ml/public/application/contexts/kibana/kibana_context.ts index aaf539322809b..5fcd7c5473d3b 100644 --- a/x-pack/legacy/plugins/ml/public/application/contexts/kibana/kibana_context.ts +++ b/x-pack/legacy/plugins/ml/public/application/contexts/kibana/kibana_context.ts @@ -10,9 +10,11 @@ import { useKibana, KibanaReactContextValue, } from '../../../../../../../../src/plugins/kibana_react/public'; +import { SecurityPluginSetup } from '../../../../../../../plugins/security/public'; interface StartPlugins { data: DataPublicPluginStart; + security: SecurityPluginSetup; } export type StartServices = CoreStart & StartPlugins; // eslint-disable-next-line react-hooks/rules-of-hooks diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts new file mode 100644 index 0000000000000..c5f23c33aeb6b --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export function createFilebeatConfig( + index: string, + results: any, + indexPatternId: string, + ingestPipelineId: string, + username: string | null +) { + const txt = [ + 'filebeat.inputs:', + `- type: ${index}`, + ' paths:', + " - ''", + ...getEncoding(results), + ...getExcludeLines(results), + ...getMultiline(results), + '', + ...getProcessors(results), + 'output.elasticsearch:', + ...getHosts(results), + ...getUserDetails(username), + ` index: ${index}`, + ` pipeline: ${ingestPipelineId}`, + ]; + return txt.join('\n'); +} + +function getEncoding(results: any) { + return results.charset !== 'UTF-8' ? [` encoding: ${results.charset}`] : []; +} + +function getExcludeLines(results: any) { + return results.exclude_lines_pattern !== undefined + ? [` exclude_lines: ['${results.exclude_lines_pattern}']`] + : []; +} + +function getMultiline(results: any) { + return results.multiline_start_pattern !== undefined + ? [ + ' multiline:', + ` pattern: '${results.multiline_start_pattern}'`, + ' match: after', + ' negate: true', + ] + : []; +} + +function getProcessors(results: any) { + return results.need_client_timezone === true ? ['processors:', '- add_locale: ~'] : []; +} + +function getHosts(results: any) { + return [' hosts:["test test"]']; +} + +function getUserDetails(username: string | null) { + if (username !== null) { + return [` username: '${username}'`, ' password: ']; + } else { + return []; + } +} diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx new file mode 100644 index 0000000000000..ffabeb8a732f7 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState, useEffect } from 'react'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { + EuiFlyout, + EuiFlyoutFooter, + EuiFlexGroup, + EuiFlexItem, + EuiButton, + EuiButtonEmpty, + EuiTitle, + EuiFlyoutBody, + EuiSpacer, + EuiTextArea, +} from '@elastic/eui'; +import { createFilebeatConfig } from './filebeat_config'; +import { useMlKibana } from '../../../../contexts/kibana'; + +const EDITOR_HEIGHT = '800px'; +export enum EDITOR_MODE { + HIDDEN, + READONLY, + EDITABLE, +} +interface Props { + index: string; + results: any; + indexPatternId: string; + ingestPipelineId: string; + closeFlyout(): void; +} +export const FilebeatConfigFlyout: FC = ({ + index, + results, + indexPatternId, + ingestPipelineId, + closeFlyout, +}) => { + const [fileBeatConfig, setFileBeatConfig] = useState(''); + const [username, setUsername] = useState(null); + const { + services: { security }, + } = useMlKibana(); + + useEffect(() => { + security.authc.getCurrentUser().then(user => { + setUsername(user.username); + }); + }, []); + + useEffect(() => { + const config = createFilebeatConfig(index, results, indexPatternId, ingestPipelineId, username); + setFileBeatConfig(config); + }, [username]); + + function onCopyToClipboard() {} + + return ( + + + + + + + + + + + + + + + + + + + + + + ); +}; + +const Contents: FC<{ + value: string; +}> = ({ value }) => { + return ( + + +
+ +
+ + + {}} /> + + ); +}; diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts new file mode 100644 index 0000000000000..9286b92c2ab97 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { FilebeatConfigFlyout } from './filebeat_config_flyout'; diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js index beb5918e277ae..bdfc27099a185 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js @@ -20,6 +20,7 @@ import { import { i18n } from '@kbn/i18n'; import { importerFactory } from './importer'; import { ResultsLinks } from '../results_links'; +import { FilebeatConfigFlyout } from '../filebeat_config_flyout'; import { ImportProgress, IMPORT_STATUS } from '../import_progress'; import { ImportErrors } from '../import_errors'; import { ImportSummary } from '../import_summary'; @@ -64,6 +65,7 @@ const DEFAULT_STATE = { indexNameError: '', indexPatternNameError: '', timeFieldName: undefined, + isFilebeatFlyoutVisible: false, }; export class ImportView extends Component { @@ -384,6 +386,16 @@ export class ImportView extends Component { }); }; + showFilebeatFlyout = () => { + this.setState({ isFilebeatFlyoutVisible: true }); + this.props.hideBottomBar(); + }; + + closeFilebeatFlyout = () => { + this.setState({ isFilebeatFlyoutVisible: false }); + this.props.showBottomBar(); + }; + async loadIndexNames() { const indices = await ml.getIndices(); const indexNames = indices.map(i => i.name); @@ -424,6 +436,7 @@ export class ImportView extends Component { indexNameError, indexPatternNameError, timeFieldName, + isFilebeatFlyoutVisible, } = this.state; const createPipeline = pipelineString !== ''; @@ -549,7 +562,18 @@ export class ImportView extends Component { indexPatternId={indexPatternId} timeFieldName={timeFieldName} createIndexPattern={createIndexPattern} + showFilebeatFlyout={this.showFilebeatFlyout} /> + + {isFilebeatFlyoutVisible && ( + + )} )} diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js index 840248817945a..c2d3ac69f0963 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js @@ -77,6 +77,11 @@ export class MessageImporter extends Importer { if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) { this.addMessage(data, message); message = ''; + } else if (data.length === 0) { + // discard everything before the first line that is considered the first line of a message + // as it could be left over partial data from a spilt or rolled over log, + // or could be a blank line after the header in a csv file + return ''; } else { message += '\n'; } diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/index.js b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/index.ts similarity index 100% rename from x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/index.js rename to x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/index.ts diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.js b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.js deleted file mode 100644 index aaebca2f58963..0000000000000 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.js +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { FormattedMessage } from '@kbn/i18n/react'; -import React, { Component } from 'react'; - -import { EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui'; - -import moment from 'moment'; - -import { ml } from '../../../../services/ml_api_service'; -import { isFullLicense } from '../../../../license/check_license'; -import { checkPermission } from '../../../../privilege/check_privilege'; -import { mlNodesAvailable } from '../../../../ml_nodes_check/check_ml_nodes'; -import { withKibana } from '../../../../../../../../../../src/plugins/kibana_react/public'; - -const RECHECK_DELAY_MS = 3000; - -class ResultsLinksUI extends Component { - constructor(props) { - super(props); - - this.state = { - from: 'now-30m', - to: 'now', - }; - - this.recheckTimeout = null; - this.showCreateJobLink = true; - } - - componentDidMount() { - this.showCreateJobLink = checkPermission('canCreateJob') && mlNodesAvailable(); - // if this data has a time field, - // find the start and end times - if (this.props.timeFieldName !== undefined) { - this.updateTimeValues(); - } - } - - componentWillUnmount() { - clearTimeout(this.recheckTimeout); - } - - async updateTimeValues(recheck = true) { - const { index, timeFieldName } = this.props; - - const { from, to } = await getFullTimeRange(index, timeFieldName); - this.setState({ - from: from === null ? this.state.from : from, - to: to === null ? this.state.to : to, - }); - - // these links may have been drawn too quickly for the index to be ready - // to give us the correct start and end times. - // especially if the data was small. - // so if the start and end were null, try again in 3s - // the timeout is cleared when this component unmounts. just in case the user - // resets the form or navigates away within 3s - if (recheck && (from === null || to === null)) { - this.recheckTimeout = setTimeout(() => { - this.updateTimeValues(false); - }, RECHECK_DELAY_MS); - } - } - - render() { - const { index, indexPatternId, timeFieldName, createIndexPattern } = this.props; - - const { from, to } = this.state; - - const _g = - this.props.timeFieldName !== undefined - ? `&_g=(time:(from:'${from}',mode:quick,to:'${to}'))` - : ''; - - const { basePath } = this.props.kibana.services.http; - return ( - - {createIndexPattern && ( - - } - title={ - - } - description="" - href={`${basePath.get()}/app/kibana#/discover?&_a=(index:'${indexPatternId}')${_g}`} - /> - - )} - - {isFullLicense() === true && - timeFieldName !== undefined && - this.showCreateJobLink && - createIndexPattern && ( - - } - title={ - - } - description="" - href={`#/jobs/new_job/step/job_type?index=${indexPatternId}${_g}`} - /> - - )} - - {createIndexPattern && ( - - } - title={ - - } - description="" - href={`#/jobs/new_job/datavisualizer?index=${indexPatternId}${_g}`} - /> - - )} - - - } - title={ - - } - description="" - href={`${basePath.get()}/app/kibana#/management/elasticsearch/index_management/indices/filter/${index}`} - /> - - - - } - title={ - - } - description="" - href={`${basePath.get()}/app/kibana#/management/kibana/index_patterns/${ - createIndexPattern ? indexPatternId : '' - }`} - /> - - - ); - } -} - -export const ResultsLinks = withKibana(ResultsLinksUI); - -async function getFullTimeRange(index, timeFieldName) { - const query = { bool: { must: [{ query_string: { analyze_wildcard: true, query: '*' } }] } }; - const resp = await ml.getTimeFieldRange({ - index, - timeFieldName, - query, - }); - - return { - from: moment(resp.start.epoch).toISOString(), - to: moment(resp.end.epoch).toISOString(), - }; -} diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx new file mode 100644 index 0000000000000..d7227a85028d7 --- /dev/null +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx @@ -0,0 +1,194 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { FC, useState, useEffect } from 'react'; +import moment from 'moment'; +import { FormattedMessage } from '@kbn/i18n/react'; +import { EuiFlexGroup, EuiFlexItem, EuiCard, EuiIcon } from '@elastic/eui'; +import { ml } from '../../../../services/ml_api_service'; +import { isFullLicense } from '../../../../license/check_license'; +import { checkPermission } from '../../../../privilege/check_privilege'; +import { mlNodesAvailable } from '../../../../ml_nodes_check/check_ml_nodes'; +import { useMlKibana } from '../../../../contexts/kibana'; + +const RECHECK_DELAY_MS = 3000; + +interface Props { + results: any; + index: string; + indexPatternId: string; + ingestPipelineId: string; + timeFieldName?: string; + createIndexPattern: boolean; + showFilebeatFlyout(): void; +} + +export const ResultsLinks: FC = ({ + results, + index, + indexPatternId, + ingestPipelineId, + timeFieldName, + createIndexPattern, + showFilebeatFlyout, +}) => { + const [duration, setDuration] = useState({ + from: 'now-30m', + to: 'now', + }); + const [showCreateJobLink, setShowCreateJobLink] = useState(false); + const [globalStateString, setGlobalStateString] = useState(''); + const { + services: { + http: { basePath }, + }, + } = useMlKibana(); + + useEffect(() => { + setShowCreateJobLink(checkPermission('canCreateJob') && mlNodesAvailable()); + updateTimeValues(); + }, []); + + useEffect(() => { + const _g = + timeFieldName !== undefined + ? `&_g=(time:(from:'${duration.from}',mode:quick,to:'${duration.to}'))` + : ''; + setGlobalStateString(_g); + }, [duration]); + + async function updateTimeValues(recheck = true) { + if (timeFieldName !== undefined) { + const { from, to } = await getFullTimeRange(index, timeFieldName); + setDuration({ + from: from === null ? duration.from : from, + to: to === null ? duration.to : to, + }); + + // these links may have been drawn too quickly for the index to be ready + // to give us the correct start and end times. + // especially if the data was small. + // so if the start and end were null, try again in 3s + if (recheck && (from === null || to === null)) { + setTimeout(() => { + updateTimeValues(false); + }, RECHECK_DELAY_MS); + } + } + } + + return ( + + {createIndexPattern && ( + + } + title={ + + } + description="" + href={`${basePath.get()}/app/kibana#/discover?&_a=(index:'${indexPatternId}')${globalStateString}`} + /> + + )} + + {isFullLicense() === true && + timeFieldName !== undefined && + showCreateJobLink && + createIndexPattern && ( + + } + title={ + + } + description="" + href={`#/jobs/new_job/step/job_type?index=${indexPatternId}${globalStateString}`} + /> + + )} + + {createIndexPattern && ( + + } + title={ + + } + description="" + href={`#/jobs/new_job/datavisualizer?index=${indexPatternId}${globalStateString}`} + /> + + )} + + + } + title={ + + } + description="" + href={`${basePath.get()}/app/kibana#/management/elasticsearch/index_management/indices/filter/${index}`} + /> + + + + } + title={ + + } + description="" + href={`${basePath.get()}/app/kibana#/management/kibana/index_patterns/${ + createIndexPattern ? indexPatternId : '' + }`} + /> + + + } + title={ + + } + description="" + onClick={showFilebeatFlyout} + /> + + + ); +}; + +async function getFullTimeRange(index: string, timeFieldName: string) { + const query = { bool: { must: [{ query_string: { analyze_wildcard: true, query: '*' } }] } }; + const resp = await ml.getTimeFieldRange({ + index, + timeFieldName, + query, + }); + + return { + from: moment(resp.start.epoch).toISOString(), + to: moment(resp.end.epoch).toISOString(), + }; +} diff --git a/x-pack/legacy/plugins/ml/public/application/util/dependency_cache.ts b/x-pack/legacy/plugins/ml/public/application/util/dependency_cache.ts index 8857485a58644..f837d90dba8fe 100644 --- a/x-pack/legacy/plugins/ml/public/application/util/dependency_cache.ts +++ b/x-pack/legacy/plugins/ml/public/application/util/dependency_cache.ts @@ -20,6 +20,7 @@ import { ChromeRecentlyAccessed, IBasePath, } from 'kibana/public'; +import { SecurityPluginSetup } from '../../../../../../plugins/security/public'; export interface DependencyCache { timefilter: TimefilterSetup | null; @@ -38,6 +39,7 @@ export interface DependencyCache { APP_URL: string | null; application: ApplicationStart | null; http: HttpStart | null; + security: SecurityPluginSetup | null; } const cache: DependencyCache = { @@ -57,6 +59,7 @@ const cache: DependencyCache = { APP_URL: null, application: null, http: null, + security: null, }; export function setDependencyCache(deps: Partial) { @@ -189,6 +192,13 @@ export function getHttp() { return cache.http; } +export function getSecurity() { + if (cache.security === null) { + throw new Error("security hasn't been initialized"); + } + return cache.security; +} + export function clearCache() { console.log('clearing dependency cache'); // eslint-disable-line no-console Object.keys(cache).forEach(k => { diff --git a/x-pack/legacy/plugins/ml/public/legacy.ts b/x-pack/legacy/plugins/ml/public/legacy.ts index bf431f0986d68..40a1afa06b5a6 100644 --- a/x-pack/legacy/plugins/ml/public/legacy.ts +++ b/x-pack/legacy/plugins/ml/public/legacy.ts @@ -6,14 +6,16 @@ import chrome from 'ui/chrome'; import { npSetup, npStart } from 'ui/new_platform'; - import { PluginInitializerContext } from 'src/core/public'; +import { SecurityPluginSetup } from '../../../../plugins/security/public'; + import { plugin } from '.'; const pluginInstance = plugin({} as PluginInitializerContext); export const setup = pluginInstance.setup(npSetup.core, { data: npStart.plugins.data, + security: ((npSetup.plugins as unknown) as { security: SecurityPluginSetup }).security, // security isn't in the PluginsSetup interface, but does exist __LEGACY: { XSRF: chrome.getXsrfToken(), // @ts-ignore getAppUrl is missing from chrome's definition diff --git a/x-pack/legacy/plugins/ml/public/plugin.ts b/x-pack/legacy/plugins/ml/public/plugin.ts index 79af300bce4ec..cb39b31a32b14 100644 --- a/x-pack/legacy/plugins/ml/public/plugin.ts +++ b/x-pack/legacy/plugins/ml/public/plugin.ts @@ -8,7 +8,7 @@ import { Plugin, CoreStart, CoreSetup } from 'src/core/public'; import { MlDependencies } from './application/app'; export class MlPlugin implements Plugin { - setup(core: CoreSetup, { data, __LEGACY }: MlDependencies) { + setup(core: CoreSetup, { data, security, __LEGACY }: MlDependencies) { core.application.register({ id: 'ml', title: 'Machine learning', @@ -21,6 +21,7 @@ export class MlPlugin implements Plugin { onAppLeave: params.onAppLeave, data, __LEGACY, + security, }); }, }); From 7c2d51ad7ad31664b6861dc86e5de8cea3d48f15 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 12:42:19 +0000 Subject: [PATCH 02/10] adding extra help text --- .../filebeat_config_flyout/filebeat_config.ts | 13 ++-- .../filebeat_config_flyout.tsx | 75 +++++++++++++++---- 2 files changed, 66 insertions(+), 22 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index c5f23c33aeb6b..3bae5745af38b 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -11,7 +11,7 @@ export function createFilebeatConfig( ingestPipelineId: string, username: string | null ) { - const txt = [ + return [ 'filebeat.inputs:', `- type: ${index}`, ' paths:', @@ -24,10 +24,9 @@ export function createFilebeatConfig( 'output.elasticsearch:', ...getHosts(results), ...getUserDetails(username), - ` index: ${index}`, - ` pipeline: ${ingestPipelineId}`, - ]; - return txt.join('\n'); + ` index: "${index}"`, + ` pipeline: "${ingestPipelineId}"`, + ].join('\n'); } function getEncoding(results: any) { @@ -56,12 +55,12 @@ function getProcessors(results: any) { } function getHosts(results: any) { - return [' hosts:["test test"]']; + return [' hosts:[""]']; } function getUserDetails(username: string | null) { if (username !== null) { - return [` username: '${username}'`, ' password: ']; + return [` username: "${username}"`, ' password: ""']; } else { return []; } diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx index ffabeb8a732f7..d4590fbc1232b 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx @@ -21,12 +21,13 @@ import { EuiTitle, EuiFlyoutBody, EuiSpacer, - EuiTextArea, + EuiCodeBlock, + EuiCode, + EuiCopy, } from '@elastic/eui'; import { createFilebeatConfig } from './filebeat_config'; import { useMlKibana } from '../../../../contexts/kibana'; -const EDITOR_HEIGHT = '800px'; export enum EDITOR_MODE { HIDDEN, READONLY, @@ -54,7 +55,7 @@ export const FilebeatConfigFlyout: FC = ({ useEffect(() => { security.authc.getCurrentUser().then(user => { - setUsername(user.username); + setUsername(user.username === undefined ? null : user.username); }); }, []); @@ -63,13 +64,11 @@ export const FilebeatConfigFlyout: FC = ({ setFileBeatConfig(config); }, [username]); - function onCopyToClipboard() {} - return ( - + @@ -83,12 +82,16 @@ export const FilebeatConfigFlyout: FC = ({ - - - + + {copy => ( + + + + )} + @@ -98,19 +101,61 @@ export const FilebeatConfigFlyout: FC = ({ const Contents: FC<{ value: string; -}> = ({ value }) => { + index: string; + username: string | null; +}> = ({ value, index, username }) => { return (
- {}} /> +

+ {index} }} + /> +

+

+ filebeat.yml }} + /> +

+ + + + {value} + + +

+ {username === null ? ( + {''}, + }} + /> + ) : ( + {username}, + password: {''}, + esUrl: {''}, + }} + /> + )} +

); }; From 8ba2cb617e5d1fe5685b96317843892af852db47 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 13:11:59 +0000 Subject: [PATCH 03/10] removing commented out code --- x-pack/legacy/plugins/ml/public/application/app.tsx | 1 - .../filebeat_config_flyout/filebeat_config_flyout.tsx | 2 +- .../file_based/components/results_links/results_links.tsx | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/app.tsx b/x-pack/legacy/plugins/ml/public/application/app.tsx index 980f8c7e6f946..add27193deb77 100644 --- a/x-pack/legacy/plugins/ml/public/application/app.tsx +++ b/x-pack/legacy/plugins/ml/public/application/app.tsx @@ -83,7 +83,6 @@ const App: FC = ({ coreStart, deps }) => { export const renderApp = (coreStart: CoreStart, depsStart: object, deps: MlDependencies) => { ReactDOM.render(, deps.element); - // const user = deps.security.authc.getCurrentUser().then(console.log); return () => ReactDOM.unmountComponentAtNode(deps.element); }; diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx index d4590fbc1232b..ff7d94ae0d36d 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx @@ -110,7 +110,7 @@ const Contents: FC<{
diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx index d7227a85028d7..777b9478b352b 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx @@ -168,7 +168,7 @@ export const ResultsLinks: FC = ({ title={ } description="" From a14d95d86f7aa39848ee81d66603e6309cd0bcb0 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 13:14:49 +0000 Subject: [PATCH 04/10] adding extra blank line to processors section --- .../components/filebeat_config_flyout/filebeat_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index 3bae5745af38b..a8930f19d2cb5 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -51,7 +51,7 @@ function getMultiline(results: any) { } function getProcessors(results: any) { - return results.need_client_timezone === true ? ['processors:', '- add_locale: ~'] : []; + return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : []; } function getHosts(results: any) { From f4d37ff2371f9c573dd6ae57212ff367b63aad27 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 13:33:59 +0000 Subject: [PATCH 05/10] cleaning up types --- .../ml/common/types/file_datavisualizer.ts | 31 +++++++++++++++++++ .../filebeat_config_flyout/filebeat_config.ts | 9 +++--- .../filebeat_config_flyout.tsx | 5 +-- .../results_links/results_links.tsx | 4 --- .../file_data_visualizer.ts | 27 ++-------------- 5 files changed, 41 insertions(+), 35 deletions(-) create mode 100644 x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts diff --git a/x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts b/x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts new file mode 100644 index 0000000000000..bc03f82673a1f --- /dev/null +++ b/x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface FindFileStructureResponse { + charset: string; + has_header_row: boolean; + has_byte_order_marker: boolean; + format: string; + field_stats: { + [fieldName: string]: { + count: number; + cardinality: number; + top_hits: Array<{ count: number; value: any }>; + }; + }; + sample_start: string; + num_messages_analyzed: number; + mappings: { + [fieldName: string]: { + type: string; + }; + }; + quote: string; + delimiter: string; + need_client_timezone: boolean; + num_lines_analyzed: number; + column_names: string[]; +} diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index a8930f19d2cb5..007388835ce22 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -4,10 +4,11 @@ * you may not use this file except in compliance with the Elastic License. */ +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; + export function createFilebeatConfig( index: string, - results: any, - indexPatternId: string, + results: FindFileStructureResponse, ingestPipelineId: string, username: string | null ) { @@ -22,7 +23,7 @@ export function createFilebeatConfig( '', ...getProcessors(results), 'output.elasticsearch:', - ...getHosts(results), + ...getHosts(), ...getUserDetails(username), ` index: "${index}"`, ` pipeline: "${ingestPipelineId}"`, @@ -54,7 +55,7 @@ function getProcessors(results: any) { return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : []; } -function getHosts(results: any) { +function getHosts() { return [' hosts:[""]']; } diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx index ff7d94ae0d36d..30fc74acbabf4 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx @@ -27,6 +27,7 @@ import { } from '@elastic/eui'; import { createFilebeatConfig } from './filebeat_config'; import { useMlKibana } from '../../../../contexts/kibana'; +import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; export enum EDITOR_MODE { HIDDEN, @@ -35,7 +36,7 @@ export enum EDITOR_MODE { } interface Props { index: string; - results: any; + results: FindFileStructureResponse; indexPatternId: string; ingestPipelineId: string; closeFlyout(): void; @@ -60,7 +61,7 @@ export const FilebeatConfigFlyout: FC = ({ }, []); useEffect(() => { - const config = createFilebeatConfig(index, results, indexPatternId, ingestPipelineId, username); + const config = createFilebeatConfig(index, results, ingestPipelineId, username); setFileBeatConfig(config); }, [username]); diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx index 777b9478b352b..debadba19051b 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/results_links/results_links.tsx @@ -17,20 +17,16 @@ import { useMlKibana } from '../../../../contexts/kibana'; const RECHECK_DELAY_MS = 3000; interface Props { - results: any; index: string; indexPatternId: string; - ingestPipelineId: string; timeFieldName?: string; createIndexPattern: boolean; showFilebeatFlyout(): void; } export const ResultsLinks: FC = ({ - results, index, indexPatternId, - ingestPipelineId, timeFieldName, createIndexPattern, showFilebeatFlyout, diff --git a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts index fd5b5221393fc..9f30f609c60b6 100644 --- a/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts +++ b/x-pack/legacy/plugins/ml/server/models/file_data_visualizer/file_data_visualizer.ts @@ -6,6 +6,7 @@ import Boom from 'boom'; import { RequestHandlerContext } from 'kibana/server'; +import { FindFileStructureResponse } from '../../../common/types/file_datavisualizer'; export type InputData = any[]; @@ -20,31 +21,7 @@ export type FormattedOverrides = InputOverrides & { }; export interface AnalysisResult { - results: { - charset: string; - has_header_row: boolean; - has_byte_order_marker: boolean; - format: string; - field_stats: { - [fieldName: string]: { - count: number; - cardinality: number; - top_hits: Array<{ count: number; value: any }>; - }; - }; - sample_start: string; - num_messages_analyzed: number; - mappings: { - [fieldName: string]: { - type: string; - }; - }; - quote: string; - delimiter: string; - need_client_timezone: boolean; - num_lines_analyzed: number; - column_names: string[]; - }; + results: FindFileStructureResponse; overrides?: FormattedOverrides; } From a58c8917f970ac4def8262f39c5c735a9f209ff6 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 13:39:35 +0000 Subject: [PATCH 06/10] moving hosts line out of function --- .../components/filebeat_config_flyout/filebeat_config.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index 007388835ce22..3428496ebf8f2 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -23,7 +23,7 @@ export function createFilebeatConfig( '', ...getProcessors(results), 'output.elasticsearch:', - ...getHosts(), + ' hosts:[""]', ...getUserDetails(username), ` index: "${index}"`, ` pipeline: "${ingestPipelineId}"`, @@ -55,10 +55,6 @@ function getProcessors(results: any) { return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : []; } -function getHosts() { - return [' hosts:[""]']; -} - function getUserDetails(username: string | null) { if (username !== null) { return [` username: "${username}"`, ' password: ""']; From 8ab3b4d139b0a929c174431e986ec4bc291e4280 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 17:20:27 +0000 Subject: [PATCH 07/10] typo in config text --- .../components/filebeat_config_flyout/filebeat_config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index 3428496ebf8f2..fbcb3bc9e25da 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -23,7 +23,7 @@ export function createFilebeatConfig( '', ...getProcessors(results), 'output.elasticsearch:', - ' hosts:[""]', + ' hosts: [""]', ...getUserDetails(username), ` index: "${index}"`, ` pipeline: "${ingestPipelineId}"`, From 96de6a22e4484f4ddb9f0b01db4e59713a4408db Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 17:46:21 +0000 Subject: [PATCH 08/10] updating config based on review --- .../filebeat_config_flyout/filebeat_config.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index fbcb3bc9e25da..e30265c3fef07 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -14,7 +14,7 @@ export function createFilebeatConfig( ) { return [ 'filebeat.inputs:', - `- type: ${index}`, + '- type: log', ' paths:', " - ''", ...getEncoding(results), @@ -27,6 +27,10 @@ export function createFilebeatConfig( ...getUserDetails(username), ` index: "${index}"`, ` pipeline: "${ingestPipelineId}"`, + '', + 'setup:', + ' template.enabled: false', + ' ilm.enabled: false', ].join('\n'); } @@ -36,7 +40,7 @@ function getEncoding(results: any) { function getExcludeLines(results: any) { return results.exclude_lines_pattern !== undefined - ? [` exclude_lines: ['${results.exclude_lines_pattern}']`] + ? [` exclude_lines: ['${results.exclude_lines_pattern.replace(/'/g, "''")}']`] : []; } @@ -44,7 +48,7 @@ function getMultiline(results: any) { return results.multiline_start_pattern !== undefined ? [ ' multiline:', - ` pattern: '${results.multiline_start_pattern}'`, + ` pattern: '${results.multiline_start_pattern.replace(/'/g, "''")}'`, ' match: after', ' negate: true', ] From aa37f4ef3b0cbd654a4aea0880b76cd282dd474a Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Mon, 24 Feb 2020 18:45:23 +0000 Subject: [PATCH 09/10] tiny refactor --- .../components/filebeat_config_flyout/filebeat_config.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index e30265c3fef07..d7423f8719a17 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -60,9 +60,5 @@ function getProcessors(results: any) { } function getUserDetails(username: string | null) { - if (username !== null) { - return [` username: "${username}"`, ' password: ""']; - } else { - return []; - } + return username !== null ? [` username: "${username}"`, ' password: ""'] : []; } From 91ad89dbdc4c02f8a32936b83e1ce549208ee886 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 25 Feb 2020 11:45:14 +0000 Subject: [PATCH 10/10] translating paths text --- .../filebeat_config_flyout/filebeat_config.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts index d7423f8719a17..3344cdf991e6b 100644 --- a/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts +++ b/x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { i18n } from '@kbn/i18n'; import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer'; export function createFilebeatConfig( @@ -15,8 +16,7 @@ export function createFilebeatConfig( return [ 'filebeat.inputs:', '- type: log', - ' paths:', - " - ''", + ...getPaths(), ...getEncoding(results), ...getExcludeLines(results), ...getMultiline(results), @@ -34,6 +34,13 @@ export function createFilebeatConfig( ].join('\n'); } +function getPaths() { + const txt = i18n.translate('xpack.ml.fileDatavisualizer.fileBeatConfig.paths', { + defaultMessage: 'add path to your files here', + }); + return [' paths:', ` - '<${txt}>'`]; +} + function getEncoding(results: any) { return results.charset !== 'UTF-8' ? [` encoding: ${results.charset}`] : []; }