[ML] Adding filebeat config to file dataviz (elastic#58152)

* [ML] Adding filebeat config to file dataviz

* adding extra help text

* removing commented out code

* adding extra blank line to processors section

* cleaning up types

* moving hosts line out of function

* typo in config text

* updating config based on review

* tiny refactor

* translating paths text

x-pack/legacy/plugins/ml/common/types/file_datavisualizer.ts

@@ -0,0 +1,31 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export interface FindFileStructureResponse {
+  charset: string;
+  has_header_row: boolean;
+  has_byte_order_marker: boolean;
+  format: string;
+  field_stats: {
+    [fieldName: string]: {
+      count: number;
+      cardinality: number;
+      top_hits: Array<{ count: number; value: any }>;
+    };
+  };
+  sample_start: string;
+  num_messages_analyzed: number;
+  mappings: {
+    [fieldName: string]: {
+      type: string;
+    };
+  };
+  quote: string;
+  delimiter: string;
+  need_client_timezone: boolean;
+  num_lines_analyzed: number;
+  column_names: string[];
+}

x-pack/legacy/plugins/ml/public/application/app.tsx

@@ -12,6 +12,7 @@ import 'ace';
 import { AppMountParameters, CoreStart } from 'kibana/public';
 
 import { DataPublicPluginStart } from 'src/plugins/data/public';
+import { SecurityPluginSetup } from '../../../../../plugins/security/public';
 
 import { KibanaContextProvider } from '../../../../../../src/plugins/kibana_react/public';
 import { setDependencyCache, clearCache } from './util/dependency_cache';
@@ -20,6 +21,7 @@ import { MlRouter } from './routing';
 
 export interface MlDependencies extends AppMountParameters {
   data: DataPublicPluginStart;
+  security: SecurityPluginSetup;
   __LEGACY: {
     XSRF: string;
     APP_URL: string;
@@ -49,6 +51,7 @@ const App: FC<AppProps> = ({ coreStart, deps }) => {
     APP_URL: deps.__LEGACY.APP_URL,
     application: coreStart.application,
     http: coreStart.http,
+    security: deps.security,
   });
   deps.onAppLeave(actions => {
     clearCache();
@@ -64,6 +67,7 @@ const App: FC<AppProps> = ({ coreStart, deps }) => {
   const services = {
     appName: 'ML',
     data: deps.data,
+    security: deps.security,
     ...coreStart,
   };
 

x-pack/legacy/plugins/ml/public/application/contexts/kibana/kibana_context.ts

@@ -10,9 +10,11 @@ import {
   useKibana,
   KibanaReactContextValue,
 } from '../../../../../../../../src/plugins/kibana_react/public';
+import { SecurityPluginSetup } from '../../../../../../../plugins/security/public';
 
 interface StartPlugins {
   data: DataPublicPluginStart;
+  security: SecurityPluginSetup;
 }
 export type StartServices = CoreStart & StartPlugins;
 // eslint-disable-next-line react-hooks/rules-of-hooks

x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config.ts

@@ -0,0 +1,71 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { i18n } from '@kbn/i18n';
+import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+
+export function createFilebeatConfig(
+  index: string,
+  results: FindFileStructureResponse,
+  ingestPipelineId: string,
+  username: string | null
+) {
+  return [
+    'filebeat.inputs:',
+    '- type: log',
+    ...getPaths(),
+    ...getEncoding(results),
+    ...getExcludeLines(results),
+    ...getMultiline(results),
+    '',
+    ...getProcessors(results),
+    'output.elasticsearch:',
+    '  hosts: ["<es_url>"]',
+    ...getUserDetails(username),
+    `  index: "${index}"`,
+    `  pipeline: "${ingestPipelineId}"`,
+    '',
+    'setup:',
+    '  template.enabled: false',
+    '  ilm.enabled: false',
+  ].join('\n');
+}
+
+function getPaths() {
+  const txt = i18n.translate('xpack.ml.fileDatavisualizer.fileBeatConfig.paths', {
+    defaultMessage: 'add path to your files here',
+  });
+  return ['  paths:', `  - '<${txt}>'`];
+}
+
+function getEncoding(results: any) {
+  return results.charset !== 'UTF-8' ? [`  encoding: ${results.charset}`] : [];
+}
+
+function getExcludeLines(results: any) {
+  return results.exclude_lines_pattern !== undefined
+    ? [`  exclude_lines: ['${results.exclude_lines_pattern.replace(/'/g, "''")}']`]
+    : [];
+}
+
+function getMultiline(results: any) {
+  return results.multiline_start_pattern !== undefined
+    ? [
+        '  multiline:',
+        `    pattern: '${results.multiline_start_pattern.replace(/'/g, "''")}'`,
+        '    match: after',
+        '    negate: true',
+      ]
+    : [];
+}
+
+function getProcessors(results: any) {
+  return results.need_client_timezone === true ? ['processors:', '- add_locale: ~', ''] : [];
+}
+
+function getUserDetails(username: string | null) {
+  return username !== null ? [`  username: "${username}"`, '  password: "<password>"'] : [];
+}

x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/filebeat_config_flyout.tsx

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import React, { FC, useState, useEffect } from 'react';
+import { FormattedMessage } from '@kbn/i18n/react';
+import {
+  EuiFlyout,
+  EuiFlyoutFooter,
+  EuiFlexGroup,
+  EuiFlexItem,
+  EuiButton,
+  EuiButtonEmpty,
+  EuiTitle,
+  EuiFlyoutBody,
+  EuiSpacer,
+  EuiCodeBlock,
+  EuiCode,
+  EuiCopy,
+} from '@elastic/eui';
+import { createFilebeatConfig } from './filebeat_config';
+import { useMlKibana } from '../../../../contexts/kibana';
+import { FindFileStructureResponse } from '../../../../../../common/types/file_datavisualizer';
+
+export enum EDITOR_MODE {
+  HIDDEN,
+  READONLY,
+  EDITABLE,
+}
+interface Props {
+  index: string;
+  results: FindFileStructureResponse;
+  indexPatternId: string;
+  ingestPipelineId: string;
+  closeFlyout(): void;
+}
+export const FilebeatConfigFlyout: FC<Props> = ({
+  index,
+  results,
+  indexPatternId,
+  ingestPipelineId,
+  closeFlyout,
+}) => {
+  const [fileBeatConfig, setFileBeatConfig] = useState('');
+  const [username, setUsername] = useState<string | null>(null);
+  const {
+    services: { security },
+  } = useMlKibana();
+
+  useEffect(() => {
+    security.authc.getCurrentUser().then(user => {
+      setUsername(user.username === undefined ? null : user.username);
+    });
+  }, []);
+
+  useEffect(() => {
+    const config = createFilebeatConfig(index, results, ingestPipelineId, username);
+    setFileBeatConfig(config);
+  }, [username]);
+
+  return (
+    <EuiFlyout onClose={closeFlyout} hideCloseButton size={'m'}>
+      <EuiFlyoutBody>
+        <EuiFlexGroup>
+          <Contents value={fileBeatConfig} username={username} index={index} />
+        </EuiFlexGroup>
+      </EuiFlyoutBody>
+      <EuiFlyoutFooter>
+        <EuiFlexGroup justifyContent="spaceBetween">
+          <EuiFlexItem grow={false}>
+            <EuiButtonEmpty iconType="cross" onClick={closeFlyout} flush="left">
+              <FormattedMessage
+                id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.closeButton"
+                defaultMessage="Close"
+              />
+            </EuiButtonEmpty>
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <EuiCopy textToCopy={fileBeatConfig}>
+              {copy => (
+                <EuiButton onClick={copy}>
+                  <FormattedMessage
+                    id="xpack.ml.fileDatavisualizer.fileBeatConfigFlyout.copyButton"
+                    defaultMessage="Copy to clipboard"
+                  />
+                </EuiButton>
+              )}
+            </EuiCopy>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiFlyoutFooter>
+    </EuiFlyout>
+  );
+};
+
+const Contents: FC<{
+  value: string;
+  index: string;
+  username: string | null;
+}> = ({ value, index, username }) => {
+  return (
+    <EuiFlexItem>
+      <EuiTitle size="s">
+        <h5>
+          <FormattedMessage
+            id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTitle"
+            defaultMessage="Filebeat configuration"
+          />
+        </h5>
+      </EuiTitle>
+      <EuiSpacer size="s" />
+      <p>
+        <FormattedMessage
+          id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText1"
+          defaultMessage="Additional data can be uploaded to the {index} index using Filebeat."
+          values={{ index: <EuiCode>{index}</EuiCode> }}
+        />
+      </p>
+      <p>
+        <FormattedMessage
+          id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigTopText2"
+          defaultMessage="Modify {filebeatYml} to set the connection information:"
+          values={{ filebeatYml: <EuiCode>filebeat.yml</EuiCode> }}
+        />
+      </p>
+
+      <EuiSpacer size="s" />
+
+      <EuiCodeBlock language="bash">{value}</EuiCodeBlock>
+
+      <EuiSpacer size="s" />
+      <p>
+        {username === null ? (
+          <FormattedMessage
+            id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomTextNoUsername"
+            defaultMessage="Where {esUrl} is the URL of Elasticsearch."
+            values={{
+              esUrl: <EuiCode>{'<es_url>'}</EuiCode>,
+            }}
+          />
+        ) : (
+          <FormattedMessage
+            id="xpack.ml.fileDatavisualizer.resultsLinks.fileBeatConfigBottomText"
+            defaultMessage="Where {password} is the password of the {user} user, {esUrl} is the URL of Elasticsearch."
+            values={{
+              user: <EuiCode>{username}</EuiCode>,
+              password: <EuiCode>{'<password>'}</EuiCode>,
+              esUrl: <EuiCode>{'<es_url>'}</EuiCode>,
+            }}
+          />
+        )}
+      </p>
+    </EuiFlexItem>
+  );
+};

x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/filebeat_config_flyout/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export { FilebeatConfigFlyout } from './filebeat_config_flyout';

x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/import_view.js

@@ -20,6 +20,7 @@ import {
 import { i18n } from '@kbn/i18n';
 import { importerFactory } from './importer';
 import { ResultsLinks } from '../results_links';
+import { FilebeatConfigFlyout } from '../filebeat_config_flyout';
 import { ImportProgress, IMPORT_STATUS } from '../import_progress';
 import { ImportErrors } from '../import_errors';
 import { ImportSummary } from '../import_summary';
@@ -64,6 +65,7 @@ const DEFAULT_STATE = {
   indexNameError: '',
   indexPatternNameError: '',
   timeFieldName: undefined,
+  isFilebeatFlyoutVisible: false,
 };
 
 export class ImportView extends Component {
@@ -384,6 +386,16 @@ export class ImportView extends Component {
     });
   };
 
+  showFilebeatFlyout = () => {
+    this.setState({ isFilebeatFlyoutVisible: true });
+    this.props.hideBottomBar();
+  };
+
+  closeFilebeatFlyout = () => {
+    this.setState({ isFilebeatFlyoutVisible: false });
+    this.props.showBottomBar();
+  };
+
   async loadIndexNames() {
     const indices = await ml.getIndices();
     const indexNames = indices.map(i => i.name);
@@ -424,6 +436,7 @@ export class ImportView extends Component {
       indexNameError,
       indexPatternNameError,
       timeFieldName,
+      isFilebeatFlyoutVisible,
     } = this.state;
 
     const createPipeline = pipelineString !== '';
@@ -549,7 +562,18 @@ export class ImportView extends Component {
                       indexPatternId={indexPatternId}
                       timeFieldName={timeFieldName}
                       createIndexPattern={createIndexPattern}
+                      showFilebeatFlyout={this.showFilebeatFlyout}
                     />
+
+                    {isFilebeatFlyoutVisible && (
+                      <FilebeatConfigFlyout
+                        index={index}
+                        results={this.props.results}
+                        indexPatternId={indexPatternId}
+                        ingestPipelineId={ingestPipelineId}
+                        closeFlyout={this.closeFilebeatFlyout}
+                      />
+                    )}
                   </React.Fragment>
                 )}
               </EuiPanel>

x-pack/legacy/plugins/ml/public/application/datavisualizer/file_based/components/import_view/importer/message_importer.js

@@ -77,6 +77,11 @@ export class MessageImporter extends Importer {
       if (this.multilineStartRegex === null || line.match(this.multilineStartRegex) !== null) {
         this.addMessage(data, message);
         message = '';
+      } else if (data.length === 0) {
+        // discard everything before the first line that is considered the first line of a message
+        // as it could be left over partial data from a spilt or rolled over log,
+        // or could be a blank line after the header in a csv file
+        return '';
       } else {
         message += '\n';
       }