[Ingest] Add support for yaml field types (#60440)

* Support yaml var type: * Change stream config model to save type and value, instead of just value * Add code editor for configuring yaml vars * Adjust tests * Account for empty yaml value * Better account for invalid yaml parsing
elastic · Mar 18, 2020 · 60d385e · 60d385e
1 parent 2d44870
commit 60d385e
Show file tree

Hide file tree

Showing 9 changed files with 158 additions and 57 deletions.
diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.test.ts
@@ -3,7 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
-import { NewDatasource } from '../types';
+import { NewDatasource, DatasourceInput } from '../types';
 import { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
 
 describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
@@ -17,21 +17,37 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
     inputs: [],
   };
 
-  const mockInput = {
+  const mockInput: DatasourceInput = {
     type: 'test-logs',
     enabled: true,
     streams: [
       {
         id: 'test-logs-foo',
         enabled: true,
         dataset: 'foo',
-        config: { fooVar: 'foo-value', fooVar2: [1, 2] },
+        config: { fooVar: { value: 'foo-value' }, fooVar2: { value: [1, 2] } },
       },
       {
         id: 'test-logs-bar',
         enabled: false,
         dataset: 'bar',
-        config: { barVar: 'bar-value', barVar2: [1, 2] },
+        config: {
+          barVar: { value: 'bar-value' },
+          barVar2: { value: [1, 2] },
+          barVar3: {
+            type: 'yaml',
+            value:
+              '- namespace: mockNamespace\n  #disabledProp: ["test"]\n  anotherProp: test\n- namespace: mockNamespace2\n  #disabledProp: ["test2"]\n  anotherProp: test2',
+          },
+          barVar4: {
+            type: 'yaml',
+            value: '',
+          },
+          barVar5: {
+            type: 'yaml',
+            value: 'testField: test\n invalidSpacing: foo',
+          },
+        },
       },
     ],
   };
@@ -91,6 +107,16 @@ describe('Ingest Manager - storedDatasourceToAgentDatasource', () => {
               dataset: 'bar',
               barVar: 'bar-value',
               barVar2: [1, 2],
+              barVar3: [
+                {
+                  namespace: 'mockNamespace',
+                  anotherProp: 'test',
+                },
+                {
+                  namespace: 'mockNamespace2',
+                  anotherProp: 'test2',
+                },
+              ],
             },
           ],
         },

diff --git a/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts b/x-pack/plugins/ingest_manager/common/services/datasource_to_agent_datasource.ts
@@ -3,6 +3,7 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { safeLoad } from 'js-yaml';
 import { Datasource, NewDatasource, FullAgentConfigDatasource } from '../types';
 import { DEFAULT_OUTPUT } from '../constants';
 
@@ -23,12 +24,26 @@ export const storedDatasourceToAgentDatasource = (
           if (stream.config) {
             const fullStream = {
               ...stream,
-              ...Object.entries(stream.config).reduce((acc, [configName, configValue]) => {
-                if (configValue !== undefined) {
-                  acc[configName] = configValue;
-                }
-                return acc;
-              }, {} as { [key: string]: any }),
+              ...Object.entries(stream.config).reduce(
+                (acc, [configName, { type: configType, value: configValue }]) => {
+                  if (configValue !== undefined) {
+                    if (configType === 'yaml') {
+                      try {
+                        const yamlValue = safeLoad(configValue);
+                        if (yamlValue) {
+                          acc[configName] = yamlValue;
+                        }
+                      } catch (e) {
+                        // Silently swallow parsing error
+                      }
+                    } else {
+                      acc[configName] = configValue;
+                    }
+                  }
+                  return acc;
+                },
+                {} as { [key: string]: any }
+              ),
             };
             delete fullStream.config;
             return fullStream;

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.test.ts
@@ -108,8 +108,14 @@ describe('Ingest Manager - packageToConfig', () => {
                 {
                   type: 'bar',
                   streams: [
-                    { dataset: 'bar', vars: [{ default: 'bar-var-value', name: 'var-name' }] },
-                    { dataset: 'bar2', vars: [{ default: 'bar2-var-value', name: 'var-name' }] },
+                    {
+                      dataset: 'bar',
+                      vars: [{ default: 'bar-var-value', name: 'var-name', type: 'text' }],
+                    },
+                    {
+                      dataset: 'bar2',
+                      vars: [{ default: 'bar2-var-value', name: 'var-name', type: 'yaml' }],
+                    },
                   ],
                 },
               ],
@@ -125,7 +131,7 @@ describe('Ingest Manager - packageToConfig', () => {
               id: 'foo-foo',
               enabled: true,
               dataset: 'foo',
-              config: { 'var-name': 'foo-var-value' },
+              config: { 'var-name': { value: 'foo-var-value' } },
             },
           ],
         },
@@ -137,13 +143,13 @@ describe('Ingest Manager - packageToConfig', () => {
               id: 'bar-bar',
               enabled: true,
               dataset: 'bar',
-              config: { 'var-name': 'bar-var-value' },
+              config: { 'var-name': { type: 'text', value: 'bar-var-value' } },
             },
             {
               id: 'bar-bar2',
               enabled: true,
               dataset: 'bar2',
-              config: { 'var-name': 'bar2-var-value' },
+              config: { 'var-name': { type: 'yaml', value: 'bar2-var-value' } },
             },
           ],
         },
@@ -204,10 +210,10 @@ describe('Ingest Manager - packageToConfig', () => {
               enabled: true,
               dataset: 'foo',
               config: {
-                'var-name': 'foo-var-value',
-                'foo-input-var-name': 'foo-input-var-value',
-                'foo-input2-var-name': 'foo-input2-var-value',
-                'foo-input3-var-name': undefined,
+                'var-name': { value: 'foo-var-value' },
+                'foo-input-var-name': { value: 'foo-input-var-value' },
+                'foo-input2-var-name': { value: 'foo-input2-var-value' },
+                'foo-input3-var-name': { value: undefined },
               },
             },
           ],
@@ -221,19 +227,19 @@ describe('Ingest Manager - packageToConfig', () => {
               enabled: true,
               dataset: 'bar',
               config: {
-                'var-name': 'bar-var-value',
-                'bar-input-var-name': ['value1', 'value2'],
-                'bar-input2-var-name': 123456,
+                'var-name': { value: 'bar-var-value' },
+                'bar-input-var-name': { value: ['value1', 'value2'] },
+                'bar-input2-var-name': { value: 123456 },
               },
             },
             {
               id: 'bar-bar2',
               enabled: true,
               dataset: 'bar2',
               config: {
-                'var-name': 'bar2-var-value',
-                'bar-input-var-name': ['value1', 'value2'],
-                'bar-input2-var-name': 123456,
+                'var-name': { value: 'bar2-var-value' },
+                'bar-input-var-name': { value: ['value1', 'value2'] },
+                'bar-input2-var-name': { value: 123456 },
               },
             },
           ],
@@ -247,7 +253,7 @@ describe('Ingest Manager - packageToConfig', () => {
               enabled: false,
               dataset: 'disabled',
               config: {
-                'var-name': [],
+                'var-name': { value: [] },
               },
             },
             {

diff --git a/x-pack/plugins/ingest_manager/common/services/package_to_config.ts b/x-pack/plugins/ingest_manager/common/services/package_to_config.ts
@@ -41,9 +41,9 @@ export const packageToConfigDatasourceInputs = (packageInfo: PackageInfo): Datas
               streamVar: RegistryVarsEntry
             ): DatasourceInputStream['config'] => {
               if (!streamVar.default && streamVar.multi) {
-                configObject![streamVar.name] = [];
+                configObject![streamVar.name] = { type: streamVar.type, value: [] };
               } else {
-                configObject![streamVar.name] = streamVar.default;
+                configObject![streamVar.name] = { type: streamVar.type, value: streamVar.default };
               }
               return configObject;
             };

diff --git a/x-pack/plugins/ingest_manager/common/types/models/datasource.ts b/x-pack/plugins/ingest_manager/common/types/models/datasource.ts
@@ -15,7 +15,13 @@ export interface DatasourceInputStream {
   enabled: boolean;
   dataset: string;
   processors?: string[];
-  config?: Record<string, any>;
+  config?: Record<
+    string,
+    {
+      type?: string;
+      value: any;
+    }
+  >;
 }
 
 export interface DatasourceInput {

diff --git a/...nager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx b/...nager/sections/agent_config/create_datasource_page/components/datasource_input_config.tsx
@@ -63,8 +63,8 @@ export const DatasourceInputConfig: React.FunctionComponent<{
       <EuiFlexItem>
         <EuiFlexGroup direction="column" gutterSize="m">
           {requiredVars.map(varDef => {
-            const varName = varDef.name;
-            const value = datasourceInput.streams[0].config![varName];
+            const { name: varName, type: varType } = varDef;
+            const value = datasourceInput.streams[0].config![varName].value;
             return (
               <EuiFlexItem key={varName}>
                 <DatasourceInputVarField
@@ -76,7 +76,10 @@ export const DatasourceInputConfig: React.FunctionComponent<{
                         ...stream,
                         config: {
                           ...stream.config,
-                          [varName]: newValue,
+                          [varName]: {
+                            type: varType,
+                            value: newValue,
+                          },
                         },
                       })),
                     });
@@ -105,8 +108,8 @@ export const DatasourceInputConfig: React.FunctionComponent<{
               </EuiFlexItem>
               {isShowingAdvanced
                 ? advancedVars.map(varDef => {
-                    const varName = varDef.name;
-                    const value = datasourceInput.streams[0].config![varName];
+                    const { name: varName, type: varType } = varDef;
+                    const value = datasourceInput.streams[0].config![varName].value;
                     return (
                       <EuiFlexItem key={varName}>
                         <DatasourceInputVarField
@@ -118,7 +121,10 @@ export const DatasourceInputConfig: React.FunctionComponent<{
                                 ...stream,
                                 config: {
                                   ...stream.config,
-                                  [varName]: newValue,
+                                  [varName]: {
+                                    type: varType,
+                                    value: newValue,
+                                  },
                                 },
                               })),
                             });

diff --git a/...ections/agent_config/create_datasource_page/components/datasource_input_stream_config.tsx b/...ections/agent_config/create_datasource_page/components/datasource_input_stream_config.tsx
@@ -65,8 +65,8 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
       <EuiFlexItem>
         <EuiFlexGroup direction="column" gutterSize="m">
           {requiredVars.map(varDef => {
-            const varName = varDef.name;
-            const value = datasourceInputStream.config![varName];
+            const { name: varName, type: varType } = varDef;
+            const value = datasourceInputStream.config![varName].value;
             return (
               <EuiFlexItem key={varName}>
                 <DatasourceInputVarField
@@ -76,7 +76,10 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
                     updateDatasourceInputStream({
                       config: {
                         ...datasourceInputStream.config,
-                        [varName]: newValue,
+                        [varName]: {
+                          type: varType,
+                          value: newValue,
+                        },
                       },
                     });
                   }}
@@ -104,8 +107,8 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
               </EuiFlexItem>
               {isShowingAdvanced
                 ? advancedVars.map(varDef => {
-                    const varName = varDef.name;
-                    const value = datasourceInputStream.config![varName];
+                    const { name: varName, type: varType } = varDef;
+                    const value = datasourceInputStream.config![varName].value;
                     return (
                       <EuiFlexItem key={varName}>
                         <DatasourceInputVarField
@@ -115,7 +118,10 @@ export const DatasourceInputStreamConfig: React.FunctionComponent<{
                             updateDatasourceInputStream({
                               config: {
                                 ...datasourceInputStream.config,
-                                [varName]: newValue,
+                                [varName]: {
+                                  type: varType,
+                                  value: newValue,
+                                },
                               },
                             });
                           }}

diff --git a/...er/sections/agent_config/create_datasource_page/components/datasource_input_var_field.tsx b/...er/sections/agent_config/create_datasource_page/components/datasource_input_var_field.tsx
@@ -6,14 +6,57 @@
 import React from 'react';
 import ReactMarkdown from 'react-markdown';
 import { FormattedMessage } from '@kbn/i18n/react';
-import { EuiFormRow, EuiFieldText, EuiComboBox, EuiText } from '@elastic/eui';
+import { EuiFormRow, EuiFieldText, EuiComboBox, EuiText, EuiCodeEditor } from '@elastic/eui';
 import { RegistryVarsEntry } from '../../../../types';
 
+import 'brace/mode/yaml';
+import 'brace/theme/textmate';
+
 export const DatasourceInputVarField: React.FunctionComponent<{
   varDef: RegistryVarsEntry;
   value: any;
   onChange: (newValue: any) => void;
 }> = ({ varDef, value, onChange }) => {
+  const renderField = () => {
+    if (varDef.multi) {
+      return (
+        <EuiComboBox
+          noSuggestions
+          selectedOptions={value.map((val: string) => ({ label: val }))}
+          onCreateOption={(newVal: any) => {
+            onChange([...value, newVal]);
+          }}
+          onChange={(newVals: any[]) => {
+            onChange(newVals.map(val => val.label));
+          }}
+        />
+      );
+    }
+    if (varDef.type === 'yaml') {
+      return (
+        <EuiCodeEditor
+          width="100%"
+          mode="yaml"
+          theme="textmate"
+          setOptions={{
+            minLines: 10,
+            maxLines: 30,
+            tabSize: 2,
+            showGutter: false,
+          }}
+          value={value}
+          onChange={newVal => onChange(newVal)}
+        />
+      );
+    }
+    return (
+      <EuiFieldText
+        value={value === undefined ? '' : value}
+        onChange={e => onChange(e.target.value)}
+      />
+    );
+  };
+
   return (
     <EuiFormRow
       label={varDef.title || varDef.name}
@@ -29,20 +72,7 @@ export const DatasourceInputVarField: React.FunctionComponent<{
       }
       helpText={<ReactMarkdown source={varDef.description} />}
     >
-      {varDef.multi ? (
-        <EuiComboBox
-          noSuggestions
-          selectedOptions={value.map((val: string) => ({ label: val }))}
-          onCreateOption={(newVal: any) => {
-            onChange([...value, newVal]);
-          }}
-          onChange={(newVals: any[]) => {
-            onChange(newVals.map(val => val.label));
-          }}
-        />
-      ) : (
-        <EuiFieldText value={value} onChange={e => onChange(e.target.value)} />
-      )}
+      {renderField()}
     </EuiFormRow>
   );
 };