Merge branch 'master' into alerting/disable-actions-plugin-ESO-tmp-key

docs/siem/siem-ui.asciidoc

@@ -33,6 +33,23 @@ investigation.
 [role="screenshot"]
 image::siem/images/network-ui.png[]
 
+[float]
+[[detections-ui]]
+=== Detections
+
+The Detections feature automatically searches for threats and creates 
+signals when they are detected. Signal detection rules define the conditions 
+for creating signals. The SIEM app comes with prebuilt rules that search for 
+suspicious activity on your network and hosts. Additionally, you can
+create your own rules.
+
+See {siem-guide}/detection-engine-overview.html[Detections] in the SIEM 
+Guide for information on managing detection rules and signals via the UI 
+or the Detections API.
+
+[role="screenshot"]
+image::siem/images/detections-ui.png[]
+
 [float]
 [[timelines-ui]]
 === Timeline

src/legacy/core_plugins/data/public/search/aggs/agg_config.ts

@@ -58,6 +58,11 @@ const unknownSchema: Schema = {
   defaults: {},
   editor: false,
   group: AggGroupNames.Metrics,
+  aggSettings: {
+    top_hits: {
+      allowStrings: true,
+    },
+  },
 };
 
 const getTypeFromRegistry = (type: string): IAggType => {
@@ -438,7 +443,7 @@ export class AggConfig {
 
     if (fieldParam) {
       // @ts-ignore
-      availableFields = fieldParam.getAvailableFields(this.getIndexPattern().fields);
+      availableFields = fieldParam.getAvailableFields(this);
     }
 
     // clear out the previous params except for a few special ones

src/legacy/core_plugins/data/public/search/aggs/metrics/metric_agg_type.ts

@@ -24,14 +24,15 @@ import { AggParamType } from '../param_types/agg';
 import { AggConfig } from '../agg_config';
 import { METRIC_TYPES } from './metric_agg_types';
 import { KBN_FIELD_TYPES } from '../../../../../../../plugins/data/public';
+import { FilterFieldTypes } from '../param_types/field';
 
 export interface IMetricAggConfig extends AggConfig {
   type: InstanceType<typeof MetricAggType>;
 }
 
 export interface MetricAggParam<TMetricAggConfig extends AggConfig>
   extends AggParamType<TMetricAggConfig> {
-  filterFieldTypes?: KBN_FIELD_TYPES | KBN_FIELD_TYPES[] | '*';
+  filterFieldTypes?: FilterFieldTypes;
   onlyAggregatable?: boolean;
 }
 

src/legacy/core_plugins/data/public/search/aggs/metrics/top_hit.ts

@@ -20,7 +20,6 @@
 import _ from 'lodash';
 import { i18n } from '@kbn/i18n';
 import { IMetricAggConfig, MetricAggType } from './metric_agg_type';
-import { aggTypeFieldFilters } from '../param_types/filter';
 import { METRIC_TYPES } from './metric_agg_types';
 import { KBN_FIELD_TYPES } from '../../../../../../../plugins/data/public';
 
@@ -33,17 +32,6 @@ const isNumericFieldSelected = (agg: IMetricAggConfig) => {
   return field && field.type && field.type === KBN_FIELD_TYPES.NUMBER;
 };
 
-aggTypeFieldFilters.addFilter((field, aggConfig) => {
-  if (
-    aggConfig.type.name !== METRIC_TYPES.TOP_HITS ||
-    _.get(aggConfig.schema, 'aggSettings.top_hits.allowStrings', false)
-  ) {
-    return true;
-  }
-
-  return field.type === KBN_FIELD_TYPES.NUMBER;
-});
-
 export const topHitMetricAgg = new MetricAggType({
   name: METRIC_TYPES.TOP_HITS,
   title: i18n.translate('data.search.aggs.metrics.topHitTitle', {
@@ -75,7 +63,10 @@ export const topHitMetricAgg = new MetricAggType({
       name: 'field',
       type: 'field',
       onlyAggregatable: false,
-      filterFieldTypes: '*',
+      filterFieldTypes: (aggConfig: IMetricAggConfig) =>
+        _.get(aggConfig.schema, 'aggSettings.top_hits.allowStrings', false)
+          ? '*'
+          : KBN_FIELD_TYPES.NUMBER,
       write(agg, output) {
         const field = agg.getParam('field');
         output.params = {};

src/legacy/core_plugins/data/public/search/aggs/param_types/field.test.ts

@@ -17,9 +17,13 @@
  * under the License.
  */
 
+import { get } from 'lodash';
 import { BaseParamType } from './base';
 import { FieldParamType } from './field';
 import { ES_FIELD_TYPES, KBN_FIELD_TYPES } from '../../../../../../../plugins/data/public';
+import { IAggConfig } from '../agg_config';
+import { IMetricAggConfig } from '../metrics/metric_agg_type';
+import { Schema } from '../schemas';
 
 jest.mock('ui/new_platform');
 
@@ -45,7 +49,11 @@ describe('Field', () => {
         searchable: true,
       },
     ],
-  } as any;
+  };
+
+  const agg = ({
+    getIndexPattern: jest.fn(() => indexPattern),
+  } as unknown) as IAggConfig;
 
   describe('constructor', () => {
     it('it is an instance of BaseParamType', () => {
@@ -65,7 +73,7 @@ describe('Field', () => {
         type: 'field',
       });
 
-      const fields = aggParam.getAvailableFields(indexPattern.fields);
+      const fields = aggParam.getAvailableFields(agg);
 
       expect(fields.length).toBe(1);
 
@@ -82,7 +90,58 @@ describe('Field', () => {
 
       aggParam.onlyAggregatable = false;
 
-      const fields = aggParam.getAvailableFields(indexPattern.fields);
+      const fields = aggParam.getAvailableFields(agg);
+
+      expect(fields.length).toBe(2);
+    });
+
+    it('should return all fields if filterFieldTypes was not specified', () => {
+      const aggParam = new FieldParamType({
+        name: 'field',
+        type: 'field',
+      });
+
+      indexPattern.fields[1].aggregatable = true;
+
+      const fields = aggParam.getAvailableFields(agg);
+
+      expect(fields.length).toBe(2);
+    });
+
+    it('should return only numeric fields if filterFieldTypes was specified as a function', () => {
+      const aggParam = new FieldParamType({
+        name: 'field',
+        type: 'field',
+        filterFieldTypes: (aggConfig: IMetricAggConfig) =>
+          get(aggConfig.schema, 'aggSettings.top_hits.allowStrings', false)
+            ? '*'
+            : KBN_FIELD_TYPES.NUMBER,
+      });
+      const fields = aggParam.getAvailableFields(agg);
+
+      expect(fields.length).toBe(1);
+      expect(fields[0].type).toBe(KBN_FIELD_TYPES.NUMBER);
+    });
+
+    it('should return all fields if filterFieldTypes was specified as a function and aggSettings allow string type fields', () => {
+      const aggParam = new FieldParamType({
+        name: 'field',
+        type: 'field',
+        filterFieldTypes: (aggConfig: IMetricAggConfig) =>
+          get(aggConfig.schema, 'aggSettings.top_hits.allowStrings', false)
+            ? '*'
+            : KBN_FIELD_TYPES.NUMBER,
+      });
+
+      agg.schema = {
+        aggSettings: {
+          top_hits: {
+            allowStrings: true,
+          },
+        },
+      } as Schema;
+
+      const fields = aggParam.getAvailableFields(agg);
 
       expect(fields.length).toBe(2);
     });

src/legacy/core_plugins/data/public/search/aggs/param_types/field.ts

@@ -17,24 +17,27 @@
  * under the License.
  */
 
-// @ts-ignore
 import { i18n } from '@kbn/i18n';
+import { isFunction } from 'lodash';
 import { npStart } from 'ui/new_platform';
-import { AggConfig } from '../agg_config';
+import { IAggConfig } from '../agg_config';
 import { SavedObjectNotFound } from '../../../../../../../plugins/kibana_utils/public';
 import { BaseParamType } from './base';
 import { propFilter } from '../filter';
-import { Field, IFieldList, isNestedField } from '../../../../../../../plugins/data/public';
+import { IMetricAggConfig } from '../metrics/metric_agg_type';
+import { Field, isNestedField, KBN_FIELD_TYPES } from '../../../../../../../plugins/data/public';
 
 const filterByType = propFilter('type');
 
+type FieldTypes = KBN_FIELD_TYPES | KBN_FIELD_TYPES[] | '*';
+export type FilterFieldTypes = ((aggConfig: IMetricAggConfig) => FieldTypes) | FieldTypes;
 // TODO need to make a more explicit interface for this
 export type IFieldParamType = FieldParamType;
 
 export class FieldParamType extends BaseParamType {
   required = true;
   scriptable = true;
-  filterFieldTypes: string;
+  filterFieldTypes: FilterFieldTypes;
   onlyAggregatable: boolean;
 
   constructor(config: Record<string, any>) {
@@ -44,7 +47,7 @@ export class FieldParamType extends BaseParamType {
     this.onlyAggregatable = config.onlyAggregatable !== false;
 
     if (!config.write) {
-      this.write = (aggConfig: AggConfig, output: Record<string, any>) => {
+      this.write = (aggConfig: IAggConfig, output: Record<string, any>) => {
         const field = aggConfig.getField();
 
         if (!field) {
@@ -73,7 +76,7 @@ export class FieldParamType extends BaseParamType {
       return field.name;
     };
 
-    this.deserialize = (fieldName: string, aggConfig?: AggConfig) => {
+    this.deserialize = (fieldName: string, aggConfig?: IAggConfig) => {
       if (!aggConfig) {
         throw new Error('aggConfig was not provided to FieldParamType deserialize function');
       }
@@ -84,9 +87,7 @@ export class FieldParamType extends BaseParamType {
       }
 
       // @ts-ignore
-      const validField = this.getAvailableFields(aggConfig.getIndexPattern().fields).find(
-        (f: any) => f.name === fieldName
-      );
+      const validField = this.getAvailableFields(aggConfig).find((f: any) => f.name === fieldName);
       if (!validField) {
         npStart.core.notifications.toasts.addDanger(
           i18n.translate(
@@ -109,7 +110,8 @@ export class FieldParamType extends BaseParamType {
   /**
    * filter the fields to the available ones
    */
-  getAvailableFields = (fields: IFieldList) => {
+  getAvailableFields = (aggConfig: IAggConfig) => {
+    const fields = aggConfig.getIndexPattern().fields;
     const filteredFields = fields.filter((field: Field) => {
       const { onlyAggregatable, scriptable, filterFieldTypes } = this;
 
@@ -120,8 +122,10 @@ export class FieldParamType extends BaseParamType {
         return false;
       }
 
-      if (!filterFieldTypes) {
-        return true;
+      if (isFunction(filterFieldTypes)) {
+        const filter = filterFieldTypes(aggConfig as IMetricAggConfig);
+
+        return filterByType([field], filter).length !== 0;
       }
 
       return filterByType([field], filterFieldTypes).length !== 0;

src/legacy/core_plugins/kibana/index.js

@@ -97,13 +97,8 @@ export default function(kibana) {
           }),
           order: -1001,
           url: `${kbnBaseUrl}#/dashboards`,
-          // The subUrlBase is the common substring of all urls for this app. If not given, it defaults to the url
-          // above. This app has to use a different subUrlBase, in addition to the url above, because "#/dashboard"
-          // routes to a page that creates a new dashboard. When we introduced a landing page, we needed to change
-          // the url above in order to preserve the original url for BWC. The subUrlBase helps the Chrome api nav
-          // to determine what url to use for the app link.
-          subUrlBase: `${kbnBaseUrl}#/dashboard`,
           euiIconType: 'dashboardApp',
+          disableSubUrlTracking: true,
           category: DEFAULT_APP_CATEGORIES.analyze,
         },
         {

src/legacy/core_plugins/kibana/public/.eslintrc.js

@@ -17,8 +17,15 @@
  * under the License.
  */
 
+const topLevelConfig = require('../../../../../.eslintrc.js');
 const path = require('path');
 
+const topLevelRestricedZones = topLevelConfig.overrides.find(
+  override =>
+    override.files[0] === '**/*.{js,ts,tsx}' &&
+    Object.keys(override.rules)[0] === '@kbn/eslint/no-restricted-paths'
+).rules['@kbn/eslint/no-restricted-paths'][1].zones;
+
 /**
  * Builds custom restricted paths configuration for the shimmed plugins within the kibana plugin.
  * These custom rules extend the default checks in the top level `eslintrc.js` by also checking two other things:
@@ -28,34 +35,37 @@ const path = require('path');
  * @returns zones configuration for the no-restricted-paths linter
  */
 function buildRestrictedPaths(shimmedPlugins) {
-  return shimmedPlugins.map(shimmedPlugin => ([{
-    target: [
-      `src/legacy/core_plugins/kibana/public/${shimmedPlugin}/np_ready/**/*`,
-    ],
-    from: [
-      'ui/**/*',
-      'src/legacy/ui/**/*',
-      'src/legacy/core_plugins/kibana/public/**/*',
-      'src/legacy/core_plugins/data/public/**/*',
-      '!src/legacy/core_plugins/data/public/index.ts',
-      `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
-    ],
-    allowSameFolder: false,
-    errorMessage: `${shimmedPlugin} is a shimmed plugin that is not allowed to import modules from the legacy platform. If you need legacy modules for the transition period, import them either in the legacy_imports, kibana_services or index module.`,
-  }, {
-    target: [
-      'src/**/*',
-      `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
-      'x-pack/**/*',
-    ],
-    from: [
-      `src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
-      `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/index.ts`,
-      `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/legacy.ts`,
-    ],
-    allowSameFolder: false,
-    errorMessage: `kibana/public/${shimmedPlugin} is behaving like a NP plugin and does not allow deep imports. If you need something from within ${shimmedPlugin} in another plugin, consider re-exporting it from the top level index module`,
-  }])).reduce((acc, part) => [...acc, ...part], []);
+  return shimmedPlugins
+    .map(shimmedPlugin => [
+      {
+        target: [`src/legacy/core_plugins/kibana/public/${shimmedPlugin}/np_ready/**/*`],
+        from: [
+          'ui/**/*',
+          'src/legacy/ui/**/*',
+          'src/legacy/core_plugins/kibana/public/**/*',
+          'src/legacy/core_plugins/data/public/**/*',
+          '!src/legacy/core_plugins/data/public/index.ts',
+          `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
+        ],
+        allowSameFolder: false,
+        errorMessage: `${shimmedPlugin} is a shimmed plugin that is not allowed to import modules from the legacy platform. If you need legacy modules for the transition period, import them either in the legacy_imports, kibana_services or index module.`,
+      },
+      {
+        target: [
+          'src/**/*',
+          `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
+          'x-pack/**/*',
+        ],
+        from: [
+          `src/legacy/core_plugins/kibana/public/${shimmedPlugin}/**/*`,
+          `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/index.ts`,
+          `!src/legacy/core_plugins/kibana/public/${shimmedPlugin}/legacy.ts`,
+        ],
+        allowSameFolder: false,
+        errorMessage: `kibana/public/${shimmedPlugin} is behaving like a NP plugin and does not allow deep imports. If you need something from within ${shimmedPlugin} in another plugin, consider re-exporting it from the top level index module`,
+      },
+    ])
+    .reduce((acc, part) => [...acc, ...part], []);
 }
 
 module.exports = {
@@ -66,7 +76,9 @@ module.exports = {
       'error',
       {
         basePath: path.resolve(__dirname, '../../../../../'),
-        zones: buildRestrictedPaths(['visualize', 'discover', 'dashboard', 'devTools', 'home']),
+        zones: topLevelRestricedZones.concat(
+          buildRestrictedPaths(['visualize', 'discover', 'dashboard', 'devTools', 'home'])
+        ),
       },
     ],
   },

src/legacy/core_plugins/kibana/public/dashboard/legacy.ts

@@ -41,6 +41,7 @@ async function getAngularDependencies(): Promise<LegacyAngularInjectedDependenci
   const instance = plugin({} as PluginInitializerContext);
   instance.setup(npSetup.core, {
     ...npSetup.plugins,
+    npData: npSetup.plugins.data,
     __LEGACY: {
       getAngularDependencies,
     },