[ML] Adding missing script fields to advanced field selects (#48812)

* [ML] Adding missing script fields to advanced field selects * refactoring select option creation * refactoring select option creation * reverting accidental change * refactoring based on review
elastic · Oct 23, 2019 · 6dd1b61 · 6dd1b61
1 parent 3d17dd3
commit 6dd1b61
Show file tree

Hide file tree

Showing 13 changed files with 217 additions and 103 deletions.
diff --git a/x-pack/legacy/plugins/ml/common/types/fields.ts b/x-pack/legacy/plugins/ml/common/types/fields.ts
@@ -10,6 +10,7 @@ import {
   KIBANA_AGGREGATION,
   ES_AGGREGATION,
 } from '../../common/constants/aggregation_types';
+import { MLCATEGORY } from '../../common/constants/field_types';
 
 export const EVENT_RATE_FIELD_ID = '__ml_event_rate_count__';
 export const METRIC_AGG_TYPE = 'metrics';
@@ -81,3 +82,10 @@ export interface AggFieldNamePair {
   };
   excludeFrequent?: string;
 }
+
+export const mlCategory: Field = {
+  id: MLCATEGORY,
+  name: MLCATEGORY,
+  type: ES_FIELD_TYPES.KEYWORD,
+  aggregatable: false,
+};
diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/advanced_job_creator.ts b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/advanced_job_creator.ts
@@ -179,7 +179,7 @@ export class AdvancedJobCreator extends JobCreator {
 
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
-    const detectors = getRichDetectors(job, datafeed, true);
+    const detectors = getRichDetectors(job, datafeed, this.scriptFields, true);
 
     // keep track of the custom rules for each detector
     const customRules = this._detectors.map(d => d.custom_rules);

diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/job_creator.ts b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/job_creator.ts
@@ -8,6 +8,7 @@ import { SavedSearch } from 'src/legacy/core_plugins/kibana/public/discover/type
 import { IndexPattern } from 'ui/index_patterns';
 import { IndexPatternTitle } from '../../../../../common/types/kibana';
 import { ML_JOB_AGGREGATION } from '../../../../../common/constants/aggregation_types';
+import { ES_FIELD_TYPES } from '../../../../../../../../../src/plugins/data/common';
 import { Job, Datafeed, Detector, JobId, DatafeedId, BucketSpan } from './configs';
 import { Aggregation, Field } from '../../../../../common/types/fields';
 import { createEmptyJob, createEmptyDatafeed } from './util/default_configs';
@@ -33,6 +34,7 @@ export class JobCreator {
   protected _subscribers: ProgressSubscriber[] = [];
   protected _aggs: Aggregation[] = [];
   protected _fields: Field[] = [];
+  protected _scriptFields: Field[] = [];
   protected _sparseData: boolean = false;
   private _stopAllRefreshPolls: {
     stop: boolean;
@@ -413,6 +415,14 @@ export class JobCreator {
     }
   }
 
+  public get indices(): string[] {
+    return this._datafeed_config.indices;
+  }
+
+  public get scriptFields(): Field[] {
+    return this._scriptFields;
+  }
+
   public get subscribers(): ProgressSubscriber[] {
     return this._subscribers;
   }
@@ -549,5 +559,16 @@ export class JobCreator {
       this.useDedicatedIndex = true;
     }
     this._sparseData = isSparseDataJob(job, datafeed);
+
+    if (this._datafeed_config.script_fields !== undefined) {
+      this._scriptFields = Object.keys(this._datafeed_config.script_fields).map(f => ({
+        id: f,
+        name: f,
+        type: ES_FIELD_TYPES.KEYWORD,
+        aggregatable: true,
+      }));
+    } else {
+      this._scriptFields = [];
+    }
   }
 }
diff --git a/.../legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/multi_metric_job_creator.ts b/.../legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/multi_metric_job_creator.ts
@@ -141,7 +141,7 @@ export class MultiMetricJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.MULTI_METRIC;
-    const detectors = getRichDetectors(job, datafeed, false);
+    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
 
     if (datafeed.aggregations !== undefined) {
       // if we've converting from a single metric job,

diff --git a/...ck/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/population_job_creator.ts b/...ck/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/population_job_creator.ts
@@ -126,7 +126,7 @@ export class PopulationJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.POPULATION;
-    const detectors = getRichDetectors(job, datafeed, false);
+    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
 
     this.removeAllDetectors();
 

diff --git a/...legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/single_metric_job_creator.ts b/...legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/single_metric_job_creator.ts
@@ -186,7 +186,7 @@ export class SingleMetricJobCreator extends JobCreator {
   public cloneFromExistingJob(job: Job, datafeed: Datafeed) {
     this._overrideConfigs(job, datafeed);
     this.createdBy = CREATED_BY_LABEL.SINGLE_METRIC;
-    const detectors = getRichDetectors(job, datafeed, false);
+    const detectors = getRichDetectors(job, datafeed, this.scriptFields, false);
 
     this.removeAllDetectors();
 

diff --git a/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/util/general.ts b/x-pack/legacy/plugins/ml/public/jobs/new_job_new/common/job_creator/util/general.ts
@@ -11,32 +11,99 @@ import {
   ML_JOB_AGGREGATION,
   SPARSE_DATA_AGGREGATIONS,
 } from '../../../../../../common/constants/aggregation_types';
-import { EVENT_RATE_FIELD_ID, AggFieldPair } from '../../../../../../common/types/fields';
+import { MLCATEGORY } from '../../../../../../common/constants/field_types';
+import {
+  EVENT_RATE_FIELD_ID,
+  Field,
+  AggFieldPair,
+  mlCategory,
+} from '../../../../../../common/types/fields';
 import { mlJobService } from '../../../../../services/job_service';
 import { JobCreatorType, isMultiMetricJobCreator, isPopulationJobCreator } from '../';
 import { CREATED_BY_LABEL, JOB_TYPE } from './constants';
 
+const getFieldByIdFactory = (scriptFields: Field[]) => (id: string) => {
+  let field = newJobCapsService.getFieldById(id);
+  // if no field could be found it may be a pretend field, like mlcategory or a script field
+  if (field === null) {
+    if (id === MLCATEGORY) {
+      field = mlCategory;
+    } else if (scriptFields.length) {
+      field = scriptFields.find(f => f.id === id) || null;
+    }
+  }
+  return field;
+};
+
 // populate the detectors with Field and Agg objects loaded from the job capabilities service
-export function getRichDetectors(job: Job, datafeed: Datafeed, advanced: boolean = false) {
+export function getRichDetectors(
+  job: Job,
+  datafeed: Datafeed,
+  scriptFields: Field[],
+  advanced: boolean = false
+) {
   const detectors = advanced ? getDetectorsAdvanced(job, datafeed) : getDetectors(job, datafeed);
+
+  const getFieldById = getFieldByIdFactory(scriptFields);
+
   return detectors.map(d => {
+    let field = null;
+    let byField = null;
+    let overField = null;
+    let partitionField = null;
+
+    if (d.field_name !== undefined) {
+      field = getFieldById(d.field_name);
+    }
+    if (d.by_field_name !== undefined) {
+      byField = getFieldById(d.by_field_name);
+    }
+    if (d.over_field_name !== undefined) {
+      overField = getFieldById(d.over_field_name);
+    }
+    if (d.partition_field_name !== undefined) {
+      partitionField = getFieldById(d.partition_field_name);
+    }
+
     return {
       agg: newJobCapsService.getAggById(d.function),
-      field: d.field_name !== undefined ? newJobCapsService.getFieldById(d.field_name) : null,
-      byField:
-        d.by_field_name !== undefined ? newJobCapsService.getFieldById(d.by_field_name) : null,
-      overField:
-        d.over_field_name !== undefined ? newJobCapsService.getFieldById(d.over_field_name) : null,
-      partitionField:
-        d.partition_field_name !== undefined
-          ? newJobCapsService.getFieldById(d.partition_field_name)
-          : null,
+      field,
+      byField,
+      overField,
+      partitionField,
       excludeFrequent: d.exclude_frequent || null,
       description: d.detector_description || null,
     };
   });
 }
 
+export function createFieldOptions(fields: Field[], filterOverride?: (f: Field) => boolean) {
+  const filter = filterOverride || (f => f.id !== EVENT_RATE_FIELD_ID);
+  return fields
+    .filter(filter)
+    .map(f => ({
+      label: f.name,
+    }))
+    .sort((a, b) => a.label.localeCompare(b.label));
+}
+
+export function createScriptFieldOptions(scriptFields: Field[]) {
+  return scriptFields.map(f => ({
+    label: f.id,
+  }));
+}
+
+export function createMlcategoryFieldOption(categorizationFieldName: string | null) {
+  if (categorizationFieldName === null) {
+    return [];
+  }
+  return [
+    {
+      label: MLCATEGORY,
+    },
+  ];
+}
+
 function getDetectorsAdvanced(job: Job, datafeed: Datafeed) {
   return processFieldlessAggs(job.analysis_config.detectors);
 }

diff --git a/...public/jobs/new_job_new/pages/components/common/json_editor_flyout/json_editor_flyout.tsx b/...public/jobs/new_job_new/pages/components/common/json_editor_flyout/json_editor_flyout.tsx
@@ -18,7 +18,7 @@ import {
   EuiFlyoutBody,
   EuiSpacer,
 } from '@elastic/eui';
-import { Job, Datafeed } from '../../../../common/job_creator/configs';
+import { Datafeed } from '../../../../common/job_creator/configs';
 import { MLJobEditor } from '../../../../../jobs_list/components/ml_job_editor';
 import { isValidJson } from '../../../../../../../common/util/validation_utils';
 import { JobCreatorContext } from '../../job_creator_context';
@@ -52,26 +52,32 @@ export const JsonEditorFlyout: FC<Props> = ({ isDisabled, jobEditorMode, datafee
   const editJsonMode =
     jobEditorMode === EDITOR_MODE.HIDDEN || datafeedEditorMode === EDITOR_MODE.HIDDEN;
   const flyOutSize = editJsonMode ? 'm' : 'l';
+  const readOnlyMode =
+    jobEditorMode === EDITOR_MODE.READONLY && datafeedEditorMode === EDITOR_MODE.READONLY;
 
   function toggleJsonFlyout() {
     setSaveable(false);
     setShowJsonFlyout(!showJsonFlyout);
   }
 
-  function updateSavable(json: string, originalConfig: Job | Datafeed) {
+  function onJobChange(json: string) {
+    setJobConfigString(json);
     const valid = isValidJson(json);
     setSaveable(valid);
   }
 
-  function onJobChange(json: string) {
-    setJobConfigString(json);
-
-    updateSavable(json, jobCreator.jobConfig);
-  }
   function onDatafeedChange(json: string) {
     setDatafeedConfigString(json);
-
-    updateSavable(json, jobCreator.datafeedConfig);
+    let valid = isValidJson(json);
+    if (valid) {
+      // ensure that the user hasn't altered the indices list in the json.
+      const { indices }: Datafeed = JSON.parse(json);
+      const originalIndices = jobCreator.indices.sort();
+      valid =
+        originalIndices.length === indices.length &&
+        originalIndices.every((value, index) => value === indices[index]);
+    }
+    setSaveable(valid);
   }
 
   function onSave() {
@@ -130,14 +136,16 @@ export const JsonEditorFlyout: FC<Props> = ({ isDisabled, jobEditorMode, datafee
                   />
                 </EuiButtonEmpty>
               </EuiFlexItem>
-              <EuiFlexItem grow={false}>
-                <EuiButton onClick={onSave} fill isDisabled={saveable === false}>
-                  <FormattedMessage
-                    id="xpack.ml.newJob.wizard.jsonFlyout.saveButton"
-                    defaultMessage="Save"
-                  />
-                </EuiButton>
-              </EuiFlexItem>
+              {readOnlyMode === false && (
+                <EuiFlexItem grow={false}>
+                  <EuiButton onClick={onSave} fill isDisabled={saveable === false}>
+                    <FormattedMessage
+                      id="xpack.ml.newJob.wizard.jsonFlyout.saveButton"
+                      defaultMessage="Save"
+                    />
+                  </EuiButton>
+                </EuiFlexItem>
+              )}
             </EuiFlexGroup>
           </EuiFlyoutFooter>
         </EuiFlyout>

diff --git a/...bs/new_job_new/pages/components/datafeed_step/components/time_field/time_field_select.tsx b/...bs/new_job_new/pages/components/datafeed_step/components/time_field/time_field_select.tsx
@@ -9,6 +9,7 @@ import { EuiComboBox, EuiComboBoxOptionProps } from '@elastic/eui';
 
 import { Field, EVENT_RATE_FIELD_ID } from '../../../../../../../../common/types/fields';
 import { ES_FIELD_TYPES } from '../../../../../../../../../../../../src/plugins/data/public';
+import { createFieldOptions } from '../../../../../common/job_creator/util/general';
 
 interface Props {
   fields: Field[];
@@ -17,12 +18,10 @@ interface Props {
 }
 
 export const TimeFieldSelect: FC<Props> = ({ fields, changeHandler, selectedField }) => {
-  const options: EuiComboBoxOptionProps[] = fields
-    .filter(f => f.id !== EVENT_RATE_FIELD_ID && f.type === ES_FIELD_TYPES.DATE)
-    .map(f => ({
-      label: f.name,
-    }))
-    .sort((a, b) => a.label.localeCompare(b.label));
+  const options: EuiComboBoxOptionProps[] = createFieldOptions(
+    fields,
+    f => f.id !== EVENT_RATE_FIELD_ID && f.type === ES_FIELD_TYPES.DATE
+  );
 
   const selection: EuiComboBoxOptionProps[] = [
     {