[ML] Move Index Data Visualizer into separate plugin (Part 1)

x-pack/plugins/file_data_visualizer/common/constants.ts

@@ -29,3 +29,5 @@ export const JOB_FIELD_TYPES = {
   TEXT: 'text',
   UNKNOWN: 'unknown',
 } as const;
+
+export const OMIT_FIELDS: string[] = ['_source', '_type', '_index', '_id', '_version', '_score'];

x-pack/plugins/file_data_visualizer/common/types/field_request_config.ts

@@ -0,0 +1,60 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import type { JobFieldType } from './index';
+
+export interface Percentile {
+  percent: number;
+  minValue: number;
+  maxValue: number;
+}
+
+export interface FieldRequestConfig {
+  fieldName?: string;
+  type: JobFieldType;
+  cardinality: number;
+}
+
+export interface DocumentCountBuckets {
+  [key: string]: number;
+}
+
+export interface DocumentCounts {
+  buckets?: DocumentCountBuckets;
+  interval?: number;
+}
+
+export interface FieldVisStats {
+  cardinality?: number;
+  count?: number;
+  sampleCount?: number;
+  trueCount?: number;
+  falseCount?: number;
+  earliest?: number;
+  latest?: number;
+  documentCounts?: {
+    buckets?: DocumentCountBuckets;
+    interval?: number;
+  };
+  avg?: number;
+  distribution?: {
+    percentiles: Percentile[];
+    maxPercentile: number;
+    minPercentile: 0;
+  };
+  fieldName?: string;
+  isTopValuesSampled?: boolean;
+  max?: number;
+  median?: number;
+  min?: number;
+  topValues?: Array<{ key: number | string; doc_count: number }>;
+  topValuesSampleSize?: number;
+  topValuesSamplerShardSize?: number;
+  examples?: Array<string | object>;
+  timeRangeEarliest?: number;
+  timeRangeLatest?: number;
+}

x-pack/plugins/file_data_visualizer/common/types.ts → x-pack/plugins/file_data_visualizer/common/types/index.ts

@@ -5,12 +5,17 @@
  * 2.0.
  */
 
-import { JOB_FIELD_TYPES } from './constants';
-
+import type { SimpleSavedObject } from 'kibana/public';
+export type { JobFieldType } from './job_field_type';
+export type {
+  FieldRequestConfig,
+  DocumentCountBuckets,
+  DocumentCounts,
+  FieldVisStats,
+  Percentile,
+} from './field_request_config';
 export type InputData = any[];
 
-export type JobFieldType = typeof JOB_FIELD_TYPES[keyof typeof JOB_FIELD_TYPES];
-
 export interface DataVisualizerTableState {
   pageSize: number;
   pageIndex: number;
@@ -20,3 +25,5 @@ export interface DataVisualizerTableState {
   visibleFieldNames: string[];
   showDistributions: boolean;
 }
+
+export type SavedSearchSavedObject = SimpleSavedObject<any>;

x-pack/plugins/ml/public/application/datavisualizer/stats_table/types/field_data_row.ts → x-pack/plugins/file_data_visualizer/common/types/indices.ts

@@ -5,8 +5,10 @@
  * 2.0.
  */
 
-import type { FieldVisConfig, FileBasedFieldVisConfig } from './field_vis_config';
+import { estypes } from '@elastic/elasticsearch';
 
-export interface FieldDataRowProps {
-  config: FieldVisConfig | FileBasedFieldVisConfig;
+export interface IndicesOptions {
+  allow_no_indices?: boolean;
+  expand_wildcards?: estypes.ExpandWildcards;
+  ignore_unavailable?: boolean;
 }

x-pack/plugins/file_data_visualizer/common/types/job_field_type.ts

@@ -0,0 +1,9 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { JOB_FIELD_TYPES } from '../constants';
+export type JobFieldType = typeof JOB_FIELD_TYPES[keyof typeof JOB_FIELD_TYPES];

x-pack/plugins/file_data_visualizer/common/types/time_field_request.ts

@@ -0,0 +1,12 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+export interface GetTimeFieldRangeResponse {
+  success: boolean;
+  start: { epoch: number; string: string };
+  end: { epoch: number; string: string };
+}

x-pack/plugins/file_data_visualizer/common/utils/datafeed_utils.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+
+export type Datafeed = estypes.Datafeed;
+export type Aggregation = Record<string, estypes.AggregationContainer>;
+
+export function getAggregations<T>(obj: any): T | undefined {
+  if (obj?.aggregations !== undefined) return obj.aggregations;
+  if (obj?.aggs !== undefined) return obj.aggs;
+  return undefined;
+}
+
+export const getDatafeedAggregations = (
+  datafeedConfig: Partial<Datafeed> | undefined
+): Aggregation | undefined => {
+  return getAggregations<Aggregation>(datafeedConfig);
+};

x-pack/plugins/file_data_visualizer/common/utils/object_utils.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/*
+ * A type guard to check record like object structures.
+ *
+ * Examples:
+ * - `isPopulatedObject({...})`
+ *      Limits type to Record<string, unknown>
+ *
+ * - `isPopulatedObject({...}, ['attribute'])`
+ *      Limits type to Record<'attribute', unknown>
+ *
+ * - `isPopulatedObject<keyof MyInterface>({...})`
+ *      Limits type to a record with keys of the given interface.
+ *      Note that you might want to add keys from the interface to the
+ *      array of requiredAttributes to satisfy runtime requirements.
+ *      Otherwise you'd just satisfy TS requirements but might still
+ *      run into runtime issues.
+ */
+export const isPopulatedObject = <U extends string = string>(
+  arg: unknown,
+  requiredAttributes: U[] = []
+): arg is Record<U, unknown> => {
+  return (
+    typeof arg === 'object' &&
+    arg !== null &&
+    Object.keys(arg).length > 0 &&
+    (requiredAttributes.length === 0 ||
+      requiredAttributes.every((d) => ({}.hasOwnProperty.call(arg, d))))
+  );
+};

x-pack/plugins/file_data_visualizer/common/utils/query_utils.ts

@@ -0,0 +1,76 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import { estypes } from '@elastic/elasticsearch';
+/*
+ * Contains utility functions for building and processing queries.
+ */
+
+// Builds the base filter criteria used in queries,
+// adding criteria for the time range and an optional query.
+export function buildBaseFilterCriteria(
+  timeFieldName?: string,
+  earliestMs?: number,
+  latestMs?: number,
+  query?: object
+) {
+  const filterCriteria = [];
+  if (timeFieldName && earliestMs && latestMs) {
+    filterCriteria.push({
+      range: {
+        [timeFieldName]: {
+          gte: earliestMs,
+          lte: latestMs,
+          format: 'epoch_millis',
+        },
+      },
+    });
+  }
+
+  if (query) {
+    filterCriteria.push(query);
+  }
+
+  return filterCriteria;
+}
+
+// Wraps the supplied aggregations in a sampler aggregation.
+// A supplied samplerShardSize (the shard_size parameter of the sampler aggregation)
+// of less than 1 indicates no sampling, and the aggs are returned as-is.
+export function buildSamplerAggregation(
+  aggs: any,
+  samplerShardSize: number
+): Record<string, estypes.AggregationContainer> {
+  if (samplerShardSize < 1) {
+    return aggs;
+  }
+
+  return {
+    sample: {
+      sampler: {
+        shard_size: samplerShardSize,
+      },
+      aggs,
+    },
+  };
+}
+
+// Returns the path of aggregations in the elasticsearch response, as an array,
+// depending on whether sampling is being used.
+// A supplied samplerShardSize (the shard_size parameter of the sampler aggregation)
+// of less than 1 indicates no sampling, and an empty array is returned.
+export function getSamplerAggregationsResponsePath(samplerShardSize: number): string[] {
+  return samplerShardSize > 0 ? ['sample'] : [];
+}
+
+// Returns a name which is safe to use in elasticsearch aggregations for the supplied
+// field name. Aggregation names must be alpha-numeric and can only contain '_' and '-' characters,
+// so if the supplied field names contains disallowed characters, the provided index
+// identifier is used to return a safe 'dummy' name in the format 'field_index' e.g. field_0, field_1
+export function getSafeAggregationName(fieldName: string, index: number): string {
+  return fieldName.match(/^[a-zA-Z0-9-_.]+$/) ? fieldName : `field_${index}`;
+}

x-pack/plugins/file_data_visualizer/common/utils/runtime_field_utils.ts

@@ -0,0 +1,28 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+import { estypes } from '@elastic/elasticsearch';
+import { isPopulatedObject } from './object_utils';
+import { RUNTIME_FIELD_TYPES } from '../../../../../src/plugins/data/common';
+
+type RuntimeType = typeof RUNTIME_FIELD_TYPES[number];
+
+export function isRuntimeField(arg: unknown): arg is estypes.RuntimeField {
+  return (
+    ((isPopulatedObject(arg, ['type']) && Object.keys(arg).length === 1) ||
+      (isPopulatedObject(arg, ['type', 'script']) &&
+        Object.keys(arg).length === 2 &&
+        (typeof arg.script === 'string' ||
+          (isPopulatedObject(arg.script, ['source']) &&
+            Object.keys(arg.script).length === 1 &&
+            typeof arg.script.source === 'string')))) &&
+    RUNTIME_FIELD_TYPES.includes(arg.type as RuntimeType)
+  );
+}
+
+export function isRuntimeMappings(arg: unknown): arg is estypes.RuntimeFields {
+  return isPopulatedObject(arg) && Object.values(arg).every((d) => isRuntimeField(d));
+}

x-pack/plugins/file_data_visualizer/common/utils/string_utils.ts

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+/**
+ * Creates a deterministic number based hash out of a string.
+ */
+export function stringHash(str: string): number {
+  let hash = 0;
+  let chr = 0;
+  if (str.length === 0) {
+    return hash;
+  }
+  for (let i = 0; i < str.length; i++) {
+    chr = str.charCodeAt(i);
+    hash = (hash << 5) - hash + chr; // eslint-disable-line no-bitwise
+    hash |= 0; // eslint-disable-line no-bitwise
+  }
+  return hash < 0 ? hash * -2 : hash;
+}

x-pack/plugins/file_data_visualizer/kibana.json

@@ -15,7 +15,8 @@
   "optionalPlugins": [
     "security",
     "maps",
-    "home"
+    "home",
+    "lens"
   ],
   "requiredBundles": [
     "kibanaReact",

x-pack/plugins/file_data_visualizer/public/api/index.ts

@@ -6,9 +6,13 @@
  */
 
 import { lazyLoadModules } from '../lazy_load_bundle';
-import { FileDataVisualizer } from '../application';
+import type { FileDataVisualizerSpec, IndexDataVisualizerSpec } from '../application';
 
-export async function getFileDataVisualizerComponent(): Promise<typeof FileDataVisualizer> {
+export async function getFileDataVisualizerComponent(): Promise<FileDataVisualizerSpec> {
   const modules = await lazyLoadModules();
   return modules.FileDataVisualizer;
 }
+export async function getIndexDataVisualizerComponent(): Promise<IndexDataVisualizerSpec> {
+  const modules = await lazyLoadModules();
+  return modules.IndexDataVisualizer;
+}

x-pack/plugins/file_data_visualizer/public/application/_index.scss

@@ -1 +1,2 @@
-@import 'components/index';
+@import 'common/components/index';
+@import 'file_data_visualizer/index';

x-pack/plugins/file_data_visualizer/public/application/common/components/_index.scss

@@ -0,0 +1,5 @@
+@import 'about_panel/index';
+@import 'embedded_map/index';
+@import 'experimental_badge/index';
+@import 'stats_table/index';
+@import 'top_values/top_values';

x-pack/plugins/file_data_visualizer/public/application/components/about_panel/welcome_content.tsx → x-pack/plugins/file_data_visualizer/public/application/common/components/about_panel/welcome_content.tsx

@@ -21,7 +21,7 @@ import {
 
 import { ExperimentalBadge } from '../experimental_badge';
 
-import { useFileDataVisualizerKibana } from '../../kibana_context';
+import { useDataVisualizerKibana } from '../../../kibana_context';
 
 export const WelcomeContent: FC = () => {
   const toolTipContent = i18n.translate(
@@ -35,7 +35,7 @@ export const WelcomeContent: FC = () => {
     services: {
       fileUpload: { getMaxBytesFormatted },
     },
-  } = useFileDataVisualizerKibana();
+  } = useDataVisualizerKibana();
   const maxFileSize = getMaxBytesFormatted();
 
   return (

x-pack/plugins/file_data_visualizer/public/application/components/combined_fields/combined_fields_form.tsx → x-pack/plugins/file_data_visualizer/public/application/common/components/combined_fields/combined_fields_form.tsx

@@ -29,7 +29,7 @@ import {
   removeCombinedFieldsFromMappings,
   removeCombinedFieldsFromPipeline,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../file_upload/common';
+import { FindFileStructureResponse } from '../../../../../../file_upload/common';
 
 interface Props {
   mappingsString: string;

x-pack/plugins/file_data_visualizer/public/application/components/combined_fields/geo_point.tsx → x-pack/plugins/file_data_visualizer/public/application/common/components/combined_fields/geo_point.tsx

@@ -29,7 +29,7 @@ import {
   getFieldNames,
   getNameCollisionMsg,
 } from './utils';
-import { FindFileStructureResponse } from '../../../../../file_upload/common';
+import { FindFileStructureResponse } from '../../../../../../file_upload/common';
 
 interface Props {
   addCombinedField: (combinedField: CombinedField) => void;