[EPM] update index patterns on install/uninstall of package (#56591)

* create index patterns functionality on install/uninstall of package

* update snapshots

x-pack/legacy/plugins/epm/common/constants.ts

@@ -21,3 +21,4 @@ export const SAVED_OBJECT_TYPE_PACKAGES = 'epm-package';
 // This is actually controled by Ingest
 // TODO: Ultimately, EPM should a) import this or b) not know about it at all
 export const SAVED_OBJECT_TYPE_DATASOURCES = 'datasources';
+export const SAVED_OBJECT_TYPE_INDEX_PATTERN = 'index-pattern';

x-pack/legacy/plugins/epm/server/lib/fields/field.ts

@@ -93,10 +93,12 @@ const isFields = (path: string) => {
  * Gets all field files, optionally filtered by dataset, extracts .yml files, merges them together
  */
 
-export const loadFieldsFromYaml = async (pkg: RegistryPackage, datasetName?: string) => {
+export const loadFieldsFromYaml = async (
+  pkg: RegistryPackage,
+  datasetName?: string
+): Promise<Field[]> => {
   // Fetch all field definition files
   const fieldDefinitionFiles = await getAssetsData(pkg, isFields, datasetName);
-
   return fieldDefinitionFiles.reduce<Field[]>((acc, file) => {
     // Make sure it is defined as it is optional. Should never happen.
     if (file.buffer) {

x-pack/legacy/plugins/epm/server/lib/kibana/index_pattern/install.test.ts

@@ -15,6 +15,8 @@ import {
   findFieldByPath,
   IndexPatternField,
   createFieldFormatMap,
+  createIndexPatternFields,
+  createIndexPattern,
 } from './install';
 import { Fields, Field } from '../../fields/field';
 
@@ -43,6 +45,16 @@ describe('creating index patterns from yaml fields', () => {
 
   const name = 'testField';
 
+  test('createIndexPatternFields function creates Kibana index pattern fields and fieldFormatMap', () => {
+    const indexPatternFields = createIndexPatternFields(fields);
+    expect(indexPatternFields).toMatchSnapshot('createIndexPatternFields');
+  });
+
+  test('createIndexPattern function creates Kibana index pattern', () => {
+    const indexPattern = createIndexPattern('logs', fields);
+    expect(indexPattern).toMatchSnapshot('createIndexPattern');
+  });
+
   test('flattenFields function flattens recursively and handles copying alias fields', () => {
     const flattened = flattenFields(fields);
     expect(flattened).toMatchSnapshot('flattenFields');

x-pack/legacy/plugins/epm/server/lib/kibana/index_pattern/install.ts

@@ -5,10 +5,11 @@
  */
 
 import { SavedObjectsClientContract } from 'kibana/server';
-import { KibanaAssetType } from '../../../../common/types';
-import { RegistryPackage, Dataset } from '../../../../common/types';
+import { SAVED_OBJECT_TYPE_INDEX_PATTERN } from '../../../../common/constants';
 import * as Registry from '../../../registry';
 import { loadFieldsFromYaml, Fields, Field } from '../../fields/field';
+import { getPackageKeysByStatus } from '../../../packages/get';
+import { InstallationStatus, RegistryPackage } from '../../../../common/types';
 
 interface FieldFormatMap {
   [key: string]: FieldFormatMapItem;
@@ -62,71 +63,99 @@ export interface IndexPatternField {
   script?: string;
   lang?: string;
 }
-interface KibanaIndexPattern {
-  [key: string]: string;
-}
-enum IndexPatternType {
+export enum IndexPatternType {
   logs = 'logs',
   metrics = 'metrics',
 }
 
 export async function installIndexPatterns(
-  pkgkey: string,
-  savedObjectsClient: SavedObjectsClientContract
+  savedObjectsClient: SavedObjectsClientContract,
+  pkgkey?: string
 ) {
-  const registryPackageInfo = await Registry.fetchInfo(pkgkey);
-  const registryDatasets = registryPackageInfo.datasets;
-  if (!registryDatasets) return;
+  // get all user installed packages
+  const installedPackages = await getPackageKeysByStatus(
+    savedObjectsClient,
+    InstallationStatus.installed
+  );
+  // add this package
+  if (pkgkey) installedPackages.push(pkgkey);
 
-  const indexPatternTypes = [IndexPatternType.logs, IndexPatternType.metrics];
+  // get each package's registry info
+  const installedPackagesFetchInfoPromise = installedPackages.map(pkg => Registry.fetchInfo(pkg));
+  const installedPackagesInfo = await Promise.all(installedPackagesFetchInfoPromise);
 
+  // for each index pattern type, create an index pattern
+  const indexPatternTypes = [IndexPatternType.logs, IndexPatternType.metrics];
   indexPatternTypes.forEach(async indexPatternType => {
-    const datasets = registryDatasets.filter(dataset => dataset.type === indexPatternType);
-    await createIndexPattern({
-      datasetType: indexPatternType,
-      datasets,
-      registryPackageInfo,
-      savedObjectsClient,
+    // if this is an update because a package is being unisntalled (no pkgkey argument passed) and no other packages are installed, remove the index pattern
+    if (!pkgkey && installedPackages.length === 0) {
+      try {
+        await savedObjectsClient.delete(
+          SAVED_OBJECT_TYPE_INDEX_PATTERN,
+          `epm-ip-${indexPatternType}`
+        );
+      } catch (err) {
+        // index pattern was probably deleted by the user already
+      }
+      return;
+    }
+
+    // get all dataset fields from all installed packages
+    const fields = await getAllDatasetFieldsByType(installedPackagesInfo, indexPatternType);
+
+    const kibanaIndexPattern = createIndexPattern(indexPatternType, fields);
+    // create or overwrite the index pattern
+    await savedObjectsClient.create(SAVED_OBJECT_TYPE_INDEX_PATTERN, kibanaIndexPattern, {
+      id: `epm-ip-${indexPatternType}`,
+      overwrite: true,
     });
   });
 }
 
-// loop through each dataset, get all the fields, create index pattern by type.
-const createIndexPattern = async ({
-  datasetType,
-  datasets,
-  registryPackageInfo,
-  savedObjectsClient,
-}: {
-  datasetType: string;
-  datasets: Dataset[];
-  registryPackageInfo: RegistryPackage;
-  savedObjectsClient: SavedObjectsClientContract;
-}) => {
-  const loadingFields = datasets.map(dataset =>
-    loadFieldsFromYaml(registryPackageInfo, dataset.name)
-  );
-  const nestedResults = await Promise.all(loadingFields);
-  const allFields = nestedResults.flat();
+// loops through all given packages and returns an array
+// of all fields from all datasets matching datasetType
+export const getAllDatasetFieldsByType = async (
+  packages: RegistryPackage[],
+  datasetType: IndexPatternType
+): Promise<Fields> => {
+  const datasetsPromises = packages.reduce<Array<Promise<Field[]>>>((acc, pkg) => {
+    if (pkg.datasets) {
+      // filter out datasets by datasetType
+      const matchingDatasets = pkg.datasets.filter(dataset => dataset.type === datasetType);
+      matchingDatasets.forEach(dataset => acc.push(loadFieldsFromYaml(pkg, dataset.name)));
+    }
+    return acc;
+  }, []);
 
-  const kibanaIndexPattern = makeKibanaIndexPattern(allFields, datasetType);
-  await savedObjectsClient.create(KibanaAssetType.indexPattern, kibanaIndexPattern);
+  // get all the datasets for each installed package into one array
+  const allDatasetFields: Fields[] = await Promise.all(datasetsPromises);
+  return allDatasetFields.flat();
 };
 
-const makeKibanaIndexPattern = (fields: Fields, datasetType: string): KibanaIndexPattern => {
-  const dedupedFields = dedupeFields(fields);
-  const flattenedFields = flattenFields(dedupedFields);
-  const fieldFormatMap = createFieldFormatMap(flattenedFields);
-  const transformedFields = flattenedFields.map(transformField);
+// creates or updates index pattern
+export const createIndexPattern = (indexPatternType: string, fields: Fields) => {
+  const { indexPatternFields, fieldFormatMap } = createIndexPatternFields(fields);
 
   return {
-    title: datasetType + '-*',
+    title: `${indexPatternType}-*`,
     timeFieldName: '@timestamp',
-    fields: JSON.stringify(transformedFields),
+    fields: JSON.stringify(indexPatternFields),
     fieldFormatMap: JSON.stringify(fieldFormatMap),
   };
 };
 
+// takes fields from yaml files and transforms into Kibana Index Pattern fields
+// and also returns the fieldFormatMap
+export const createIndexPatternFields = (
+  fields: Fields
+): { indexPatternFields: IndexPatternField[]; fieldFormatMap: FieldFormatMap } => {
+  const dedupedFields = dedupeFields(fields);
+  const flattenedFields = flattenFields(dedupedFields);
+  const fieldFormatMap = createFieldFormatMap(flattenedFields);
+  const transformedFields = flattenedFields.map(transformField);
+  return { indexPatternFields: transformedFields, fieldFormatMap };
+};
+
 export const dedupeFields = (fields: Fields) => {
   const uniqueObj = fields.reduce<{ [name: string]: Field }>((acc, field) => {
     if (!acc[field.name]) {

x-pack/legacy/plugins/epm/server/lib/kibana/index_pattern/tests/coredns.logs.yml

@@ -0,0 +1,71 @@
+- name: coredns
+  type: group
+  description: >
+    coredns fields after normalization
+  fields:
+    - name: id
+      type: keyword
+      description: >
+        id of the DNS transaction
+
+    - name: allParams
+      type: integer
+      format: bytes
+      pattern: patternValQueryWeight
+      input_format: inputFormatVal,
+      output_format: outputFormalVal,
+      output_precision: 3,
+      label_template: labelTemplateVal,
+      url_template: urlTemplateVal,
+      openLinkInCurrentTab: true,
+      description: >
+        weight of the DNS query
+
+    - name: query.length
+      type: integer
+      pattern: patternValQueryLength
+      description: >
+        length of the DNS query
+
+    - name: query.size
+      type: integer
+      format: bytes
+      pattern: patternValQuerySize
+      description: >
+        size of the DNS query
+
+    - name: query.class
+      type: keyword
+      description: >
+        DNS query class
+
+    - name: query.name
+      type: keyword
+      description: >
+        DNS query name
+
+    - name: query.type
+      type: keyword
+      description: >
+        DNS query type
+
+    - name: response.code
+      type: keyword
+      description: >
+        DNS response code
+
+    - name: response.flags
+      type: keyword
+      description: >
+        DNS response flags
+
+    - name: response.size
+      type: integer
+      format: bytes
+      description: >
+        size of the DNS response
+
+    - name: dnssec_ok
+      type: boolean
+      description: >
+        dnssec flag

x-pack/legacy/plugins/epm/server/packages/get.ts

@@ -6,7 +6,7 @@
 
 import { SavedObjectsClientContract } from 'src/core/server/';
 import { SAVED_OBJECT_TYPE_PACKAGES } from '../../common/constants';
-import { Installation, Installed, NotInstalled } from '../../common/types';
+import { Installation, Installed, NotInstalled, InstallationStatus } from '../../common/types';
 import * as Registry from '../registry';
 import { createInstallableFrom } from './index';
 
@@ -48,6 +48,19 @@ export async function getPackages(
   return packageList;
 }
 
+export async function getPackageKeysByStatus(
+  savedObjectsClient: SavedObjectsClientContract,
+  status: InstallationStatus
+) {
+  const allPackages = await getPackages({ savedObjectsClient });
+  return allPackages.reduce<string[]>((acc, pkg) => {
+    if (pkg.status === status) {
+      acc.push(`${pkg.name}-${pkg.version}`);
+    }
+    return acc;
+  }, []);
+}
+
 export async function getPackageInfo(options: {
   savedObjectsClient: SavedObjectsClientContract;
   pkgkey: string;

x-pack/legacy/plugins/epm/server/packages/install.ts

@@ -18,14 +18,16 @@ export async function installPackage(options: {
 }): Promise<AssetReference[]> {
   const { savedObjectsClient, pkgkey } = options;
 
-  const toSave = await installAssets({
+  const installIndexPatternsPromise = installIndexPatterns(savedObjectsClient, pkgkey);
+
+  const installAssetsPromise = installAssets({
     savedObjectsClient,
     pkgkey,
   });
 
-  // Setup basic index patterns
-  // TODO: This should be updated and not overwritten in the future
-  await installIndexPatterns(pkgkey, savedObjectsClient);
+  const res = await Promise.all([installIndexPatternsPromise, installAssetsPromise]);
+  // save the response of assets that were installed and return
+  const toSave = res[1];
 
   // Save those references in the package manager's state saved object
   await saveInstallationReferences({

x-pack/legacy/plugins/epm/server/packages/remove.ts

@@ -9,6 +9,7 @@ import { SAVED_OBJECT_TYPE_PACKAGES } from '../../common/constants';
 import { AssetReference, AssetType, ElasticsearchAssetType } from '../../common/types';
 import { CallESAsCurrentUser } from '../lib/cluster_access';
 import { getInstallation, savedObjectTypes } from './index';
+import { installIndexPatterns } from '../lib/kibana/index_pattern/install';
 
 export async function removeInstallation(options: {
   savedObjectsClient: SavedObjectsClientContract;
@@ -23,6 +24,9 @@ export async function removeInstallation(options: {
   // could also update with [] or some other state
   await savedObjectsClient.delete(SAVED_OBJECT_TYPE_PACKAGES, pkgkey);
 
+  // recreate or delete index patterns when a package is uninstalled
+  const indexPatternPromise = installIndexPatterns(savedObjectsClient);
+
   // Delete the installed assets
   const deletePromises = installedObjects.map(async ({ id, type }) => {
     const assetType = type as AssetType;
@@ -34,7 +38,7 @@ export async function removeInstallation(options: {
       deleteTemplate(callCluster, id);
     }
   });
-  await Promise.all(deletePromises);
+  await Promise.all([...deletePromises, indexPatternPromise]);
 
   // successful delete's in SO client return {}. return something more useful
   return installedObjects;