[EPM] /packages/{package} endpoint to support upgrades (#63629)

* install template after pipeline creation

* return installed pkg if this pkg version is already installed

* remove pipelines after templates are updated

* remove kibana saved objects assets before installing

* update current write indices

* add back removal of merging previous references lost in rebase

* improve some typing names, consolidate, fix bad merges

* update query to use aggregate on _index

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/ingest_manager/common/types/models/epm.ts

@@ -256,10 +256,19 @@ export enum DefaultPackages {
   endpoint = 'endpoint',
 }
 
+export interface IndexTemplateMappings {
+  properties: any;
+}
+
 export interface IndexTemplate {
   order: number;
   index_patterns: string[];
   settings: any;
   mappings: object;
   aliases: object;
 }
+
+export interface TemplateRef {
+  templateName: string;
+  indexTemplate: IndexTemplate;
+}

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/install.ts

@@ -4,13 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import {
-  AssetReference,
-  Dataset,
-  RegistryPackage,
-  IngestAssetType,
-  ElasticsearchAssetType,
-} from '../../../../types';
+import { Dataset, RegistryPackage, ElasticsearchAssetType, TemplateRef } from '../../../../types';
 import { CallESAsCurrentUser } from '../../../../types';
 import { Field, loadFieldsFromYaml, processFields } from '../../fields/field';
 import { getPipelineNameForInstallation } from '../ingest_pipeline/install';
@@ -22,15 +16,15 @@ export const installTemplates = async (
   callCluster: CallESAsCurrentUser,
   pkgName: string,
   pkgVersion: string
-) => {
+): Promise<TemplateRef[]> => {
   // install any pre-built index template assets,
   // atm, this is only the base package's global template
   installPreBuiltTemplates(pkgName, pkgVersion, callCluster);
 
   // build templates per dataset from yml files
   const datasets = registryPackage.datasets;
   if (datasets) {
-    const templates = datasets.reduce<Array<Promise<AssetReference>>>((acc, dataset) => {
+    const installTemplatePromises = datasets.reduce<Array<Promise<TemplateRef>>>((acc, dataset) => {
       acc.push(
         installTemplateForDataset({
           pkg: registryPackage,
@@ -40,7 +34,9 @@ export const installTemplates = async (
       );
       return acc;
     }, []);
-    return Promise.all(templates).then(results => results.flat());
+
+    const res = await Promise.all(installTemplatePromises);
+    return res.flat();
   }
   return [];
 };
@@ -84,7 +80,7 @@ export async function installTemplateForDataset({
   pkg: RegistryPackage;
   callCluster: CallESAsCurrentUser;
   dataset: Dataset;
-}): Promise<AssetReference> {
+}): Promise<TemplateRef> {
   const fields = await loadFieldsFromYaml(pkg, dataset.path);
   return installTemplate({
     callCluster,
@@ -104,7 +100,7 @@ export async function installTemplate({
   fields: Field[];
   dataset: Dataset;
   packageVersion: string;
-}): Promise<AssetReference> {
+}): Promise<TemplateRef> {
   const mappings = generateMappings(processFields(fields));
   const templateName = generateTemplateName(dataset);
   let pipelineName;
@@ -122,6 +118,8 @@ export async function installTemplate({
     body: template,
   });
 
-  // The id of a template is its name
-  return { id: templateName, type: IngestAssetType.IndexTemplate };
+  return {
+    templateName,
+    indexTemplate: template,
+  };
 }

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -5,24 +5,30 @@
  */
 
 import { Field, Fields } from '../../fields/field';
-import { Dataset, IndexTemplate } from '../../../../types';
+import {
+  Dataset,
+  CallESAsCurrentUser,
+  TemplateRef,
+  IndexTemplate,
+  IndexTemplateMappings,
+} from '../../../../types';
 import { getDatasetAssetBaseName } from '../index';
 
 interface Properties {
   [key: string]: any;
 }
-interface Mappings {
-  properties: any;
-}
-
-interface Mapping {
-  [key: string]: any;
-}
 
 interface MultiFields {
   [key: string]: object;
 }
 
+export interface IndexTemplateMapping {
+  [key: string]: any;
+}
+export interface CurrentIndex {
+  indexName: string;
+  indexTemplate: IndexTemplate;
+}
 const DEFAULT_SCALING_FACTOR = 1000;
 const DEFAULT_IGNORE_ABOVE = 1024;
 
@@ -34,7 +40,7 @@ const DEFAULT_IGNORE_ABOVE = 1024;
 export function getTemplate(
   type: string,
   templateName: string,
-  mappings: Mappings,
+  mappings: IndexTemplateMappings,
   pipelineName?: string | undefined
 ): IndexTemplate {
   const template = getBaseTemplate(type, templateName, mappings);
@@ -52,7 +58,7 @@ export function getTemplate(
  *
  * @param fields
  */
-export function generateMappings(fields: Field[]): Mappings {
+export function generateMappings(fields: Field[]): IndexTemplateMappings {
   const props: Properties = {};
   // TODO: this can happen when the fields property in fields.yml is present but empty
   // Maybe validation should be moved to fields/field.ts
@@ -140,8 +146,8 @@ function generateMultiFields(fields: Fields): MultiFields {
   return multiFields;
 }
 
-function generateKeywordMapping(field: Field): Mapping {
-  const mapping: Mapping = {
+function generateKeywordMapping(field: Field): IndexTemplateMapping {
+  const mapping: IndexTemplateMapping = {
     ignore_above: DEFAULT_IGNORE_ABOVE,
   };
   if (field.ignore_above) {
@@ -150,8 +156,8 @@ function generateKeywordMapping(field: Field): Mapping {
   return mapping;
 }
 
-function generateTextMapping(field: Field): Mapping {
-  const mapping: Mapping = {};
+function generateTextMapping(field: Field): IndexTemplateMapping {
+  const mapping: IndexTemplateMapping = {};
   if (field.analyzer) {
     mapping.analyzer = field.analyzer;
   }
@@ -200,7 +206,11 @@ export function generateESIndexPatterns(datasets: Dataset[] | undefined): Record
   return patterns;
 }
 
-function getBaseTemplate(type: string, templateName: string, mappings: Mappings): IndexTemplate {
+function getBaseTemplate(
+  type: string,
+  templateName: string,
+  mappings: IndexTemplateMappings
+): IndexTemplate {
   return {
     // We need to decide which order we use for the templates
     order: 1,
@@ -234,10 +244,6 @@ function getBaseTemplate(type: string, templateName: string, mappings: Mappings)
       },
     },
     mappings: {
-      // To be filled with interesting information about this specific index
-      _meta: {
-        package: 'foo',
-      },
       // All the dynamic field mappings
       dynamic_templates: [
         // This makes sure all mappings are keywords by default
@@ -261,3 +267,112 @@ function getBaseTemplate(type: string, templateName: string, mappings: Mappings)
     aliases: {},
   };
 }
+
+export const updateCurrentWriteIndices = async (
+  callCluster: CallESAsCurrentUser,
+  templates: TemplateRef[]
+): Promise<void> => {
+  if (!templates) return;
+
+  const allIndices = await queryIndicesFromTemplates(callCluster, templates);
+  return updateAllIndices(allIndices, callCluster);
+};
+
+const queryIndicesFromTemplates = async (
+  callCluster: CallESAsCurrentUser,
+  templates: TemplateRef[]
+): Promise<CurrentIndex[]> => {
+  const indexPromises = templates.map(template => {
+    return getIndices(callCluster, template);
+  });
+  const indexObjects = await Promise.all(indexPromises);
+  return indexObjects.filter(item => item !== undefined).flat();
+};
+
+const getIndices = async (
+  callCluster: CallESAsCurrentUser,
+  template: TemplateRef
+): Promise<CurrentIndex[] | undefined> => {
+  const { templateName, indexTemplate } = template;
+  const res = await callCluster('search', getIndexQuery(templateName));
+  const indices: any[] = res?.aggregations?.index.buckets;
+  if (indices) {
+    return indices.map(index => ({
+      indexName: index.key,
+      indexTemplate,
+    }));
+  }
+};
+
+const updateAllIndices = async (
+  indexNameWithTemplates: CurrentIndex[],
+  callCluster: CallESAsCurrentUser
+): Promise<void> => {
+  const updateIndexPromises = indexNameWithTemplates.map(({ indexName, indexTemplate }) => {
+    return updateExistingIndex({ indexName, callCluster, indexTemplate });
+  });
+  await Promise.all(updateIndexPromises);
+};
+const updateExistingIndex = async ({
+  indexName,
+  callCluster,
+  indexTemplate,
+}: {
+  indexName: string;
+  callCluster: CallESAsCurrentUser;
+  indexTemplate: IndexTemplate;
+}) => {
+  const { settings, mappings } = indexTemplate;
+  // try to update the mappings first
+  // for now we assume updates are compatible
+  try {
+    await callCluster('indices.putMapping', {
+      index: indexName,
+      body: mappings,
+    });
+  } catch (err) {
+    throw new Error('incompatible mappings update');
+  }
+  // update settings after mappings was successful to ensure
+  // pointing to theme new pipeline is safe
+  // for now, only update the pipeline
+  if (!settings.index.default_pipeline) return;
+  try {
+    await callCluster('indices.putSettings', {
+      index: indexName,
+      body: { index: { default_pipeline: settings.index.default_pipeline } },
+    });
+  } catch (err) {
+    throw new Error('incompatible settings update');
+  }
+};
+
+const getIndexQuery = (templateName: string) => ({
+  index: `${templateName}-*`,
+  size: 0,
+  body: {
+    query: {
+      bool: {
+        must: [
+          {
+            exists: {
+              field: 'stream.namespace',
+            },
+          },
+          {
+            exists: {
+              field: 'stream.dataset',
+            },
+          },
+        ],
+      },
+    },
+    aggs: {
+      index: {
+        terms: {
+          field: '_index',
+        },
+      },
+    },
+  },
+});