[Fleet] Remove upgradePackage and consolidate it with `installPacka…

…ge`, optimize calls to create index patterns (#94490)

* Add data plugin to server app context

* First attempt at switching to indexPatternService for EPM index pattern creation & deletion, instead of interacting directly with index pattern SOs

* Simplify bulk install package, remove upgradePackage() method in favor of consolidating with installPackage(), use installPackage() for bulk install instead

* Update tests

* Change cache logging of objects to trace level

* Install index patterns as a post-package installation operation, for bulk package installation, install index pattern only if one or more packages are actually new installs, add debug logging

* Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true

* Allow `getFieldsForWildcard` to return fields saved on index pattern when allowNoIndices is true

* Bring back passing custom ID for index pattern SO

* Fix tests

* Revert "Merge branch 'index-pattern/allow-no-index' into epm/missing-index-patterns"

This reverts commit 8e712e9, reversing
changes made to af0fb0e.

* Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true

(cherry picked from commit 69b93da)

* Update API docs

* Run post-install ops for install by upload too

* Remove allowedInstallTypes param

* Adjust force install conditions

* Revert "Update API docs"

This reverts commit b9770fd.

* Revert "Allow getAsSavedObjectBody to return non-scripted fields when allowNoIndex is true"

This reverts commit afc91ce.

* Go back to using SO client for index patterns :(

* Stringify attributes again for SO client saving

* Fix condition for reinstall same pkg version

* Remove unused type

* Adjust comment

* Update snapshot

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/fleet/common/constants/epm.ts

@@ -7,7 +7,6 @@
 
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const ASSETS_SAVED_OBJECT_TYPE = 'epm-packages-assets';
-export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const MAX_TIME_COMPLETE_INSTALL = 60000;
 
 export const FLEET_SERVER_PACKAGE = 'fleet_server';

x-pack/plugins/fleet/common/types/rest_spec/epm.ts

@@ -82,12 +82,15 @@ export interface IBulkInstallPackageHTTPError {
   error: string | Error;
 }
 
+export interface InstallResult {
+  assets: AssetReference[];
+  status: 'installed' | 'already_installed';
+}
+
 export interface BulkInstallPackageInfo {
   name: string;
-  newVersion: string;
-  // this will be null if no package was present before the upgrade (aka it was an install)
-  oldVersion: string | null;
-  assets: AssetReference[];
+  version: string;
+  result: InstallResult;
 }
 
 export interface BulkInstallPackagesResponse {

x-pack/plugins/fleet/server/constants/index.ts

@@ -43,7 +43,6 @@ export {
   OUTPUT_SAVED_OBJECT_TYPE,
   PACKAGES_SAVED_OBJECT_TYPE,
   ASSETS_SAVED_OBJECT_TYPE,
-  INDEX_PATTERN_SAVED_OBJECT_TYPE,
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
   GLOBAL_SETTINGS_SAVED_OBJECT_TYPE,
   // Defaults

x-pack/plugins/fleet/server/mocks/index.ts

@@ -10,6 +10,7 @@ import {
   savedObjectsServiceMock,
   coreMock,
 } from '../../../../../src/core/server/mocks';
+import { dataPluginMock } from '../../../../../src/plugins/data/server/mocks';
 import { licensingMock } from '../../../../plugins/licensing/server/mocks';
 import { encryptedSavedObjectsMock } from '../../../encrypted_saved_objects/server/mocks';
 import { securityMock } from '../../../security/server/mocks';
@@ -23,6 +24,7 @@ export * from '../services/artifacts/mocks';
 export const createAppContextStartContractMock = (): FleetAppContext => {
   return {
     elasticsearch: elasticsearchServiceMock.createStart(),
+    data: dataPluginMock.createStartContract(),
     encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(),
     savedObjects: savedObjectsServiceMock.createStartContract(),
     security: securityMock.createStart(),

x-pack/plugins/fleet/server/plugin.ts

@@ -23,6 +23,7 @@ import type {
 import type { UsageCollectionSetup } from 'src/plugins/usage_collection/server';
 
 import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
+import type { PluginStart as DataPluginStart } from '../../../../src/plugins/data/server';
 import type { LicensingPluginSetup, ILicense } from '../../licensing/server';
 import type {
   EncryptedSavedObjectsPluginStart,
@@ -100,12 +101,14 @@ export interface FleetSetupDeps {
 }
 
 export interface FleetStartDeps {
+  data: DataPluginStart;
   encryptedSavedObjects: EncryptedSavedObjectsPluginStart;
   security?: SecurityPluginStart;
 }
 
 export interface FleetAppContext {
   elasticsearch: ElasticsearchServiceStart;
+  data: DataPluginStart;
   encryptedSavedObjectsStart?: EncryptedSavedObjectsPluginStart;
   encryptedSavedObjectsSetup?: EncryptedSavedObjectsPluginSetup;
   security?: SecurityPluginStart;
@@ -293,6 +296,7 @@ export class FleetPlugin
   public async start(core: CoreStart, plugins: FleetStartDeps): Promise<FleetStartContract> {
     await appContextService.start({
       elasticsearch: core.elasticsearch,
+      data: plugins.data,
       encryptedSavedObjectsStart: plugins.encryptedSavedObjects,
       encryptedSavedObjectsSetup: this.encryptedSavedObjectsSetup,
       security: plugins.security,

x-pack/plugins/fleet/server/routes/epm/handlers.ts

@@ -40,12 +40,10 @@ import {
   getPackages,
   getFile,
   getPackageInfo,
-  handleInstallPackageFailure,
   isBulkInstallError,
   installPackage,
   removeInstallation,
   getLimitedPackages,
-  getInstallationObject,
   getInstallation,
 } from '../../services/epm/packages';
 import type { BulkInstallResponse } from '../../services/epm/packages';
@@ -228,15 +226,7 @@ export const installPackageFromRegistryHandler: RequestHandler<
   const savedObjectsClient = context.core.savedObjects.client;
   const esClient = context.core.elasticsearch.client.asCurrentUser;
   const { pkgkey } = request.params;
-
-  let pkgName: string | undefined;
-  let pkgVersion: string | undefined;
-
   try {
-    const parsedPkgKey = splitPkgKey(pkgkey);
-    pkgName = parsedPkgKey.pkgName;
-    pkgVersion = parsedPkgKey.pkgVersion;
-
     const res = await installPackage({
       installSource: 'registry',
       savedObjectsClient,
@@ -245,24 +235,11 @@ export const installPackageFromRegistryHandler: RequestHandler<
       force: request.body?.force,
     });
     const body: InstallPackageResponse = {
-      response: res,
+      response: res.assets,
     };
     return response.ok({ body });
   } catch (e) {
-    const defaultResult = await defaultIngestErrorHandler({ error: e, response });
-    if (pkgName && pkgVersion) {
-      const installedPkg = await getInstallationObject({ savedObjectsClient, pkgName });
-      await handleInstallPackageFailure({
-        savedObjectsClient,
-        error: e,
-        pkgName,
-        pkgVersion,
-        installedPkg,
-        esClient,
-      });
-    }
-
-    return defaultResult;
+    return await defaultIngestErrorHandler({ error: e, response });
   }
 };
 
@@ -291,7 +268,7 @@ export const bulkInstallPackagesFromRegistryHandler: RequestHandler<
   const bulkInstalledResponses = await bulkInstallPackages({
     savedObjectsClient,
     esClient,
-    packagesToUpgrade: request.body.packages,
+    packagesToInstall: request.body.packages,
   });
   const payload = bulkInstalledResponses.map(bulkInstallServiceResponseToHttpEntry);
   const body: BulkInstallPackagesResponse = {
@@ -324,7 +301,7 @@ export const installPackageByUploadHandler: RequestHandler<
       contentType,
     });
     const body: InstallPackageResponse = {
-      response: res,
+      response: res.assets,
     };
     return response.ok({ body });
   } catch (error) {

x-pack/plugins/fleet/server/services/app_context.ts

@@ -17,6 +17,7 @@ import type {
   Logger,
 } from 'src/core/server';
 
+import type { PluginStart as DataPluginStart } from '../../../../../src/plugins/data/server';
 import type {
   EncryptedSavedObjectsClient,
   EncryptedSavedObjectsPluginSetup,
@@ -29,6 +30,7 @@ import type { CloudSetup } from '../../../cloud/server';
 class AppContextService {
   private encryptedSavedObjects: EncryptedSavedObjectsClient | undefined;
   private encryptedSavedObjectsSetup: EncryptedSavedObjectsPluginSetup | undefined;
+  private data: DataPluginStart | undefined;
   private esClient: ElasticsearchClient | undefined;
   private security: SecurityPluginStart | undefined;
   private config$?: Observable<FleetConfigType>;
@@ -43,6 +45,7 @@ class AppContextService {
   private externalCallbacks: ExternalCallbacksStorage = new Map();
 
   public async start(appContext: FleetAppContext) {
+    this.data = appContext.data;
     this.esClient = appContext.elasticsearch.client.asInternalUser;
     this.encryptedSavedObjects = appContext.encryptedSavedObjectsStart?.getClient();
     this.encryptedSavedObjectsSetup = appContext.encryptedSavedObjectsSetup;
@@ -67,6 +70,13 @@ class AppContextService {
     this.externalCallbacks.clear();
   }
 
+  public getData() {
+    if (!this.data) {
+      throw new Error('Data start service not set.');
+    }
+    return this.data;
+  }
+
   public getEncryptedSavedObjects() {
     if (!this.encryptedSavedObjects) {
       throw new Error('Encrypted saved object start service not set.');

x-pack/plugins/fleet/server/services/epm/archive/cache.ts

@@ -28,13 +28,9 @@ export const getArchiveFilelist = (keyArgs: SharedKey) =>
   archiveFilelistCache.get(sharedKey(keyArgs));
 
 export const setArchiveFilelist = (keyArgs: SharedKey, paths: string[]) => {
-  appContextService
-    .getLogger()
-    .debug(
-      `setting file list to the cache for ${keyArgs.name}-${keyArgs.version}:\n${JSON.stringify(
-        paths
-      )}`
-    );
+  const logger = appContextService.getLogger();
+  logger.debug(`setting file list to the cache for ${keyArgs.name}-${keyArgs.version}`);
+  logger.trace(JSON.stringify(paths));
   return archiveFilelistCache.set(sharedKey(keyArgs), paths);
 };
 
@@ -63,12 +59,10 @@ export const setPackageInfo = ({
   version,
   packageInfo,
 }: SharedKey & { packageInfo: ArchivePackage | RegistryPackage }) => {
+  const logger = appContextService.getLogger();
   const key = sharedKey({ name, version });
-  appContextService
-    .getLogger()
-    .debug(
-      `setting package info to the cache for ${name}-${version}:\n${JSON.stringify(packageInfo)}`
-    );
+  logger.debug(`setting package info to the cache for ${name}-${version}`);
+  logger.trace(JSON.stringify(packageInfo));
   return packageInfoCache.set(key, packageInfo);
 };
 

x-pack/plugins/fleet/server/services/epm/kibana/index_pattern/install.test.ts

@@ -11,6 +11,8 @@ import { readFileSync } from 'fs';
 import glob from 'glob';
 import { safeLoad } from 'js-yaml';
 
+import type { FieldSpec } from 'src/plugins/data/common';
+
 import type { Fields, Field } from '../../fields/field';
 
 import {
@@ -22,7 +24,6 @@ import {
   createIndexPatternFields,
   createIndexPattern,
 } from './install';
-import type { IndexPatternField } from './install';
 import { dupeFields } from './tests/test_data';
 
 // Add our own serialiser to just do JSON.stringify
@@ -93,7 +94,6 @@ describe('creating index patterns from yaml fields', () => {
       const mergedField = deduped.find((field) => field.name === '1');
       expect(mergedField?.searchable).toBe(true);
       expect(mergedField?.aggregatable).toBe(true);
-      expect(mergedField?.analyzed).toBe(true);
       expect(mergedField?.count).toBe(0);
     });
   });
@@ -156,7 +156,7 @@ describe('creating index patterns from yaml fields', () => {
       { fields: [{ name: 'testField', type: 'short' }], expect: 'number' },
       { fields: [{ name: 'testField', type: 'byte' }], expect: 'number' },
       { fields: [{ name: 'testField', type: 'keyword' }], expect: 'string' },
-      { fields: [{ name: 'testField', type: 'invalidType' }], expect: undefined },
+      { fields: [{ name: 'testField', type: 'invalidType' }], expect: 'string' },
       { fields: [{ name: 'testField', type: 'text' }], expect: 'string' },
       { fields: [{ name: 'testField', type: 'date' }], expect: 'date' },
       { fields: [{ name: 'testField', type: 'geo_point' }], expect: 'geo_point' },
@@ -171,7 +171,7 @@ describe('creating index patterns from yaml fields', () => {
 
   test('transformField changes values based on other values', () => {
     interface TestWithAttr extends Test {
-      attr: keyof IndexPatternField;
+      attr: keyof FieldSpec;
     }
 
     const tests: TestWithAttr[] = [
@@ -211,43 +211,6 @@ describe('creating index patterns from yaml fields', () => {
         attr: 'aggregatable',
       },
 
-      // analyzed
-      { fields: [{ name }], expect: false, attr: 'analyzed' },
-      { fields: [{ name, analyzed: true }], expect: true, attr: 'analyzed' },
-      { fields: [{ name, analyzed: false }], expect: false, attr: 'analyzed' },
-      { fields: [{ name, type: 'binary' }], expect: false, attr: 'analyzed' },
-      { fields: [{ name, analyzed: true, type: 'binary' }], expect: false, attr: 'analyzed' },
-      {
-        fields: [{ name, analyzed: true, type: 'object', enabled: false }],
-        expect: false,
-        attr: 'analyzed',
-      },
-
-      // doc_values always set to true except for meta fields
-      { fields: [{ name }], expect: true, attr: 'doc_values' },
-      { fields: [{ name, doc_values: true }], expect: true, attr: 'doc_values' },
-      { fields: [{ name, doc_values: false }], expect: false, attr: 'doc_values' },
-      { fields: [{ name, script: 'doc[]' }], expect: false, attr: 'doc_values' },
-      { fields: [{ name, doc_values: true, script: 'doc[]' }], expect: false, attr: 'doc_values' },
-      { fields: [{ name, type: 'binary' }], expect: false, attr: 'doc_values' },
-      { fields: [{ name, doc_values: true, type: 'binary' }], expect: true, attr: 'doc_values' },
-      {
-        fields: [{ name, doc_values: true, type: 'object', enabled: false }],
-        expect: false,
-        attr: 'doc_values',
-      },
-
-      // enabled - only applies to objects (and only if set)
-      { fields: [{ name, type: 'binary', enabled: false }], expect: undefined, attr: 'enabled' },
-      { fields: [{ name, type: 'binary', enabled: true }], expect: undefined, attr: 'enabled' },
-      { fields: [{ name, type: 'object', enabled: true }], expect: true, attr: 'enabled' },
-      { fields: [{ name, type: 'object', enabled: false }], expect: false, attr: 'enabled' },
-      {
-        fields: [{ name, type: 'object', enabled: false }],
-        expect: false,
-        attr: 'doc_values',
-      },
-
       // indexed
       { fields: [{ name, type: 'binary' }], expect: false, attr: 'indexed' },
       {